def autoscaling_status( kube_client: kubernetes_tools.KubeClient, job_config: LongRunningServiceConfig, namespace: str, ): status = {} hpa = kube_client.autoscaling.read_namespaced_horizontal_pod_autoscaler( name=job_config.get_sanitised_deployment_name(), namespace=namespace ) status["min_instances"] = hpa.spec.min_replicas status["max_instances"] = hpa.spec.max_replicas # Parse metrics sources, based on # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta1ExternalMetricSource.md#v2beta1externalmetricsource metric_stats = [] parser = HPAMetricsParser(hpa) if hpa.spec.metrics is not None: for metric_spec in hpa.spec.metrics: metric_stats.append(parser.parse_target(metric_spec)) if hpa.status.current_metrics is not None: for metric_spec in hpa.status.current_metrics: metric_stats.append(parser.parse_current(metric_spec)) status["metrics"] = metric_stats status["desired_replicas"] = hpa.status.desired_replicas status["last_scale_time"] = ( hpa.status.last_scale_time.replace(tzinfo=pytz.UTC).isoformat() if getattr(hpa.status, "last_scale_time") else "N/A" ) return status
async def job_status( kstatus: MutableMapping[str, Any], client: kubernetes_tools.KubeClient, job_config: LongRunningServiceConfig, pod_list: Sequence[V1Pod], replicaset_list: Sequence[V1ReplicaSet], verbose: int, namespace: str, ) -> None: app_id = job_config.get_sanitised_deployment_name() kstatus["app_id"] = app_id kstatus["pods"] = [] kstatus["replicasets"] = [] if verbose > 0: num_tail_lines = calculate_tail_lines(verbose) kstatus["pods"] = await asyncio.gather( *[pod_info(pod, client, num_tail_lines) for pod in pod_list]) for replicaset in replicaset_list: try: ready_replicas = replicaset.status.ready_replicas if ready_replicas is None: ready_replicas = 0 except AttributeError: ready_replicas = 0 kstatus["replicasets"].append({ "name": replicaset.metadata.name, "replicas": replicaset.spec.replicas, "ready_replicas": ready_replicas, "create_timestamp": replicaset.metadata.creation_timestamp.timestamp(), }) kstatus["expected_instance_count"] = job_config.get_instances() app = kubernetes_tools.get_kubernetes_app_by_name(name=app_id, kube_client=client, namespace=namespace) desired_instances = (job_config.get_instances() if job_config.get_desired_state() != "stop" else 0) deploy_status, message = await kubernetes_tools.get_kubernetes_app_deploy_status( app=app, kube_client=client, desired_instances=desired_instances, ) kstatus[ "deploy_status"] = kubernetes_tools.KubernetesDeployStatus.tostring( deploy_status) kstatus["deploy_status_message"] = message kstatus["running_instance_count"] = (app.status.ready_replicas if app.status.ready_replicas else 0) kstatus["create_timestamp"] = app.metadata.creation_timestamp.timestamp() kstatus["namespace"] = app.metadata.namespace
async def autoscaling_status( kube_client: kubernetes_tools.KubeClient, job_config: LongRunningServiceConfig, namespace: str, ) -> KubernetesAutoscalingStatusDict: hpa = await kubernetes_tools.get_hpa( kube_client, name=job_config.get_sanitised_deployment_name(), namespace=namespace, ) if hpa is None: return KubernetesAutoscalingStatusDict( min_instances=-1, max_instances=-1, metrics=[], desired_replicas=-1, last_scale_time="unknown (could not find HPA object)", ) # Parse metrics sources, based on # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta2ExternalMetricSource.md#v2beta2externalmetricsource parser = HPAMetricsParser(hpa) # https://github.com/python/mypy/issues/7217 metrics_by_name: DefaultDict[str, HPAMetricsDict] = defaultdict( lambda: HPAMetricsDict() ) if hpa.spec.metrics is not None: for metric_spec in hpa.spec.metrics: parsed = parser.parse_target(metric_spec) metrics_by_name[parsed["name"]].update(parsed) if hpa.status.current_metrics is not None: for metric_spec in hpa.status.current_metrics: parsed = parser.parse_current(metric_spec) if parsed is not None: metrics_by_name[parsed["name"]].update(parsed) metric_stats = list(metrics_by_name.values()) last_scale_time = ( hpa.status.last_scale_time.replace(tzinfo=pytz.UTC).isoformat() if getattr(hpa.status, "last_scale_time") else "N/A" ) return KubernetesAutoscalingStatusDict( min_instances=hpa.spec.min_replicas, max_instances=hpa.spec.max_replicas, metrics=metric_stats, desired_replicas=hpa.status.desired_replicas, last_scale_time=last_scale_time, )
def autoscaling_status( kube_client: kubernetes_tools.KubeClient, job_config: LongRunningServiceConfig, namespace: str, ) -> KubernetesAutoscalingStatusDict: try: hpa = kube_client.autoscaling.read_namespaced_horizontal_pod_autoscaler( name=job_config.get_sanitised_deployment_name(), namespace=namespace) except ApiException as e: if e.status == 404: return KubernetesAutoscalingStatusDict( min_instances=-1, max_instances=-1, metrics=[], desired_replicas=-1, last_scale_time="unknown (could not find HPA object)", ) else: raise # Parse metrics sources, based on # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta1ExternalMetricSource.md#v2beta1externalmetricsource metric_stats = [] parser = HPAMetricsParser(hpa) if hpa.spec.metrics is not None: for metric_spec in hpa.spec.metrics: metric_stats.append(parser.parse_target(metric_spec)) if hpa.status.current_metrics is not None: for metric_spec in hpa.status.current_metrics: metric_stats.append(parser.parse_current(metric_spec)) last_scale_time = (hpa.status.last_scale_time.replace( tzinfo=pytz.UTC).isoformat() if getattr(hpa.status, "last_scale_time") else "N/A") return KubernetesAutoscalingStatusDict( min_instances=hpa.spec.min_replicas, max_instances=hpa.spec.max_replicas, metrics=metric_stats, desired_replicas=hpa.status.desired_replicas, last_scale_time=last_scale_time, )
async def job_status( kstatus: MutableMapping[str, Any], client: kubernetes_tools.KubeClient, job_config: LongRunningServiceConfig, pod_list: Sequence[V1Pod], replicaset_list: Sequence[V1ReplicaSet], verbose: int, namespace: str, ) -> None: app_id = job_config.get_sanitised_deployment_name() kstatus["app_id"] = app_id kstatus["pods"] = [] kstatus["replicasets"] = [] if verbose > 0: num_tail_lines = calculate_tail_lines(verbose) for pod in pod_list: container_statuses = pod.status.container_statuses or [] containers = [ dict( name=container.name, tail_lines=await get_tail_lines_for_kubernetes_container( client, pod, container, num_tail_lines, ), ) for container in container_statuses ] kstatus["pods"].append( { "name": pod.metadata.name, "host": kubernetes_tools.get_pod_hostname(client, pod), "deployed_timestamp": pod.metadata.creation_timestamp.timestamp(), "phase": pod.status.phase, "ready": kubernetes_tools.is_pod_ready(pod), "containers": containers, "reason": pod.status.reason, "message": pod.status.message, } ) for replicaset in replicaset_list: try: ready_replicas = replicaset.status.ready_replicas if ready_replicas is None: ready_replicas = 0 except AttributeError: ready_replicas = 0 kstatus["replicasets"].append( { "name": replicaset.metadata.name, "replicas": replicaset.spec.replicas, "ready_replicas": ready_replicas, "create_timestamp": replicaset.metadata.creation_timestamp.timestamp(), } ) kstatus["expected_instance_count"] = job_config.get_instances() app = kubernetes_tools.get_kubernetes_app_by_name( name=app_id, kube_client=client, namespace=namespace ) deploy_status = kubernetes_tools.get_kubernetes_app_deploy_status( app=app, desired_instances=job_config.get_instances() ) kstatus["deploy_status"] = kubernetes_tools.KubernetesDeployStatus.tostring( deploy_status ) kstatus["running_instance_count"] = ( app.status.ready_replicas if app.status.ready_replicas else 0 ) kstatus["create_timestamp"] = app.metadata.creation_timestamp.timestamp() kstatus["namespace"] = app.metadata.namespace