def autoscaling_status(
kube_client: kubernetes_tools.KubeClient,
job_config: LongRunningServiceConfig,
namespace: str,
):
status = {}
hpa = kube_client.autoscaling.read_namespaced_horizontal_pod_autoscaler(
name=job_config.get_sanitised_deployment_name(), namespace=namespace
)
status["min_instances"] = hpa.spec.min_replicas
status["max_instances"] = hpa.spec.max_replicas
# Parse metrics sources, based on
# https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta1ExternalMetricSource.md#v2beta1externalmetricsource
metric_stats = []
parser = HPAMetricsParser(hpa)
if hpa.spec.metrics is not None:
for metric_spec in hpa.spec.metrics:
metric_stats.append(parser.parse_target(metric_spec))
if hpa.status.current_metrics is not None:
for metric_spec in hpa.status.current_metrics:
metric_stats.append(parser.parse_current(metric_spec))
status["metrics"] = metric_stats
status["desired_replicas"] = hpa.status.desired_replicas
status["last_scale_time"] = (
hpa.status.last_scale_time.replace(tzinfo=pytz.UTC).isoformat()
if getattr(hpa.status, "last_scale_time")
else "N/A"
)
return status
async def job_status(
kstatus: MutableMapping[str, Any],
client: kubernetes_tools.KubeClient,
job_config: LongRunningServiceConfig,
pod_list: Sequence[V1Pod],
replicaset_list: Sequence[V1ReplicaSet],
verbose: int,
namespace: str,
) -> None:
app_id = job_config.get_sanitised_deployment_name()
kstatus["app_id"] = app_id
kstatus["pods"] = []
kstatus["replicasets"] = []
if verbose > 0:
num_tail_lines = calculate_tail_lines(verbose)
kstatus["pods"] = await asyncio.gather(
*[pod_info(pod, client, num_tail_lines) for pod in pod_list])
for replicaset in replicaset_list:
try:
ready_replicas = replicaset.status.ready_replicas
if ready_replicas is None:
ready_replicas = 0
except AttributeError:
ready_replicas = 0
kstatus["replicasets"].append({
"name":
replicaset.metadata.name,
"replicas":
replicaset.spec.replicas,
"ready_replicas":
ready_replicas,
"create_timestamp":
replicaset.metadata.creation_timestamp.timestamp(),
})
kstatus["expected_instance_count"] = job_config.get_instances()
app = kubernetes_tools.get_kubernetes_app_by_name(name=app_id,
kube_client=client,
namespace=namespace)
desired_instances = (job_config.get_instances()
if job_config.get_desired_state() != "stop" else 0)
deploy_status, message = await kubernetes_tools.get_kubernetes_app_deploy_status(
app=app,
kube_client=client,
desired_instances=desired_instances,
)
kstatus[
"deploy_status"] = kubernetes_tools.KubernetesDeployStatus.tostring(
deploy_status)
kstatus["deploy_status_message"] = message
kstatus["running_instance_count"] = (app.status.ready_replicas
if app.status.ready_replicas else 0)
kstatus["create_timestamp"] = app.metadata.creation_timestamp.timestamp()
kstatus["namespace"] = app.metadata.namespace
async def autoscaling_status(
kube_client: kubernetes_tools.KubeClient,
job_config: LongRunningServiceConfig,
namespace: str,
) -> KubernetesAutoscalingStatusDict:
hpa = await kubernetes_tools.get_hpa(
kube_client,
name=job_config.get_sanitised_deployment_name(),
namespace=namespace,
)
if hpa is None:
return KubernetesAutoscalingStatusDict(
min_instances=-1,
max_instances=-1,
metrics=[],
desired_replicas=-1,
last_scale_time="unknown (could not find HPA object)",
)
# Parse metrics sources, based on
# https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta2ExternalMetricSource.md#v2beta2externalmetricsource
parser = HPAMetricsParser(hpa)
# https://github.com/python/mypy/issues/7217
metrics_by_name: DefaultDict[str, HPAMetricsDict] = defaultdict(
lambda: HPAMetricsDict()
)
if hpa.spec.metrics is not None:
for metric_spec in hpa.spec.metrics:
parsed = parser.parse_target(metric_spec)
metrics_by_name[parsed["name"]].update(parsed)
if hpa.status.current_metrics is not None:
for metric_spec in hpa.status.current_metrics:
parsed = parser.parse_current(metric_spec)
if parsed is not None:
metrics_by_name[parsed["name"]].update(parsed)
metric_stats = list(metrics_by_name.values())
last_scale_time = (
hpa.status.last_scale_time.replace(tzinfo=pytz.UTC).isoformat()
if getattr(hpa.status, "last_scale_time")
else "N/A"
)
return KubernetesAutoscalingStatusDict(
min_instances=hpa.spec.min_replicas,
max_instances=hpa.spec.max_replicas,
metrics=metric_stats,
desired_replicas=hpa.status.desired_replicas,
last_scale_time=last_scale_time,
)
def autoscaling_status(
kube_client: kubernetes_tools.KubeClient,
job_config: LongRunningServiceConfig,
namespace: str,
) -> KubernetesAutoscalingStatusDict:
try:
hpa = kube_client.autoscaling.read_namespaced_horizontal_pod_autoscaler(
name=job_config.get_sanitised_deployment_name(),
namespace=namespace)
except ApiException as e:
if e.status == 404:
return KubernetesAutoscalingStatusDict(
min_instances=-1,
max_instances=-1,
metrics=[],
desired_replicas=-1,
last_scale_time="unknown (could not find HPA object)",
)
else:
raise
# Parse metrics sources, based on
# https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V2beta1ExternalMetricSource.md#v2beta1externalmetricsource
metric_stats = []
parser = HPAMetricsParser(hpa)
if hpa.spec.metrics is not None:
for metric_spec in hpa.spec.metrics:
metric_stats.append(parser.parse_target(metric_spec))
if hpa.status.current_metrics is not None:
for metric_spec in hpa.status.current_metrics:
metric_stats.append(parser.parse_current(metric_spec))
last_scale_time = (hpa.status.last_scale_time.replace(
tzinfo=pytz.UTC).isoformat()
if getattr(hpa.status, "last_scale_time") else "N/A")
return KubernetesAutoscalingStatusDict(
min_instances=hpa.spec.min_replicas,
max_instances=hpa.spec.max_replicas,
metrics=metric_stats,
desired_replicas=hpa.status.desired_replicas,
last_scale_time=last_scale_time,
)
async def job_status(
kstatus: MutableMapping[str, Any],
client: kubernetes_tools.KubeClient,
job_config: LongRunningServiceConfig,
pod_list: Sequence[V1Pod],
replicaset_list: Sequence[V1ReplicaSet],
verbose: int,
namespace: str,
) -> None:
app_id = job_config.get_sanitised_deployment_name()
kstatus["app_id"] = app_id
kstatus["pods"] = []
kstatus["replicasets"] = []
if verbose > 0:
num_tail_lines = calculate_tail_lines(verbose)
for pod in pod_list:
container_statuses = pod.status.container_statuses or []
containers = [
dict(
name=container.name,
tail_lines=await get_tail_lines_for_kubernetes_container(
client, pod, container, num_tail_lines,
),
)
for container in container_statuses
]
kstatus["pods"].append(
{
"name": pod.metadata.name,
"host": kubernetes_tools.get_pod_hostname(client, pod),
"deployed_timestamp": pod.metadata.creation_timestamp.timestamp(),
"phase": pod.status.phase,
"ready": kubernetes_tools.is_pod_ready(pod),
"containers": containers,
"reason": pod.status.reason,
"message": pod.status.message,
}
)
for replicaset in replicaset_list:
try:
ready_replicas = replicaset.status.ready_replicas
if ready_replicas is None:
ready_replicas = 0
except AttributeError:
ready_replicas = 0
kstatus["replicasets"].append(
{
"name": replicaset.metadata.name,
"replicas": replicaset.spec.replicas,
"ready_replicas": ready_replicas,
"create_timestamp": replicaset.metadata.creation_timestamp.timestamp(),
}
)
kstatus["expected_instance_count"] = job_config.get_instances()
app = kubernetes_tools.get_kubernetes_app_by_name(
name=app_id, kube_client=client, namespace=namespace
)
deploy_status = kubernetes_tools.get_kubernetes_app_deploy_status(
app=app, desired_instances=job_config.get_instances()
)
kstatus["deploy_status"] = kubernetes_tools.KubernetesDeployStatus.tostring(
deploy_status
)
kstatus["running_instance_count"] = (
app.status.ready_replicas if app.status.ready_replicas else 0
)
kstatus["create_timestamp"] = app.metadata.creation_timestamp.timestamp()
kstatus["namespace"] = app.metadata.namespace