def update_job_containers(event: Mapping, status: str,
job_container_name: str) -> None:
if JobLifeCycle.is_done(status):
# Remove the job monitoring
job_uuid = event['metadata']['labels']['job_uuid']
logger.info('Stop monitoring job_uuid: %s', job_uuid)
RedisJobContainers.remove_job(job_uuid)
if event['status']['container_statuses'] is None:
return
def get_container_id(container_id):
if not container_id:
return None
if container_id.startswith('docker://'):
return container_id[len('docker://'):]
return container_id
for container_status in event['status']['container_statuses']:
if container_status['name'] != job_container_name:
continue
container_id = get_container_id(container_status['container_id'])
if container_id:
job_uuid = event['metadata']['labels']['job_uuid']
if container_status['state']['running'] is not None:
logger.info('Monitoring (container_id, job_uuid): (%s, %s)',
container_id, job_uuid)
RedisJobContainers.monitor(container_id=container_id,
job_uuid=job_uuid)
else:
RedisJobContainers.remove_container(container_id=container_id)
def get_container_resources(node, container, gpu_resources):
# Check if the container is running
if container.status != ContainerStatuses.RUNNING:
logger.debug("`%s` container is not running", container.name)
RedisJobContainers.remove_container(container.id)
return
job_uuid, experiment_uuid = RedisJobContainers.get_job(container.id)
if not job_uuid:
logger.debug("`%s` container is not recognised", container.name)
return
logger.debug(
"Streaming resources for container %s in (job, experiment) (`%s`, `%s`) ",
container.id, job_uuid, experiment_uuid)
try:
stats = container.stats(decode=True, stream=False)
except json.decoder.JSONDecodeError:
logger.info("Error streaming states for `%s`", container.name)
except NotFound:
logger.debug("`%s` was not found", container.name)
RedisJobContainers.remove_container(container.id)
return
except requests.ReadTimeout:
return
precpu_stats = stats['precpu_stats']
cpu_stats = stats['cpu_stats']
pre_total_usage = float(precpu_stats['cpu_usage']['total_usage'])
total_usage = float(cpu_stats['cpu_usage']['total_usage'])
delta_total_usage = total_usage - pre_total_usage
pre_system_cpu_usage = float(precpu_stats['system_cpu_usage'])
system_cpu_usage = float(cpu_stats['system_cpu_usage'])
delta_system_cpu_usage = system_cpu_usage - pre_system_cpu_usage
percpu_usage = cpu_stats['cpu_usage']['percpu_usage']
num_cpu_cores = len(percpu_usage)
if num_cpu_cores >= node.cpu * 1.5:
logger.warning('Docker reporting num cpus `%s` and kubernetes reporting `%s`',
num_cpu_cores, node.cpu)
num_cpu_cores = node.cpu
cpu_percentage = 0.
percpu_percentage = [0.] * num_cpu_cores
if delta_total_usage > 0 and delta_system_cpu_usage > 0:
cpu_percentage = (delta_total_usage / delta_system_cpu_usage) * num_cpu_cores * 100.0
percpu_percentage = [cpu_usage / total_usage * cpu_percentage for cpu_usage in percpu_usage]
memory_used = int(stats['memory_stats']['usage'])
memory_limit = int(stats['memory_stats']['limit'])
container_gpu_resources = None
if gpu_resources:
gpu_indices = get_container_gpu_indices(container)
container_gpu_resources = [gpu_resources[gpu_indice] for gpu_indice in gpu_indices]
return ContainerResourcesConfig.from_dict({
'job_uuid': job_uuid,
'job_name': job_uuid, # it will be updated during the streaming
'experiment_uuid': experiment_uuid,
'container_id': container.id,
'cpu_percentage': cpu_percentage,
'n_cpus': num_cpu_cores,
'percpu_percentage': percpu_percentage,
'memory_used': memory_used,
'memory_limit': memory_limit,
'gpu_resources': container_gpu_resources
})