def run(containers, node, persist):
container_ids = RedisJobContainers.get_containers()
gpu_resources = get_gpu_resources()
if gpu_resources:
gpu_resources = {
gpu_resource['index']: gpu_resource
for gpu_resource in gpu_resources
}
# update cluster and current node
update_cluster(gpu_resources)
for container_id in container_ids:
container = get_container(containers, container_id)
if not container:
continue
payload = get_container_resources(node, containers[container_id],
gpu_resources)
if payload:
payload = payload.to_dict()
logger.info("Publishing resources event")
handle_events_resources.delay(payload=payload, persist=persist)
job_uuid = payload['job_uuid']
# Check if we should stream the payload
# Check if we have this container already in place
experiment_uuid = RedisJobContainers.get_experiment_for_job(
job_uuid)
if (RedisToStream.is_monitored_job_resources(job_uuid)
or RedisToStream.is_monitored_experiment_resources(
experiment_uuid)):
RedisToStream.set_latest_job_resources(job_uuid, payload)
def run(containers, node, persist):
container_ids = RedisJobContainers.get_containers()
gpu_resources = get_gpu_resources()
if gpu_resources:
gpu_resources = {
gpu_resource['index']: gpu_resource
for gpu_resource in gpu_resources
}
update_cluster_node(gpu_resources)
for container_id in container_ids:
container = get_container(containers, container_id)
if not container:
continue
payload = get_container_resources(node, containers[container_id],
gpu_resources)
if payload:
payload = payload.to_dict()
logger.debug("Publishing resources event")
celery_app.send_task(EventsCeleryTasks.EVENTS_HANDLE_RESOURCES,
kwargs={
'payload': payload,
'persist': persist
})
job_uuid = payload['job_uuid']
# Check if we should stream the payload
# Check if we have this container already in place
experiment_uuid = RedisJobContainers.get_experiment_for_job(
job_uuid)
set_last_resources_cond = (
RedisToStream.is_monitored_job_resources(job_uuid)
or RedisToStream.is_monitored_experiment_resources(
experiment_uuid))
if set_last_resources_cond:
RedisToStream.set_latest_job_resources(job_uuid, payload)
def run(containers, node, persist):
container_ids = RedisJobContainers.get_containers()
gpu_resources = get_gpu_resources()
if gpu_resources:
gpu_resources = {gpu_resource['index']: gpu_resource for gpu_resource in gpu_resources}
# update cluster and current node
update_cluster(gpu_resources)
for container_id in container_ids:
container = get_container(containers, container_id)
if not container:
continue
payload = get_container_resources(node, containers[container_id], gpu_resources)
if payload:
payload = payload.to_dict()
logger.info("Publishing resources event")
celery_app.send_task(
EventsCeleryTasks.EVENTS_HANDLE_RESOURCES,
kwargs={'payload': payload, 'persist': persist})
job_uuid = payload['job_uuid']
# Check if we should stream the payload
# Check if we have this container already in place
experiment_uuid = RedisJobContainers.get_experiment_for_job(job_uuid)
if (RedisToStream.is_monitored_job_resources(job_uuid) or
RedisToStream.is_monitored_experiment_resources(experiment_uuid)):
RedisToStream.set_latest_job_resources(job_uuid, payload)
def test_update_job_containers(self):
update_job_containers(event=status_raw_event_with_conditions['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.JOB_CONTAINER_NAME)
# Assert it's still 0 because no job was created with that job_uuid
assert len(RedisJobContainers.get_containers()) == 0
# Create a job with a specific uuid
labels = status_raw_event_with_conditions['object']['metadata']['labels']
ExperimentJobFactory(uuid=labels['job_uuid'])
job = ExperimentJob.objects.get(uuid=labels['job_uuid'])
update_job_containers(event=status_raw_event_with_conditions['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.JOB_CONTAINER_NAME)
# Assert now it has started monitoring the container
assert len(RedisJobContainers.get_containers()) == 1
container_id = '539e6a6f4209997094802b0657f90576fe129b7f81697120172836073d9bbd75'
assert RedisJobContainers.get_containers() == [container_id]
job_uuid, experiment_uuid = RedisJobContainers.get_job(container_id)
assert job.uuid.hex == job_uuid
assert job.experiment.uuid.hex == experiment_uuid
def test_update_job_containers(self):
update_job_containers(event=status_experiment_job_event_with_conditions['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.CONTAINER_NAME_EXPERIMENT_JOB)
# Assert it's still 0 because no job was created with that job_uuid
assert len(RedisJobContainers.get_containers()) == 0 # pylint:disable=len-as-condition
# Create a job with a specific uuid
labels = status_experiment_job_event_with_conditions['object']['metadata']['labels']
ExperimentJobFactory(uuid=labels['job_uuid'])
job = ExperimentJob.objects.get(uuid=labels['job_uuid'])
update_job_containers(event=status_experiment_job_event_with_conditions['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.CONTAINER_NAME_EXPERIMENT_JOB)
# Assert now it has started monitoring the container
assert len(RedisJobContainers.get_containers()) == 1
container_id = '539e6a6f4209997094802b0657f90576fe129b7f81697120172836073d9bbd75'
assert RedisJobContainers.get_containers() == [container_id]
job_uuid, experiment_uuid = RedisJobContainers.get_job(container_id)
assert job.uuid.hex == job_uuid
assert job.experiment.uuid.hex == experiment_uuid
def test_update_job_containers_with_no_container_statuses(self):
update_job_containers(event=status_experiment_job_event['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.CONTAINER_NAME_JOB)
assert len(RedisJobContainers.get_containers()) == 0
def test_update_job_containers_with_no_container_statuses(self):
update_job_containers(event=status_experiment_job_event['object'],
status=JobLifeCycle.BUILDING,
job_container_name=settings.CONTAINER_NAME_EXPERIMENT_JOB)
assert len(RedisJobContainers.get_containers()) == 0 # pylint:disable=len-as-condition