def run(containers, node, persist): container_ids = RedisJobContainers.get_containers() gpu_resources = get_gpu_resources() if gpu_resources: gpu_resources = { gpu_resource['index']: gpu_resource for gpu_resource in gpu_resources } update_cluster_node(gpu_resources) for container_id in container_ids: container = get_container(containers, container_id) if not container: continue payload = get_container_resources(node, containers[container_id], gpu_resources) if payload: payload = payload.to_dict() logger.debug("Publishing resources event") celery_app.send_task(EventsCeleryTasks.EVENTS_HANDLE_RESOURCES, kwargs={ 'payload': payload, 'persist': persist }) job_uuid = payload['job_uuid'] # Check if we should stream the payload # Check if we have this container already in place experiment_uuid = RedisJobContainers.get_experiment_for_job( job_uuid) set_last_resources_cond = ( RedisToStream.is_monitored_job_resources(job_uuid) or RedisToStream.is_monitored_experiment_resources( experiment_uuid)) if set_last_resources_cond: RedisToStream.set_latest_job_resources(job_uuid, payload)
def run(containers, node, persist): container_ids = RedisJobContainers.get_containers() gpu_resources = get_gpu_resources() if gpu_resources: gpu_resources = { gpu_resource['index']: gpu_resource for gpu_resource in gpu_resources } # update cluster and current node update_cluster(gpu_resources) for container_id in container_ids: container = get_container(containers, container_id) if not container: continue payload = get_container_resources(node, containers[container_id], gpu_resources) if payload: payload = payload.to_dict() logger.info("Publishing resources event") handle_events_resources.delay(payload=payload, persist=persist) job_uuid = payload['job_uuid'] # Check if we should stream the payload # Check if we have this container already in place experiment_uuid = RedisJobContainers.get_experiment_for_job( job_uuid) if (RedisToStream.is_monitored_job_resources(job_uuid) or RedisToStream.is_monitored_experiment_resources( experiment_uuid)): RedisToStream.set_latest_job_resources(job_uuid, payload)
def run(containers, node, persist): container_ids = RedisJobContainers.get_containers() gpu_resources = get_gpu_resources() if gpu_resources: gpu_resources = {gpu_resource['index']: gpu_resource for gpu_resource in gpu_resources} # update cluster and current node update_cluster(gpu_resources) for container_id in container_ids: container = get_container(containers, container_id) if not container: continue payload = get_container_resources(node, containers[container_id], gpu_resources) if payload: payload = payload.to_dict() logger.info("Publishing resources event") celery_app.send_task( EventsCeleryTasks.EVENTS_HANDLE_RESOURCES, kwargs={'payload': payload, 'persist': persist}) job_uuid = payload['job_uuid'] # Check if we should stream the payload # Check if we have this container already in place experiment_uuid = RedisJobContainers.get_experiment_for_job(job_uuid) if (RedisToStream.is_monitored_job_resources(job_uuid) or RedisToStream.is_monitored_experiment_resources(experiment_uuid)): RedisToStream.set_latest_job_resources(job_uuid, payload)