def update_job_containers(event: Mapping, status: str,
job_container_name: str) -> None:
if JobLifeCycle.is_done(status):
# Remove the job monitoring
job_uuid = event['metadata']['labels']['job_uuid']
logger.info('Stop monitoring job_uuid: %s', job_uuid)
RedisJobContainers.remove_job(job_uuid)
if event['status']['container_statuses'] is None:
return
def get_container_id(container_id):
if not container_id:
return None
if container_id.startswith('docker://'):
return container_id[len('docker://'):]
return container_id
for container_status in event['status']['container_statuses']:
if container_status['name'] != job_container_name:
continue
container_id = get_container_id(container_status['container_id'])
if container_id:
job_uuid = event['metadata']['labels']['job_uuid']
if container_status['state']['running'] is not None:
logger.info('Monitoring (container_id, job_uuid): (%s, %s)',
container_id, job_uuid)
RedisJobContainers.monitor(container_id=container_id,
job_uuid=job_uuid)
else:
RedisJobContainers.remove_container(container_id=container_id)
def experiment_job_status_post_save(sender, **kwargs):
instance = kwargs['instance']
job = instance.job
job.status = instance
set_job_started_at(instance=job, status=instance.status)
set_job_finished_at(instance=job, status=instance.status)
job.save(update_fields=['status', 'started_at', 'finished_at'])
# check if the new status is done to remove the containers from the monitors
if job.is_done:
from db.redis.containers import RedisJobContainers
RedisJobContainers.remove_job(job.uuid.hex)
# Check if we need to change the experiment status
auditor.record(event_type=EXPERIMENT_JOB_NEW_STATUS, instance=job)
def experiment_job_status_post_save(sender, **kwargs):
instance = kwargs['instance']
job = instance.job
job.status = instance
set_job_started_at(instance=job, status=instance.status)
set_job_finished_at(instance=job, status=instance.status)
job.save(update_fields=['status', 'started_at', 'finished_at'])
# check if the new status is done to remove the containers from the monitors
if job.is_done:
from db.redis.containers import RedisJobContainers
RedisJobContainers.remove_job(job.uuid.hex)
# Check if we need to change the experiment status
experiment = instance.job.experiment
if experiment.is_done:
return
celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_CHECK_STATUS,
kwargs={'experiment_id': experiment.id},
countdown=1)