def projects_notebook_build(notebook_job_id):
notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook_job:
_logger.warning('Notebook %s does not exist anymore.', notebook_job_id)
return None
if not JobLifeCycle.can_transition(status_from=notebook_job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Notebook `%s` cannot transition from `%s` to `%s`.',
notebook_job, notebook_job.last_status, JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=notebook_job.user,
project=notebook_job.project,
config=notebook_job.specification.build,
code_reference=notebook_job.code_reference)
notebook_job.build_job = build_job
notebook_job.save()
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'notebook_job_id': notebook_job_id})
return
if not build_status:
notebook_job.set_status(JobLifeCycle.FAILED, message='Could not start build process.')
return
# Update job status to show that its building docker image
notebook_job.set_status(JobLifeCycle.BUILDING, message='Building container')
def projects_notebook_start(notebook_job_id):
notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook_job:
_logger.warning('Notebook %s does not exist anymore.', notebook_job_id)
return None
if not JobLifeCycle.can_transition(status_from=notebook_job.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Notebook `%s` cannot transition from `%s` to `%s`.',
notebook_job.unique_name, notebook_job.last_status, JobLifeCycle.SCHEDULED)
notebook_scheduler.start_notebook(notebook_job)
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
_logger.warning('Tensorboard %s does not exist anymore.', tensorboard_job_id)
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status, JobLifeCycle.SCHEDULED)
tensorboard_scheduler.start_tensorboard(tensorboard)
def projects_notebook_start(notebook_job_id):
notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook_job:
return None
if not JobLifeCycle.can_transition(status_from=notebook_job.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Notebook `%s` cannot transition from `%s` to `%s`.',
notebook_job.unique_name, notebook_job.last_status,
JobLifeCycle.SCHEDULED)
notebook_scheduler.start_notebook(notebook_job)
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status,
JobLifeCycle.SCHEDULED)
tensorboard_scheduler.start_tensorboard(tensorboard)
def jobs_start(job_id):
job = get_valid_job(job_id=job_id)
if not job:
return None
if job.last_status == JobLifeCycle.RUNNING:
_logger.warning('Job is already running.')
return None
if not JobLifeCycle.can_transition(status_from=job.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Job `%s` cannot transition from `%s` to `%s`.',
job.unique_name, job.last_status, JobLifeCycle.SCHEDULED)
return None
job_scheduler.start_job(job)
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status, JobLifeCycle.SCHEDULED)
try:
tensorboard_scheduler.start_tensorboard(tensorboard)
except VolumeNotFoundError:
tensorboard.set_status(status=JobLifeCycle.FAILED,
message='Tensorboard failed to start, '
'the outputs volume/storage was not found.')
def _set_status(self, status_model, logger, status, message=None, details=None):
current_status = self.last_status
if self.is_done:
# We should not update statuses anymore
logger.info(
'Received a new status `{}` for job `{}`. '
'But the job is already done with status `{}`'.format(
status, self.unique_name, current_status))
return False
if JobLifeCycle.can_transition(status_from=current_status, status_to=status):
# Add new status to the job
status_model.objects.create(job=self,
status=status,
message=message,
details=details)
return True
return False
def _set_status(self, status_model, logger, status, message=None, details=None):
current_status = self.last_status
if self.is_done:
# We should not update statuses anymore
logger.info(
'Received a new status `{}` for job `{}`. '
'But the job is already done with status `{}`'.format(
status, self.unique_name, current_status))
return False
if JobLifeCycle.can_transition(status_from=current_status, status_to=status):
# Add new status to the job
status_model.objects.create(job=self,
status=status,
message=message,
details=details)
return True
return False
def projects_notebook_build(notebook_job_id):
notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook_job:
return None
if not JobLifeCycle.can_transition(status_from=notebook_job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Notebook `%s` cannot transition from `%s` to `%s`.',
notebook_job, notebook_job.last_status,
JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=notebook_job.user,
project=notebook_job.project,
config=notebook_job.specification.build,
configmap_refs=notebook_job.specification.configmap_refs,
secret_refs=notebook_job.specification.secret_refs,
code_reference=notebook_job.code_reference)
notebook_job.build_job = build_job
notebook_job.save(update_fields=['build_job'])
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'notebook_job_id': notebook_job_id},
countdown=conf.get('GLOBAL_COUNTDOWN'))
return
if not build_status:
notebook_job.set_status(JobLifeCycle.FAILED,
message='Could not start build process.')
return
# Update job status to show that its building docker image
notebook_job.set_status(JobLifeCycle.BUILDING,
message='Building container')
