def projects_notebook_build(project_id):
project = get_valid_project(project_id=project_id)
if not project or not project.notebook:
_logger.warning('Project does not have a notebook.')
return None
job = project.notebook
if not JobLifeCycle.can_transition(status_from=job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Notebook for project id `%s` cannot transition from `%s` to `%s`.',
project_id, job.last_status, JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=job.user,
project=job.project,
config=job.specification.run_exec,
code_reference=job.code_reference)
job.build_job = build_job
job.save()
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'project_id': project_id})
return
if not build_status:
job.set_status(JobLifeCycle.FAILED, message='Could not start build process.')
return
# Update job status to show that its building docker image
job.set_status(JobLifeCycle.BUILDING, message='Building container')
def projects_tensorboard_start(project_id):
project = get_valid_project(project_id=project_id)
if not project or not project.tensorboard:
_logger.warning('Project does not have a tensorboard.')
return None
if project.tensorboard.last_status == JobLifeCycle.RUNNING:
_logger.warning('Tensorboard is already running.')
return None
tensorboard_scheduler.start_tensorboard(project.tensorboard)
def projects_notebook_start(project_id):
project = get_valid_project(project_id=project_id)
if not project or not project.notebook:
_logger.warning('Project does not have a notebook.')
return None
if project.notebook.last_status == JobLifeCycle.RUNNING:
_logger.warning('Tensorboard is already running.')
return None
try:
image_name, image_tag = get_notebook_image_info(project=project, job=project.notebook)
except ValueError as e:
_logger.warning('Could not start the notebook, %s', e)
return
job_docker_image = '{}:{}'.format(image_name, image_tag)
_logger.info('Start notebook with built image `%s`', job_docker_image)
notebook_scheduler.start_notebook(project.notebook, image=job_docker_image)
def build_project_notebook(project_id):
project = get_valid_project(project_id)
if not project or not project.notebook:
return None
notebook_job = project.notebook
# Update job status to show that its building docker image
notebook_job.set_status(JobLifeCycle.BUILDING, message='Building container')
# Building the docker image
try:
status = notebooks_builder.build_notebook_job(project=project, job=project.notebook)
except DockerException as e:
_logger.warning('Failed to build notebook %s', e)
notebook_job.set_status(
JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
except Repo.DoesNotExist:
_logger.warning('No code was found for this project')
notebook_job.set_status(
JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
except Exception as e: # Other exceptions
_logger.warning('Failed to build notebook %s', e)
notebook_job.set_status(JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
if not status:
return
# Now we can start the notebook
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'notebook_job_id': notebook_job.id})
def build_project_notebook(project_id):
project = get_valid_project(project_id)
if not project or not project.notebook:
return None
job = project.notebook
# Update job status to show that its building docker image
job.set_status(JobLifeCycle.BUILDING, message='Building container')
# Building the docker image
try:
status = notebooks_builder.build_notebook_job(project=project, job=project.notebook)
except DockerException as e:
_logger.warning('Failed to build notebook %s', e)
job.set_status(
JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
except Repo.DoesNotExist:
_logger.warning('No code was found for this project')
job.set_status(
JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
except Exception as e: # Other exceptions
_logger.warning('Failed to build notebook %s', e)
job.set_status(JobLifeCycle.FAILED,
message='Failed to build image for notebook.')
return
if not status:
return
# Now we can start the notebook
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'project_id': project_id})
def projects_schedule_deletion(project_id, immediate=False):
project = get_valid_project(project_id=project_id, include_deleted=True)
if not project:
# No need to check this project
return
project.archive()
message = 'Project is scheduled for deletion.'
groups = project.all_experiment_groups.exclude(
status__status__in=ExperimentGroupLifeCycle.DONE_STATUS).distinct()
for group in groups.values_list('id', flat=True):
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS,
kwargs={
'experiment_group_id': group,
'pending': False,
'collect_logs': False,
'message': message,
})
experiments = project.all_experiments.exclude(
experiment_group__isnull=True,
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
for experiment in experiments:
if experiment.is_running:
celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={
'project_name':
experiment.project.unique_name,
'project_uuid':
experiment.project.uuid.hex,
'experiment_name': experiment.unique_name,
'experiment_uuid': experiment.uuid.hex,
'experiment_group_name': None,
'experiment_group_uuid': None,
'specification': experiment.config,
'update_status': True,
'collect_logs': False,
'message': message,
})
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
jobs = project.all_jobs.exclude(
status__status__in=JobLifeCycle.DONE_STATUS).distinct()
for job in jobs.values_list('id', flat=True):
celery_app.send_task(SchedulerCeleryTasks.JOBS_SCHEDULE_DELETION,
kwargs={'job_id': job})
builds = project.all_build_jobs.exclude(
status__status__in=JobLifeCycle.DONE_STATUS).distinct()
for build in builds.values_list('id', flat=True):
celery_app.send_task(SchedulerCeleryTasks.BUILD_JOBS_SCHEDULE_DELETION,
kwargs={'build_job_id': build})
notebooks = project.all_notebook_jobs.exclude(
status__status__in=JobLifeCycle.DONE_STATUS).distinct()
for notebook in notebooks.values_list('id', flat=True):
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_SCHEDULE_DELETION,
kwargs={'notebook_job_id': notebook})
tensorboards = project.all_tensorboard_jobs.exclude(
status__status__in=JobLifeCycle.DONE_STATUS).distinct()
for tensorboard in tensorboards.values_list('id', flat=True):
celery_app.send_task(
SchedulerCeleryTasks.TENSORBOARDS_SCHEDULE_DELETION,
kwargs={'tensorboard_job_id': tensorboard})
if immediate:
celery_app.send_task(SchedulerCeleryTasks.DELETE_ARCHIVED_PROJECT,
kwargs={
'project_id': project_id,
})
def projects_tensorboard_stop(project_id):
project = get_valid_project(project_id=project_id)
if not project:
return None
tensorboard_scheduler.stop_tensorboard(project.tensorboard,
update_status=True)
def projects_notebook_stop(project_id):
project = get_valid_project(project_id=project_id)
if not project:
return None
notebook_scheduler.stop_notebook(project.notebook, update_status=True)