def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(project.notebook, update_status=False)
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(project.tensorboard, update_status=False)
def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard, update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def _clean():
for job in NotebookJob.objects.filter(
status__status__in=JobLifeCycle.RUNNING_STATUS):
notebook_scheduler.stop_notebook(
project_name=job.project.unique_name,
project_uuid=job.project.uuid.hex,
notebook_job_name=job.unique_name,
notebook_job_uuid=job.unique_name)
job.set_status(JobLifeCycle.STOPPED, message='Cleanup')
def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard,
update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(project.notebook,
update_status=False)
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(project.tensorboard,
update_status=False)
def _clean() -> None:
for job in NotebookJob.objects.filter(
status__status__in=JobLifeCycle.RUNNING_STATUS):
notebook_scheduler.stop_notebook(
project_name=job.project.unique_name,
project_uuid=job.project.uuid.hex,
notebook_job_name=job.unique_name,
notebook_job_uuid=job.uuid.hex)
job.set_status(JobLifeCycle.STOPPED,
message='Stop triggered by the cleaning hook.')
def projects_notebook_stop(self,
project_name,
project_uuid,
notebook_job_name,
notebook_job_uuid,
update_status=True,
collect_logs=False,
message=None):
deleted = notebook_scheduler.stop_notebook(
project_name=project_name,
project_uuid=project_uuid,
notebook_job_name=notebook_job_name,
notebook_job_uuid=notebook_job_uuid)
if not deleted and self.request.retries < 2:
_logger.info('Trying again to delete job `%s`.', notebook_job_name)
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
if not update_status:
return
notebook = get_valid_notebook(notebook_job_uuid=notebook_job_uuid, include_deleted=True)
if not notebook:
return None
# Update notebook status to show that its stopped
notebook.set_status(status=JobLifeCycle.STOPPED,
message=message or 'Notebook was stopped')
def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(
project_name=project.unique_name,
project_uuid=project.uuid.hex,
notebook_job_name=project.notebook.unique_name,
notebook_job_uuid=project.notebook.uuid.hex)
project.notebook.set_status(status=JobLifeCycle.STOPPED,
message='Cleanup')
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(
project_name=project.unique_name,
project_uuid=project.uuid.hex,
tensorboard_job_name=project.tensorboard.unique_name,
tensorboard_job_uuid=project.tensorboard.uuid.hex)
project.tensorboard.set_status(status=JobLifeCycle.STOPPED,
message='Cleanup')
def projects_notebook_stop(project_name,
project_uuid,
notebook_job_name,
notebook_job_uuid,
update_status=True):
notebook_scheduler.stop_notebook(project_name=project_name,
project_uuid=project_uuid,
notebook_job_name=notebook_job_name,
notebook_job_uuid=notebook_job_uuid)
if not update_status:
return
notebook = get_valid_notebook(notebook_job_uuid=notebook_job_uuid)
if not notebook:
return None
# Update notebook status to show that its stopped
notebook.set_status(status=JobLifeCycle.STOPPED,
message='Notebook was stopped')
def projects_notebook_stop(notebook_job_id):
notebook = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook:
return None
notebook_scheduler.stop_notebook(notebook, update_status=True)
def projects_notebook_stop(project_id):
project = get_valid_project(project_id=project_id)
if not project:
return None
notebook_scheduler.stop_notebook(project.notebook, update_status=True)
