def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(project.notebook, update_status=False)
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(project.tensorboard, update_status=False)
def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard, update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def _clean():
for job in TensorboardJob.objects.filter(
status__status__in=JobLifeCycle.RUNNING_STATUS):
tensorboard_scheduler.stop_tensorboard(
project_name=job.project.unique_name,
project_uuid=job.project.uuid.hex,
tensorboard_job_name=job.unique_name,
tensorboard_job_uuid=job.unique_name)
job.set_status(JobLifeCycle.STOPPED, message='Cleanup')
def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard,
update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def _clean() -> None:
for job in TensorboardJob.objects.filter(
status__status__in=JobLifeCycle.RUNNING_STATUS):
tensorboard_scheduler.stop_tensorboard(
project_name=job.project.unique_name,
project_uuid=job.project.uuid.hex,
tensorboard_job_name=job.unique_name,
tensorboard_job_uuid=job.uuid.hex)
job.set_status(JobLifeCycle.STOPPED, message='Stop triggered by the cleaning hook.')
def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(project.notebook,
update_status=False)
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(project.tensorboard,
update_status=False)
def tensorboards_stop(self,
project_name,
project_uuid,
tensorboard_job_name,
tensorboard_job_uuid,
update_status=True,
collect_logs=False,
message=None):
deleted = tensorboard_scheduler.stop_tensorboard(
project_name=project_name,
project_uuid=project_uuid,
tensorboard_job_name=tensorboard_job_name,
tensorboard_job_uuid=tensorboard_job_uuid)
if not deleted and self.request.retries < 2:
_logger.info('Trying again to delete job `%s`.', tensorboard_job_name)
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
if not update_status:
return
tensorboard = get_valid_tensorboard(
tensorboard_job_uuid=tensorboard_job_uuid, include_deleted=True)
if not tensorboard:
return None
# Update tensorboard status to show that its stopped
tensorboard.set_status(status=JobLifeCycle.STOPPED,
message=message or 'Tensorboard was stopped')
def _clean():
filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None)
for project in Project.objects.exclude(filters):
if project.has_notebook:
notebook_scheduler.stop_notebook(
project_name=project.unique_name,
project_uuid=project.uuid.hex,
notebook_job_name=project.notebook.unique_name,
notebook_job_uuid=project.notebook.uuid.hex)
project.notebook.set_status(status=JobLifeCycle.STOPPED,
message='Cleanup')
if project.has_tensorboard:
tensorboard_scheduler.stop_tensorboard(
project_name=project.unique_name,
project_uuid=project.uuid.hex,
tensorboard_job_name=project.tensorboard.unique_name,
tensorboard_job_uuid=project.tensorboard.uuid.hex)
project.tensorboard.set_status(status=JobLifeCycle.STOPPED,
message='Cleanup')
def tensorboards_stop(project_name,
project_uuid,
tensorboard_job_name,
tensorboard_job_uuid,
update_status=True):
tensorboard_scheduler.stop_tensorboard(
project_name=project_name,
project_uuid=project_uuid,
tensorboard_job_name=tensorboard_job_name,
tensorboard_job_uuid=tensorboard_job_uuid)
if not update_status:
return
tensorboard = get_valid_tensorboard(
tensorboard_job_uuid=tensorboard_job_uuid)
if not tensorboard:
return None
# Update tensorboard status to show that its stopped
tensorboard.set_status(status=JobLifeCycle.STOPPED,
message='Tensorboard was stopped')
def tensorboards_stop(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
tensorboard_scheduler.stop_tensorboard(tensorboard, update_status=True)
def projects_tensorboard_stop(project_id):
project = get_valid_project(project_id=project_id)
if not project:
return None
tensorboard_scheduler.stop_tensorboard(project.tensorboard,
update_status=True)
def tensorboards_stop(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
tensorboard_scheduler.stop_tensorboard(tensorboard, update_status=True)
