def _clean(): filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None) for project in Project.objects.exclude(filters): if project.has_notebook: notebook_scheduler.stop_notebook(project.notebook, update_status=False) if project.has_tensorboard: tensorboard_scheduler.stop_tensorboard(project.tensorboard, update_status=False)
def project_stop_jobs(sender, **kwargs): from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler instance = kwargs['instance'] tensorboard_scheduler.stop_tensorboard(instance.tensorboard, update_status=False) notebook_scheduler.stop_notebook(instance.notebook, update_status=False) for build_job in instance.build_jobs.all(): dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def _clean(): for job in TensorboardJob.objects.filter( status__status__in=JobLifeCycle.RUNNING_STATUS): tensorboard_scheduler.stop_tensorboard( project_name=job.project.unique_name, project_uuid=job.project.uuid.hex, tensorboard_job_name=job.unique_name, tensorboard_job_uuid=job.unique_name) job.set_status(JobLifeCycle.STOPPED, message='Cleanup')
def _clean() -> None: for job in TensorboardJob.objects.filter( status__status__in=JobLifeCycle.RUNNING_STATUS): tensorboard_scheduler.stop_tensorboard( project_name=job.project.unique_name, project_uuid=job.project.uuid.hex, tensorboard_job_name=job.unique_name, tensorboard_job_uuid=job.uuid.hex) job.set_status(JobLifeCycle.STOPPED, message='Stop triggered by the cleaning hook.')
def tensorboards_stop(self, project_name, project_uuid, tensorboard_job_name, tensorboard_job_uuid, update_status=True, collect_logs=False, message=None): deleted = tensorboard_scheduler.stop_tensorboard( project_name=project_name, project_uuid=project_uuid, tensorboard_job_name=tensorboard_job_name, tensorboard_job_uuid=tensorboard_job_uuid) if not deleted and self.request.retries < 2: _logger.info('Trying again to delete job `%s`.', tensorboard_job_name) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return if not update_status: return tensorboard = get_valid_tensorboard( tensorboard_job_uuid=tensorboard_job_uuid, include_deleted=True) if not tensorboard: return None # Update tensorboard status to show that its stopped tensorboard.set_status(status=JobLifeCycle.STOPPED, message=message or 'Tensorboard was stopped')
def _clean(): filters = Q(tensorboard_jobs=None) | Q(notebook_jobs=None) for project in Project.objects.exclude(filters): if project.has_notebook: notebook_scheduler.stop_notebook( project_name=project.unique_name, project_uuid=project.uuid.hex, notebook_job_name=project.notebook.unique_name, notebook_job_uuid=project.notebook.uuid.hex) project.notebook.set_status(status=JobLifeCycle.STOPPED, message='Cleanup') if project.has_tensorboard: tensorboard_scheduler.stop_tensorboard( project_name=project.unique_name, project_uuid=project.uuid.hex, tensorboard_job_name=project.tensorboard.unique_name, tensorboard_job_uuid=project.tensorboard.uuid.hex) project.tensorboard.set_status(status=JobLifeCycle.STOPPED, message='Cleanup')
def tensorboards_stop(project_name, project_uuid, tensorboard_job_name, tensorboard_job_uuid, update_status=True): tensorboard_scheduler.stop_tensorboard( project_name=project_name, project_uuid=project_uuid, tensorboard_job_name=tensorboard_job_name, tensorboard_job_uuid=tensorboard_job_uuid) if not update_status: return tensorboard = get_valid_tensorboard( tensorboard_job_uuid=tensorboard_job_uuid) if not tensorboard: return None # Update tensorboard status to show that its stopped tensorboard.set_status(status=JobLifeCycle.STOPPED, message='Tensorboard was stopped')
def tensorboards_stop(tensorboard_job_id): tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id) if not tensorboard: return None tensorboard_scheduler.stop_tensorboard(tensorboard, update_status=True)
def projects_tensorboard_stop(project_id): project = get_valid_project(project_id=project_id) if not project: return None tensorboard_scheduler.stop_tensorboard(project.tensorboard, update_status=True)