def project_stop_jobs(sender, **kwargs): from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler instance = kwargs['instance'] tensorboard_scheduler.stop_tensorboard(instance.tensorboard, update_status=False) notebook_scheduler.stop_notebook(instance.notebook, update_status=False) for build_job in instance.build_jobs.all(): dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def build_jobs_stop(build_job_id, update_status=True): build_job = get_valid_build_job(build_job_id=build_job_id) if not build_job: _logger.info('Something went wrong, ' 'the BuildJob `%s` does not exist anymore.', build_job_id) return dockerizer_scheduler.stop_dockerizer(build_job, update_status=update_status)
def build_jobs_stop(self, project_name, project_uuid, build_job_name, build_job_uuid, update_status=True): deleted = dockerizer_scheduler.stop_dockerizer( project_name=project_name, project_uuid=project_uuid, build_job_name=build_job_name, build_job_uuid=build_job_uuid) if not deleted and self.request.retries < 2: _logger.info('Trying again to delete build `%s`.', build_job_name) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return if not update_status: return build_job = get_valid_build_job(build_job_uuid=build_job_uuid) if not build_job: _logger.info( 'Something went wrong, ' 'the BuildJob `%s` does not exist anymore.', build_job_uuid) return # Update build job status to show that its stopped build_job.set_status(status=JobLifeCycle.STOPPED, message='BuildJob was stopped.')
def build_jobs_stop(project_name, project_uuid, build_job_name, build_job_uuid, update_status=True): dockerizer_scheduler.stop_dockerizer( project_name=project_name, project_uuid=project_uuid, build_job_name=build_job_name, build_job_uuid=build_job_uuid) if not update_status: return build_job = get_valid_build_job(build_job_uuid=build_job_uuid) if not build_job: _logger.info('Something went wrong, ' 'the BuildJob `%s` does not exist anymore.', build_job_uuid) return # Update build job status to show that its stopped build_job.set_status(status=JobLifeCycle.STOPPED, message='BuildJob was stopped.')
def build_jobs_stop(self, project_name, project_uuid, build_job_name, build_job_uuid, update_status=True, collect_logs=True, is_managed=True, message=None): if collect_logs and is_managed: try: logs_collect_build_job(build_uuid=build_job_uuid) except (OSError, VolumeNotFoundError, PolyaxonStoresException): _logger.warning( 'Scheduler could not collect the logs for build `%s`.', build_job_name) if is_managed: deleted = dockerizer_scheduler.stop_dockerizer( project_name=project_name, project_uuid=project_uuid, build_job_name=build_job_name, build_job_uuid=build_job_uuid) else: deleted = True if not deleted and self.request.retries < 2: _logger.info('Trying again to delete build `%s`.', build_job_name) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return if not update_status: return build_job = get_valid_build_job(build_job_uuid=build_job_uuid, include_deleted=True) if not build_job: _logger.info( 'Something went wrong, ' 'the BuildJob `%s` does not exist anymore.', build_job_uuid) return # Update build job status to show that its stopped build_job.set_status(status=JobLifeCycle.STOPPED, message=message or 'BuildJob was stopped.')