def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard, update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def build_jobs_stop(build_job_id, update_status=True):
build_job = get_valid_build_job(build_job_id=build_job_id)
if not build_job:
_logger.info('Something went wrong, '
'the BuildJob `%s` does not exist anymore.', build_job_id)
return
dockerizer_scheduler.stop_dockerizer(build_job, update_status=update_status)
def build_jobs_stop(build_job_id, update_status=True):
build_job = get_valid_build_job(build_job_id=build_job_id)
if not build_job:
_logger.info('Something went wrong, '
'the BuildJob `%s` does not exist anymore.', build_job_id)
return
dockerizer_scheduler.stop_dockerizer(build_job, update_status=update_status)
def project_stop_jobs(sender, **kwargs):
from scheduler import dockerizer_scheduler, notebook_scheduler, tensorboard_scheduler
instance = kwargs['instance']
tensorboard_scheduler.stop_tensorboard(instance.tensorboard,
update_status=False)
notebook_scheduler.stop_notebook(instance.notebook, update_status=False)
for build_job in instance.build_jobs.all():
dockerizer_scheduler.stop_dockerizer(build_job, update_status=False)
def build_jobs_stop(self,
project_name,
project_uuid,
build_job_name,
build_job_uuid,
update_status=True):
deleted = dockerizer_scheduler.stop_dockerizer(
project_name=project_name,
project_uuid=project_uuid,
build_job_name=build_job_name,
build_job_uuid=build_job_uuid)
if not deleted and self.request.retries < 2:
_logger.info('Trying again to delete build `%s`.', build_job_name)
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
if not update_status:
return
build_job = get_valid_build_job(build_job_uuid=build_job_uuid)
if not build_job:
_logger.info(
'Something went wrong, '
'the BuildJob `%s` does not exist anymore.', build_job_uuid)
return
# Update build job status to show that its stopped
build_job.set_status(status=JobLifeCycle.STOPPED,
message='BuildJob was stopped.')
def build_jobs_stop(project_name, project_uuid, build_job_name, build_job_uuid, update_status=True):
dockerizer_scheduler.stop_dockerizer(
project_name=project_name,
project_uuid=project_uuid,
build_job_name=build_job_name,
build_job_uuid=build_job_uuid)
if not update_status:
return
build_job = get_valid_build_job(build_job_uuid=build_job_uuid)
if not build_job:
_logger.info('Something went wrong, '
'the BuildJob `%s` does not exist anymore.', build_job_uuid)
return
# Update build job status to show that its stopped
build_job.set_status(status=JobLifeCycle.STOPPED,
message='BuildJob was stopped.')
def build_jobs_stop(self,
project_name,
project_uuid,
build_job_name,
build_job_uuid,
update_status=True,
collect_logs=True,
is_managed=True,
message=None):
if collect_logs and is_managed:
try:
logs_collect_build_job(build_uuid=build_job_uuid)
except (OSError, VolumeNotFoundError, PolyaxonStoresException):
_logger.warning(
'Scheduler could not collect the logs for build `%s`.',
build_job_name)
if is_managed:
deleted = dockerizer_scheduler.stop_dockerizer(
project_name=project_name,
project_uuid=project_uuid,
build_job_name=build_job_name,
build_job_uuid=build_job_uuid)
else:
deleted = True
if not deleted and self.request.retries < 2:
_logger.info('Trying again to delete build `%s`.', build_job_name)
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
if not update_status:
return
build_job = get_valid_build_job(build_job_uuid=build_job_uuid,
include_deleted=True)
if not build_job:
_logger.info(
'Something went wrong, '
'the BuildJob `%s` does not exist anymore.', build_job_uuid)
return
# Update build job status to show that its stopped
build_job.set_status(status=JobLifeCycle.STOPPED,
message=message or 'BuildJob was stopped.')
