def tensorboards_schedule_deletion(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id,
include_deleted=True)
if not tensorboard:
return None
tensorboard.archive()
if tensorboard.is_stoppable:
project = tensorboard.project
celery_app.send_task(SchedulerCeleryTasks.TENSORBOARDS_STOP,
kwargs={
'project_name':
project.unique_name,
'project_uuid':
project.uuid.hex,
'tensorboard_job_name':
tensorboard.unique_name,
'tensorboard_job_uuid':
tensorboard.uuid.hex,
'update_status':
True,
'collect_logs':
False,
'message':
'Tensorboard is scheduled for deletion.'
})
def tensorboards_stop(self,
project_name,
project_uuid,
tensorboard_job_name,
tensorboard_job_uuid,
update_status=True,
collect_logs=False,
message=None):
deleted = tensorboard_scheduler.stop_tensorboard(
project_name=project_name,
project_uuid=project_uuid,
tensorboard_job_name=tensorboard_job_name,
tensorboard_job_uuid=tensorboard_job_uuid)
if not deleted and self.request.retries < 2:
_logger.info('Trying again to delete job `%s`.', tensorboard_job_name)
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
if not update_status:
return
tensorboard = get_valid_tensorboard(
tensorboard_job_uuid=tensorboard_job_uuid, include_deleted=True)
if not tensorboard:
return None
# Update tensorboard status to show that its stopped
tensorboard.set_status(status=JobLifeCycle.STOPPED,
message=message or 'Tensorboard was stopped')
def tensorboards_schedule_deletion(tensorboard_job_id, immediate=False):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id,
include_deleted=True)
if not tensorboard:
return None
tensorboard.archive()
if tensorboard.is_stoppable:
project = tensorboard.project
workers.send(
SchedulerCeleryTasks.TENSORBOARDS_STOP,
kwargs={
'project_name': project.unique_name,
'project_uuid': project.uuid.hex,
'tensorboard_job_name': tensorboard.unique_name,
'tensorboard_job_uuid': tensorboard.uuid.hex,
'update_status': True,
'collect_logs': False,
'is_managed': tensorboard.is_managed,
'message': 'Tensorboard is scheduled for deletion.'
})
if immediate:
workers.send(
SchedulerCeleryTasks.DELETE_ARCHIVED_TENSORBOARD_JOB,
kwargs={
'job_id': tensorboard_job_id,
},
countdown=conf.get(SCHEDULER_GLOBAL_COUNTDOWN_DELAYED))
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status,
JobLifeCycle.SCHEDULED)
tensorboard_scheduler.start_tensorboard(tensorboard)
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
_logger.warning('Tensorboard %s does not exist anymore.', tensorboard_job_id)
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status, JobLifeCycle.SCHEDULED)
tensorboard_scheduler.start_tensorboard(tensorboard)
def tensorboards_start(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
if not JobLifeCycle.can_transition(status_from=tensorboard.last_status,
status_to=JobLifeCycle.SCHEDULED):
_logger.info('Tensorboard `%s` cannot transition from `%s` to `%s`.',
tensorboard.unique_name, tensorboard.last_status, JobLifeCycle.SCHEDULED)
try:
tensorboard_scheduler.start_tensorboard(tensorboard)
except StoreNotFoundError:
tensorboard.set_status(status=JobLifeCycle.FAILED,
message='Tensorboard failed to start, '
'the outputs volume/storage was not found.')
def tensorboards_stop(project_name,
project_uuid,
tensorboard_job_name,
tensorboard_job_uuid,
update_status=True):
tensorboard_scheduler.stop_tensorboard(
project_name=project_name,
project_uuid=project_uuid,
tensorboard_job_name=tensorboard_job_name,
tensorboard_job_uuid=tensorboard_job_uuid)
if not update_status:
return
tensorboard = get_valid_tensorboard(
tensorboard_job_uuid=tensorboard_job_uuid)
if not tensorboard:
return None
# Update tensorboard status to show that its stopped
tensorboard.set_status(status=JobLifeCycle.STOPPED,
message='Tensorboard was stopped')
def tensorboards_stop(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
tensorboard_scheduler.stop_tensorboard(tensorboard, update_status=True)
def tensorboards_stop(tensorboard_job_id):
tensorboard = get_valid_tensorboard(tensorboard_job_id=tensorboard_job_id)
if not tensorboard:
return None
tensorboard_scheduler.stop_tensorboard(tensorboard, update_status=True)
