def experiments_group_stop_experiments(experiment_group_id,
pending,
message=None):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if pending:
for experiment in experiment_group.pending_experiments:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
else:
experiments = experiment_group.experiments.exclude(
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
for experiment in experiments:
if experiment.is_running:
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={
'project_name': experiment.project.unique_name,
'project_uuid': experiment.project.uuid.hex,
'experiment_name': experiment.unique_name,
'experiment_uuid': experiment.uuid.hex,
'experiment_group_name': experiment_group.unique_name,
'experiment_group_uuid': experiment_group.uuid.hex,
'specification': experiment.config,
'update_status': True
})
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED, message=message)
def hp_random_search_create(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group=experiment_group)
def hp_hyperband_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
iteration_manager.reduce_configs()
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id)
def experiments_group_stop_experiments(experiment_group_id,
pending,
message=None):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if pending:
for experiment in experiment_group.pending_experiments:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
else:
experiments = experiment_group.experiments.exclude(
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
for experiment in experiments:
if experiment.is_running:
celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={'experiment_id': experiment.id})
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)
def hp_bo_create(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group=experiment_group)
def hp_hyperband_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
iteration_manager.reduce_configs()
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id, auto_retry=False):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
if auto_retry:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration):
celery_app.send_task(
HPCeleryTasks.HP_BO_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id,
auto_retry=auto_retry)
def hp_start(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
start(experiment_group)
def hp_bo_start(self, experiment_group_id, auto_retry=False):
if not base.should_group_start(experiment_group_id=experiment_group_id,
task=HPCeleryTasks.HP_BO_START,
auto_retry=auto_retry):
return
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(
experiment_group=experiment_group)
if should_retry:
if auto_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
workers.send(HPCeleryTasks.HP_BO_ITERATE,
kwargs={
'experiment_group_id': experiment_group_id,
'auto_retry': auto_retry
},
countdown=None)
def hp_create(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group and self.request.retries < 2:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
create(experiment_group)
def experiments_group_stop_experiments(experiment_group_id,
pending,
collect_logs=True,
update_status=True,
message=None):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id, include_deleted=True)
if not experiment_group:
return
if pending:
# this won't work for archived groups anyways!
for experiment in experiment_group.pending_experiments.iterator():
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
else:
experiments = experiment_group.all_experiments.exclude(
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct(
).iterator()
for experiment in experiments:
if experiment.is_stoppable:
celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={
'project_name':
experiment.project.unique_name,
'project_uuid':
experiment.project.uuid.hex,
'experiment_name':
experiment.unique_name,
'experiment_uuid':
experiment.uuid.hex,
'experiment_group_name':
experiment_group.unique_name,
'experiment_group_uuid':
experiment_group.uuid.hex,
'specification':
experiment.content,
'update_status':
True,
'collect_logs':
collect_logs,
'is_managed':
experiment.is_managed,
},
countdown=conf.get('GLOBAL_COUNTDOWN'))
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
if update_status:
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED,
message=message)
def hp_grid_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id)
def hp_random_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id)
def hp_bo_create_experiments(experiment_group_id, suggestions):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
try:
experiments = base.create_group_experiments(experiment_group=experiment_group,
suggestions=suggestions)
except ExperimentGroupException: # The experiments will be stopped
return
experiment_group.iteration_manager.add_iteration_experiments(
experiment_ids=[xp.id for xp in experiments])
def hp_hyperband_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_ITERATE,
kwargs={'experiment_group_id': experiment_group_id})
def hp_grid_search_create_experiments(experiment_group_id, suggestions):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
experiments = base.create_group_experiments(
experiment_group=experiment_group, suggestions=suggestions)
experiment_group.iteration_manager.add_iteration_experiments(
experiment_ids=[xp.id for xp in experiments])
celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_START,
kwargs={'experiment_group_id': experiment_group.id},
countdown=1)
def hp_hyperband_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(
experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
celery_app.send_task(HPCeleryTasks.HP_HYPERBAND_ITERATE,
kwargs={'experiment_group_id': experiment_group_id})
def hp_bo_create_experiments(experiment_group_id, suggestions):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
try:
experiments = base.create_group_experiments(experiment_group=experiment_group,
suggestions=suggestions)
except ExperimentGroupException: # The experiments will be stopped
return
experiment_group.iteration_manager.add_iteration_experiments(
experiment_ids=[xp.id for xp in experiments])
celery_app.send_task(
HPCeleryTasks.HP_BO_START,
kwargs={'experiment_group_id': experiment_group.id},
countdown=1)
def experiments_group_stop(experiment_group_id,
collect_logs=True,
update_status=True,
message=None):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id, include_deleted=True)
if not experiment_group:
return
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING)
workers.send(SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS,
kwargs={
'experiment_group_id': experiment_group_id,
'pending': False,
'collect_logs': collect_logs,
'update_status': update_status,
'message': message
})
def experiments_group_stop(experiment_group_id,
collect_logs=True,
message=None):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id,
include_deleted=True)
if not experiment_group:
return
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING)
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS,
kwargs={
'experiment_group_id': experiment_group_id,
'pending': False,
'collect_logs': collect_logs,
'message': message
},
countdown=conf.get('GLOBAL_COUNTDOWN'))
def hp_grid_search_start(self, experiment_group_id, auto_retry=False):
if not base.should_group_start(experiment_group_id=experiment_group_id,
task=HPCeleryTasks.HP_GRID_SEARCH_START,
auto_retry=auto_retry):
return
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
if auto_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_done(experiment_group_id, auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id, auto_retry=False):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
if auto_retry:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
try:
iteration_manager.reduce_configs()
except ExperimentGroupException:
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new iteration.')
return
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id,
auto_retry=auto_retry)
def experiments_group_stop_experiments(experiment_group_id, pending, message=None):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if pending:
for experiment in experiment_group.pending_experiments:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
else:
experiments = experiment_group.experiments.exclude(
status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
for experiment in experiments:
if experiment.is_running:
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_STOP,
kwargs={'experiment_id': experiment.id})
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)
def hp_create(experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
create(experiment_group)
def hp_grid_search_create(experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def hp_hyperband_create(experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def hp_create(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
create(experiment_group)