def create(experiment_group):
# This is a bit different since hyperband requires an iteration to work correctly
# May search manager's needs to be updated.
experiment_group.iteration_manager.create_iteration()
suggestions = base.get_suggestions(experiment_group=experiment_group)
if not suggestions:
logger.error('Experiment group `%s` could not create any suggestion.',
experiment_group.id)
experiment_group.set_status(ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new suggestions.')
return
experiment_group.iteration_manager.update_iteration_num_suggestions(
num_suggestions=len(suggestions))
def send_chunk():
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE_EXPERIMENTS,
kwargs={'experiment_group_id': experiment_group.id,
'suggestions': chunk_suggestions},
countdown=1)
chunk_suggestions = []
for suggestion in suggestions:
chunk_suggestions.append(suggestion)
if len(chunk_suggestions) == conf.get('GROUP_CHUNKS'):
send_chunk()
chunk_suggestions = []
if chunk_suggestions:
send_chunk()
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group.id, 'auto_retry': True},
countdown=1)
def create(experiment_group):
suggestions = base.get_suggestions(experiment_group=experiment_group)
if not suggestions:
logger.error('Experiment group `%s` could not create any suggestion.',
experiment_group.id)
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new suggestions.')
return
experiment_group.iteration_manager.create_iteration(
num_suggestions=len(suggestions))
def send_chunk():
workers.send(HPCeleryTasks.HP_BO_CREATE_EXPERIMENTS,
kwargs={
'experiment_group_id': experiment_group.id,
'suggestions': chunk_suggestions
})
chunk_suggestions = []
for suggestion in suggestions:
chunk_suggestions.append(suggestion)
if len(chunk_suggestions) == conf.get(GROUPS_CHUNKS):
send_chunk()
chunk_suggestions = []
if chunk_suggestions:
send_chunk()
workers.send(HPCeleryTasks.HP_BO_START,
kwargs={
'experiment_group_id': experiment_group.id,
'auto_retry': True
})
def get_suggestions(experiment_group):
# Parse polyaxonfile content and create the experiments
specification = experiment_group.specification
suggestions = experiment_group.get_suggestions()
if not suggestions:
logger.error('Search algorithm `%s` could not make any suggestions.',
specification.search_algorithm,
extra={'stack': True})
return
return suggestions
def get_suggestions(experiment_group):
# Parse polyaxonfile content and create the experiments
specification = experiment_group.specification
suggestions = experiment_group.get_suggestions()
if not suggestions:
logger.error('Search algorithm `%s` could not make any suggestions.',
specification.search_algorithm,
extra={'stack': True})
return
# We sanitize numpy types to be able to jsonify and split the scheduling of different tasks
return [{k: sanitize_np_types(v) for k, v in suggestion.items()} for suggestion in suggestions]
def create(experiment_group):
experiment_group.iteration_manager.create_iteration()
experiments = base.create_group_experiments(
experiment_group=experiment_group)
if not experiments:
logger.error('Experiment group `%s` could not create any experiment.',
experiment_group.id)
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new suggestions.')
return
experiment_group.iteration_manager.add_iteration_experiments(
experiment_ids=[xp.id for xp in experiments])
celery_app.send_task(HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group.id},
countdown=1)
def create(experiment_group):
suggestions = base.get_suggestions(experiment_group=experiment_group)
if not suggestions:
logger.error('Experiment group `%s` could not create any suggestion.',
experiment_group.id)
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new suggestions.')
return
experiment_group.iteration_manager.create_iteration(
num_suggestions=len(suggestions))
def send_chunk():
celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_CREATE_EXPERIMENTS,
kwargs={
'experiment_group_id': experiment_group.id,
'suggestions': chunk_suggestions
},
countdown=1)
chunk_suggestions = []
for suggestion in suggestions:
chunk_suggestions.append(suggestion)
if len(chunk_suggestions) == settings.GROUP_CHUNKS:
send_chunk()
chunk_suggestions = []
if chunk_suggestions:
send_chunk()
celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_START,
kwargs={
'experiment_group_id': experiment_group.id,
'auto_retry': True
},
countdown=1)
def create_group_experiments(experiment_group):
# Parse polyaxonfile content and create the experiments
specification = experiment_group.specification
suggestions = experiment_group.get_suggestions()
if not suggestions:
logger.error('Search algorithm `%s` could not make any suggestions.',
specification.search_algorithm,
extra={'stack': True})
return
experiments = []
for suggestion in suggestions:
# We need to check if we should create or restart
experiment = Experiment.objects.create(
project_id=experiment_group.project_id,
user_id=experiment_group.user_id,
experiment_group=experiment_group,
config=specification.get_experiment_spec(
matrix_declaration=suggestion).parsed_data,
code_reference_id=experiment_group.code_reference_id)
experiments.append(experiment)
return experiments
