def get_reduced_configs(self):
"""Reduce the experiments to restart."""
iteration_config = self.experiment_group.iteration_config
if iteration_config is None:
logger.error(
'Experiment group `%s` attempt to update iteration, but has no iteration',
self.experiment_group.id,
extra={'stack': True})
return
search_manager = self.experiment_group.search_manager
# Get the number of experiments to keep
n_configs_to_keep = search_manager.get_n_config_to_keep_for_iteration(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration)
# Get the last group's experiments metrics
experiments_metrics = self.experiment_group.iteration_config.experiments_metrics
if not experiments_metrics:
raise ExperimentGroupException()
# Order the experiments
reverse = Optimization.maximize(
self.experiment_group.hptuning_config.hyperband.metric.optimization)
experiments_metrics = sorted(experiments_metrics, key=lambda x: x[1], reverse=reverse)
# Keep n experiments
return [xp[0] for xp in experiments_metrics[:n_configs_to_keep]]
def create_group_experiments(experiment_group, suggestions):
# Parse polyaxonfile content and create the experiments
specification = experiment_group.specification
experiments = []
for suggestion in suggestions:
# We need to check if we should create or restart
try:
experiment = Experiment.objects.create(
project_id=experiment_group.project_id,
user_id=experiment_group.user_id,
experiment_group=experiment_group,
content=specification.get_experiment_spec(
matrix_declaration=suggestion).raw_data,
code_reference_id=experiment_group.code_reference_id)
except ValidationError:
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create experiments, '
'encountered a validation error.',
traceback=traceback.format_exc())
raise ExperimentGroupException()
experiments.append(experiment)
return experiments