def create(experiment_group): # This is a bit different since hyperband requires an iteration to work correctly # May search manager's needs to be updated. experiment_group.iteration_manager.create_iteration() suggestions = base.get_suggestions(experiment_group=experiment_group) if not suggestions: logger.error('Experiment group `%s` could not create any suggestion.', experiment_group.id) experiment_group.set_status(ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new suggestions.') return experiment_group.iteration_manager.update_iteration_num_suggestions( num_suggestions=len(suggestions)) def send_chunk(): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE_EXPERIMENTS, kwargs={'experiment_group_id': experiment_group.id, 'suggestions': chunk_suggestions}, countdown=1) chunk_suggestions = [] for suggestion in suggestions: chunk_suggestions.append(suggestion) if len(chunk_suggestions) == conf.get('GROUP_CHUNKS'): send_chunk() chunk_suggestions = [] if chunk_suggestions: send_chunk() celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group.id, 'auto_retry': True}, countdown=1)
def create(experiment_group): suggestions = base.get_suggestions(experiment_group=experiment_group) if not suggestions: logger.error('Experiment group `%s` could not create any suggestion.', experiment_group.id) experiment_group.set_status( ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new suggestions.') return experiment_group.iteration_manager.create_iteration( num_suggestions=len(suggestions)) def send_chunk(): workers.send(HPCeleryTasks.HP_BO_CREATE_EXPERIMENTS, kwargs={ 'experiment_group_id': experiment_group.id, 'suggestions': chunk_suggestions }) chunk_suggestions = [] for suggestion in suggestions: chunk_suggestions.append(suggestion) if len(chunk_suggestions) == conf.get(GROUPS_CHUNKS): send_chunk() chunk_suggestions = [] if chunk_suggestions: send_chunk() workers.send(HPCeleryTasks.HP_BO_START, kwargs={ 'experiment_group_id': experiment_group.id, 'auto_retry': True })
def get_suggestions(experiment_group): # Parse polyaxonfile content and create the experiments specification = experiment_group.specification suggestions = experiment_group.get_suggestions() if not suggestions: logger.error('Search algorithm `%s` could not make any suggestions.', specification.search_algorithm, extra={'stack': True}) return return suggestions
def get_suggestions(experiment_group): # Parse polyaxonfile content and create the experiments specification = experiment_group.specification suggestions = experiment_group.get_suggestions() if not suggestions: logger.error('Search algorithm `%s` could not make any suggestions.', specification.search_algorithm, extra={'stack': True}) return # We sanitize numpy types to be able to jsonify and split the scheduling of different tasks return [{k: sanitize_np_types(v) for k, v in suggestion.items()} for suggestion in suggestions]
def create(experiment_group): experiment_group.iteration_manager.create_iteration() experiments = base.create_group_experiments( experiment_group=experiment_group) if not experiments: logger.error('Experiment group `%s` could not create any experiment.', experiment_group.id) experiment_group.set_status( ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new suggestions.') return experiment_group.iteration_manager.add_iteration_experiments( experiment_ids=[xp.id for xp in experiments]) celery_app.send_task(HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group.id}, countdown=1)
def create(experiment_group): suggestions = base.get_suggestions(experiment_group=experiment_group) if not suggestions: logger.error('Experiment group `%s` could not create any suggestion.', experiment_group.id) experiment_group.set_status( ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new suggestions.') return experiment_group.iteration_manager.create_iteration( num_suggestions=len(suggestions)) def send_chunk(): celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_CREATE_EXPERIMENTS, kwargs={ 'experiment_group_id': experiment_group.id, 'suggestions': chunk_suggestions }, countdown=1) chunk_suggestions = [] for suggestion in suggestions: chunk_suggestions.append(suggestion) if len(chunk_suggestions) == settings.GROUP_CHUNKS: send_chunk() chunk_suggestions = [] if chunk_suggestions: send_chunk() celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_START, kwargs={ 'experiment_group_id': experiment_group.id, 'auto_retry': True }, countdown=1)
def create_group_experiments(experiment_group): # Parse polyaxonfile content and create the experiments specification = experiment_group.specification suggestions = experiment_group.get_suggestions() if not suggestions: logger.error('Search algorithm `%s` could not make any suggestions.', specification.search_algorithm, extra={'stack': True}) return experiments = [] for suggestion in suggestions: # We need to check if we should create or restart experiment = Experiment.objects.create( project_id=experiment_group.project_id, user_id=experiment_group.user_id, experiment_group=experiment_group, config=specification.get_experiment_spec( matrix_declaration=suggestion).parsed_data, code_reference_id=experiment_group.code_reference_id) experiments.append(experiment) return experiments