def experiments_group_stop_experiments(experiment_group_id, pending, message=None): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if pending: for experiment in experiment_group.pending_experiments: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) else: experiments = experiment_group.experiments.exclude( status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct() for experiment in experiments: if experiment.is_running: celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={ 'project_name': experiment.project.unique_name, 'project_uuid': experiment.project.uuid.hex, 'experiment_name': experiment.unique_name, 'experiment_uuid': experiment.uuid.hex, 'experiment_group_name': experiment_group.unique_name, 'experiment_group_uuid': experiment_group.uuid.hex, 'specification': experiment.config, 'update_status': True }) else: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED, message=message)
def hp_random_search_create(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group=experiment_group)
def hp_hyperband_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): iteration_manager.reduce_configs() celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id)
def experiments_group_stop_experiments(experiment_group_id, pending, message=None): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if pending: for experiment in experiment_group.pending_experiments: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) else: experiments = experiment_group.experiments.exclude( status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct() for experiment in experiments: if experiment.is_running: celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={'experiment_id': experiment.id}) else: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)
def hp_bo_create(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group=experiment_group)
def hp_hyperband_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): iteration_manager.reduce_configs() celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id, auto_retry=False): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: if auto_retry: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration): celery_app.send_task( HPCeleryTasks.HP_BO_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)
def hp_start(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return start(experiment_group)
def hp_bo_start(self, experiment_group_id, auto_retry=False): if not base.should_group_start(experiment_group_id=experiment_group_id, task=HPCeleryTasks.HP_BO_START, auto_retry=auto_retry): return experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments( experiment_group=experiment_group) if should_retry: if auto_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return workers.send(HPCeleryTasks.HP_BO_ITERATE, kwargs={ 'experiment_group_id': experiment_group_id, 'auto_retry': auto_retry }, countdown=None)
def hp_create(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group and self.request.retries < 2: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return create(experiment_group)
def experiments_group_stop_experiments(experiment_group_id, pending, collect_logs=True, update_status=True, message=None): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id, include_deleted=True) if not experiment_group: return if pending: # this won't work for archived groups anyways! for experiment in experiment_group.pending_experiments.iterator(): # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) else: experiments = experiment_group.all_experiments.exclude( status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct( ).iterator() for experiment in experiments: if experiment.is_stoppable: celery_app.send_task(SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={ 'project_name': experiment.project.unique_name, 'project_uuid': experiment.project.uuid.hex, 'experiment_name': experiment.unique_name, 'experiment_uuid': experiment.uuid.hex, 'experiment_group_name': experiment_group.unique_name, 'experiment_group_uuid': experiment_group.uuid.hex, 'specification': experiment.content, 'update_status': True, 'collect_logs': collect_logs, 'is_managed': experiment.is_managed, }, countdown=conf.get('GLOBAL_COUNTDOWN')) else: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) if update_status: experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED, message=message)
def hp_grid_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id)
def hp_random_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id)
def hp_bo_create_experiments(experiment_group_id, suggestions): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return try: experiments = base.create_group_experiments(experiment_group=experiment_group, suggestions=suggestions) except ExperimentGroupException: # The experiments will be stopped return experiment_group.iteration_manager.add_iteration_experiments( experiment_ids=[xp.id for xp in experiments])
def hp_hyperband_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_ITERATE, kwargs={'experiment_group_id': experiment_group_id})
def hp_grid_search_create_experiments(experiment_group_id, suggestions): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return experiments = base.create_group_experiments( experiment_group=experiment_group, suggestions=suggestions) experiment_group.iteration_manager.add_iteration_experiments( experiment_ids=[xp.id for xp in experiments]) celery_app.send_task(HPCeleryTasks.HP_GRID_SEARCH_START, kwargs={'experiment_group_id': experiment_group.id}, countdown=1)
def hp_hyperband_start(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments( experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return celery_app.send_task(HPCeleryTasks.HP_HYPERBAND_ITERATE, kwargs={'experiment_group_id': experiment_group_id})
def hp_bo_create_experiments(experiment_group_id, suggestions): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return try: experiments = base.create_group_experiments(experiment_group=experiment_group, suggestions=suggestions) except ExperimentGroupException: # The experiments will be stopped return experiment_group.iteration_manager.add_iteration_experiments( experiment_ids=[xp.id for xp in experiments]) celery_app.send_task( HPCeleryTasks.HP_BO_START, kwargs={'experiment_group_id': experiment_group.id}, countdown=1)
def experiments_group_stop(experiment_group_id, collect_logs=True, update_status=True, message=None): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id, include_deleted=True) if not experiment_group: return experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING) workers.send(SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS, kwargs={ 'experiment_group_id': experiment_group_id, 'pending': False, 'collect_logs': collect_logs, 'update_status': update_status, 'message': message })
def experiments_group_stop(experiment_group_id, collect_logs=True, message=None): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id, include_deleted=True) if not experiment_group: return experiment_group.set_status(ExperimentGroupLifeCycle.STOPPING) celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_GROUP_STOP_EXPERIMENTS, kwargs={ 'experiment_group_id': experiment_group_id, 'pending': False, 'collect_logs': collect_logs, 'message': message }, countdown=conf.get('GLOBAL_COUNTDOWN'))
def hp_grid_search_start(self, experiment_group_id, auto_retry=False): if not base.should_group_start(experiment_group_id=experiment_group_id, task=HPCeleryTasks.HP_GRID_SEARCH_START, auto_retry=auto_retry): return experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: if auto_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_done(experiment_group_id, auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id, auto_retry=False): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: if auto_retry: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): try: iteration_manager.reduce_configs() except ExperimentGroupException: experiment_group.set_status( ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new iteration.') return celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)
def experiments_group_stop_experiments(experiment_group_id, pending, message=None): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if pending: for experiment in experiment_group.pending_experiments: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) else: experiments = experiment_group.experiments.exclude( status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct() for experiment in experiments: if experiment.is_running: celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_STOP, kwargs={'experiment_id': experiment.id}) else: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)
def hp_create(experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) create(experiment_group)
def hp_grid_search_create(experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group)
def hp_hyperband_create(experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group)
def hp_create(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) create(experiment_group)