def test_stopping_group_stops_iteration(self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) self.assertEqual( mock_fct.call_count, math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1) ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) # Mark group as stopped ExperimentGroupStatusFactory(experiment_group=experiment_group, status=ExperimentGroupLifeCycle.STOPPED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 0
def test_stopping_group_stops_iteration(self): # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) # Mark group as stopped ExperimentGroupStatusFactory(experiment_group=experiment_group, status=ExperimentGroupLifeCycle.STOPPED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 0
def test_hyperband_rescheduling(self): with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: ExperimentGroupFactory(content=experiment_group_spec_content_hyperband) assert mock_fct.call_count == 1 with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async') as mock_fct1: with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_fct2: ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct1.call_count == 1 assert mock_fct2.call_count == 9 # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 1 # Fake reduce with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 assert experiment_group.non_done_experiments.count() == 9 # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct2: with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3: hp_hyperband_start(experiment_group.id) assert mock_fct2.call_count == 1 assert mock_fct3.call_count == 1
def test_hyperband_rescheduling(self): with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: ExperimentGroupFactory(content=experiment_group_spec_content_hyperband) assert mock_fct.call_count == 1 with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async') as mock_fct1: with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_fct2: ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct1.call_count == 1 assert mock_fct2.call_count == 9 # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 1 # Fake reduce with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 assert experiment_group.non_done_experiments.count() == 9 # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct2: with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3: hp_hyperband_start(experiment_group.id) assert mock_fct2.call_count == 1 assert mock_fct3.call_count == 1
def test_stopping_group_stops_iteration(self): # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) # Mark group as stopped ExperimentGroupStatusFactory(experiment_group=experiment_group, status=ExperimentGroupLifeCycle.STOPPED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 0
def test_hyperband_rescheduling(self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: ExperimentGroupFactory( content=experiment_group_spec_content_hyperband) assert mock_fct.call_count == 2 with patch.object(GroupChecks, 'is_checked') as mock_is_check: with patch( 'hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async' ) as mock_fct1: with patch('scheduler.tasks.experiments.' 'experiments_build.apply_async') as mock_fct2: mock_is_check.return_value = False experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule ) assert experiment_group.iteration_config.num_suggestions == 9 assert mock_fct1.call_count == 2 # 9 experiments, but since we are mocking the scheduling function, it's ~ 3 x calls, # every call to start tries to schedule again, but in reality it's just 9 calls assert mock_fct2.call_count >= 9 * 2 # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) self.assertEqual( mock_fct.call_count, math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1) ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21, 'num_suggestions': 9 }) experiment_group.iteration.experiments.set( experiment_group.experiments.values_list('id', flat=True)) # Mark experiments as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.hyperband.' 'hp_hyperband_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory( experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count( ) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 1 # Fake reduce with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) self.assertEqual( mock_fct.call_count, math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1) assert experiment_group.non_done_experiments.count() == 9 # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.hyperband.' 'hp_hyperband_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory( experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count( ) with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct2: with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3: hp_hyperband_start(experiment_group.id) assert mock_fct2.call_count == 1 assert mock_fct3.call_count == 1