def test_stopping_group_stops_iteration(self): # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) # Mark group as stopped ExperimentGroupStatusFactory(experiment_group=experiment_group, status=ExperimentGroupLifeCycle.STOPPED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 0
def test_stop_all_experiments(self): with patch('hpsearch.tasks.random.hp_random_search_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_early_stopping) assert mock_fct.call_count == 1 # Add a running experiment experiment = ExperimentFactory(experiment_group=experiment_group) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 1 assert experiment_group.experiments.count() == 3 assert experiment_group.stopped_experiments.count() == 0 with patch('scheduler.experiment_scheduler.stop_experiment' ) as spawner_mock_fct: experiments_group_stop_experiments( experiment_group_id=experiment_group.id, pending=False) assert experiment_group.pending_experiments.count() == 0 assert experiment_group.running_experiments.count() == 0 assert spawner_mock_fct.call_count == 1 # Should be stopped with this function assert experiment_group.stopped_experiments.count() == 3
def test_bo_rescheduling(self): with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct: ExperimentGroupFactory(content=experiment_group_spec_content_bo) assert mock_fct.call_count == 2 with patch.object(GroupChecks, 'is_checked') as mock_is_check: with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async') as mock_fct1: with patch('scheduler.tasks.experiments.' 'experiments_build.apply_async') as mock_fct2: mock_is_check.return_value = False ExperimentGroupFactory( content=experiment_group_spec_content_bo) assert mock_fct1.call_count == 2 # 2 experiments, but since we are mocking the scheduling function, it's 4 calls, # every call to start tries to schedule again, but in reality it's just 2 calls assert mock_fct2.call_count == 4 # Fake with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_bo) assert mock_fct.call_count == 2 assert experiment_group.non_done_experiments.count() == 2 # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count() with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async') as mock_fct1: hp_bo_start(experiment_group.id) assert mock_fct1.call_count == 1 # Mark experiment as done with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count() GroupChecks(group=experiment_group.id).clear() with patch('hpsearch.tasks.bo.hp_bo_create.apply_async') as mock_fct1: hp_bo_start(experiment_group.id) assert mock_fct1.call_count == 1
def test_heartbeat_experiments(self): experiment1 = ExperimentFactory() ExperimentStatusFactory(experiment=experiment1, status=ExperimentLifeCycle.SCHEDULED) experiment2 = ExperimentFactory() ExperimentStatusFactory(experiment=experiment2, status=ExperimentLifeCycle.CREATED) experiment3 = ExperimentFactory() ExperimentStatusFactory(experiment=experiment3, status=ExperimentLifeCycle.FAILED) experiment4 = ExperimentFactory() ExperimentStatusFactory(experiment=experiment4, status=ExperimentLifeCycle.STARTING) experiment5 = ExperimentFactory() ExperimentStatusFactory(experiment=experiment5, status=ExperimentLifeCycle.RUNNING) with patch('scheduler.tasks.experiments' '.experiments_check_heartbeat.apply_async') as mock_fct: heartbeat_experiments() assert mock_fct.call_count == 1
def test_experiments_sync_jobs_statuses(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiments = [ExperimentFactory() for _ in range(3)] done_xp, no_jobs_xp, xp_with_jobs = experiments # Set done status with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa ExperimentStatusFactory(experiment=done_xp, status=JobLifeCycle.FAILED) # Create jobs for xp_with_jobs and update status, and do not update the xp status with patch.object(Experiment, 'set_status') as _: # noqa job = ExperimentJobFactory(experiment=xp_with_jobs) ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING) xp_with_jobs.refresh_from_db() assert xp_with_jobs.last_status is None # Mock sync experiments and jobs constants with patch( 'scheduler.tasks.experiments.' 'experiments_check_status.apply_async') as check_status_mock: experiments_sync_jobs_statuses() assert check_status_mock.call_count == 1 # Call sync experiments and jobs constants with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as build_mock: ExperimentStatusFactory(experiment=xp_with_jobs, status=JobLifeCycle.CREATED) assert build_mock.call_count == 1 experiments_sync_jobs_statuses() done_xp.refresh_from_db() no_jobs_xp.refresh_from_db() xp_with_jobs.refresh_from_db() assert done_xp.last_status == ExperimentLifeCycle.FAILED assert no_jobs_xp.last_status is None assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING
def test_bo_rescheduling(self): with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct: ExperimentGroupFactory(content=experiment_group_spec_content_bo) assert mock_fct.call_count == 1 with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async') as mock_fct1: with patch( 'scheduler.tasks.experiments.experiments_build.apply_async' ) as mock_fct2: ExperimentGroupFactory( content=experiment_group_spec_content_bo) assert mock_fct1.call_count == 1 assert mock_fct2.call_count == 2 # Fake with patch('hpsearch.tasks.bo.hp_bo_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_bo) assert mock_fct.call_count == 1 assert experiment_group.non_done_experiments.count() == 2 # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.bo.hp_bo_iterate.apply_async') as mock_fct1: hp_bo_start(experiment_group.id) assert mock_fct1.call_count == 1 # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.bo.hp_bo_create.apply_async') as mock_fct1: hp_bo_start(experiment_group.id) assert mock_fct1.call_count == 1
def test_spec_creation_triggers_experiments_creations_and_scheduling(self): with patch('runner.hp_search.grid.hp_grid_search_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory() assert Experiment.objects.filter(experiment_group=experiment_group).count() == 2 assert mock_fct.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 0 experiment = Experiment.objects.filter(experiment_group=experiment_group).first() ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert experiment_group.pending_experiments.count() == 1 assert experiment_group.running_experiments.count() == 1 with patch('runner.schedulers.experiment_scheduler.stop_experiment') as _: # noqa ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.SUCCEEDED) assert experiment_group.pending_experiments.count() == 1 assert experiment_group.running_experiments.count() == 0 assert experiment_group.succeeded_experiments.count() == 1 experiment.resume() assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 0 assert experiment_group.succeeded_experiments.count() == 0
def test_spec_creation_triggers_experiments_creations_and_scheduling(self): with patch('experiment_groups.tasks.start_group_experiments.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory() assert Experiment.objects.filter(experiment_group=experiment_group).count() == 2 assert mock_fct.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 0 experiment = Experiment.objects.filter(experiment_group=experiment_group).first() ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert experiment_group.pending_experiments.count() == 1 assert experiment_group.running_experiments.count() == 1
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('projects.tasks.start_group_experiments.apply_async') as _: self.object = self.factory_class(project=project) # Add a running experiment experiment = ExperimentFactory(experiment_group=self.object) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) self.url = '/{}/{}/{}/groups/{}/stop'.format(API_V1, project.user.username, project.name, self.object.sequence)
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async') as mock_fct: self.object = self.factory_class(project=project) assert mock_fct.call_count == 2 # Add a running experiment experiment = ExperimentFactory(experiment_group=self.object) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) self.url = '/{}/{}/{}/groups/{}/stop'.format( API_V1, project.user.username, project.name, self.object.id)
def test_spec_creation_triggers_experiments_creations_and_scheduling( self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory() assert Experiment.objects.filter( experiment_group=experiment_group).count() == 2 assert mock_fct.call_count == 2 assert experiment_group.iteration_config.num_suggestions == 2 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 0 experiment = Experiment.objects.filter( experiment_group=experiment_group).first() ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert experiment_group.pending_experiments.count() == 1 assert experiment_group.running_experiments.count() == 1 with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.SUCCEEDED) assert experiment_group.pending_experiments.count() == 1 assert experiment_group.running_experiments.count() == 0 assert experiment_group.succeeded_experiments.count() == 1 with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as start_build: experiment.resume() assert start_build.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 0 assert experiment_group.succeeded_experiments.count() == 1
def test_stop_pending_experiments(self): with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_early_stopping) experiment = ExperimentFactory(experiment_group=experiment_group) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert mock_fct.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 1 experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True) assert experiment_group.pending_experiments.count() == 0 assert experiment_group.running_experiments.count() == 1
def test_stop_pending_experiments(self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_early_stopping) experiment = ExperimentFactory(experiment_group=experiment_group) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert mock_fct.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 1 experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True) assert experiment_group.pending_experiments.count() == 0 assert experiment_group.running_experiments.count() == 1
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async') as mock_fct: with patch('scheduler.dockerizer_scheduler.create_build_job') as mock_start: build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) mock_start.return_value = build, True, True self.object = self.factory_class(project=project) assert mock_fct.call_count == 1 # Add a running experiment experiment = ExperimentFactory(experiment_group=self.object) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) self.url = '/{}/{}/{}/groups/{}/stop'.format( API_V1, project.user.username, project.name, self.object.id)
def test_sync_experiments_and_jobs_statuses(self): with patch('runner.tasks.experiments.start_experiment.delay' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiments = [ExperimentFactory() for _ in range(3)] done_xp, no_jobs_xp, xp_with_jobs = experiments # Set done status with patch('runner.schedulers.experiment_scheduler.stop_experiment' ) as _: # noqa ExperimentStatusFactory(experiment=done_xp, status=JobLifeCycle.FAILED) # Create jobs for xp_with_jobs and update status, and do not update the xp status with patch.object(Experiment, 'set_status') as _: # noqa job = ExperimentJobFactory(experiment=xp_with_jobs) ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING) xp_with_jobs.refresh_from_db() assert xp_with_jobs.last_status is None # Mock sync experiments and jobs statuses with patch('experiments.tasks.check_experiment_status.delay' ) as check_status_mock: sync_experiments_and_jobs_statuses() assert check_status_mock.call_count == 1 # Call sync experiments and jobs statuses sync_experiments_and_jobs_statuses() done_xp.refresh_from_db() no_jobs_xp.refresh_from_db() xp_with_jobs.refresh_from_db() assert done_xp.last_status == ExperimentLifeCycle.FAILED assert no_jobs_xp.last_status is None assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING
def test_resume(self): experiment = ExperimentFactory() count_experiment = Experiment.objects.count() ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.STOPPED) assert experiment.last_status == ExperimentLifeCycle.STOPPED config = experiment.config declarations = experiment.declarations # Resume with same config experiment.resume() experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations == declarations assert Experiment.objects.count() == count_experiment + 1 assert experiment.clones.count() == 1 # Resume with different config new_declarations = { 'lr': 0.1, 'dropout': 0.5 } new_experiment = experiment.resume(declarations=new_declarations) experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 2 assert experiment.clones.count() == 2 # Resuming a resumed experiment with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa resumed = new_experiment.resume() ExperimentStatusFactory(experiment=resumed, status=ExperimentLifeCycle.CREATED) experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment_new = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment_new.original_experiment.pk != last_resumed_experiment.pk assert (last_resumed_experiment_new.original_experiment.pk == last_resumed_experiment.original_experiment.pk) assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 3 assert experiment.clones.count() == 3 # Deleting a resumed experiment does not delete other experiments last_resumed_experiment_new.set_status(ExperimentLifeCycle.SCHEDULED) ExperimentJobFactory(experiment=last_resumed_experiment_new) with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop: last_resumed_experiment_new.delete() assert experiment.clones.count() == 2 assert mock_stop.call_count == 1 # Deleting original experiment deletes all with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop: experiment.delete() assert Experiment.objects.count() == 0 assert mock_stop.call_count == 0 # No running experiment
def test_hyperband_rescheduling(self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: ExperimentGroupFactory( content=experiment_group_spec_content_hyperband) assert mock_fct.call_count == 2 with patch.object(GroupChecks, 'is_checked') as mock_is_check: with patch( 'hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async' ) as mock_fct1: with patch('scheduler.tasks.experiments.' 'experiments_build.apply_async') as mock_fct2: mock_is_check.return_value = False experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule ) assert experiment_group.iteration_config.num_suggestions == 9 assert mock_fct1.call_count == 2 # 9 experiments, but since we are mocking the scheduling function, it's ~ 3 x calls, # every call to start tries to schedule again, but in reality it's just 9 calls assert mock_fct2.call_count >= 9 * 2 # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) self.assertEqual( mock_fct.call_count, math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1) ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21, 'num_suggestions': 9 }) experiment_group.iteration.experiments.set( experiment_group.experiments.values_list('id', flat=True)) # Mark experiments as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.hyperband.' 'hp_hyperband_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory( experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count( ) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 1 # Fake reduce with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) self.assertEqual( mock_fct.call_count, math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1) assert experiment_group.non_done_experiments.count() == 9 # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa with patch('hpsearch.tasks.hyperband.' 'hp_hyperband_start.apply_async') as xp_trigger_start: for xp in experiment_group.experiments.all(): ExperimentStatusFactory( experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) assert xp_trigger_start.call_count == experiment_group.experiments.count( ) with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct2: with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3: hp_hyperband_start(experiment_group.id) assert mock_fct2.call_count == 1 assert mock_fct3.call_count == 1
def test_hyperband_rescheduling(self): with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: ExperimentGroupFactory( content=experiment_group_spec_content_hyperband) assert mock_fct.call_count == 1 with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async' ) as mock_fct1: with patch( 'scheduler.tasks.experiments.experiments_build.apply_async' ) as mock_fct2: ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct1.call_count == 1 assert mock_fct2.call_count == 9 # Fake reschedule with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 ExperimentGroupIteration.objects.create( experiment_group=experiment_group, data={ 'iteration': 0, 'bracket_iteration': 21 }) # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async' ) as mock_fct1: hp_hyperband_start(experiment_group.id) assert mock_fct1.call_count == 1 # Fake reduce with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content= experiment_group_spec_content_hyperband_trigger_reschedule) assert mock_fct.call_count == 1 assert experiment_group.non_done_experiments.count() == 9 # Mark experiment as done with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa for xp in experiment_group.experiments.all(): ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED) with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async' ) as mock_fct2: with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3: hp_hyperband_start(experiment_group.id) assert mock_fct2.call_count == 1 assert mock_fct3.call_count == 1