def test_stop_all_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
# Add a running experiment
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment,
status=ExperimentLifeCycle.RUNNING)
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
assert experiment_group.experiments.count() == 3
assert experiment_group.stopped_experiments.count() == 0
with patch('scheduler.experiment_scheduler.stop_experiment'
) as spawner_mock_fct:
experiments_group_stop_experiments(
experiment_group_id=experiment_group.id, pending=False)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 0
assert spawner_mock_fct.call_count == 1 # Should be stopped with this function
assert experiment_group.stopped_experiments.count() == 3
def test_stop_all_experiments(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 2
# Add a running experiment
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment,
status=ExperimentLifeCycle.RUNNING)
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
assert experiment_group.experiments.count() == 3
assert experiment_group.stopped_experiments.count() == 0
with patch('scheduler.experiment_scheduler.stop_experiment'
) as spawner_mock_fct:
with patch(
'logs_handlers.collectors.'
'logs_collect_experiment_jobs') as logs_collector_mock_fct:
experiments_group_stop_experiments(
experiment_group_id=experiment_group.id, pending=False)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 0
assert spawner_mock_fct.call_count == 1 # Should be stopped with this function
assert logs_collector_mock_fct.call_count == 1
assert experiment_group.stopped_experiments.count() == 3
def _clean():
for experiment_group in ExperimentGroup.objects.filter(
status__status__in=ExperimentGroupLifeCycle.RUNNING_STATUS):
experiments_group_stop_experiments(
experiment_group_id=experiment_group.id,
pending=True,
message='Stop triggered by the cleaning hook.')
def _clean():
for experiment_group in ExperimentGroup.objects.filter(
status__status__in=ExperimentGroupLifeCycle.RUNNING_STATUS):
experiments_group_stop_experiments(
experiment_group_id=experiment_group.id,
pending=True,
message='Cleanup')
def test_stop_pending_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True)
assert experiment_group.pending_experiments.count() == 0
def test_stop_pending_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True)
assert experiment_group.pending_experiments.count() == 0
def test_stop_pending_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 1
def test_stop_pending_experiments(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
experiments_group_stop_experiments(experiment_group_id=experiment_group.id, pending=True)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 1
def test_stop_all_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
# Add a running experiment
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING)
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
assert experiment_group.experiments.count() == 3
assert experiment_group.stopped_experiments.count() == 0
with patch('scheduler.experiment_scheduler.stop_experiment') as spawner_mock_fct:
experiments_group_stop_experiments(experiment_group_id=experiment_group.id,
pending=False)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 1
assert spawner_mock_fct.call_count == 1 # Should be stopped with ths function
assert experiment_group.stopped_experiments.count() == 2
