def test_experiment_group_deletion_triggers_stopping_for_running_experiment(
self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory()
assert mock_fct.call_count == 2
experiment = ExperimentFactory(project=experiment_group.project,
experiment_group=experiment_group)
# Set this experiment to scheduled
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert Experiment.objects.filter(
experiment_group=experiment_group).count() == 3
with patch('scheduler.tasks.experiments.experiments_stop.apply_async'
) as mock_fct:
experiment_group.delete()
assert mock_fct.call_count == 1 # Only one experiment was stopped
assert Experiment.objects.filter(
experiment_group=experiment_group).count() == 0
def test_delete_triggers_stopping_of_experiments(self):
assert self.queryset.count() == 1
assert ExperimentGroup.objects.count() == 2
# Add experiment
experiment = ExperimentFactory(project=self.object)
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert Experiment.objects.count() == 3
with patch('scheduler.tasks.experiments.experiments_stop.apply_async'
) as xp_mock_stop:
resp = self.auth_client.delete(self.url)
assert xp_mock_stop.call_count == 1
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.queryset.count() == 0
assert ExperimentGroup.objects.count() == 0
assert Experiment.objects.count() == 0
# Delete does not work for other project public and private
resp = self.auth_client.delete(self.url_other)
assert resp.status_code in (status.HTTP_401_UNAUTHORIZED,
status.HTTP_403_FORBIDDEN)
resp = self.auth_client.delete(self.url_private)
assert resp.status_code in (status.HTTP_401_UNAUTHORIZED,
status.HTTP_403_FORBIDDEN)
def test_experiment_group_deletion_triggers_stopping_for_running_experiment(
self):
with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory()
assert mock_fct.call_count == 1
experiment = ExperimentFactory(project=experiment_group.project,
experiment_group=experiment_group)
# Set this experiment to scheduled
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert Experiment.objects.filter(
experiment_group=experiment_group).count() == 3
with patch('scheduler.tasks.experiments.experiments_stop.apply_async'
) as mock_fct:
experiment_group.delete()
assert mock_fct.call_count == 1 # Only one experiment was stopped
assert Experiment.objects.filter(
experiment_group=experiment_group).count() == 0
def test_delete_experiment_triggers_experiment_stop_mocks(self, delete_path):
experiment = ExperimentFactory()
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert delete_path.call_count == 2 # outputs + logs
with patch('scheduler.experiment_scheduler.stop_experiment') as mock_fct:
experiment.delete()
assert delete_path.call_count == 2 + 2 # outputs + logs
assert mock_fct.call_count == 1
def test_status_update_results_in_new_updated_at_datetime_experiment(self):
experiment = ExperimentFactory()
updated_at = experiment.updated_at
# Create new status
ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.STARTING)
experiment.refresh_from_db()
assert updated_at < experiment.updated_at
updated_at = experiment.updated_at
# Create status Using set_status
experiment.set_status(ExperimentLifeCycle.FAILED)
experiment.refresh_from_db()
assert updated_at < experiment.updated_at
def test_delete_triggers_stopping_of_experiments(self):
assert self.queryset.count() == 1
assert ExperimentGroup.objects.count() == 2
# Add experiment
experiment = ExperimentFactory(project=self.object)
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert Experiment.objects.count() == 3
with patch('scheduler.tasks.experiments.experiments_stop.apply_async') as xp_mock_stop:
resp = self.auth_client.delete(self.url)
assert xp_mock_stop.called
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.queryset.count() == 0
assert Experiment.all.count() == 0
def test_delete_archives_and_schedules_deletion(self):
assert self.model_class.objects.count() == 1
experiment = ExperimentFactory(project=self.object.project,
experiment_group=self.object)
assert Experiment.objects.count() == 3
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
with patch('scheduler.tasks.experiment_groups.'
'experiments_group_schedule_deletion.apply_async') as scheduler_mock:
resp = self.auth_client.delete(self.url)
assert scheduler_mock.call_count == 1
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.model_class.objects.count() == 0
assert self.model_class.all.count() == 1
assert Experiment.all.count() == 3
assert Experiment.objects.count() == 0
def test_archive_schedule_deletion(self):
assert self.model_class.objects.count() == 1
experiment = ExperimentFactory(project=self.object.project,
experiment_group=self.object)
assert Experiment.objects.count() == 3
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
with patch('scheduler.tasks.experiment_groups.'
'experiments_group_schedule_deletion.apply_async') as spawner_mock_stop:
resp = self.auth_client.post(self.url + 'archive/')
assert resp.status_code == status.HTTP_200_OK
assert spawner_mock_stop.call_count == 1
assert self.model_class.objects.count() == 1
assert self.model_class.all.count() == 1
assert Experiment.all.count() == 3
assert Experiment.objects.count() == 3
def test_delete(self):
assert self.model_class.objects.count() == 1
assert Experiment.objects.count() == 2
experiment = ExperimentFactory(project=self.object.project,
experiment_group=self.object)
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
with patch('scheduler.tasks.experiments.experiments_stop.apply_async') as scheduler_mock:
with patch('libs.paths.experiments.delete_path') as outputs_mock_stop:
resp = self.auth_client.delete(self.url)
assert outputs_mock_stop.call_count == 6 # Outputs and Logs * 3
assert scheduler_mock.call_count == 1
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.model_class.objects.count() == 0
assert Experiment.objects.count() == 0
def test_archive_triggers_stopping_of_experiment_groups(self):
assert self.queryset.count() == 1
assert ExperimentGroup.objects.count() == 2
experiment_group = ExperimentGroup.objects.first()
# Add running experiment
experiment = ExperimentFactory(project=experiment_group.project,
experiment_group=experiment_group)
# Set one experiment to running with one job
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert Experiment.objects.count() == 3
with patch('scheduler.tasks.experiments.experiments_stop.apply_async') as xp_mock_stop:
resp = self.auth_client.post(self.url + 'archive/')
assert xp_mock_stop.call_count == 1
assert resp.status_code == status.HTTP_200_OK
assert self.queryset.count() == 0
assert ExperimentGroup.objects.count() == 0
assert ExperimentGroup.all.count() == 2
assert Experiment.objects.count() == 0
assert Experiment.all.count() == 3
