def test_delete_experiment_does_not_trigger_experiment_stop_if_not_running(self, delete_path): experiment = ExperimentFactory() assert delete_path.call_count == 2 # outputs + logs with patch('scheduler.experiment_scheduler.stop_experiment') as mock_fct: experiment.delete() assert delete_path.call_count == 2 + 2 # outputs + logs assert mock_fct.call_count == 0
def test_delete_experiment_triggers_experiment_stop_mocks_runner(self): experiment = ExperimentFactory() with patch('runner.schedulers.experiment_scheduler.stop_experiment' ) as mock_fct: experiment.delete() assert mock_fct.call_count == 1
def test_resume(self): experiment = ExperimentFactory() count_experiment = Experiment.objects.count() ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.STOPPED) assert experiment.last_status == ExperimentLifeCycle.STOPPED config = experiment.config declarations = experiment.declarations # Resume with same config experiment.resume() experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations == declarations assert Experiment.objects.count() == count_experiment + 1 assert experiment.clones.count() == 1 # Resume with different config new_declarations = {'lr': 0.1, 'dropout': 0.5} new_experiment = experiment.resume(declarations=new_declarations) experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 2 assert experiment.clones.count() == 2 # Resuming a resumed experiment new_experiment.resume() experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment_new = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment_new.original_experiment.pk != last_resumed_experiment.pk assert (last_resumed_experiment_new.original_experiment.pk == last_resumed_experiment.original_experiment.pk) assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 3 assert experiment.clones.count() == 3 # Deleting a resumed experiment does not delete other experiments last_resumed_experiment_new.delete() assert experiment.clones.count() == 2 # Deleting original experiment deletes all experiment.delete() assert Experiment.objects.count() == 0
def test_delete_experiment_triggers_experiment_stop_mocks(self, delete_path): experiment = ExperimentFactory() experiment.set_status(ExperimentLifeCycle.SCHEDULED) # Add job ExperimentJobFactory(experiment=experiment) assert delete_path.call_count == 2 # outputs + logs with patch('scheduler.experiment_scheduler.stop_experiment') as mock_fct: experiment.delete() assert delete_path.call_count == 2 + 2 # outputs + logs assert mock_fct.call_count == 1
def test_resume(self): experiment = ExperimentFactory() count_experiment = Experiment.objects.count() ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.STOPPED) assert experiment.last_status == ExperimentLifeCycle.STOPPED config = experiment.config declarations = experiment.declarations # Resume with same config experiment.resume() experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations == declarations assert Experiment.objects.count() == count_experiment + 1 assert experiment.clones.count() == 1 # Resume with different config new_declarations = { 'lr': 0.1, 'dropout': 0.5 } new_experiment = experiment.resume(declarations=new_declarations) experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 2 assert experiment.clones.count() == 2 # Resuming a resumed experiment with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa resumed = new_experiment.resume() ExperimentStatusFactory(experiment=resumed, status=ExperimentLifeCycle.CREATED) experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STOPPED last_resumed_experiment_new = experiment.clones.filter( cloning_strategy=CloningStrategy.RESUME).last() assert last_resumed_experiment_new.original_experiment.pk != last_resumed_experiment.pk assert (last_resumed_experiment_new.original_experiment.pk == last_resumed_experiment.original_experiment.pk) assert last_resumed_experiment.config == config assert last_resumed_experiment.declarations != declarations assert last_resumed_experiment.declarations == new_declarations assert Experiment.objects.count() == count_experiment + 3 assert experiment.clones.count() == 3 # Deleting a resumed experiment does not delete other experiments last_resumed_experiment_new.set_status(ExperimentLifeCycle.SCHEDULED) ExperimentJobFactory(experiment=last_resumed_experiment_new) with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop: last_resumed_experiment_new.delete() assert experiment.clones.count() == 2 assert mock_stop.call_count == 1 # Deleting original experiment deletes all with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop: experiment.delete() assert Experiment.objects.count() == 0 assert mock_stop.call_count == 0 # No running experiment
def test_delete_experiment_triggers_experiment_stop_mocks( self, delete_path): experiment = ExperimentFactory() assert delete_path.call_count == 2 # outputs + logs experiment.delete() assert delete_path.call_count == 2 + 2 # outputs + logs