def test_resume(self):
experiment = ExperimentFactory()
count_experiment = Experiment.objects.count()
ExperimentStatus.objects.create(experiment=experiment,
status=ExperimentLifeCycle.STOPPED)
assert experiment.last_status == ExperimentLifeCycle.STOPPED
config = experiment.config
declarations = experiment.declarations
# Resume with same config
experiment.resume()
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.RESUMING
assert experiment.config == config
assert experiment.declarations == declarations
assert Experiment.objects.count() == count_experiment
ExperimentStatus.objects.create(experiment=experiment,
status=ExperimentLifeCycle.STOPPED)
assert experiment.last_status == ExperimentLifeCycle.STOPPED
# Resume with different config
new_declarations = {'lr': 0.1, 'dropout': 0.5}
experiment.resume(declarations=new_declarations)
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.RESUMING
assert Experiment.objects.count() == count_experiment
assert experiment.config == config
assert experiment.declarations != declarations
assert experiment.declarations == new_declarations
def test_resume(self):
experiment = ExperimentFactory()
count_experiment = Experiment.objects.count()
ExperimentStatus.objects.create(experiment=experiment,
status=ExperimentLifeCycle.STOPPED)
assert experiment.last_status == ExperimentLifeCycle.STOPPED
config = experiment.config
declarations = experiment.declarations
# Resume with same config
experiment.resume()
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations == declarations
assert Experiment.objects.count() == count_experiment + 1
assert experiment.clones.count() == 1
# Resume with different config
new_declarations = {'lr': 0.1, 'dropout': 0.5}
new_experiment = experiment.resume(declarations=new_declarations)
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations != declarations
assert last_resumed_experiment.declarations == new_declarations
assert Experiment.objects.count() == count_experiment + 2
assert experiment.clones.count() == 2
# Resuming a resumed experiment
new_experiment.resume()
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment_new = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment_new.original_experiment.pk != last_resumed_experiment.pk
assert (last_resumed_experiment_new.original_experiment.pk ==
last_resumed_experiment.original_experiment.pk)
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations != declarations
assert last_resumed_experiment.declarations == new_declarations
assert Experiment.objects.count() == count_experiment + 3
assert experiment.clones.count() == 3
# Deleting a resumed experiment does not delete other experiments
last_resumed_experiment_new.delete()
assert experiment.clones.count() == 2
# Deleting original experiment deletes all
experiment.delete()
assert Experiment.objects.count() == 0
def test_resume(self):
experiment = ExperimentFactory()
count_experiment = Experiment.objects.count()
ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.STOPPED)
assert experiment.last_status == ExperimentLifeCycle.STOPPED
config = experiment.config
declarations = experiment.declarations
# Resume with same config
experiment.resume()
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations == declarations
assert Experiment.objects.count() == count_experiment + 1
assert experiment.clones.count() == 1
# Resume with different config
new_declarations = {
'lr': 0.1,
'dropout': 0.5
}
new_experiment = experiment.resume(declarations=new_declarations)
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations != declarations
assert last_resumed_experiment.declarations == new_declarations
assert Experiment.objects.count() == count_experiment + 2
assert experiment.clones.count() == 2
# Resuming a resumed experiment
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa
resumed = new_experiment.resume()
ExperimentStatusFactory(experiment=resumed, status=ExperimentLifeCycle.CREATED)
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STOPPED
last_resumed_experiment_new = experiment.clones.filter(
cloning_strategy=CloningStrategy.RESUME).last()
assert last_resumed_experiment_new.original_experiment.pk != last_resumed_experiment.pk
assert (last_resumed_experiment_new.original_experiment.pk ==
last_resumed_experiment.original_experiment.pk)
assert last_resumed_experiment.config == config
assert last_resumed_experiment.declarations != declarations
assert last_resumed_experiment.declarations == new_declarations
assert Experiment.objects.count() == count_experiment + 3
assert experiment.clones.count() == 3
# Deleting a resumed experiment does not delete other experiments
last_resumed_experiment_new.set_status(ExperimentLifeCycle.SCHEDULED)
ExperimentJobFactory(experiment=last_resumed_experiment_new)
with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop:
last_resumed_experiment_new.delete()
assert experiment.clones.count() == 2
assert mock_stop.call_count == 1
# Deleting original experiment deletes all
with patch('scheduler.experiment_scheduler.stop_experiment') as mock_stop:
experiment.delete()
assert Experiment.objects.count() == 0
assert mock_stop.call_count == 0 # No running experiment
