def test_experiment_group_deletion_triggers_experiments_deletion(self):
with patch('projects.tasks.start_group_experiments.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory()
assert mock_fct.call_count == 1
assert Experiment.objects.filter(experiment_group=experiment_group).count() == 2
with patch('spawner.scheduler.stop_experiment') as mock_fct:
experiment_group.delete()
assert mock_fct.call_count == 2
assert Experiment.objects.filter(experiment_group=experiment_group).count() == 0
def test_get(self):
resp = self.auth_client.get(self.url)
assert resp.status_code == status.HTTP_200_OK
assert resp.data['next'] is None
assert resp.data['count'] == len(self.objects)
data = resp.data['results']
assert len(data) == self.queryset.count()
assert data == self.serializer_class(self.queryset, many=True).data
# Test other
resp = self.auth_client.get(self.other_url)
assert resp.status_code == status.HTTP_200_OK
independent_count = self.queryset.count()
# Create group to test independent filter
group = ExperimentGroupFactory(project=self.project)
all_experiment_count = self.queryset.all().count()
assert all_experiment_count == independent_count + group.experiments.count(
)
# Getting all experiments
resp = self.auth_client.get(self.url)
assert resp.status_code == status.HTTP_200_OK
assert resp.data['count'] == all_experiment_count
# Getting only independent experiments
resp = self.auth_client.get(self.url + '?independent=true')
assert resp.status_code == status.HTTP_200_OK
assert resp.data['count'] == independent_count
def test_stop_all_experiments(self):
with patch('projects.tasks.start_group_experiments.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
# Add a running experiment
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment,
status=ExperimentLifeCycle.RUNNING)
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
assert experiment_group.experiments.count() == 3
assert experiment_group.stopped_experiments.count() == 0
with patch('schedulers.experiment_scheduler.stop_experiment'
) as spawner_mock_fct:
stop_group_experiments(experiment_group_id=experiment_group.id,
pending=False)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 1
assert spawner_mock_fct.call_count == 1 # Should be stopped with ths function
assert experiment_group.stopped_experiments.count() == 2
def test_experiment_creation_triggers_status_creation(self):
with patch('projects.tasks.start_group_experiments.apply_async') as _:
experiment_group = ExperimentGroupFactory()
experiment = ExperimentFactory(experiment_group=experiment_group)
assert ExperimentStatus.objects.filter(experiment=experiment).count() == 1
assert experiment.last_status == ExperimentLifeCycle.CREATED
def test_spec_creation_triggers_experiments_planning(self):
with patch('projects.tasks.create_group_experiments.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory()
assert Experiment.objects.filter(
experiment_group=experiment_group).count() == 0
assert mock_fct.call_count == 1
def test_experiment_create_a_max_of_experiments(self):
with patch('projects.tasks.start_group_experiments.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
assert experiment_group.specification.matrix_space == 3
assert experiment_group.experiments.count() == 2
def test_experiment_creation_triggers_status_creation_mocks(self):
with patch('projects.tasks.start_group_experiments.apply_async') as _:
experiment_group = ExperimentGroupFactory()
with patch('experiments.tasks.start_experiment.delay') as mock_fct:
with patch.object(Experiment, 'set_status') as mock_fct2:
ExperimentFactory(experiment_group=experiment_group)
assert mock_fct.call_count == 0
assert mock_fct2.call_count == 1
def setUp(self):
super().setUp()
content = """---
version: 1
project:
name: project1
matrix:
lr:
linspace: '1.:3.:3'
model:
model_type: regressor
loss:
MeanSquaredError:
optimizer:
Adam:
learning_rate: "{{ lr }}"
graph:
input_layers: images
layers:
- Conv2D:
filters: 64
kernel_size: [3, 3]
strides: [1, 1]
activation: relu
kernel_initializer: Ones
- MaxPooling2D:
kernels: 2
- Flatten:
- Dense:
units: 10
activation: softmax
train:
data_pipeline:
TFRecordImagePipeline:
batch_size: 64
num_epochs: 1
shuffle: true
dynamic_pad: false
data_files: ["../data/mnist/mnist_train.tfrecord"]
meta_data_file: "../data/mnist/meta_data.json"
"""
with patch('projects.tasks.start_group_experiments.retry') as _:
self.experiment_group = ExperimentGroupFactory(content=content)
assert self.experiment_group.specification.matrix_space == 3
self.url = '/{}/{}/{}/groups/{}/experiments/'.format(
API_V1, self.experiment_group.project.user,
self.experiment_group.project.name, self.experiment_group.sequence)
# one object that does not belong to the filter
self.factory_class()
self.queryset = self.model_class.objects.filter(
experiment_group=self.experiment_group)
def test_spec_creation_triggers_experiments_creations_and_scheduling(self):
with patch('projects.tasks.start_group_experiments.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory()
assert Experiment.objects.filter(experiment_group=experiment_group).count() == 2
assert mock_fct.call_count == 1
assert len(experiment_group.pending_experiments) == 2
assert len(experiment_group.running_experiments) == 0
experiment = Experiment.objects.filter(experiment_group=experiment_group).first()
ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING)
assert len(experiment_group.pending_experiments) == 1
assert len(experiment_group.running_experiments) == 1
def test_stop_pending_experiments(self):
with patch('projects.tasks.start_group_experiments.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
stop_group_experiments(experiment_group_id=experiment_group.id,
pending=True)
assert experiment_group.pending_experiments.count() == 0
def setUp(self):
super().setUp()
self.object = self.factory_class(user=self.auth_client.user)
self.url = '/{}/{}/{}/'.format(API_V1, self.object.user.username, self.object.name)
self.queryset = self.model_class.objects.filter(user=self.object.user)
# Create related fields
for i in range(2):
ExperimentGroupFactory(project=self.object)
# creating the default factory should trigger the creation of 2 experiments per group
assert Experiment.objects.count() == 4
# Other user objects
self.other_object = self.factory_class()
self.url_other = '/{}/{}/{}/'.format(API_V1,
self.other_object.user.username,
self.other_object.name)
# One private project
self.private = self.factory_class(is_public=False)
self.url_private = '/{}/{}/{}/'.format(API_V1,
self.private.user.username,
self.private.name)
def test_experiment_group_should_stop_early(self):
with patch('projects.tasks.start_group_experiments.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
assert mock_fct.call_count == 1
assert experiment_group.should_stop_early() is False
assert experiment_group.pending_experiments.count() == 2
# Make a metric for one of the experiments
experiment1, experiment2 = list(experiment_group.experiments.all())
metric1 = ExperimentMetric.objects.create(experiment=experiment1,
values={'precision': 0.99})
# Check again that early stopping works
assert experiment_group.should_stop_early() is True
# Add another metric
metric2 = ExperimentMetric.objects.create(experiment=experiment2,
values={'loss': 0.01})
# Check again that early stopping still works
assert experiment_group.should_stop_early() is True
# Delete metric1
metric1.delete()
# Check again that early stopping still works
assert experiment_group.should_stop_early() is True
# Delete metric2
metric2.delete()
# Check again that early stopping still works
assert experiment_group.should_stop_early() is False
def setUp(self):
super().setUp()
with patch('projects.tasks.start_group_experiments.apply_async') as _:
self.experiment_group = ExperimentGroupFactory()
self.project = self.experiment_group.project