def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async'
) as mock_fct:
self.object = self.factory_class(project=project)
assert mock_fct.call_count == 1
# Add a running experiment
experiment = ExperimentFactory(experiment_group=self.object)
ExperimentStatusFactory(experiment=experiment,
status=ExperimentLifeCycle.RUNNING)
self.url = '/{}/{}/{}/groups/{}/stop'.format(API_V1,
project.user.username,
project.name,
self.object.id)
def setUp(self):
super().setUp()
with patch.object(Experiment, 'set_status') as _: # noqa
with patch('runner.tasks.experiments.start_experiment.delay'
) as _: # noqa
project = ProjectFactory(user=self.auth_client.user)
self.experiment = ExperimentFactory(project=project)
self.url = '/{}/{}/{}/experiments/{}/metrics/'.format(
API_V1, project.user.username, project.name,
self.experiment.sequence)
self.objects = [
self.factory_class(experiment=self.experiment,
values={'accuracy': i / 10})
for i in range(self.num_objects)
]
self.queryset = self.model_class.objects.all()
def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
with patch('runner.hp_search.grid.hp_grid_search_start.apply_async') as mock_fct:
self.object = self.factory_class(project=project)
assert mock_fct.call_count == 1
self.url = '/{}/{}/{}/groups/{}/'.format(API_V1,
project.user.username,
project.name,
self.object.sequence)
self.queryset = self.model_class.objects.all()
# Add 2 more experiments
for _ in range(2):
ExperimentFactory(experiment_group=self.object)
def test_non_independent_experiment_creation_doesnt_trigger_start(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.hp_create.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory()
assert mock_fct.call_count == 1
with patch('scheduler.tasks.experiments.experiments_start.apply_async') as mock_fct:
with patch.object(Experiment, 'set_status') as mock_fct2:
ExperimentFactory(experiment_group=experiment_group)
assert mock_fct.call_count == 0
assert mock_fct2.call_count == 1
def setUp(self):
super().setUp()
with patch.object(Experiment, 'set_status') as _:
with patch('runner.tasks.experiments.start_experiment.delay'
) as _: # noqa
project = ProjectFactory(user=self.auth_client.user)
self.experiment = ExperimentFactory(project=project)
self.url = '/{}/{}/{}/experiments/{}/statuses/'.format(
API_V1, project.user.username, project.name,
self.experiment.sequence)
self.objects = [
self.factory_class(experiment=self.experiment,
status=ExperimentLifeCycle.CHOICES[i][0])
for i in range(self.num_objects)
]
self.queryset = self.model_class.objects.all()
def setUp(self):
super().setUp()
with patch('experiments.tasks.start_experiment.delay') as _:
with patch.object(ExperimentJob, 'set_status') as _:
project = ProjectFactory(user=self.auth_client.user)
experiment = ExperimentFactory(project=project)
self.experiment_job = ExperimentJobFactory(experiment=experiment)
self.object = self.factory_class(job=self.experiment_job)
self.url = '/{}/{}/{}/experiments/{}/jobs/{}/statuses/{}'.format(
API_V1,
project.user.username,
project.name,
experiment.sequence,
self.experiment_job.sequence,
self.object.uuid.hex)
self.queryset = self.model_class.objects.filter(job=self.experiment_job)
def test_restore(self):
ExperimentFactory(project=self.object.project, experiment_group=self.object)
assert Experiment.objects.count() == 3
self.object.archive()
assert self.model_class.objects.count() == 0
assert self.model_class.all.count() == 1
assert Experiment.all.count() == 3
assert Experiment.objects.count() == 0
resp = self.auth_client.post(self.url + 'restore/')
assert resp.status_code == status.HTTP_200_OK
assert self.model_class.objects.count() == 1
assert self.model_class.all.count() == 1
assert Experiment.all.count() == 3
assert Experiment.objects.count() == 3
def test_delete_remove_paths(self, delete_outputs_path, delete_logs_path):
project = ProjectFactory()
for _ in range(2):
ExperimentGroupFactory(project=project)
ExperimentFactory(project=project)
assert ExperimentGroup.objects.count() == 2
assert Experiment.objects.count() == 2
with patch('libs.paths.projects.delete_path'
) as delete_path_project_mock_stop:
project.delete()
# 1 repo
assert delete_path_project_mock_stop.call_count == 1
# 1 project + 2 * groups + 2 experiments
assert delete_outputs_path.call_count == 5
assert delete_logs_path.call_count == 5
def test_experiment_group_outputs_path_creation_deletion(self):
with patch('experiments.tasks.build_experiment.apply_async') as _:
experiment = ExperimentFactory(
user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
create_experiment_outputs_path(experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(
experiment.unique_name)
experiment_group_outputs_path = get_experiment_group_outputs_path(
self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(experiment_group_outputs_path) is True
delete_experiment_group_outputs(self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(experiment_group_outputs_path) is False
def test_experiment_group_logs_path_creation_deletion(self):
with patch('experiments.tasks.build_experiment.apply_async') as _:
experiment = ExperimentFactory(
user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
experiment_logs_path = get_experiment_logs_path(experiment.unique_name)
open(experiment_logs_path, '+w')
experiment_group_logs_path = get_experiment_group_logs_path(
self.experiment_group.unique_name)
# Should be true, created by the signal
assert os.path.exists(experiment_logs_path) is True
assert os.path.exists(experiment_group_logs_path) is True
delete_experiment_group_logs(self.experiment_group.unique_name)
assert os.path.exists(experiment_logs_path) is False
assert os.path.exists(experiment_group_logs_path) is False
def test_project_logs_path_creation_deletion(self):
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa
experiment = ExperimentFactory(user=self.project.user, project=self.project)
experiment_logs_path = get_experiment_logs_path(experiment.unique_name)
create_experiment_logs_path(experiment.unique_name)
open(experiment_logs_path, '+w')
project_logs_path = get_project_logs_path(self.project.unique_name)
project_repos_path = get_project_logs_path(self.project.unique_name)
# Should be true, created by the signal
assert os.path.exists(experiment_logs_path) is True
assert os.path.exists(project_logs_path) is True
assert os.path.exists(project_repos_path) is True
delete_project_logs(self.project.unique_name)
assert os.path.exists(experiment_logs_path) is False
assert os.path.exists(project_logs_path) is False
assert os.path.exists(project_repos_path) is False
def test_set_job_definition(self):
experiment = ExperimentFactory()
job_uuid = uuid.uuid4().hex
create_job(job_uuid=job_uuid,
experiment=experiment)
job = ExperimentJob.objects.last()
assert job.role == TaskType.MASTER
assert job.resources is None
assert job.node_selector is None
assert job.affinity is None
assert job.tolerations is None
assert job.definition == {}
definition = {'spec': {}}
set_job_definition(job_uuid=job_uuid, definition=definition)
job = ExperimentJob.objects.last()
assert job.definition == definition
def test_delete_experiment_triggers_experiment_stop_mocks(self, delete_path):
experiment = ExperimentFactory()
experiment.set_status(ExperimentLifeCycle.SCHEDULED)
# Add job
ExperimentJobFactory(experiment=experiment)
assert delete_path.call_count == 2 # outputs + logs
with patch('scheduler.experiment_scheduler.stop_experiment') as mock_fct:
experiment.delete()
assert delete_path.call_count == 2 + 2 # outputs + logs
assert mock_fct.call_count == 1
def setUp(self):
super().setUp()
with patch('experiments.tasks.start_experiment.delay') as _:
with patch.object(ExperimentJob, 'set_status') as _:
project = ProjectFactory(user=self.auth_client.user)
experiment = ExperimentFactory(project=project)
self.experiment_job = ExperimentJobFactory(experiment=experiment)
self.url = '/{}/{}/{}/experiments/{}/jobs/{}/statuses/'.format(
API_V1,
project.user.username,
project.name,
experiment.sequence,
self.experiment_job.sequence)
self.objects = [self.factory_class(job=self.experiment_job,
status=JobLifeCycle.CHOICES[i][0])
for i in range(self.num_objects)]
self.queryset = self.model_class.objects.filter(job=self.experiment_job)
def test_build_job_statuses(self):
assert BuildJobStatus.objects.count() == 0
experiment = ExperimentFactory(project=self.project)
build_job, rebuild = BuildJob.create(
user=experiment.user,
project=experiment.project,
config=experiment.specification.build,
code_reference=self.code_reference)
self.assertEqual(rebuild, True)
assert build_job.last_status == JobLifeCycle.CREATED
assert BuildJobStatus.objects.count() == 1
build_job.set_status(JobLifeCycle.FAILED)
assert BuildJobStatus.objects.count() == 2
build_job.set_status(JobLifeCycle.SUCCEEDED)
assert BuildJobStatus.objects.count() == 2
def test_archive(self):
project = ProjectFactory()
ExperimentGroupFactory(project=project)
ExperimentFactory(project=project)
JobFactory(project=project)
BuildJobFactory(project=project)
NotebookJobFactory(project=project)
TensorboardJobFactory(project=project)
assert project.deleted is False
assert project.experiments.count() == 1
assert project.experiment_groups.count() == 1
assert project.jobs.count() == 1
assert project.build_jobs.count() == 1
assert project.notebook_jobs.count() == 1
assert project.tensorboard_jobs.count() == 1
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
project.archive()
assert project.deleted is True
assert project.experiments.count() == 0
assert project.experiment_groups.count() == 0
assert project.jobs.count() == 0
assert project.build_jobs.count() == 0
assert project.notebook_jobs.count() == 0
assert project.tensorboard_jobs.count() == 0
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
project.unarchive()
assert project.deleted is False
assert project.experiments.count() == 1
assert project.experiment_groups.count() == 1
assert project.jobs.count() == 1
assert project.build_jobs.count() == 1
assert project.notebook_jobs.count() == 1
assert project.tensorboard_jobs.count() == 1
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
def test_experiment_group_logs_path_creation_deletion(self):
experiment = ExperimentFactory(user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
experiment_logs_path = stores.get_experiment_logs_path(
experiment_name=experiment.unique_name,
temp=False)
stores.create_experiment_logs_path(experiment_name=experiment.unique_name, temp=False)
open(experiment_logs_path, '+w')
experiment_group_logs_path = stores.get_experiment_group_logs_path(
experiment_group_name=self.experiment_group.unique_name)
# Should be true, created by the signal
assert os.path.exists(experiment_logs_path) is True
assert os.path.exists(experiment_group_logs_path) is True
stores_schedule_logs_deletion(persistence=None, subpath=self.experiment_group.subpath)
assert os.path.exists(experiment_logs_path) is False
assert os.path.exists(experiment_group_logs_path) is False
def test_delete_remove_paths(self):
project = ProjectFactory()
for _ in range(2):
ExperimentGroupFactory(project=project)
ExperimentFactory(project=project)
assert ExperimentGroup.objects.count() == 2
assert Experiment.objects.count() == 2
with patch('libs.paths.projects.delete_path') as delete_path_project_mock_stop:
with patch('libs.paths.experiment_groups.delete_path') as delete_path_group_mock_stop:
with patch('libs.paths.experiments.delete_path') as delete_path_xp_mock_stop:
project.delete()
# 2 * project + 1 repo
assert delete_path_project_mock_stop.call_count == 3
# 2 * 2 * groups
assert delete_path_group_mock_stop.call_count
assert delete_path_xp_mock_stop.call_count == 4 # 2 * 2 * groups
def setUp(self):
super().setUp()
self.user = UserFactory()
activitylogs.validate()
activitylogs.setup()
self.project = ProjectFactory()
activitylogs.record(event_type=PROJECT_DELETED_TRIGGERED,
instance=self.project,
actor_id=self.user.id)
self.experiment = ExperimentFactory()
activitylogs.record(event_type=EXPERIMENT_DELETED_TRIGGERED,
instance=self.experiment,
actor_id=self.user.id)
self.job = JobFactory()
activitylogs.record(event_type=JOB_VIEWED,
instance=self.job,
actor_id=self.user.id)
def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
experiment = ExperimentFactory(project=project)
self.url = '/{}/{}/{}/experiments/{}/logs'.format(
API_V1, project.user.username, project.name, experiment.sequence)
log_path = get_experiment_logs_path(experiment.unique_name)
create_experiment_logs_path(experiment_name=experiment.unique_name)
fake = Faker()
self.logs = []
for _ in range(self.num_log_lines):
self.logs.append(fake.sentence())
with open(log_path, 'w') as file:
for line in self.logs:
file.write(line)
file.write('\n')
def test_get_experiments_metrics(self):
experiment_group = ExperimentGroupFactory()
assert len(experiment_group.get_experiments_metrics( # pylint:disable=len-as-condition
experiment_ids=[],
metric='precision'
)) == 0
experiments = []
experiment_ids = []
for _ in range(5):
experiment = ExperimentFactory(experiment_group=experiment_group)
experiments.append(experiment)
experiment_ids.append(experiment.id)
ExperimentMetric.objects.create(experiment=experiment,
values={'precision': random.random()})
for experiment in experiments[:3]:
ExperimentMetric.objects.create(experiment=experiment,
values={'loss': random.random()})
experiment_metrics = experiment_group.get_experiments_metrics(
experiment_ids=experiment_ids,
metric='precision'
)
assert len(experiment_metrics) == 5
metrics = [m[1] for m in experiment_metrics if m[1] is not None]
assert len(metrics) == 2
experiment_metrics = experiment_group.get_experiments_metrics(
experiment_ids=experiment_ids,
metric='loss'
)
assert len(experiment_metrics) == 5
metrics = [m[1] for m in experiment_metrics if m[1] is not None]
assert len(metrics) == 3
experiment_metrics = experiment_group.get_experiments_metrics(
experiment_ids=experiment_ids,
metric='accuracy'
)
assert len(experiment_metrics) == 5
assert len( # pylint:disable=len-as-condition
[m for m in experiment_metrics if m[1] is not None]) == 0
def test_create_with_valid_group(self):
resp = self.auth_client.post(self.url)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
content = """---
version: 1
kind: group
hptuning:
concurrency: 3
matrix:
lr:
values: [0.1, 0.2, 0.3]
run:
cmd: my_command --lr={{ lr }}"""
data = {'content': content, 'description': 'new-deep'}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_201_CREATED
assert self.queryset.count() == self.num_objects + 1
last_object = self.model_class.objects.last()
assert last_object.project == self.project
assert last_object.content == data['content']
assert last_object.hptuning is not None
assert last_object.hptuning['concurrency'] == 3
assert last_object.hptuning['matrix']['lr'] is not None
assert last_object.is_study is True
# Creating a study with selection ignores selection
experiments = [
ExperimentFactory(project=self.project) for _ in range(2)
]
experiment_ids = [xp.id for xp in experiments]
data = {
'content': content,
'description': 'new-deep',
'experiment_ids': experiment_ids
}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_201_CREATED
assert self.queryset.count() == self.num_objects + 2
last_object = self.model_class.objects.last()
assert last_object.is_study is True
assert last_object.selection_experiments.count() == 0
def test_should_stop_early(self, _):
# Experiment group with no early stopping
experiment_group = ExperimentGroupFactory()
assert experiment_group.should_stop_early() is False
# Experiment group with early stopping
experiment_group = ExperimentGroupFactory(content=None,
hptuning={
'concurrency':
2,
'random_search': {
'n_experiments': 10
},
'early_stopping': [{
'metric':
'precision',
'value':
0.9,
'optimization':
'maximize'
}],
'matrix': {
'lr': {
'values':
[1, 2, 3]
}
}
})
assert experiment_group.should_stop_early() is False
# Create experiments and metrics
experiments = [
ExperimentFactory(experiment_group=experiment_group)
for _ in range(2)
]
ExperimentMetric.objects.create(experiment=experiments[0],
values={'precision': 0.8})
assert experiment_group.should_stop_early() is False
# Create a metric that triggers early stopping
ExperimentMetric.objects.create(experiment=experiments[0],
values={'precision': 0.91})
assert experiment_group.should_stop_early() is True
def test_redirects_to_proxy_protected_url(self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
experiment = ExperimentFactory(project=project)
tensorboard = TensorboardJobFactory(project=project,
experiment=experiment)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = JOB_NAME.format(job_uuid=tensorboard.uuid.hex,
name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
response = self.auth_client.get(self._get_url(project, experiment))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/'.format(service_url)
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER],
proxy_url)
def test_stop_pending_experiments(self):
with patch('hpsearch.tasks.random.hp_random_search_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_early_stopping)
experiment = ExperimentFactory(experiment_group=experiment_group)
ExperimentStatusFactory(experiment=experiment,
status=ExperimentLifeCycle.RUNNING)
assert mock_fct.call_count == 1
assert experiment_group.pending_experiments.count() == 2
assert experiment_group.running_experiments.count() == 1
experiments_group_stop_experiments(
experiment_group_id=experiment_group.id, pending=True)
assert experiment_group.pending_experiments.count() == 0
assert experiment_group.running_experiments.count() == 1
def test_from_event_data(self):
instance = ExperimentFactory()
event = ExperimentSucceededEvent.from_instance(instance=instance,
actor_id=1,
actor_name='user')
assert event.ref_id is None
event_serialized = event.serialize(dumps=False, include_instance_info=True)
assert event_serialized.get('ref_id') is None
new_event = ExperimentSucceededEvent.from_event_data(event_data=event_serialized)
assert new_event.serialize(include_instance_info=True) == event_serialized
# Add ref id
event.ref_id = uuid1()
event_serialized = event.serialize(dumps=False, include_instance_info=True)
assert event_serialized['ref_id'] == event.ref_id.hex
new_event = ExperimentSucceededEvent.from_event_data(event_data=event_serialized)
assert new_event.ref_id == event.ref_id
assert new_event.serialize(include_instance_info=True) == event_serialized
def test_experiment_group_outputs_path_creation_deletion(self):
experiment = ExperimentFactory(user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
stores.create_experiment_outputs_path(
persistence=experiment.persistence_outputs,
experiment_name=experiment.unique_name)
experiment_outputs_path = stores.get_experiment_outputs_path(
persistence=experiment.persistence_outputs,
experiment_name=experiment.unique_name)
experiment_group_outputs_path = stores.get_experiment_group_outputs_path(
persistence=self.experiment_group.persistence_outputs,
experiment_group_name=self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(experiment_group_outputs_path) is True
stores_schedule_outputs_deletion(persistence=None, subpath=self.experiment_group.subpath)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(experiment_group_outputs_path) is False
def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async') as mock_fct:
with patch('scheduler.dockerizer_scheduler.create_build_job') as mock_start:
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
mock_start.return_value = build, True, True
self.object = self.factory_class(project=project)
assert mock_fct.call_count == 1
# Add a running experiment
experiment = ExperimentFactory(experiment_group=self.object)
ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING)
self.url = '/{}/{}/{}/groups/{}/stop'.format(
API_V1,
project.user.username,
project.name,
self.object.id)
def test_experiments_sync_jobs_statuses(self):
with patch('scheduler.tasks.experiments.experiments_build.apply_async'
) as _: # noqa
with patch.object(Experiment, 'set_status') as _: # noqa
experiments = [ExperimentFactory() for _ in range(3)]
done_xp, no_jobs_xp, xp_with_jobs = experiments
# Set done status
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
ExperimentStatusFactory(experiment=done_xp,
status=JobLifeCycle.FAILED)
# Create jobs for xp_with_jobs and update status, and do not update the xp status
with patch.object(Experiment, 'set_status') as _: # noqa
job = ExperimentJobFactory(experiment=xp_with_jobs)
ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING)
xp_with_jobs.refresh_from_db()
assert xp_with_jobs.last_status is None
# Mock sync experiments and jobs constants
with patch(
'scheduler.tasks.experiments.'
'experiments_check_status.apply_async') as check_status_mock:
experiments_sync_jobs_statuses()
assert check_status_mock.call_count == 1
# Call sync experiments and jobs constants
with patch('scheduler.tasks.experiments.experiments_build.apply_async'
) as build_mock:
ExperimentStatusFactory(experiment=xp_with_jobs,
status=JobLifeCycle.CREATED)
assert build_mock.call_count == 1
experiments_sync_jobs_statuses()
done_xp.refresh_from_db()
no_jobs_xp.refresh_from_db()
xp_with_jobs.refresh_from_db()
assert done_xp.last_status == ExperimentLifeCycle.FAILED
assert no_jobs_xp.last_status is None
assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING
def test_experiment_group_outputs_path_creation_deletion(self):
experiment = ExperimentFactory(user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
create_experiment_outputs_path(persistence_outputs=experiment.persistence_outputs,
experiment_name=experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(
persistence_outputs=experiment.persistence_outputs,
experiment_name=experiment.unique_name)
experiment_group_outputs_path = get_experiment_group_outputs_path(
self.experiment_group.persistence_outputs,
self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(experiment_group_outputs_path) is True
delete_experiment_group_outputs(
persistence_outputs=self.experiment_group.persistence_outputs,
experiment_group_name=self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(experiment_group_outputs_path) is False
