def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async' ) as mock_fct: self.object = self.factory_class(project=project) assert mock_fct.call_count == 1 # Add a running experiment experiment = ExperimentFactory(experiment_group=self.object) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) self.url = '/{}/{}/{}/groups/{}/stop'.format(API_V1, project.user.username, project.name, self.object.id)
def setUp(self): super().setUp() with patch.object(Experiment, 'set_status') as _: # noqa with patch('runner.tasks.experiments.start_experiment.delay' ) as _: # noqa project = ProjectFactory(user=self.auth_client.user) self.experiment = ExperimentFactory(project=project) self.url = '/{}/{}/{}/experiments/{}/metrics/'.format( API_V1, project.user.username, project.name, self.experiment.sequence) self.objects = [ self.factory_class(experiment=self.experiment, values={'accuracy': i / 10}) for i in range(self.num_objects) ] self.queryset = self.model_class.objects.all()
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('runner.hp_search.grid.hp_grid_search_start.apply_async') as mock_fct: self.object = self.factory_class(project=project) assert mock_fct.call_count == 1 self.url = '/{}/{}/{}/groups/{}/'.format(API_V1, project.user.username, project.name, self.object.sequence) self.queryset = self.model_class.objects.all() # Add 2 more experiments for _ in range(2): ExperimentFactory(experiment_group=self.object)
def test_non_independent_experiment_creation_doesnt_trigger_start(self, create_build_job): build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) create_build_job.return_value = build, True, True with patch('hpsearch.tasks.hp_create.apply_async') as mock_fct: experiment_group = ExperimentGroupFactory() assert mock_fct.call_count == 1 with patch('scheduler.tasks.experiments.experiments_start.apply_async') as mock_fct: with patch.object(Experiment, 'set_status') as mock_fct2: ExperimentFactory(experiment_group=experiment_group) assert mock_fct.call_count == 0 assert mock_fct2.call_count == 1
def setUp(self): super().setUp() with patch.object(Experiment, 'set_status') as _: with patch('runner.tasks.experiments.start_experiment.delay' ) as _: # noqa project = ProjectFactory(user=self.auth_client.user) self.experiment = ExperimentFactory(project=project) self.url = '/{}/{}/{}/experiments/{}/statuses/'.format( API_V1, project.user.username, project.name, self.experiment.sequence) self.objects = [ self.factory_class(experiment=self.experiment, status=ExperimentLifeCycle.CHOICES[i][0]) for i in range(self.num_objects) ] self.queryset = self.model_class.objects.all()
def setUp(self): super().setUp() with patch('experiments.tasks.start_experiment.delay') as _: with patch.object(ExperimentJob, 'set_status') as _: project = ProjectFactory(user=self.auth_client.user) experiment = ExperimentFactory(project=project) self.experiment_job = ExperimentJobFactory(experiment=experiment) self.object = self.factory_class(job=self.experiment_job) self.url = '/{}/{}/{}/experiments/{}/jobs/{}/statuses/{}'.format( API_V1, project.user.username, project.name, experiment.sequence, self.experiment_job.sequence, self.object.uuid.hex) self.queryset = self.model_class.objects.filter(job=self.experiment_job)
def test_restore(self): ExperimentFactory(project=self.object.project, experiment_group=self.object) assert Experiment.objects.count() == 3 self.object.archive() assert self.model_class.objects.count() == 0 assert self.model_class.all.count() == 1 assert Experiment.all.count() == 3 assert Experiment.objects.count() == 0 resp = self.auth_client.post(self.url + 'restore/') assert resp.status_code == status.HTTP_200_OK assert self.model_class.objects.count() == 1 assert self.model_class.all.count() == 1 assert Experiment.all.count() == 3 assert Experiment.objects.count() == 3
def test_delete_remove_paths(self, delete_outputs_path, delete_logs_path): project = ProjectFactory() for _ in range(2): ExperimentGroupFactory(project=project) ExperimentFactory(project=project) assert ExperimentGroup.objects.count() == 2 assert Experiment.objects.count() == 2 with patch('libs.paths.projects.delete_path' ) as delete_path_project_mock_stop: project.delete() # 1 repo assert delete_path_project_mock_stop.call_count == 1 # 1 project + 2 * groups + 2 experiments assert delete_outputs_path.call_count == 5 assert delete_logs_path.call_count == 5
def test_experiment_group_outputs_path_creation_deletion(self): with patch('experiments.tasks.build_experiment.apply_async') as _: experiment = ExperimentFactory( user=self.project.user, project=self.project, experiment_group=self.experiment_group) create_experiment_outputs_path(experiment.unique_name) experiment_outputs_path = get_experiment_outputs_path( experiment.unique_name) experiment_group_outputs_path = get_experiment_group_outputs_path( self.experiment_group.unique_name) assert os.path.exists(experiment_outputs_path) is True assert os.path.exists(experiment_group_outputs_path) is True delete_experiment_group_outputs(self.experiment_group.unique_name) assert os.path.exists(experiment_outputs_path) is False assert os.path.exists(experiment_group_outputs_path) is False
def test_experiment_group_logs_path_creation_deletion(self): with patch('experiments.tasks.build_experiment.apply_async') as _: experiment = ExperimentFactory( user=self.project.user, project=self.project, experiment_group=self.experiment_group) experiment_logs_path = get_experiment_logs_path(experiment.unique_name) open(experiment_logs_path, '+w') experiment_group_logs_path = get_experiment_group_logs_path( self.experiment_group.unique_name) # Should be true, created by the signal assert os.path.exists(experiment_logs_path) is True assert os.path.exists(experiment_group_logs_path) is True delete_experiment_group_logs(self.experiment_group.unique_name) assert os.path.exists(experiment_logs_path) is False assert os.path.exists(experiment_group_logs_path) is False
def test_project_logs_path_creation_deletion(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa experiment = ExperimentFactory(user=self.project.user, project=self.project) experiment_logs_path = get_experiment_logs_path(experiment.unique_name) create_experiment_logs_path(experiment.unique_name) open(experiment_logs_path, '+w') project_logs_path = get_project_logs_path(self.project.unique_name) project_repos_path = get_project_logs_path(self.project.unique_name) # Should be true, created by the signal assert os.path.exists(experiment_logs_path) is True assert os.path.exists(project_logs_path) is True assert os.path.exists(project_repos_path) is True delete_project_logs(self.project.unique_name) assert os.path.exists(experiment_logs_path) is False assert os.path.exists(project_logs_path) is False assert os.path.exists(project_repos_path) is False
def test_set_job_definition(self): experiment = ExperimentFactory() job_uuid = uuid.uuid4().hex create_job(job_uuid=job_uuid, experiment=experiment) job = ExperimentJob.objects.last() assert job.role == TaskType.MASTER assert job.resources is None assert job.node_selector is None assert job.affinity is None assert job.tolerations is None assert job.definition == {} definition = {'spec': {}} set_job_definition(job_uuid=job_uuid, definition=definition) job = ExperimentJob.objects.last() assert job.definition == definition
def test_delete_experiment_triggers_experiment_stop_mocks(self, delete_path): experiment = ExperimentFactory() experiment.set_status(ExperimentLifeCycle.SCHEDULED) # Add job ExperimentJobFactory(experiment=experiment) assert delete_path.call_count == 2 # outputs + logs with patch('scheduler.experiment_scheduler.stop_experiment') as mock_fct: experiment.delete() assert delete_path.call_count == 2 + 2 # outputs + logs assert mock_fct.call_count == 1
def setUp(self): super().setUp() with patch('experiments.tasks.start_experiment.delay') as _: with patch.object(ExperimentJob, 'set_status') as _: project = ProjectFactory(user=self.auth_client.user) experiment = ExperimentFactory(project=project) self.experiment_job = ExperimentJobFactory(experiment=experiment) self.url = '/{}/{}/{}/experiments/{}/jobs/{}/statuses/'.format( API_V1, project.user.username, project.name, experiment.sequence, self.experiment_job.sequence) self.objects = [self.factory_class(job=self.experiment_job, status=JobLifeCycle.CHOICES[i][0]) for i in range(self.num_objects)] self.queryset = self.model_class.objects.filter(job=self.experiment_job)
def test_build_job_statuses(self): assert BuildJobStatus.objects.count() == 0 experiment = ExperimentFactory(project=self.project) build_job, rebuild = BuildJob.create( user=experiment.user, project=experiment.project, config=experiment.specification.build, code_reference=self.code_reference) self.assertEqual(rebuild, True) assert build_job.last_status == JobLifeCycle.CREATED assert BuildJobStatus.objects.count() == 1 build_job.set_status(JobLifeCycle.FAILED) assert BuildJobStatus.objects.count() == 2 build_job.set_status(JobLifeCycle.SUCCEEDED) assert BuildJobStatus.objects.count() == 2
def test_archive(self): project = ProjectFactory() ExperimentGroupFactory(project=project) ExperimentFactory(project=project) JobFactory(project=project) BuildJobFactory(project=project) NotebookJobFactory(project=project) TensorboardJobFactory(project=project) assert project.deleted is False assert project.experiments.count() == 1 assert project.experiment_groups.count() == 1 assert project.jobs.count() == 1 assert project.build_jobs.count() == 1 assert project.notebook_jobs.count() == 1 assert project.tensorboard_jobs.count() == 1 assert project.all_experiments.count() == 1 assert project.all_experiment_groups.count() == 1 assert project.all_notebook_jobs.count() == 1 assert project.all_tensorboard_jobs.count() == 1 project.archive() assert project.deleted is True assert project.experiments.count() == 0 assert project.experiment_groups.count() == 0 assert project.jobs.count() == 0 assert project.build_jobs.count() == 0 assert project.notebook_jobs.count() == 0 assert project.tensorboard_jobs.count() == 0 assert project.all_experiments.count() == 1 assert project.all_experiment_groups.count() == 1 assert project.all_notebook_jobs.count() == 1 assert project.all_tensorboard_jobs.count() == 1 project.unarchive() assert project.deleted is False assert project.experiments.count() == 1 assert project.experiment_groups.count() == 1 assert project.jobs.count() == 1 assert project.build_jobs.count() == 1 assert project.notebook_jobs.count() == 1 assert project.tensorboard_jobs.count() == 1 assert project.all_experiments.count() == 1 assert project.all_experiment_groups.count() == 1 assert project.all_notebook_jobs.count() == 1 assert project.all_tensorboard_jobs.count() == 1
def test_experiment_group_logs_path_creation_deletion(self): experiment = ExperimentFactory(user=self.project.user, project=self.project, experiment_group=self.experiment_group) experiment_logs_path = stores.get_experiment_logs_path( experiment_name=experiment.unique_name, temp=False) stores.create_experiment_logs_path(experiment_name=experiment.unique_name, temp=False) open(experiment_logs_path, '+w') experiment_group_logs_path = stores.get_experiment_group_logs_path( experiment_group_name=self.experiment_group.unique_name) # Should be true, created by the signal assert os.path.exists(experiment_logs_path) is True assert os.path.exists(experiment_group_logs_path) is True stores_schedule_logs_deletion(persistence=None, subpath=self.experiment_group.subpath) assert os.path.exists(experiment_logs_path) is False assert os.path.exists(experiment_group_logs_path) is False
def test_delete_remove_paths(self): project = ProjectFactory() for _ in range(2): ExperimentGroupFactory(project=project) ExperimentFactory(project=project) assert ExperimentGroup.objects.count() == 2 assert Experiment.objects.count() == 2 with patch('libs.paths.projects.delete_path') as delete_path_project_mock_stop: with patch('libs.paths.experiment_groups.delete_path') as delete_path_group_mock_stop: with patch('libs.paths.experiments.delete_path') as delete_path_xp_mock_stop: project.delete() # 2 * project + 1 repo assert delete_path_project_mock_stop.call_count == 3 # 2 * 2 * groups assert delete_path_group_mock_stop.call_count assert delete_path_xp_mock_stop.call_count == 4 # 2 * 2 * groups
def setUp(self): super().setUp() self.user = UserFactory() activitylogs.validate() activitylogs.setup() self.project = ProjectFactory() activitylogs.record(event_type=PROJECT_DELETED_TRIGGERED, instance=self.project, actor_id=self.user.id) self.experiment = ExperimentFactory() activitylogs.record(event_type=EXPERIMENT_DELETED_TRIGGERED, instance=self.experiment, actor_id=self.user.id) self.job = JobFactory() activitylogs.record(event_type=JOB_VIEWED, instance=self.job, actor_id=self.user.id)
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) experiment = ExperimentFactory(project=project) self.url = '/{}/{}/{}/experiments/{}/logs'.format( API_V1, project.user.username, project.name, experiment.sequence) log_path = get_experiment_logs_path(experiment.unique_name) create_experiment_logs_path(experiment_name=experiment.unique_name) fake = Faker() self.logs = [] for _ in range(self.num_log_lines): self.logs.append(fake.sentence()) with open(log_path, 'w') as file: for line in self.logs: file.write(line) file.write('\n')
def test_get_experiments_metrics(self): experiment_group = ExperimentGroupFactory() assert len(experiment_group.get_experiments_metrics( # pylint:disable=len-as-condition experiment_ids=[], metric='precision' )) == 0 experiments = [] experiment_ids = [] for _ in range(5): experiment = ExperimentFactory(experiment_group=experiment_group) experiments.append(experiment) experiment_ids.append(experiment.id) ExperimentMetric.objects.create(experiment=experiment, values={'precision': random.random()}) for experiment in experiments[:3]: ExperimentMetric.objects.create(experiment=experiment, values={'loss': random.random()}) experiment_metrics = experiment_group.get_experiments_metrics( experiment_ids=experiment_ids, metric='precision' ) assert len(experiment_metrics) == 5 metrics = [m[1] for m in experiment_metrics if m[1] is not None] assert len(metrics) == 2 experiment_metrics = experiment_group.get_experiments_metrics( experiment_ids=experiment_ids, metric='loss' ) assert len(experiment_metrics) == 5 metrics = [m[1] for m in experiment_metrics if m[1] is not None] assert len(metrics) == 3 experiment_metrics = experiment_group.get_experiments_metrics( experiment_ids=experiment_ids, metric='accuracy' ) assert len(experiment_metrics) == 5 assert len( # pylint:disable=len-as-condition [m for m in experiment_metrics if m[1] is not None]) == 0
def test_create_with_valid_group(self): resp = self.auth_client.post(self.url) assert resp.status_code == status.HTTP_400_BAD_REQUEST content = """--- version: 1 kind: group hptuning: concurrency: 3 matrix: lr: values: [0.1, 0.2, 0.3] run: cmd: my_command --lr={{ lr }}""" data = {'content': content, 'description': 'new-deep'} resp = self.auth_client.post(self.url, data) assert resp.status_code == status.HTTP_201_CREATED assert self.queryset.count() == self.num_objects + 1 last_object = self.model_class.objects.last() assert last_object.project == self.project assert last_object.content == data['content'] assert last_object.hptuning is not None assert last_object.hptuning['concurrency'] == 3 assert last_object.hptuning['matrix']['lr'] is not None assert last_object.is_study is True # Creating a study with selection ignores selection experiments = [ ExperimentFactory(project=self.project) for _ in range(2) ] experiment_ids = [xp.id for xp in experiments] data = { 'content': content, 'description': 'new-deep', 'experiment_ids': experiment_ids } resp = self.auth_client.post(self.url, data) assert resp.status_code == status.HTTP_201_CREATED assert self.queryset.count() == self.num_objects + 2 last_object = self.model_class.objects.last() assert last_object.is_study is True assert last_object.selection_experiments.count() == 0
def test_should_stop_early(self, _): # Experiment group with no early stopping experiment_group = ExperimentGroupFactory() assert experiment_group.should_stop_early() is False # Experiment group with early stopping experiment_group = ExperimentGroupFactory(content=None, hptuning={ 'concurrency': 2, 'random_search': { 'n_experiments': 10 }, 'early_stopping': [{ 'metric': 'precision', 'value': 0.9, 'optimization': 'maximize' }], 'matrix': { 'lr': { 'values': [1, 2, 3] } } }) assert experiment_group.should_stop_early() is False # Create experiments and metrics experiments = [ ExperimentFactory(experiment_group=experiment_group) for _ in range(2) ] ExperimentMetric.objects.create(experiment=experiments[0], values={'precision': 0.8}) assert experiment_group.should_stop_early() is False # Create a metric that triggers early stopping ExperimentMetric.objects.create(experiment=experiments[0], values={'precision': 0.91}) assert experiment_group.should_stop_early() is True
def test_redirects_to_proxy_protected_url(self, spawner_mock): project = ProjectFactory(user=self.auth_client.user) experiment = ExperimentFactory(project=project) tensorboard = TensorboardJobFactory(project=project, experiment=experiment) tensorboard.set_status(status=JobLifeCycle.RUNNING) deployment_name = JOB_NAME.format(job_uuid=tensorboard.uuid.hex, name=self.plugin_app) service_url = self._get_service_url(deployment_name=deployment_name) mock_instance = spawner_mock.return_value mock_instance.get_tensorboard_url.return_value = service_url response = self.auth_client.get(self._get_url(project, experiment)) assert response.status_code == 200 self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response) proxy_url = '{}/'.format(service_url) self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER], proxy_url)
def test_stop_pending_experiments(self): with patch('hpsearch.tasks.random.hp_random_search_start.apply_async' ) as mock_fct: experiment_group = ExperimentGroupFactory( content=experiment_group_spec_content_early_stopping) experiment = ExperimentFactory(experiment_group=experiment_group) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) assert mock_fct.call_count == 1 assert experiment_group.pending_experiments.count() == 2 assert experiment_group.running_experiments.count() == 1 experiments_group_stop_experiments( experiment_group_id=experiment_group.id, pending=True) assert experiment_group.pending_experiments.count() == 0 assert experiment_group.running_experiments.count() == 1
def test_from_event_data(self): instance = ExperimentFactory() event = ExperimentSucceededEvent.from_instance(instance=instance, actor_id=1, actor_name='user') assert event.ref_id is None event_serialized = event.serialize(dumps=False, include_instance_info=True) assert event_serialized.get('ref_id') is None new_event = ExperimentSucceededEvent.from_event_data(event_data=event_serialized) assert new_event.serialize(include_instance_info=True) == event_serialized # Add ref id event.ref_id = uuid1() event_serialized = event.serialize(dumps=False, include_instance_info=True) assert event_serialized['ref_id'] == event.ref_id.hex new_event = ExperimentSucceededEvent.from_event_data(event_data=event_serialized) assert new_event.ref_id == event.ref_id assert new_event.serialize(include_instance_info=True) == event_serialized
def test_experiment_group_outputs_path_creation_deletion(self): experiment = ExperimentFactory(user=self.project.user, project=self.project, experiment_group=self.experiment_group) stores.create_experiment_outputs_path( persistence=experiment.persistence_outputs, experiment_name=experiment.unique_name) experiment_outputs_path = stores.get_experiment_outputs_path( persistence=experiment.persistence_outputs, experiment_name=experiment.unique_name) experiment_group_outputs_path = stores.get_experiment_group_outputs_path( persistence=self.experiment_group.persistence_outputs, experiment_group_name=self.experiment_group.unique_name) assert os.path.exists(experiment_outputs_path) is True assert os.path.exists(experiment_group_outputs_path) is True stores_schedule_outputs_deletion(persistence=None, subpath=self.experiment_group.subpath) assert os.path.exists(experiment_outputs_path) is False assert os.path.exists(experiment_group_outputs_path) is False
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async') as mock_fct: with patch('scheduler.dockerizer_scheduler.create_build_job') as mock_start: build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) mock_start.return_value = build, True, True self.object = self.factory_class(project=project) assert mock_fct.call_count == 1 # Add a running experiment experiment = ExperimentFactory(experiment_group=self.object) ExperimentStatusFactory(experiment=experiment, status=ExperimentLifeCycle.RUNNING) self.url = '/{}/{}/{}/groups/{}/stop'.format( API_V1, project.user.username, project.name, self.object.id)
def test_experiments_sync_jobs_statuses(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiments = [ExperimentFactory() for _ in range(3)] done_xp, no_jobs_xp, xp_with_jobs = experiments # Set done status with patch( 'scheduler.experiment_scheduler.stop_experiment') as _: # noqa ExperimentStatusFactory(experiment=done_xp, status=JobLifeCycle.FAILED) # Create jobs for xp_with_jobs and update status, and do not update the xp status with patch.object(Experiment, 'set_status') as _: # noqa job = ExperimentJobFactory(experiment=xp_with_jobs) ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING) xp_with_jobs.refresh_from_db() assert xp_with_jobs.last_status is None # Mock sync experiments and jobs constants with patch( 'scheduler.tasks.experiments.' 'experiments_check_status.apply_async') as check_status_mock: experiments_sync_jobs_statuses() assert check_status_mock.call_count == 1 # Call sync experiments and jobs constants with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as build_mock: ExperimentStatusFactory(experiment=xp_with_jobs, status=JobLifeCycle.CREATED) assert build_mock.call_count == 1 experiments_sync_jobs_statuses() done_xp.refresh_from_db() no_jobs_xp.refresh_from_db() xp_with_jobs.refresh_from_db() assert done_xp.last_status == ExperimentLifeCycle.FAILED assert no_jobs_xp.last_status is None assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING
def test_experiment_group_outputs_path_creation_deletion(self): experiment = ExperimentFactory(user=self.project.user, project=self.project, experiment_group=self.experiment_group) create_experiment_outputs_path(persistence_outputs=experiment.persistence_outputs, experiment_name=experiment.unique_name) experiment_outputs_path = get_experiment_outputs_path( persistence_outputs=experiment.persistence_outputs, experiment_name=experiment.unique_name) experiment_group_outputs_path = get_experiment_group_outputs_path( self.experiment_group.persistence_outputs, self.experiment_group.unique_name) assert os.path.exists(experiment_outputs_path) is True assert os.path.exists(experiment_group_outputs_path) is True delete_experiment_group_outputs( persistence_outputs=self.experiment_group.persistence_outputs, experiment_group_name=self.experiment_group.unique_name) assert os.path.exists(experiment_outputs_path) is False assert os.path.exists(experiment_group_outputs_path) is False