def setUp(self): super().setUp() self.user = UserFactory() self.project = ProjectFactory(user=self.user) self.project_job = JobFactory(project=self.project, name='unique') self.job = JobFactory(name='unique2') self.project_experiment = ExperimentFactory(project=self.project, name='unique') self.experiment = ExperimentFactory(name='unique2') self.instance_mock = InstanceSpec(self.user.id, self.project.id) self.wrong_instance_mock = InstanceSpec(-1, -1)
def test_heartbeat_jobs(self): job1 = JobFactory() JobStatusFactory(job=job1, status=JobLifeCycle.SCHEDULED) job2 = JobFactory() JobStatusFactory(job=job2, status=JobLifeCycle.CREATED) job3 = JobFactory() JobStatusFactory(job=job3, status=JobLifeCycle.FAILED) job4 = JobFactory() JobStatusFactory(job=job4, status=JobLifeCycle.RUNNING) with patch('scheduler.tasks.jobs.jobs_check_heartbeat.apply_async') as mock_fct: heartbeat_jobs() assert mock_fct.call_count == 1
def test_jobs_check_heartbeat(self): job1 = JobFactory() JobStatusFactory(job=job1, status=JobLifeCycle.RUNNING) RedisHeartBeat.job_ping(job_id=job1.id) job2 = JobFactory() JobStatusFactory(job=job2, status=JobLifeCycle.RUNNING) jobs_check_heartbeat(job1.id) job1.refresh_from_db() self.assertEqual(job1.last_status, JobLifeCycle.RUNNING) jobs_check_heartbeat(job2.id) job2.refresh_from_db() self.assertEqual(job2.last_status, JobLifeCycle.FAILED)
def test_delete_jobs(self): project1 = ProjectFactory() JobFactory(project=project1) project1.archive() project2 = ProjectFactory() job2 = JobFactory(project=project2) job2.archive() assert Job.all.count() == 2 conf.set(CLEANING_INTERVALS_ARCHIVES, -10) delete_archived_jobs() # Although the other entity is archived it's not deleted because of project1 assert Job.all.count() == 1
def test_delete_jobs(self): project1 = ProjectFactory() JobFactory(project=project1) project1.archive() project2 = ProjectFactory() job2 = JobFactory(project=project2) job2.archive() assert Job.all.count() == 2 CleaningIntervals.ARCHIVED = -10 delete_archived_jobs() # Although the other entity is archived it's not deleted because of project1 assert Job.all.count() == 1
def setUp(self): super().setUp() self.user = UserFactory() activitylogs.validate() activitylogs.setup() self.project = ProjectFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=USER_ACTIVATED, instance=self.user, actor_id=self.user.id, actor_name=self.user.username) activitylogs.record(ref_id=uuid.uuid4(), event_type=PROJECT_DELETED_TRIGGERED, instance=self.project, actor_id=self.user.id, actor_name=self.user.username) self.experiment = ExperimentFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=EXPERIMENT_DELETED_TRIGGERED, instance=self.experiment, actor_id=self.user.id, actor_name=self.user.username) self.job = JobFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=JOB_VIEWED, instance=self.job, actor_id=self.user.id, actor_name=self.user.username)
def test_job_creation_triggers_scheduling_mocks(self): with patch('scheduler.tasks.jobs.jobs_build.apply_async') as mock_fct: with patch.object(Job, 'set_status') as mock_fct2: JobFactory() assert mock_fct.call_count == 0 assert mock_fct2.call_count == 1
def test_experiment_group_outputs_path_creation_deletion(self): job = JobFactory() create_job_outputs_path(job.unique_name) job_outputs_path = get_job_outputs_path(job.unique_name) assert os.path.exists(job_outputs_path) is True delete_job_outputs(job.unique_name) assert os.path.exists(job_outputs_path) is False
def test_job_creation_with_already_built_triggers_scheduling(self): # Create a repo for the project repo = RepoFactory() with patch( 'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build: job = JobFactory(project=repo.project) assert mock_build.call_count == 1 assert job.project.repo is not None assert JobStatus.objects.filter(job=job).count() == 1 assert list( JobStatus.objects.filter(job=job).values_list( 'status', flat=True)) == [JobLifeCycle.CREATED] with patch('scheduler.dockerizer_scheduler.create_build_job' ) as mock_start: build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) mock_start.return_value = build, True, True jobs_build(job_id=job.id) assert mock_start.call_count == 1 assert JobStatus.objects.filter(job=job).count() == 2 assert list( JobStatus.objects.filter(job=job).values_list( 'status', flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.SCHEDULED] job.refresh_from_db() assert job.last_status == JobLifeCycle.SCHEDULED
def test_job_creation_triggers_build(self): # Create a repo for the project repo = RepoFactory() with patch( 'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build: job = JobFactory(project=repo.project) assert mock_build.call_count == 1 assert job.project.repo is not None assert JobStatus.objects.filter(job=job).count() == 1 assert list( JobStatus.objects.filter(job=job).values_list( 'status', flat=True)) == [JobLifeCycle.CREATED] with patch('scheduler.dockerizer_scheduler.start_dockerizer' ) as mock_start: mock_start.return_value = True jobs_build(job_id=job.id) assert mock_start.call_count == 1 assert JobStatus.objects.filter(job=job).count() == 2 assert list( JobStatus.objects.filter(job=job).values_list( 'status', flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.BUILDING] job.refresh_from_db() assert job.last_status == JobLifeCycle.BUILDING
def test_patch(self): new_description = 'updated_xp_name' data = {'description': new_description} assert self.object.description != data['description'] resp = self.auth_client.patch(self.url, data=data) assert resp.status_code == status.HTTP_200_OK new_object = self.model_class.objects.get(id=self.object.id) assert new_object.user == self.object.user assert new_object.description != self.object.description assert new_object.description == new_description # Update original job assert new_object.is_clone is False new_job = JobFactory() data = {'original_job': new_job.id} resp = self.auth_client.patch(self.url, data=data) assert resp.status_code == status.HTTP_200_OK new_object = self.model_class.objects.get(id=self.object.id) assert new_object.user == self.object.user assert new_object.description == new_description assert new_object.is_clone is True assert new_object.original_job == new_job # Update name data = {'name': 'new_name'} assert new_object.name is None resp = self.auth_client.patch(self.url, data=data) assert resp.status_code == status.HTTP_200_OK new_object = self.model_class.objects.get(id=self.object.id) assert new_object.name == data['name']
def test_delete_job_does_not_trigger_job_stop_if_not_running(self, delete_path): job = JobFactory() assert delete_path.call_count == 2 # outputs + logs with patch('scheduler.job_scheduler.stop_job') as mock_fct: job.delete() assert delete_path.call_count == 2 + 2 # outputs + logs assert mock_fct.call_count == 0
def setUp(self): super().setUp() self.job = JobFactory(project=ProjectFactory()) self.tested_events = { job_events.JOB_CREATED, job_events.JOB_UPDATED, job_events.JOB_STARTED, job_events.JOB_STARTED_TRIGGERED, job_events.JOB_CLEANED_TRIGGERED, job_events.JOB_DELETED, job_events.JOB_DELETED_TRIGGERED, job_events.JOB_STOPPED, job_events.JOB_STOPPED_TRIGGERED, job_events.JOB_VIEWED, job_events.JOB_ARCHIVED, job_events.JOB_RESTORED, job_events.JOB_BOOKMARKED, job_events.JOB_UNBOOKMARKED, job_events.JOB_NEW_STATUS, job_events.JOB_FAILED, job_events.JOB_SUCCEEDED, job_events.JOB_DONE, job_events.JOB_LOGS_VIEWED, job_events.JOB_RESTARTED, job_events.JOB_RESTARTED_TRIGGERED, job_events.JOB_STATUSES_VIEWED, job_events.JOB_OUTPUTS_DOWNLOADED, }
def setUp(self): super().setUp() self.project = ProjectFactory() self.experiment = ExperimentFactory() self.group = ExperimentGroupFactory() self.job = JobFactory() self.build = BuildJobFactory()
def set_objects(self): self.user = self.auth_client.user self.project = ProjectFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=PROJECT_DELETED_TRIGGERED, instance=self.project, actor_id=self.user.id, actor_name=self.user.username) self.experiment = ExperimentFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=EXPERIMENT_DELETED_TRIGGERED, instance=self.experiment, actor_id=self.user.id, actor_name=self.user.username) self.job = JobFactory() activitylogs.record(ref_id=uuid.uuid4(), event_type=JOB_CREATED, instance=self.job, actor_id=self.user.id, actor_name=self.user.username) activitylogs.record(ref_id=uuid.uuid4(), event_type=JOB_VIEWED, instance=self.job, actor_id=self.user.id, actor_name=self.user.username)
def setUp(self): super().setUp() self.project = ProjectFactory() self.build_job = BuildJobFactory(project=self.project) self.notebook = NotebookJobFactory(project=self.project, build_job=self.build_job) self.tensorboard = TensorboardJobFactory(project=self.project, build_job=self.build_job) self.job = JobFactory(project=self.project, build_job=self.build_job) self.experiment = ExperimentFactory(project=self.project, build_job=self.build_job)
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) self.job = JobFactory(project=project) self.logs = [] self.url = '/{}/{}/{}/jobs/{}/logs'.format(API_V1, project.user.username, project.name, self.job.id)
def test_delete_job_triggers_job_stop_mock(self, delete_path): job = JobFactory() job.set_status(JobLifeCycle.SCHEDULED) assert delete_path.call_count == 2 # outputs + logs with patch('scheduler.job_scheduler.stop_job') as mock_fct: job.delete() assert delete_path.call_count == 2 + 2 # outputs + logs assert mock_fct.call_count == 1
def test_job_logs_path_creation_deletion(self): job = JobFactory() job_logs_path = get_job_logs_path(job.unique_name) create_job_logs_path(job.unique_name) open(job_logs_path, '+w') # Should be true, created by the signal assert os.path.exists(job_logs_path) is True delete_job_logs(job.unique_name) assert os.path.exists(job_logs_path) is False
def test_job_logs_path_creation_deletion(self): job = JobFactory() job_logs_path = stores.get_job_logs_path(job_name=job.unique_name, temp=False) stores.create_job_logs_path(job_name=job.unique_name, temp=False) open(job_logs_path, '+w') # Should be true, created by the signal assert os.path.exists(job_logs_path) is True stores_schedule_logs_deletion(persistence=None, subpath=job.subpath) assert os.path.exists(job_logs_path) is False
def test_job_outputs_path_creation_deletion(self): job = JobFactory() stores.create_job_outputs_path(persistence=job.persistence_outputs, job_name=job.unique_name) job_outputs_path = stores.get_job_outputs_path( persistence=job.persistence_outputs, job_name=job.unique_name) assert os.path.exists(job_outputs_path) is True stores_schedule_outputs_deletion(persistence='outputs', subpath=job.subpath) assert os.path.exists(job_outputs_path) is False
def test_job_outputs_path_creation_deletion(self): job = JobFactory() create_job_outputs_path(persistence_outputs=job.persistence_outputs, job_name=job.unique_name) job_outputs_path = get_job_outputs_path( persistence_outputs=job.persistence_outputs, job_name=job.unique_name) assert os.path.exists(job_outputs_path) is True delete_job_outputs(persistence_outputs=job.persistence_outputs, job_name=job.unique_name) assert os.path.exists(job_outputs_path) is False
def setUp(self): super().setUp() with patch.object(Job, 'set_status') as _: # noqa with patch('scheduler.tasks.jobs.jobs_build.apply_async' ) as _: # noqa self.job = JobFactory() self.object = self.factory_class(job=self.job) self.url = '/{}/{}/{}/jobs/{}/statuses/{}/'.format( API_V1, self.job.project.user.username, self.job.project.name, self.job.id, self.object.uuid.hex) self.queryset = self.model_class.objects.all()
def setUp(self): super().setUp() project = ProjectFactory(user=self.auth_client.user) job = JobFactory(project=project) self.url = '/{}/{}/{}/jobs/{}/outputs/files'.format( API_V1, project.user.username, project.name, job.id) outputs_path = stores.get_job_outputs_path( persistence=job.persistence_outputs, job_name=job.unique_name) stores.create_job_outputs_path(persistence=job.persistence_outputs, job_name=job.unique_name) self.create_paths(path=outputs_path, url=self.url)
def test_status_update_results_in_new_updated_at_datetime(self): job = JobFactory() updated_at = job.updated_at # Create new status JobStatus.objects.create(job=job, status=JobLifeCycle.BUILDING) job.refresh_from_db() assert updated_at < job.updated_at updated_at = job.updated_at # Create status Using set_status job.set_status(JobLifeCycle.RUNNING) job.refresh_from_db() assert updated_at < job.updated_at
def test_set_outputs(self): experiment_outputs = [ '{}'.format(self.experiment.id), '{}'.format(self.project_experiment.id), self.project_experiment.name, '{}.{}'.format(self.project.name, self.project_experiment.name), '{}/{}'.format(self.project.name, self.project_experiment.name), '{}.{}.{}'.format(self.user.username, self.project.name, self.project_experiment.name), '{}/{}/{}'.format(self.user.username, self.project.name, self.project_experiment.name), ] job_outputs = [ '{}'.format(self.job.id), '{}'.format(self.project_job.id), self.project_job.name, '{}.{}'.format(self.project.name, self.project_job.name), '{}/{}'.format(self.project.name, self.project_job.name), '{}.{}.{}'.format(self.user.username, self.project.name, self.project_job.name), '{}/{}/{}'.format(self.user.username, self.project.name, self.project_job.name), ] outputs_config = OutputsConfig( jobs=job_outputs, experiments=experiment_outputs).to_dict() experiment = ExperimentFactory(user=self.user, project=self.project) assert experiment.outputs is None experiment.outputs = outputs_config assert experiment.outputs is not None assert len(experiment.outputs_experiments) == len(experiment_outputs) assert len(experiment.outputs_jobs) == len(job_outputs) set_outputs(instance=experiment) del experiment.outputs_config del experiment.outputs_experiments del experiment.outputs_jobs assert len(experiment.outputs_experiments) == 2 assert len(experiment.outputs_jobs) == 2 job = JobFactory(user=self.user, project=self.project) assert job.outputs is None job.outputs = outputs_config assert job.outputs is not None assert job.outputs is not None assert len(job.outputs_experiments) == len(experiment_outputs) assert len(job.outputs_jobs) == len(job_outputs) set_outputs(instance=job) del job.outputs_config del job.outputs_experiments del job.outputs_jobs assert len(job.outputs_experiments) == 2 assert len(job.outputs_jobs) == 2
def test_delete_triggers_stopping_of_jobs(self): assert self.queryset.count() == 1 for _ in range(2): job = JobFactory(project=self.object) job.set_status(JobLifeCycle.SCHEDULED) assert Job.objects.count() == 2 with patch('scheduler.tasks.jobs.jobs_stop.apply_async') as job_mock_stop: resp = self.auth_client.delete(self.url) assert job_mock_stop.called assert resp.status_code == status.HTTP_204_NO_CONTENT assert self.queryset.count() == 0 assert Job.all.count() == 0
def test_archive_schedules_deletion(self, xp_group_scheduler_mock, xp_scheduler_mock, job_scheduler_mock, build_scheduler_mock, notebook_scheduler_mock, tensorboard_scheduler_mock): for _ in range(2): JobFactory(project=self.object) BuildJobFactory(project=self.object) TensorboardJobFactory(project=self.object) NotebookJobFactory(project=self.object) self.object.experiment_groups.first().set_status(ExperimentGroupLifeCycle.RUNNING) self.object.experiments.first().set_status(ExperimentLifeCycle.RUNNING) self.object.jobs.first().set_status(JobLifeCycle.RUNNING) self.object.build_jobs.first().set_status(JobLifeCycle.RUNNING) self.object.notebook_jobs.first().set_status(JobLifeCycle.RUNNING) self.object.tensorboard_jobs.first().set_status(JobLifeCycle.RUNNING) assert self.queryset.count() == 1 assert ExperimentGroup.objects.count() == 2 assert Experiment.objects.count() == 2 assert Job.objects.count() == 2 assert BuildJob.objects.count() == 2 assert NotebookJob.objects.count() == 2 assert TensorboardJob.objects.count() == 2 resp = self.auth_client.post(self.url + 'archive/') assert xp_group_scheduler_mock.call_count == 2 assert xp_scheduler_mock.call_count == 1 assert job_scheduler_mock.call_count == 1 assert build_scheduler_mock.call_count == 1 assert notebook_scheduler_mock.call_count == 1 assert tensorboard_scheduler_mock.call_count == 1 assert resp.status_code == status.HTTP_200_OK assert self.queryset.count() == 0 assert Project.all.filter(user=self.object.user).count() == 1 assert ExperimentGroup.objects.count() == 0 assert ExperimentGroup.all.count() == 2 assert Experiment.objects.count() == 0 assert Experiment.all.count() == 2 assert Job.objects.count() == 0 assert Job.all.count() == 2 assert BuildJob.objects.count() == 0 assert BuildJob.all.count() == 2 assert TensorboardJob.objects.count() == 0 assert TensorboardJob.all.count() == 2 assert NotebookJob.objects.count() == 0 assert NotebookJob.all.count() == 2
def set_objects(self): self.user = self.auth_client.user self.project = ProjectFactory(user=self.user) activitylogs.record(event_type=PROJECT_DELETED_TRIGGERED, instance=self.project, actor_id=self.user.id) self.experiment = ExperimentFactory(project=self.project) activitylogs.record(event_type=EXPERIMENT_DELETED_TRIGGERED, instance=self.experiment, actor_id=self.user.id) self.job = JobFactory(project=self.project) activitylogs.record(event_type=JOB_VIEWED, instance=self.job, actor_id=self.user.id)
def setUp(self): super().setUp() with patch.object(Job, 'set_status') as _: with patch('scheduler.tasks.jobs.jobs_build.apply_async' ) as _: # noqa project = ProjectFactory(user=self.auth_client.user) self.job = JobFactory(project=project) self.url = '/{}/{}/{}/jobs/{}/statuses/'.format( API_V1, project.user.username, project.name, self.job.id) self.objects = [ self.factory_class(job=self.job, status=JobLifeCycle.CHOICES[i][0]) for i in range(self.num_objects) ] self.queryset = self.model_class.objects.all()