def test_delete_triggers_stopping_of_plugin_jobs(self):
assert self.queryset.count() == 1
notebook_job = NotebookJobFactory(project=self.object)
notebook_job.set_status(JobLifeCycle.SCHEDULED)
tensorboard_job = TensorboardJobFactory(project=self.object)
tensorboard_job.set_status(JobLifeCycle.SCHEDULED)
assert NotebookJob.objects.count() == 1
assert TensorboardJob.objects.count() == 1
with patch('scheduler.tasks.notebooks.'
'projects_notebook_stop.apply_async') as notebook_mock_stop:
with patch(
'scheduler.tasks.tensorboards.'
'tensorboards_stop.apply_async') as tensorboard_mock_stop:
resp = self.auth_client.delete(self.url)
assert notebook_mock_stop.call_count == 1
assert tensorboard_mock_stop.call_count == 1
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.queryset.count() == 0
assert ExperimentGroup.objects.count() == 0
assert Experiment.objects.count() == 0
# Delete does not work for other project public and private
resp = self.auth_client.delete(self.url_other)
assert resp.status_code in (status.HTTP_401_UNAUTHORIZED,
status.HTTP_403_FORBIDDEN)
resp = self.auth_client.delete(self.url_private)
assert resp.status_code in (status.HTTP_401_UNAUTHORIZED,
status.HTTP_403_FORBIDDEN)
def test_archive_triggers_stopping_of_plugin_jobs(self):
assert self.queryset.count() == 1
notebook_job = NotebookJobFactory(project=self.object)
notebook_job.set_status(JobLifeCycle.SCHEDULED)
tensorboard_job = TensorboardJobFactory(project=self.object)
tensorboard_job.set_status(JobLifeCycle.SCHEDULED)
assert NotebookJob.objects.count() == 1
assert TensorboardJob.objects.count() == 1
with patch('scheduler.tasks.notebooks.'
'projects_notebook_stop.apply_async') as notebook_mock_stop:
with patch(
'scheduler.tasks.tensorboards.'
'tensorboards_stop.apply_async') as tensorboard_mock_stop:
resp = self.auth_client.post(self.url + 'archive/')
assert notebook_mock_stop.call_count == 1
assert tensorboard_mock_stop.call_count == 1
assert resp.status_code == status.HTTP_200_OK
assert self.queryset.count() == 0
assert TensorboardJob.objects.count() == 0
assert NotebookJob.objects.count() == 0
assert TensorboardJob.all.count() == 1
assert NotebookJob.all.count() == 1
def test_redirects_to_proxy_protected_url_with_extra_path(
self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = DEPLOYMENT_NAME.format(project_uuid=project.uuid.hex,
name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
# To `tree?`
response = self.auth_client.get(self._get_url(project, 'tree?'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(service_url, 'tree/')
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER],
proxy_url)
# To static files
response = self.auth_client.get(
self._get_url(project, 'static/components/something?v=4.7.0'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(service_url,
'static/components/something?v=4.7.0')
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER],
proxy_url)
def test_redirects_to_proxy_protected_url_with_extra_path(
self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
with patch('scheduler.tasks.experiment_groups.'
'experiments_group_create.apply_async') as _: # noqa
group = ExperimentGroupFactory(project=project)
tensorboard = TensorboardJobFactory(project=project,
experiment_group=group)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = JOB_NAME.format(job_uuid=tensorboard.uuid.hex,
name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
# To `tree?`
response = self.auth_client.get(self._get_url(project, group, 'tree?'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(service_url, 'tree/')
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER],
proxy_url)
# To static files
response = self.auth_client.get(
self._get_url(project, group,
'static/components/something?v=4.7.0'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(service_url,
'static/components/something?v=4.7.0')
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER],
proxy_url)
def test_redirects_to_proxy_protected_url_with_extra_path(self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = JOB_NAME.format(
job_uuid=tensorboard.uuid.hex, name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
# To `tree?`
response = self.auth_client.get(self._get_url(project, 'tree?'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(
service_url,
'tree/'
)
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER], proxy_url)
# To static files
response = self.auth_client.get(
self._get_url(project, 'static/components/something?v=4.7.0'))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/{}'.format(
service_url,
'static/components/something?v=4.7.0'
)
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER], proxy_url)
def setUp(self):
super().setUp()
self.object = self.factory_class(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=self.object)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
self.url = '/{}/{}/{}/tensorboard/stop'.format(
API_V1, self.object.user.username, self.object.name)
self.queryset = TensorboardJob.objects.all()
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.build_job = BuildJobFactory(project=self.project)
self.notebook = NotebookJobFactory(project=self.project, build_job=self.build_job)
self.tensorboard = TensorboardJobFactory(project=self.project, build_job=self.build_job)
self.job = JobFactory(project=self.project, build_job=self.build_job)
self.experiment = ExperimentFactory(project=self.project, build_job=self.build_job)
def test_project_requests_tensorboard_url(self):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
with patch('scheduler.tensorboard_scheduler.get_tensorboard_url') as mock_fct:
response = self.auth_client.get(self._get_url(project))
assert mock_fct.call_count == 1
assert response.status_code == 200
def test_project_requests_tensorboard_url(self):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
with patch('scheduler.tensorboard_scheduler.get_tensorboard_url') as mock_fct:
response = self.auth_client.get(self._get_url(project))
assert mock_fct.call_count == 1
assert response.status_code == 200
def setUp(self):
super().setUp()
self.object = self.factory_class(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=self.object)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
self.url = '/{}/{}/{}/tensorboard/stop'.format(
API_V1,
self.object.user.username,
self.object.name)
self.queryset = TensorboardJob.objects.all()
def test_project_requests_tensorboard_url(self):
project = ProjectFactory(user=self.auth_client.user)
with patch('scheduler.tasks.experiment_groups.'
'experiments_group_create.apply_async') as _: # noqa
group = ExperimentGroupFactory(project=project)
tensorboard = TensorboardJobFactory(project=project,
experiment_group=group)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
with patch('scheduler.tensorboard_scheduler.get_tensorboard_url'
) as mock_fct:
response = self.auth_client.get(self._get_url(project, group))
assert mock_fct.call_count == 1
assert response.status_code == 200
def setUp(self):
super().setUp()
self.project = ProjectFactory(user=self.auth_client.user)
with patch('scheduler.tasks.experiment_groups.'
'experiments_group_create.apply_async') as _: # noqa
self.object = self.factory_class(user=self.auth_client.user,
project=self.project)
tensorboard = TensorboardJobFactory(project=self.project,
experiment_group=self.object)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
self.url = '/{}/{}/{}/groups/{}/tensorboard/stop'.format(
API_V1, self.project.user.username, self.project.name,
self.object.id)
self.queryset = TensorboardJob.objects.all()
def test_delete_tensorboard_jobs(self):
project1 = ProjectFactory()
TensorboardJobFactory(project=project1)
project1.archive()
project2 = ProjectFactory()
job2 = TensorboardJobFactory(project=project2)
job2.archive()
assert TensorboardJob.all.count() == 2
CleaningIntervals.ARCHIVED = -10
delete_archived_tensorboard_jobs()
# Although the other entity is archived it's not deleted because of project1
assert TensorboardJob.all.count() == 1
def test_delete_tensorboard_jobs(self):
project1 = ProjectFactory()
TensorboardJobFactory(project=project1)
project1.archive()
project2 = ProjectFactory()
job2 = TensorboardJobFactory(project=project2)
job2.archive()
assert TensorboardJob.all.count() == 2
conf.set(CLEANING_INTERVALS_ARCHIVES, -10)
delete_archived_tensorboard_jobs()
# Although the other entity is archived it's not deleted because of project1
assert TensorboardJob.all.count() == 1
def test_redirects_to_proxy_protected_url(self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = DEPLOYMENT_NAME.format(
job_uuid=tensorboard.uuid.hex, name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
response = self.auth_client.get(self._get_url(project))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/'.format(service_url)
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER], proxy_url)
def test_redirects_to_proxy_protected_url(self, spawner_mock):
project = ProjectFactory(user=self.auth_client.user)
tensorboard = TensorboardJobFactory(project=project)
tensorboard.set_status(status=JobLifeCycle.RUNNING)
deployment_name = JOB_NAME.format(
job_uuid=tensorboard.uuid.hex, name=self.plugin_app)
service_url = self._get_service_url(deployment_name=deployment_name)
mock_instance = spawner_mock.return_value
mock_instance.get_tensorboard_url.return_value = service_url
response = self.auth_client.get(self._get_url(project))
assert response.status_code == 200
self.assertTrue(ProtectedView.NGINX_REDIRECT_HEADER in response)
proxy_url = '{}/'.format(service_url)
self.assertEqual(response[ProtectedView.NGINX_REDIRECT_HEADER], proxy_url)
def test_tensorboard_creation_triggers_status_creation(self):
assert TensorboardJobStatus.objects.count() == 0
project = ProjectFactory()
TensorboardJobFactory(project=project)
assert TensorboardJobStatus.objects.count() == 1
assert project.tensorboard.last_status == JobLifeCycle.CREATED
def setUp(self):
self.tensorboard = TensorboardJobFactory(project=ProjectFactory())
auditor.validate()
auditor.setup()
tracker.validate()
tracker.setup()
activitylogs.validate()
activitylogs.setup()
super(AuditorTensorboardTest, self).setUp()
def test_project_deletion_cascade_to_tensorboard_job(self):
assert TensorboardJob.objects.count() == 0
project = ProjectFactory()
TensorboardJobFactory(project=project)
assert TensorboardJob.objects.count() == 1
with patch('scheduler.tensorboard_scheduler.stop_tensorboard') as _: # noqa
with patch('scheduler.notebook_scheduler.stop_notebook') as _: # noqa
project.delete()
assert TensorboardJob.objects.count() == 0
def test_status_update_results_in_new_updated_at_datetime_tensorboard(self):
project = ProjectFactory()
job = TensorboardJobFactory(project=project)
updated_at = job.updated_at
# Create new status
TensorboardJobStatus.objects.create(job=job, status=JobLifeCycle.BUILDING)
job.refresh_from_db()
assert updated_at < job.updated_at
updated_at = job.updated_at
# Create status Using set_status
job.set_status(JobLifeCycle.RUNNING)
job.refresh_from_db()
assert updated_at < job.updated_at
def test_archive(self):
project = ProjectFactory()
notebook_job = NotebookJobFactory(project=project)
tensorboard_job = TensorboardJobFactory(project=project)
assert notebook_job.deleted is False
assert tensorboard_job.deleted is False
assert NotebookJob.objects.count() == 1
assert TensorboardJob.all.count() == 1
notebook_job.archive()
tensorboard_job.archive()
assert notebook_job.deleted is True
assert tensorboard_job.deleted is True
assert NotebookJob.objects.count() == 0
assert TensorboardJob.objects.count() == 0
assert NotebookJob.all.count() == 1
assert TensorboardJob.all.count() == 1
notebook_job.restore()
tensorboard_job.restore()
assert notebook_job.deleted is False
assert tensorboard_job.deleted is False
assert NotebookJob.objects.count() == 1
assert TensorboardJob.objects.count() == 1
assert NotebookJob.all.count() == 1
assert TensorboardJob.all.count() == 1
def test_archive_schedules_deletion(self,
xp_group_scheduler_mock,
xp_scheduler_mock,
job_scheduler_mock,
build_scheduler_mock,
notebook_scheduler_mock,
tensorboard_scheduler_mock):
for _ in range(2):
JobFactory(project=self.object)
BuildJobFactory(project=self.object)
TensorboardJobFactory(project=self.object)
NotebookJobFactory(project=self.object)
self.object.experiment_groups.first().set_status(ExperimentGroupLifeCycle.RUNNING)
self.object.experiments.first().set_status(ExperimentLifeCycle.RUNNING)
self.object.jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.build_jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.notebook_jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.tensorboard_jobs.first().set_status(JobLifeCycle.RUNNING)
assert self.queryset.count() == 1
assert ExperimentGroup.objects.count() == 2
assert Experiment.objects.count() == 2
assert Job.objects.count() == 2
assert BuildJob.objects.count() == 2
assert NotebookJob.objects.count() == 2
assert TensorboardJob.objects.count() == 2
resp = self.auth_client.post(self.url + 'archive/')
assert xp_group_scheduler_mock.call_count == 2
assert xp_scheduler_mock.call_count == 1
assert job_scheduler_mock.call_count == 1
assert build_scheduler_mock.call_count == 1
assert notebook_scheduler_mock.call_count == 1
assert tensorboard_scheduler_mock.call_count == 1
assert resp.status_code == status.HTTP_200_OK
assert self.queryset.count() == 0
assert Project.all.filter(user=self.object.user).count() == 1
assert ExperimentGroup.objects.count() == 0
assert ExperimentGroup.all.count() == 2
assert Experiment.objects.count() == 0
assert Experiment.all.count() == 2
assert Job.objects.count() == 0
assert Job.all.count() == 2
assert BuildJob.objects.count() == 0
assert BuildJob.all.count() == 2
assert TensorboardJob.objects.count() == 0
assert TensorboardJob.all.count() == 2
assert NotebookJob.objects.count() == 0
assert NotebookJob.all.count() == 2
def test_archive(self):
project = ProjectFactory()
ExperimentGroupFactory(project=project)
ExperimentFactory(project=project)
JobFactory(project=project)
BuildJobFactory(project=project)
NotebookJobFactory(project=project)
TensorboardJobFactory(project=project)
assert project.deleted is False
assert project.experiments.count() == 1
assert project.experiment_groups.count() == 1
assert project.jobs.count() == 1
assert project.build_jobs.count() == 1
assert project.notebook_jobs.count() == 1
assert project.tensorboard_jobs.count() == 1
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
project.archive()
assert project.deleted is True
assert project.experiments.count() == 0
assert project.experiment_groups.count() == 0
assert project.jobs.count() == 0
assert project.build_jobs.count() == 0
assert project.notebook_jobs.count() == 0
assert project.tensorboard_jobs.count() == 0
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
project.unarchive()
assert project.deleted is False
assert project.experiments.count() == 1
assert project.experiment_groups.count() == 1
assert project.jobs.count() == 1
assert project.build_jobs.count() == 1
assert project.notebook_jobs.count() == 1
assert project.tensorboard_jobs.count() == 1
assert project.all_experiments.count() == 1
assert project.all_experiment_groups.count() == 1
assert project.all_notebook_jobs.count() == 1
assert project.all_tensorboard_jobs.count() == 1
def setUp(self):
self.tensorboard = TensorboardJobFactory(project=ProjectFactory())
super().setUp()
self.tested_events = {
tensorboard_events.TENSORBOARD_STARTED,
tensorboard_events.TENSORBOARD_STARTED_TRIGGERED,
tensorboard_events.TENSORBOARD_STOPPED,
tensorboard_events.TENSORBOARD_STOPPED_TRIGGERED,
tensorboard_events.TENSORBOARD_CLEANED_TRIGGERED,
tensorboard_events.TENSORBOARD_VIEWED,
tensorboard_events.TENSORBOARD_UNBOOKMARKED,
tensorboard_events.TENSORBOARD_BOOKMARKED,
tensorboard_events.TENSORBOARD_NEW_STATUS,
tensorboard_events.TENSORBOARD_FAILED,
tensorboard_events.TENSORBOARD_SUCCEEDED,
tensorboard_events.TENSORBOARD_STATUSES_VIEWED,
tensorboard_events.TENSORBOARD_UPDATED,
tensorboard_events.TENSORBOARD_DELETED,
tensorboard_events.TENSORBOARD_DELETED_TRIGGERED,
tensorboard_events.TENSORBOARD_ARCHIVED,
tensorboard_events.TENSORBOARD_RESTORED,
}
def test_creation_with_bad_config(self):
with self.assertRaises(ValidationError):
NotebookJobFactory(content='foo')
with self.assertRaises(ValidationError):
TensorboardJobFactory(content='foo')
class TestBuildJobStatuses(BaseTest):
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.build_job = BuildJobFactory(project=self.project)
self.notebook = NotebookJobFactory(project=self.project,
build_job=self.build_job)
self.tensorboard = TensorboardJobFactory(project=self.project,
build_job=self.build_job)
self.job = JobFactory(project=self.project, build_job=self.build_job)
self.experiment = ExperimentFactory(project=self.project,
build_job=self.build_job)
def test_build_job_failed_sets_dependency_to_failed(self):
assert self.build_job.last_status != JobLifeCycle.FAILED
assert self.notebook.last_status != JobLifeCycle.FAILED
assert self.tensorboard.last_status != JobLifeCycle.FAILED
assert self.job.last_status != JobLifeCycle.FAILED
assert self.experiment.last_status != ExperimentLifeCycle.FAILED
self.build_job.set_status(JobLifeCycle.FAILED)
assert self.build_job.last_status == JobLifeCycle.FAILED
self.notebook.refresh_from_db()
assert self.notebook.last_status == JobLifeCycle.FAILED
self.tensorboard.refresh_from_db()
assert self.tensorboard.last_status == JobLifeCycle.FAILED
self.job.refresh_from_db()
assert self.job.last_status == JobLifeCycle.FAILED
self.experiment.refresh_from_db()
assert self.experiment.last_status == ExperimentLifeCycle.FAILED
def test_build_job_stopped_sets_dependency_to_stopped(self):
assert self.build_job.last_status != JobLifeCycle.STOPPED
assert self.notebook.last_status != JobLifeCycle.STOPPED
assert self.tensorboard.last_status != JobLifeCycle.STOPPED
assert self.job.last_status != JobLifeCycle.STOPPED
assert self.experiment.last_status != ExperimentLifeCycle.STOPPED
self.build_job.set_status(JobLifeCycle.STOPPED)
assert self.build_job.last_status == JobLifeCycle.STOPPED
self.notebook.refresh_from_db()
assert self.notebook.last_status == JobLifeCycle.STOPPED
self.tensorboard.refresh_from_db()
assert self.tensorboard.last_status == JobLifeCycle.STOPPED
self.job.refresh_from_db()
assert self.job.last_status == JobLifeCycle.STOPPED
self.experiment.refresh_from_db()
assert self.experiment.last_status == ExperimentLifeCycle.STOPPED
def test_build_job_succeeded_starts_dependency(self):
assert self.build_job.last_status != JobLifeCycle.SUCCEEDED
assert self.notebook.last_status != JobLifeCycle.SUCCEEDED
assert self.tensorboard.last_status != JobLifeCycle.SUCCEEDED
assert self.job.last_status != JobLifeCycle.SUCCEEDED
assert self.experiment.last_status != ExperimentLifeCycle.SUCCEEDED
with patch('scheduler.notebook_scheduler.start_notebook'
) as mock_notebook:
with patch('scheduler.tensorboard_scheduler.start_tensorboard'
) as mock_tensorboard:
with patch('scheduler.experiment_scheduler.start_experiment'
) as mock_experiment:
with patch(
'scheduler.job_scheduler.start_job') as mock_job:
self.build_job.set_status(JobLifeCycle.SUCCEEDED)
assert self.build_job.last_status == JobLifeCycle.SUCCEEDED
assert mock_notebook.call_count == 1
assert mock_tensorboard.call_count == 1
assert mock_experiment.call_count == 1
assert mock_job.call_count == 1
def get_job_object(self, job_state):
project_uuid = job_state.details.labels.project_uuid.hex
project = ProjectFactory(uuid=project_uuid)
return TensorboardJobFactory(project=project)
def get_job_object(self, job_state):
project_uuid = job_state.details.labels.project_uuid.hex
return ProjectFactory(uuid=project_uuid,
tensorboard=TensorboardJobFactory()).tensorboard
def setUp(self):
self.tensorboard = TensorboardJobFactory(project=ProjectFactory())
super().setUp()
