def setUp(self):
super().setUp()
self.user = UserFactory()
self.project = ProjectFactory(user=self.user)
self.project_job = JobFactory(project=self.project, name='unique')
self.job = JobFactory(name='unique2')
self.project_experiment = ExperimentFactory(project=self.project,
name='unique')
self.experiment = ExperimentFactory(name='unique2')
self.instance_mock = InstanceSpec(self.user.id, self.project.id)
self.wrong_instance_mock = InstanceSpec(-1, -1)
def test_heartbeat_jobs(self):
job1 = JobFactory()
JobStatusFactory(job=job1, status=JobLifeCycle.SCHEDULED)
job2 = JobFactory()
JobStatusFactory(job=job2, status=JobLifeCycle.CREATED)
job3 = JobFactory()
JobStatusFactory(job=job3, status=JobLifeCycle.FAILED)
job4 = JobFactory()
JobStatusFactory(job=job4, status=JobLifeCycle.RUNNING)
with patch('scheduler.tasks.jobs.jobs_check_heartbeat.apply_async') as mock_fct:
heartbeat_jobs()
assert mock_fct.call_count == 1
def test_jobs_check_heartbeat(self):
job1 = JobFactory()
JobStatusFactory(job=job1, status=JobLifeCycle.RUNNING)
RedisHeartBeat.job_ping(job_id=job1.id)
job2 = JobFactory()
JobStatusFactory(job=job2, status=JobLifeCycle.RUNNING)
jobs_check_heartbeat(job1.id)
job1.refresh_from_db()
self.assertEqual(job1.last_status, JobLifeCycle.RUNNING)
jobs_check_heartbeat(job2.id)
job2.refresh_from_db()
self.assertEqual(job2.last_status, JobLifeCycle.FAILED)
def test_delete_jobs(self):
project1 = ProjectFactory()
JobFactory(project=project1)
project1.archive()
project2 = ProjectFactory()
job2 = JobFactory(project=project2)
job2.archive()
assert Job.all.count() == 2
conf.set(CLEANING_INTERVALS_ARCHIVES, -10)
delete_archived_jobs()
# Although the other entity is archived it's not deleted because of project1
assert Job.all.count() == 1
def test_delete_jobs(self):
project1 = ProjectFactory()
JobFactory(project=project1)
project1.archive()
project2 = ProjectFactory()
job2 = JobFactory(project=project2)
job2.archive()
assert Job.all.count() == 2
CleaningIntervals.ARCHIVED = -10
delete_archived_jobs()
# Although the other entity is archived it's not deleted because of project1
assert Job.all.count() == 1
def setUp(self):
super().setUp()
self.user = UserFactory()
activitylogs.validate()
activitylogs.setup()
self.project = ProjectFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=USER_ACTIVATED,
instance=self.user,
actor_id=self.user.id,
actor_name=self.user.username)
activitylogs.record(ref_id=uuid.uuid4(),
event_type=PROJECT_DELETED_TRIGGERED,
instance=self.project,
actor_id=self.user.id,
actor_name=self.user.username)
self.experiment = ExperimentFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=EXPERIMENT_DELETED_TRIGGERED,
instance=self.experiment,
actor_id=self.user.id,
actor_name=self.user.username)
self.job = JobFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=JOB_VIEWED,
instance=self.job,
actor_id=self.user.id,
actor_name=self.user.username)
def test_job_creation_triggers_scheduling_mocks(self):
with patch('scheduler.tasks.jobs.jobs_build.apply_async') as mock_fct:
with patch.object(Job, 'set_status') as mock_fct2:
JobFactory()
assert mock_fct.call_count == 0
assert mock_fct2.call_count == 1
def test_experiment_group_outputs_path_creation_deletion(self):
job = JobFactory()
create_job_outputs_path(job.unique_name)
job_outputs_path = get_job_outputs_path(job.unique_name)
assert os.path.exists(job_outputs_path) is True
delete_job_outputs(job.unique_name)
assert os.path.exists(job_outputs_path) is False
def test_job_creation_with_already_built_triggers_scheduling(self):
# Create a repo for the project
repo = RepoFactory()
with patch(
'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build:
job = JobFactory(project=repo.project)
assert mock_build.call_count == 1
assert job.project.repo is not None
assert JobStatus.objects.filter(job=job).count() == 1
assert list(
JobStatus.objects.filter(job=job).values_list(
'status', flat=True)) == [JobLifeCycle.CREATED]
with patch('scheduler.dockerizer_scheduler.create_build_job'
) as mock_start:
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED,
job=build)
mock_start.return_value = build, True, True
jobs_build(job_id=job.id)
assert mock_start.call_count == 1
assert JobStatus.objects.filter(job=job).count() == 2
assert list(
JobStatus.objects.filter(job=job).values_list(
'status',
flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.SCHEDULED]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.SCHEDULED
def test_job_creation_triggers_build(self):
# Create a repo for the project
repo = RepoFactory()
with patch(
'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build:
job = JobFactory(project=repo.project)
assert mock_build.call_count == 1
assert job.project.repo is not None
assert JobStatus.objects.filter(job=job).count() == 1
assert list(
JobStatus.objects.filter(job=job).values_list(
'status', flat=True)) == [JobLifeCycle.CREATED]
with patch('scheduler.dockerizer_scheduler.start_dockerizer'
) as mock_start:
mock_start.return_value = True
jobs_build(job_id=job.id)
assert mock_start.call_count == 1
assert JobStatus.objects.filter(job=job).count() == 2
assert list(
JobStatus.objects.filter(job=job).values_list(
'status',
flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.BUILDING]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.BUILDING
def test_patch(self):
new_description = 'updated_xp_name'
data = {'description': new_description}
assert self.object.description != data['description']
resp = self.auth_client.patch(self.url, data=data)
assert resp.status_code == status.HTTP_200_OK
new_object = self.model_class.objects.get(id=self.object.id)
assert new_object.user == self.object.user
assert new_object.description != self.object.description
assert new_object.description == new_description
# Update original job
assert new_object.is_clone is False
new_job = JobFactory()
data = {'original_job': new_job.id}
resp = self.auth_client.patch(self.url, data=data)
assert resp.status_code == status.HTTP_200_OK
new_object = self.model_class.objects.get(id=self.object.id)
assert new_object.user == self.object.user
assert new_object.description == new_description
assert new_object.is_clone is True
assert new_object.original_job == new_job
# Update name
data = {'name': 'new_name'}
assert new_object.name is None
resp = self.auth_client.patch(self.url, data=data)
assert resp.status_code == status.HTTP_200_OK
new_object = self.model_class.objects.get(id=self.object.id)
assert new_object.name == data['name']
def test_delete_job_does_not_trigger_job_stop_if_not_running(self, delete_path):
job = JobFactory()
assert delete_path.call_count == 2 # outputs + logs
with patch('scheduler.job_scheduler.stop_job') as mock_fct:
job.delete()
assert delete_path.call_count == 2 + 2 # outputs + logs
assert mock_fct.call_count == 0
def setUp(self):
super().setUp()
self.job = JobFactory(project=ProjectFactory())
self.tested_events = {
job_events.JOB_CREATED,
job_events.JOB_UPDATED,
job_events.JOB_STARTED,
job_events.JOB_STARTED_TRIGGERED,
job_events.JOB_CLEANED_TRIGGERED,
job_events.JOB_DELETED,
job_events.JOB_DELETED_TRIGGERED,
job_events.JOB_STOPPED,
job_events.JOB_STOPPED_TRIGGERED,
job_events.JOB_VIEWED,
job_events.JOB_ARCHIVED,
job_events.JOB_RESTORED,
job_events.JOB_BOOKMARKED,
job_events.JOB_UNBOOKMARKED,
job_events.JOB_NEW_STATUS,
job_events.JOB_FAILED,
job_events.JOB_SUCCEEDED,
job_events.JOB_DONE,
job_events.JOB_LOGS_VIEWED,
job_events.JOB_RESTARTED,
job_events.JOB_RESTARTED_TRIGGERED,
job_events.JOB_STATUSES_VIEWED,
job_events.JOB_OUTPUTS_DOWNLOADED,
}
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.experiment = ExperimentFactory()
self.group = ExperimentGroupFactory()
self.job = JobFactory()
self.build = BuildJobFactory()
def set_objects(self):
self.user = self.auth_client.user
self.project = ProjectFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=PROJECT_DELETED_TRIGGERED,
instance=self.project,
actor_id=self.user.id,
actor_name=self.user.username)
self.experiment = ExperimentFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=EXPERIMENT_DELETED_TRIGGERED,
instance=self.experiment,
actor_id=self.user.id,
actor_name=self.user.username)
self.job = JobFactory()
activitylogs.record(ref_id=uuid.uuid4(),
event_type=JOB_CREATED,
instance=self.job,
actor_id=self.user.id,
actor_name=self.user.username)
activitylogs.record(ref_id=uuid.uuid4(),
event_type=JOB_VIEWED,
instance=self.job,
actor_id=self.user.id,
actor_name=self.user.username)
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.build_job = BuildJobFactory(project=self.project)
self.notebook = NotebookJobFactory(project=self.project, build_job=self.build_job)
self.tensorboard = TensorboardJobFactory(project=self.project, build_job=self.build_job)
self.job = JobFactory(project=self.project, build_job=self.build_job)
self.experiment = ExperimentFactory(project=self.project, build_job=self.build_job)
def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
self.job = JobFactory(project=project)
self.logs = []
self.url = '/{}/{}/{}/jobs/{}/logs'.format(API_V1,
project.user.username,
project.name, self.job.id)
def test_delete_job_triggers_job_stop_mock(self, delete_path):
job = JobFactory()
job.set_status(JobLifeCycle.SCHEDULED)
assert delete_path.call_count == 2 # outputs + logs
with patch('scheduler.job_scheduler.stop_job') as mock_fct:
job.delete()
assert delete_path.call_count == 2 + 2 # outputs + logs
assert mock_fct.call_count == 1
def test_job_logs_path_creation_deletion(self):
job = JobFactory()
job_logs_path = get_job_logs_path(job.unique_name)
create_job_logs_path(job.unique_name)
open(job_logs_path, '+w')
# Should be true, created by the signal
assert os.path.exists(job_logs_path) is True
delete_job_logs(job.unique_name)
assert os.path.exists(job_logs_path) is False
def test_job_logs_path_creation_deletion(self):
job = JobFactory()
job_logs_path = stores.get_job_logs_path(job_name=job.unique_name, temp=False)
stores.create_job_logs_path(job_name=job.unique_name, temp=False)
open(job_logs_path, '+w')
# Should be true, created by the signal
assert os.path.exists(job_logs_path) is True
stores_schedule_logs_deletion(persistence=None, subpath=job.subpath)
assert os.path.exists(job_logs_path) is False
def test_job_outputs_path_creation_deletion(self):
job = JobFactory()
stores.create_job_outputs_path(persistence=job.persistence_outputs,
job_name=job.unique_name)
job_outputs_path = stores.get_job_outputs_path(
persistence=job.persistence_outputs, job_name=job.unique_name)
assert os.path.exists(job_outputs_path) is True
stores_schedule_outputs_deletion(persistence='outputs',
subpath=job.subpath)
assert os.path.exists(job_outputs_path) is False
def test_job_outputs_path_creation_deletion(self):
job = JobFactory()
create_job_outputs_path(persistence_outputs=job.persistence_outputs,
job_name=job.unique_name)
job_outputs_path = get_job_outputs_path(
persistence_outputs=job.persistence_outputs,
job_name=job.unique_name)
assert os.path.exists(job_outputs_path) is True
delete_job_outputs(persistence_outputs=job.persistence_outputs,
job_name=job.unique_name)
assert os.path.exists(job_outputs_path) is False
def setUp(self):
super().setUp()
with patch.object(Job, 'set_status') as _: # noqa
with patch('scheduler.tasks.jobs.jobs_build.apply_async'
) as _: # noqa
self.job = JobFactory()
self.object = self.factory_class(job=self.job)
self.url = '/{}/{}/{}/jobs/{}/statuses/{}/'.format(
API_V1, self.job.project.user.username, self.job.project.name,
self.job.id, self.object.uuid.hex)
self.queryset = self.model_class.objects.all()
def setUp(self):
super().setUp()
project = ProjectFactory(user=self.auth_client.user)
job = JobFactory(project=project)
self.url = '/{}/{}/{}/jobs/{}/outputs/files'.format(
API_V1, project.user.username, project.name, job.id)
outputs_path = stores.get_job_outputs_path(
persistence=job.persistence_outputs, job_name=job.unique_name)
stores.create_job_outputs_path(persistence=job.persistence_outputs,
job_name=job.unique_name)
self.create_paths(path=outputs_path, url=self.url)
def test_status_update_results_in_new_updated_at_datetime(self):
job = JobFactory()
updated_at = job.updated_at
# Create new status
JobStatus.objects.create(job=job, status=JobLifeCycle.BUILDING)
job.refresh_from_db()
assert updated_at < job.updated_at
updated_at = job.updated_at
# Create status Using set_status
job.set_status(JobLifeCycle.RUNNING)
job.refresh_from_db()
assert updated_at < job.updated_at
def test_set_outputs(self):
experiment_outputs = [
'{}'.format(self.experiment.id),
'{}'.format(self.project_experiment.id),
self.project_experiment.name,
'{}.{}'.format(self.project.name, self.project_experiment.name),
'{}/{}'.format(self.project.name, self.project_experiment.name),
'{}.{}.{}'.format(self.user.username, self.project.name,
self.project_experiment.name),
'{}/{}/{}'.format(self.user.username, self.project.name,
self.project_experiment.name),
]
job_outputs = [
'{}'.format(self.job.id),
'{}'.format(self.project_job.id),
self.project_job.name,
'{}.{}'.format(self.project.name, self.project_job.name),
'{}/{}'.format(self.project.name, self.project_job.name),
'{}.{}.{}'.format(self.user.username, self.project.name,
self.project_job.name),
'{}/{}/{}'.format(self.user.username, self.project.name,
self.project_job.name),
]
outputs_config = OutputsConfig(
jobs=job_outputs, experiments=experiment_outputs).to_dict()
experiment = ExperimentFactory(user=self.user, project=self.project)
assert experiment.outputs is None
experiment.outputs = outputs_config
assert experiment.outputs is not None
assert len(experiment.outputs_experiments) == len(experiment_outputs)
assert len(experiment.outputs_jobs) == len(job_outputs)
set_outputs(instance=experiment)
del experiment.outputs_config
del experiment.outputs_experiments
del experiment.outputs_jobs
assert len(experiment.outputs_experiments) == 2
assert len(experiment.outputs_jobs) == 2
job = JobFactory(user=self.user, project=self.project)
assert job.outputs is None
job.outputs = outputs_config
assert job.outputs is not None
assert job.outputs is not None
assert len(job.outputs_experiments) == len(experiment_outputs)
assert len(job.outputs_jobs) == len(job_outputs)
set_outputs(instance=job)
del job.outputs_config
del job.outputs_experiments
del job.outputs_jobs
assert len(job.outputs_experiments) == 2
assert len(job.outputs_jobs) == 2
def test_delete_triggers_stopping_of_jobs(self):
assert self.queryset.count() == 1
for _ in range(2):
job = JobFactory(project=self.object)
job.set_status(JobLifeCycle.SCHEDULED)
assert Job.objects.count() == 2
with patch('scheduler.tasks.jobs.jobs_stop.apply_async') as job_mock_stop:
resp = self.auth_client.delete(self.url)
assert job_mock_stop.called
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.queryset.count() == 0
assert Job.all.count() == 0
def test_archive_schedules_deletion(self,
xp_group_scheduler_mock,
xp_scheduler_mock,
job_scheduler_mock,
build_scheduler_mock,
notebook_scheduler_mock,
tensorboard_scheduler_mock):
for _ in range(2):
JobFactory(project=self.object)
BuildJobFactory(project=self.object)
TensorboardJobFactory(project=self.object)
NotebookJobFactory(project=self.object)
self.object.experiment_groups.first().set_status(ExperimentGroupLifeCycle.RUNNING)
self.object.experiments.first().set_status(ExperimentLifeCycle.RUNNING)
self.object.jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.build_jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.notebook_jobs.first().set_status(JobLifeCycle.RUNNING)
self.object.tensorboard_jobs.first().set_status(JobLifeCycle.RUNNING)
assert self.queryset.count() == 1
assert ExperimentGroup.objects.count() == 2
assert Experiment.objects.count() == 2
assert Job.objects.count() == 2
assert BuildJob.objects.count() == 2
assert NotebookJob.objects.count() == 2
assert TensorboardJob.objects.count() == 2
resp = self.auth_client.post(self.url + 'archive/')
assert xp_group_scheduler_mock.call_count == 2
assert xp_scheduler_mock.call_count == 1
assert job_scheduler_mock.call_count == 1
assert build_scheduler_mock.call_count == 1
assert notebook_scheduler_mock.call_count == 1
assert tensorboard_scheduler_mock.call_count == 1
assert resp.status_code == status.HTTP_200_OK
assert self.queryset.count() == 0
assert Project.all.filter(user=self.object.user).count() == 1
assert ExperimentGroup.objects.count() == 0
assert ExperimentGroup.all.count() == 2
assert Experiment.objects.count() == 0
assert Experiment.all.count() == 2
assert Job.objects.count() == 0
assert Job.all.count() == 2
assert BuildJob.objects.count() == 0
assert BuildJob.all.count() == 2
assert TensorboardJob.objects.count() == 0
assert TensorboardJob.all.count() == 2
assert NotebookJob.objects.count() == 0
assert NotebookJob.all.count() == 2
def set_objects(self):
self.user = self.auth_client.user
self.project = ProjectFactory(user=self.user)
activitylogs.record(event_type=PROJECT_DELETED_TRIGGERED,
instance=self.project,
actor_id=self.user.id)
self.experiment = ExperimentFactory(project=self.project)
activitylogs.record(event_type=EXPERIMENT_DELETED_TRIGGERED,
instance=self.experiment,
actor_id=self.user.id)
self.job = JobFactory(project=self.project)
activitylogs.record(event_type=JOB_VIEWED,
instance=self.job,
actor_id=self.user.id)
def setUp(self):
super().setUp()
with patch.object(Job, 'set_status') as _:
with patch('scheduler.tasks.jobs.jobs_build.apply_async'
) as _: # noqa
project = ProjectFactory(user=self.auth_client.user)
self.job = JobFactory(project=project)
self.url = '/{}/{}/{}/jobs/{}/statuses/'.format(
API_V1, project.user.username, project.name, self.job.id)
self.objects = [
self.factory_class(job=self.job, status=JobLifeCycle.CHOICES[i][0])
for i in range(self.num_objects)
]
self.queryset = self.model_class.objects.all()