def test_job_creation_triggers_build(self):
# Create a repo for the project
repo = RepoFactory()
with patch(
'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build:
job = JobFactory(project=repo.project)
assert mock_build.call_count == 1
assert job.project.repo is not None
assert JobStatus.objects.filter(job=job).count() == 1
assert list(
JobStatus.objects.filter(job=job).values_list(
'status', flat=True)) == [JobLifeCycle.CREATED]
with patch('scheduler.dockerizer_scheduler.start_dockerizer'
) as mock_start:
mock_start.return_value = True
jobs_build(job_id=job.id)
assert mock_start.call_count == 1
assert JobStatus.objects.filter(job=job).count() == 2
assert list(
JobStatus.objects.filter(job=job).values_list(
'status',
flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.BUILDING]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.BUILDING
def test_job_creation_with_already_built_triggers_scheduling(self):
# Create a repo for the project
repo = RepoFactory()
with patch(
'scheduler.tasks.jobs.jobs_build.apply_async') as mock_build:
job = JobFactory(project=repo.project)
assert mock_build.call_count == 1
assert job.project.repo is not None
assert JobStatus.objects.filter(job=job).count() == 1
assert list(
JobStatus.objects.filter(job=job).values_list(
'status', flat=True)) == [JobLifeCycle.CREATED]
with patch('scheduler.dockerizer_scheduler.create_build_job'
) as mock_start:
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED,
job=build)
mock_start.return_value = build, True, True
jobs_build(job_id=job.id)
assert mock_start.call_count == 1
assert JobStatus.objects.filter(job=job).count() == 2
assert list(
JobStatus.objects.filter(job=job).values_list(
'status',
flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.SCHEDULED]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.SCHEDULED
def test_status_update_results_in_new_updated_at_datetime(self):
job = JobFactory()
updated_at = job.updated_at
# Create new status
JobStatus.objects.create(job=job, status=JobLifeCycle.BUILDING)
job.refresh_from_db()
assert updated_at < job.updated_at
updated_at = job.updated_at
# Create status Using set_status
job.set_status(JobLifeCycle.RUNNING)
job.refresh_from_db()
assert updated_at < job.updated_at
def test_jobs_check_heartbeat(self):
job1 = JobFactory()
JobStatusFactory(job=job1, status=JobLifeCycle.RUNNING)
RedisHeartBeat.job_ping(job_id=job1.id)
job2 = JobFactory()
JobStatusFactory(job=job2, status=JobLifeCycle.RUNNING)
jobs_check_heartbeat(job1.id)
job1.refresh_from_db()
self.assertEqual(job1.last_status, JobLifeCycle.RUNNING)
jobs_check_heartbeat(job2.id)
job2.refresh_from_db()
self.assertEqual(job2.last_status, JobLifeCycle.FAILED)
def test_create_job_with_resources_spec(self, spawner_mock):
config = JobSpecification.read(job_spec_resources_content)
mock_instance = spawner_mock.return_value
mock_instance.start_job.return_value = {'pod': 'pod_content'}
mock_instance.spec = config
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_start.return_value = False
mock_check.return_value = True
job = JobFactory(config=config.parsed_data)
assert JobStatus.objects.filter(job=job).count() == 2
assert list(JobStatus.objects.filter(job=job).values_list(
'status', flat=True)) == [JobLifeCycle.CREATED,
JobLifeCycle.SCHEDULED]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.SCHEDULED
def test_create_job_with_valid_spec(self, spawner_mock):
config = JobSpecification.read(job_spec_content)
mock_instance = spawner_mock.return_value
mock_instance.start_job.return_value = {'pod': 'pod_content'}
mock_instance.spec = config
with patch('scheduler.dockerizer_scheduler.create_build_job'
) as mock_start:
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED,
job=build)
mock_start.return_value = build, True, True
job = JobFactory(config=config.parsed_data)
assert JobStatus.objects.filter(job=job).count() == 2
assert list(
JobStatus.objects.filter(job=job).values_list(
'status',
flat=True)) == [JobLifeCycle.CREATED, JobLifeCycle.SCHEDULED]
job.refresh_from_db()
assert job.last_status == JobLifeCycle.SCHEDULED
class TestBuildJobStatuses(BaseTest):
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.build_job = BuildJobFactory(project=self.project)
self.notebook = NotebookJobFactory(project=self.project,
build_job=self.build_job)
self.tensorboard = TensorboardJobFactory(project=self.project,
build_job=self.build_job)
self.job = JobFactory(project=self.project, build_job=self.build_job)
self.experiment = ExperimentFactory(project=self.project,
build_job=self.build_job)
def test_build_job_failed_sets_dependency_to_failed(self):
assert self.build_job.last_status != JobLifeCycle.FAILED
assert self.notebook.last_status != JobLifeCycle.FAILED
assert self.tensorboard.last_status != JobLifeCycle.FAILED
assert self.job.last_status != JobLifeCycle.FAILED
assert self.experiment.last_status != ExperimentLifeCycle.FAILED
self.build_job.set_status(JobLifeCycle.FAILED)
assert self.build_job.last_status == JobLifeCycle.FAILED
self.notebook.refresh_from_db()
assert self.notebook.last_status == JobLifeCycle.FAILED
self.tensorboard.refresh_from_db()
assert self.tensorboard.last_status == JobLifeCycle.FAILED
self.job.refresh_from_db()
assert self.job.last_status == JobLifeCycle.FAILED
self.experiment.refresh_from_db()
assert self.experiment.last_status == ExperimentLifeCycle.FAILED
def test_build_job_stopped_sets_dependency_to_stopped(self):
assert self.build_job.last_status != JobLifeCycle.STOPPED
assert self.notebook.last_status != JobLifeCycle.STOPPED
assert self.tensorboard.last_status != JobLifeCycle.STOPPED
assert self.job.last_status != JobLifeCycle.STOPPED
assert self.experiment.last_status != ExperimentLifeCycle.STOPPED
self.build_job.set_status(JobLifeCycle.STOPPED)
assert self.build_job.last_status == JobLifeCycle.STOPPED
self.notebook.refresh_from_db()
assert self.notebook.last_status == JobLifeCycle.STOPPED
self.tensorboard.refresh_from_db()
assert self.tensorboard.last_status == JobLifeCycle.STOPPED
self.job.refresh_from_db()
assert self.job.last_status == JobLifeCycle.STOPPED
self.experiment.refresh_from_db()
assert self.experiment.last_status == ExperimentLifeCycle.STOPPED
def test_build_job_succeeded_starts_dependency(self):
assert self.build_job.last_status != JobLifeCycle.SUCCEEDED
assert self.notebook.last_status != JobLifeCycle.SUCCEEDED
assert self.tensorboard.last_status != JobLifeCycle.SUCCEEDED
assert self.job.last_status != JobLifeCycle.SUCCEEDED
assert self.experiment.last_status != ExperimentLifeCycle.SUCCEEDED
with patch('scheduler.notebook_scheduler.start_notebook'
) as mock_notebook:
with patch('scheduler.tensorboard_scheduler.start_tensorboard'
) as mock_tensorboard:
with patch('scheduler.experiment_scheduler.start_experiment'
) as mock_experiment:
with patch(
'scheduler.job_scheduler.start_job') as mock_job:
self.build_job.set_status(JobLifeCycle.SUCCEEDED)
assert self.build_job.last_status == JobLifeCycle.SUCCEEDED
assert mock_notebook.call_count == 1
assert mock_tensorboard.call_count == 1
assert mock_experiment.call_count == 1
assert mock_job.call_count == 1
