def test_scheduler_create_build_job_of_already_done_job(self):
"""Check the case when the job is already done and
we need to create a new job."""
config = {'image': 'busybox:tag'}
build_job = BuildJobFactory(project=self.project,
user=self.project.user,
code_reference=self.code_reference,
config=BuildSpecification.create_specification(config))
build_job.set_status(JobLifeCycle.STOPPED)
assert BuildJob.objects.count() == 1
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_start.return_value = True
mock_check.return_value = False
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config=config,
code_reference=self.code_reference
)
assert mock_start.call_count == 1
assert mock_check.call_count == 1
assert image_exists is False
assert build_status is True
assert BuildJob.objects.count() == 2
def projects_notebook_build(notebook_job_id):
notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id)
if not notebook_job:
_logger.warning('Notebook %s does not exist anymore.', notebook_job_id)
return None
if not JobLifeCycle.can_transition(status_from=notebook_job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Notebook `%s` cannot transition from `%s` to `%s`.',
notebook_job, notebook_job.last_status, JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=notebook_job.user,
project=notebook_job.project,
config=notebook_job.specification.build,
code_reference=notebook_job.code_reference)
notebook_job.build_job = build_job
notebook_job.save()
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'notebook_job_id': notebook_job_id})
return
if not build_status:
notebook_job.set_status(JobLifeCycle.FAILED, message='Could not start build process.')
return
# Update job status to show that its building docker image
notebook_job.set_status(JobLifeCycle.BUILDING, message='Building container')
def test_scheduler_create_build_job_of_already_done_job(self):
"""Check the case when the job is already done and
we need to create a new job."""
config = {'image': 'busybox:tag'}
build_job = BuildJobFactory(project=self.project,
user=self.project.user,
code_reference=self.code_reference,
config=BuildSpecification.create_specification(config))
build_job.set_status(JobLifeCycle.STOPPED)
assert BuildJob.objects.count() == 1
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_start.return_value = True
mock_check.return_value = False
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config=config,
code_reference=self.code_reference
)
assert mock_start.call_count == 1
assert mock_check.call_count == 1
assert image_exists is False
assert build_status is True
assert BuildJob.objects.count() == 2
def jobs_build(job_id):
job = get_valid_job(job_id=job_id)
if not job:
return None
if not JobLifeCycle.can_transition(status_from=job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Job id `%s` cannot transition from `%s` to `%s`.',
job_id, job.last_status, JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=job.user,
project=job.project,
config=job.specification.build,
configmap_refs=job.specification.configmap_refs,
secret_refs=job.specification.secret_refs,
code_reference=job.code_reference)
job.build_job = build_job
job.save(update_fields=['build_job'])
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.JOBS_START,
kwargs={'job_id': job_id},
countdown=conf.get('GLOBAL_COUNTDOWN'))
return
if not build_status:
job.set_status(JobLifeCycle.FAILED, message='Could not start build process.')
return
# Update job status to show that its building docker image
job.set_status(JobLifeCycle.BUILDING, message='Building container')
def projects_notebook_build(project_id):
project = get_valid_project(project_id=project_id)
if not project or not project.notebook:
_logger.warning('Project does not have a notebook.')
return None
job = project.notebook
if not JobLifeCycle.can_transition(status_from=job.last_status,
status_to=JobLifeCycle.BUILDING):
_logger.info('Notebook for project id `%s` cannot transition from `%s` to `%s`.',
project_id, job.last_status, JobLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=job.user,
project=job.project,
config=job.specification.run_exec,
code_reference=job.code_reference)
job.build_job = build_job
job.save()
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START,
kwargs={'project_id': project_id})
return
if not build_status:
job.set_status(JobLifeCycle.FAILED, message='Could not start build process.')
return
# Update job status to show that its building docker image
job.set_status(JobLifeCycle.BUILDING, message='Building container')
def test_scheduler_create_build_job(self):
"""Test the case when the job needs to be built and started."""
assert BuildJob.objects.count() == 0
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
mock_start.return_value = True
_, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config={'image': 'bar:foo'},
code_reference=self.code_reference
)
assert mock_start.call_count == 1
assert image_exists is False
assert build_status is True
assert BuildJob.objects.count() == 1
def experiments_group_create(self, experiment_group_id):
experiment_group = _get_group_or_retry(
experiment_group_id=experiment_group_id, task=self)
if not experiment_group:
return
last_status = experiment_group.last_status
if not ExperimentGroupLifeCycle.can_transition(
status_from=last_status,
status_to=ExperimentGroupLifeCycle.RUNNING):
_logger.info(
'Experiment group id `%s` cannot transition from `%s` to `%s`.',
experiment_group_id, last_status, ExperimentGroupLifeCycle.RUNNING)
return
def hp_create():
experiment_group.set_status(ExperimentGroupLifeCycle.RUNNING)
celery_app.send_task(
HPCeleryTasks.HP_CREATE,
kwargs={'experiment_group_id': experiment_group_id},
countdown=1)
# We start first by creating a build if necessary
# No need to build the image, start the experiment directly
if not experiment_group.specification.build:
hp_create()
return
_, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=experiment_group.user,
project=experiment_group.project,
config=experiment_group.specification.build,
configmap_refs=experiment_group.specification.configmap_refs,
secret_refs=experiment_group.specification.secret_refs,
code_reference=experiment_group.code_reference)
if image_exists:
# The image already exists, so we can start the experiment right away
hp_create()
return
if not build_status:
experiment_group.set_status(ExperimentGroupLifeCycle.FAILED,
message='Could not start build process.')
return
# We start the group process
hp_create()
def test_scheduler_create_build_job(self):
"""Test the case when the job needs to be built and started."""
assert BuildJob.objects.count() == 0
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_start.return_value = True
mock_check.return_value = False
_, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config={'image': 'bar:foo'},
code_reference=self.code_reference
)
assert mock_start.call_count == 1
assert mock_check.call_count == 1
assert image_exists is False
assert build_status is True
assert BuildJob.objects.count() == 1
def experiments_build(experiment_id):
experiment = get_valid_experiment(experiment_id=experiment_id)
if not experiment:
return
# No need to build the image, start the experiment directly
if not (experiment.specification.build and experiment.specification.run):
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_START,
kwargs={'experiment_id': experiment_id},
countdown=conf.get(SCHEDULER_GLOBAL_COUNTDOWN))
return
last_status = experiment.last_status
if not ExperimentLifeCycle.can_transition(status_from=last_status,
status_to=ExperimentLifeCycle.BUILDING):
_logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.',
experiment_id, last_status, ExperimentLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=experiment.user,
project=experiment.project,
config=experiment.specification.build,
configmap_refs=experiment.specification.configmap_refs,
secret_refs=experiment.specification.secret_refs,
code_reference=experiment.code_reference)
experiment.build_job = build_job
experiment.save(update_fields=['build_job'])
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_START,
kwargs={'experiment_id': experiment_id},
countdown=conf.get(SCHEDULER_GLOBAL_COUNTDOWN))
return
if not build_status:
experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.')
return
# Update experiment status to show that its building
experiment.set_status(ExperimentLifeCycle.BUILDING)
def test_scheduler_create_build_job_of_already_running_job(self):
"""Check the case when the job is already running and
we just set the requesting service to running."""
config = {'image': 'busybox:tag'}
build_job = BuildJobFactory(project=self.project,
user=self.project.user,
code_reference=self.code_reference,
config=BuildSpecification.create_specification(config))
build_job.set_status(JobLifeCycle.RUNNING)
assert BuildJob.objects.count() == 1
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config=config,
code_reference=self.code_reference
)
assert mock_start.call_count == 0
assert image_exists is False
assert build_status is True
assert BuildJob.objects.count() == 1
def experiments_build(experiment_id):
experiment = get_valid_experiment(experiment_id=experiment_id)
if not experiment:
return
# No need to build the image, start the experiment directly
if not (experiment.specification.build and experiment.specification.run):
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_START,
kwargs={'experiment_id': experiment_id})
return
if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status,
status_to=ExperimentLifeCycle.BUILDING):
_logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.',
experiment_id, experiment.last_status, ExperimentLifeCycle.BUILDING)
return
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=experiment.user,
project=experiment.project,
config=experiment.specification.build,
code_reference=experiment.code_reference)
experiment.build_job = build_job
experiment.save()
if image_exists:
# The image already exists, so we can start the experiment right away
celery_app.send_task(
SchedulerCeleryTasks.EXPERIMENTS_START,
kwargs={'experiment_id': experiment_id})
return
if not build_status:
experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.')
return
# Update experiment status to show that its building
experiment.set_status(ExperimentLifeCycle.BUILDING)
def test_scheduler_create_build_job_image_already_exists(self):
"""Check the case when the image is already built."""
config = {'image': 'busybox:tag'}
BuildJobFactory(project=self.project,
user=self.project.user,
code_reference=self.code_reference,
config=BuildSpecification.create_specification(config))
assert BuildJob.objects.count() == 1
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_check.return_value = True
build_job, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config=config,
code_reference=self.code_reference
)
assert mock_start.call_count == 0
assert mock_check.call_count == 1
assert image_exists is True
assert build_status is False
assert BuildJob.objects.count() == 1
def test_scheduler_create_build_job_image_already_exists(self):
"""Check the case when the image is already built."""
config = {'image': 'busybox:tag'}
BuildJobFactory(project=self.project,
user=self.project.user,
code_reference=self.code_reference,
config=BuildSpecification.create_specification(config))
assert BuildJob.objects.count() == 1
with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start:
with patch('scheduler.dockerizer_scheduler.check_image') as mock_check:
mock_check.return_value = True
_, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=self.project.user,
project=self.project,
config=config,
code_reference=self.code_reference
)
assert mock_start.call_count == 0
assert mock_check.call_count == 1
assert image_exists is True
assert build_status is False
assert BuildJob.objects.count() == 1
