def test_scheduler_create_build_job_of_already_done_job(self): """Check the case when the job is already done and we need to create a new job.""" config = {'image': 'busybox:tag'} build_job = BuildJobFactory(project=self.project, user=self.project.user, code_reference=self.code_reference, config=BuildSpecification.create_specification(config)) build_job.set_status(JobLifeCycle.STOPPED) assert BuildJob.objects.count() == 1 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: with patch('scheduler.dockerizer_scheduler.check_image') as mock_check: mock_start.return_value = True mock_check.return_value = False build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config=config, code_reference=self.code_reference ) assert mock_start.call_count == 1 assert mock_check.call_count == 1 assert image_exists is False assert build_status is True assert BuildJob.objects.count() == 2
def projects_notebook_build(notebook_job_id): notebook_job = get_valid_notebook(notebook_job_id=notebook_job_id) if not notebook_job: _logger.warning('Notebook %s does not exist anymore.', notebook_job_id) return None if not JobLifeCycle.can_transition(status_from=notebook_job.last_status, status_to=JobLifeCycle.BUILDING): _logger.info('Notebook `%s` cannot transition from `%s` to `%s`.', notebook_job, notebook_job.last_status, JobLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=notebook_job.user, project=notebook_job.project, config=notebook_job.specification.build, code_reference=notebook_job.code_reference) notebook_job.build_job = build_job notebook_job.save() if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START, kwargs={'notebook_job_id': notebook_job_id}) return if not build_status: notebook_job.set_status(JobLifeCycle.FAILED, message='Could not start build process.') return # Update job status to show that its building docker image notebook_job.set_status(JobLifeCycle.BUILDING, message='Building container')
def jobs_build(job_id): job = get_valid_job(job_id=job_id) if not job: return None if not JobLifeCycle.can_transition(status_from=job.last_status, status_to=JobLifeCycle.BUILDING): _logger.info('Job id `%s` cannot transition from `%s` to `%s`.', job_id, job.last_status, JobLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=job.user, project=job.project, config=job.specification.build, configmap_refs=job.specification.configmap_refs, secret_refs=job.specification.secret_refs, code_reference=job.code_reference) job.build_job = build_job job.save(update_fields=['build_job']) if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.JOBS_START, kwargs={'job_id': job_id}, countdown=conf.get('GLOBAL_COUNTDOWN')) return if not build_status: job.set_status(JobLifeCycle.FAILED, message='Could not start build process.') return # Update job status to show that its building docker image job.set_status(JobLifeCycle.BUILDING, message='Building container')
def projects_notebook_build(project_id): project = get_valid_project(project_id=project_id) if not project or not project.notebook: _logger.warning('Project does not have a notebook.') return None job = project.notebook if not JobLifeCycle.can_transition(status_from=job.last_status, status_to=JobLifeCycle.BUILDING): _logger.info('Notebook for project id `%s` cannot transition from `%s` to `%s`.', project_id, job.last_status, JobLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=job.user, project=job.project, config=job.specification.run_exec, code_reference=job.code_reference) job.build_job = build_job job.save() if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.PROJECTS_NOTEBOOK_START, kwargs={'project_id': project_id}) return if not build_status: job.set_status(JobLifeCycle.FAILED, message='Could not start build process.') return # Update job status to show that its building docker image job.set_status(JobLifeCycle.BUILDING, message='Building container')
def test_scheduler_create_build_job(self): """Test the case when the job needs to be built and started.""" assert BuildJob.objects.count() == 0 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: mock_start.return_value = True _, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config={'image': 'bar:foo'}, code_reference=self.code_reference ) assert mock_start.call_count == 1 assert image_exists is False assert build_status is True assert BuildJob.objects.count() == 1
def experiments_group_create(self, experiment_group_id): experiment_group = _get_group_or_retry( experiment_group_id=experiment_group_id, task=self) if not experiment_group: return last_status = experiment_group.last_status if not ExperimentGroupLifeCycle.can_transition( status_from=last_status, status_to=ExperimentGroupLifeCycle.RUNNING): _logger.info( 'Experiment group id `%s` cannot transition from `%s` to `%s`.', experiment_group_id, last_status, ExperimentGroupLifeCycle.RUNNING) return def hp_create(): experiment_group.set_status(ExperimentGroupLifeCycle.RUNNING) celery_app.send_task( HPCeleryTasks.HP_CREATE, kwargs={'experiment_group_id': experiment_group_id}, countdown=1) # We start first by creating a build if necessary # No need to build the image, start the experiment directly if not experiment_group.specification.build: hp_create() return _, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment_group.user, project=experiment_group.project, config=experiment_group.specification.build, configmap_refs=experiment_group.specification.configmap_refs, secret_refs=experiment_group.specification.secret_refs, code_reference=experiment_group.code_reference) if image_exists: # The image already exists, so we can start the experiment right away hp_create() return if not build_status: experiment_group.set_status(ExperimentGroupLifeCycle.FAILED, message='Could not start build process.') return # We start the group process hp_create()
def test_scheduler_create_build_job(self): """Test the case when the job needs to be built and started.""" assert BuildJob.objects.count() == 0 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: with patch('scheduler.dockerizer_scheduler.check_image') as mock_check: mock_start.return_value = True mock_check.return_value = False _, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config={'image': 'bar:foo'}, code_reference=self.code_reference ) assert mock_start.call_count == 1 assert mock_check.call_count == 1 assert image_exists is False assert build_status is True assert BuildJob.objects.count() == 1
def experiments_build(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}, countdown=conf.get(SCHEDULER_GLOBAL_COUNTDOWN)) return last_status = experiment.last_status if not ExperimentLifeCycle.can_transition(status_from=last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, last_status, ExperimentLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment.user, project=experiment.project, config=experiment.specification.build, configmap_refs=experiment.specification.configmap_refs, secret_refs=experiment.specification.secret_refs, code_reference=experiment.code_reference) experiment.build_job = build_job experiment.save(update_fields=['build_job']) if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}, countdown=conf.get(SCHEDULER_GLOBAL_COUNTDOWN)) return if not build_status: experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.') return # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING)
def test_scheduler_create_build_job_of_already_running_job(self): """Check the case when the job is already running and we just set the requesting service to running.""" config = {'image': 'busybox:tag'} build_job = BuildJobFactory(project=self.project, user=self.project.user, code_reference=self.code_reference, config=BuildSpecification.create_specification(config)) build_job.set_status(JobLifeCycle.RUNNING) assert BuildJob.objects.count() == 1 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config=config, code_reference=self.code_reference ) assert mock_start.call_count == 0 assert image_exists is False assert build_status is True assert BuildJob.objects.count() == 1
def experiments_build(experiment_id): experiment = get_valid_experiment(experiment_id=experiment_id) if not experiment: return # No need to build the image, start the experiment directly if not (experiment.specification.build and experiment.specification.run): celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not ExperimentLifeCycle.can_transition(status_from=experiment.last_status, status_to=ExperimentLifeCycle.BUILDING): _logger.info('Experiment id `%s` cannot transition from `%s` to `%s`.', experiment_id, experiment.last_status, ExperimentLifeCycle.BUILDING) return build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment.user, project=experiment.project, config=experiment.specification.build, code_reference=experiment.code_reference) experiment.build_job = build_job experiment.save() if image_exists: # The image already exists, so we can start the experiment right away celery_app.send_task( SchedulerCeleryTasks.EXPERIMENTS_START, kwargs={'experiment_id': experiment_id}) return if not build_status: experiment.set_status(ExperimentLifeCycle.FAILED, message='Could not start build process.') return # Update experiment status to show that its building experiment.set_status(ExperimentLifeCycle.BUILDING)
def test_scheduler_create_build_job_image_already_exists(self): """Check the case when the image is already built.""" config = {'image': 'busybox:tag'} BuildJobFactory(project=self.project, user=self.project.user, code_reference=self.code_reference, config=BuildSpecification.create_specification(config)) assert BuildJob.objects.count() == 1 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: with patch('scheduler.dockerizer_scheduler.check_image') as mock_check: mock_check.return_value = True build_job, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config=config, code_reference=self.code_reference ) assert mock_start.call_count == 0 assert mock_check.call_count == 1 assert image_exists is True assert build_status is False assert BuildJob.objects.count() == 1
def test_scheduler_create_build_job_image_already_exists(self): """Check the case when the image is already built.""" config = {'image': 'busybox:tag'} BuildJobFactory(project=self.project, user=self.project.user, code_reference=self.code_reference, config=BuildSpecification.create_specification(config)) assert BuildJob.objects.count() == 1 with patch('scheduler.dockerizer_scheduler.start_dockerizer') as mock_start: with patch('scheduler.dockerizer_scheduler.check_image') as mock_check: mock_check.return_value = True _, image_exists, build_status = dockerizer_scheduler.create_build_job( user=self.project.user, project=self.project, config=config, code_reference=self.code_reference ) assert mock_start.call_count == 0 assert mock_check.call_count == 1 assert image_exists is True assert build_status is False assert BuildJob.objects.count() == 1