def build_job(project, job, job_builder, image_tag=None): """Build necessary code for a job to run""" job_spec = job.specification build_info = get_job_repo_info(project, job) # Build the image docker_builder = job_builder(project_id=project.id, project_name=project.unique_name, repo_path=build_info['repo_path'], from_image=job_spec.run_exec.image, image_name=build_info['image_name'], image_tag=image_tag or build_info['image_tag'], build_steps=job_spec.run_exec.build_steps, env_vars=job_spec.run_exec.env_vars) docker_builder.login(registry_user=settings.REGISTRY_USER, registry_password=settings.REGISTRY_PASSWORD, registry_host=get_registry_host()) if not docker_builder.build(): docker_builder.clean() return False if not docker_builder.push(): docker_builder.clean() try: job.set_status(JobLifeCycle.FAILED, message='The docker image could not be pushed.') except ObjectDoesNotExist: pass return False docker_builder.clean() return True
def get_experiment_repo_info(experiment): """Returns information required to create a build for an experiment.""" project_name = experiment.project.name experiment_spec = experiment.specification if experiment_spec.run_exec.git: # We need to fetch the repo first repo, is_created = ExternalRepo.objects.get_or_create( project=experiment.project, git_url=experiment_spec.run_exec.git) if not is_created: # If the repo already exist, we just need to refetch it git.fetch(git_url=repo.git_url, repo_path=repo.path) if not experiment.code_reference.commit: # Update experiment commit if not set already code_reference, _ = CodeReference.objects.get_or_create( repo=repo, commit=repo.last_commit[0]) experiment.code_reference = code_reference experiment.save() repo_path = repo.path repo_name = repo.name else: repo_path = experiment.project.repo.path repo_name = project_name image_name = '{}/{}'.format(get_registry_host(), repo_name) image_tag = experiment.code_reference.commit if not image_tag: raise Repo.DoesNotExist return { 'repo_path': repo_path, 'image_name': image_name, 'image_tag': image_tag }
def get_job_repo_info(project, job): project_name = project.name job_spec = job.specification if job_spec.run_exec.git: # We need to fetch the repo first repo, is_created = ExternalRepo.objects.get_or_create( project=project, git_url=job_spec.run_exec.git) if not is_created: # If the repo already exist, we just need to refetch it git.fetch(git_url=repo.git_url, repo_path=repo.path) repo_path = repo.path repo_name = repo.name last_commit = repo.last_commit else: repo_path = project.repo.path last_commit = project.repo.last_commit repo_name = project_name image_name = '{}/{}'.format(get_registry_host(), repo_name) if not last_commit: raise Repo.DoesNotExist image_tag = last_commit[0] return { 'repo_path': repo_path, 'image_name': image_name, 'image_tag': image_tag }
def build_experiment(experiment, image_tag=None): """Build necessary code for an experiment to run""" experiment_spec = experiment.specification build_info = get_experiment_repo_info(experiment) # Build the image docker_builder = ExperimentDockerBuilder( experiment_name=experiment.unique_name, experiment_uuid=experiment.uuid.hex, repo_path=build_info['repo_path'], from_image=experiment_spec.run_exec.image, image_name=build_info['image_name'], image_tag=image_tag or build_info['image_tag'], build_steps=experiment_spec.run_exec.build_steps, env_vars=experiment_spec.run_exec.env_vars) docker_builder.login(registry_user=settings.REGISTRY_USER, registry_password=settings.REGISTRY_PASSWORD, registry_host=get_registry_host()) if not docker_builder.build(): docker_builder.clean() return False if not docker_builder.push(): docker_builder.clean() try: experiment.set_status( ExperimentLifeCycle.FAILED, message='The docker image could not be pushed.') except ObjectDoesNotExist: pass return False docker_builder.clean() return True
def build_job(project, job, job_builder, image_tag=None): """Build necessary code for a job to run""" job_spec = job.compiled_spec build_info = get_job_repo_info(project, job) # Build the image docker_builder = job_builder(project_id=project.id, project_name=project.unique_name, repo_path=build_info['repo_path'], from_image=job_spec.run_exec.image, image_name=build_info['image_name'], image_tag=image_tag or build_info['image_tag'], steps=job_spec.run_exec.steps, env_vars=job_spec.run_exec.env_vars) docker_builder.login(registry_user=settings.REGISTRY_USER, registry_password=settings.REGISTRY_PASSWORD, registry_host=get_registry_host()) if not docker_builder.build(): docker_builder.clean() return False if not docker_builder.push(): docker_builder.clean() return False docker_builder.clean() return True
def build_experiment(experiment, image_tag=None): """Build necessary code for an experiment to run""" experiment_spec = experiment.compiled_spec build_info = get_experiment_repo_info(experiment) # Build the image docker_builder = ExperimentDockerBuilder( experiment_name=experiment.unique_name, experiment_uuid=experiment.uuid.hex, repo_path=build_info['repo_path'], from_image=experiment_spec.run_exec.image, image_name=build_info['image_name'], image_tag=image_tag or build_info['image_tag'], steps=experiment_spec.run_exec.steps, env_vars=experiment_spec.run_exec.env_vars) docker_builder.login(registry_user=settings.REGISTRY_USER, registry_password=settings.REGISTRY_PASSWORD, registry_host=get_registry_host()) if not docker_builder.build(): docker_builder.clean() return False if not docker_builder.push(): docker_builder.clean() return False docker_builder.clean() return True
def get_job_image_info(project, job): """Return the image name and image tag for a job""" project_name = project.name job_spec = job.compiled_spec if job_spec.run_exec.git: try: repo = ExternalRepo.objects.get(project=project, git_url=job_spec.run_exec.git) except ExternalRepo.DoesNotExist: logger.error('Something went wrong, ' 'the external repo `{}` was not found'.format( job_spec.run_exec.git)) raise ValueError('Repo was not found for `{}`.'.format( job_spec.run_exec.git)) repo_name = repo.name last_commit = repo.last_commit else: repo_name = project_name last_commit = project.repo.last_commit image_name = '{}/{}'.format(get_registry_host(), repo_name) if not last_commit: raise ValueError( 'Repo was not found for project `{}`.'.format(project)) return image_name, last_commit[0]
def get_experiment_image_info(experiment): """Return the image name and image tag for an experiment""" project_name = experiment.project.name experiment_spec = experiment.specification if experiment_spec.run_exec.git: try: repo = ExternalRepo.objects.get(project=experiment.project, git_url=experiment_spec.run_exec.git) except ExternalRepo.DoesNotExist: logger.error( 'Something went wrong, the external repo `%s` was not found', experiment_spec.run_exec.git) raise ValueError('Repo was not found for `{}`.'.format(experiment_spec.run_exec.git)) repo_name = repo.name else: repo_name = project_name image_name = '{}/{}'.format(get_registry_host(), repo_name) image_tag = experiment.code_reference.commit return image_name, image_tag