def get_job_repo_info(project, job):
project_name = project.name
job_spec = job.specification
if job_spec.run_exec.git: # We need to fetch the repo first
repo, is_created = ExternalRepo.objects.get_or_create(
project=project, git_url=job_spec.run_exec.git)
if not is_created:
# If the repo already exist, we just need to refetch it
git.fetch(git_url=repo.git_url, repo_path=repo.path)
repo_path = repo.path
repo_name = repo.name
last_commit = repo.last_commit
else:
repo_path = project.repo.path
last_commit = project.repo.last_commit
repo_name = project_name
image_name = '{}/{}'.format(get_registry_host(), repo_name)
if not last_commit:
raise Repo.DoesNotExist
image_tag = last_commit[0]
return {
'repo_path': repo_path,
'image_name': image_name,
'image_tag': image_tag
}
def test_external_repo_creation_and_deletion(self):
repo_path = '{}/{}/{}/{}'.format(settings.REPOS_ROOT,
self.project.user.username,
self.project.name,
'empty')
self.assertFalse(os.path.exists(repo_path))
git_url = 'https://github.com/polyaxon/empty.git'
# Create repo
repo = ExternalRepo(project=self.project, git_url=git_url)
repo.save()
assert repo.path == repo_path
assert repo.name == 'empty'
self.assertTrue(os.path.exists(repo_path))
git_file_path = '{}/.git'.format(repo_path)
self.assertTrue(os.path.exists(git_file_path))
# Test fetch works correctly
git.fetch(repo.git_url, repo.path)
# Test query model works
assert repo == ExternalRepo.objects.get(project=self.project, git_url=git_url)
# Check last commit
assert repo.last_commit is None
# Delete repo
repo.delete()
self.assertFalse(os.path.exists(repo_path))
def get_experiment_repo_info(experiment):
"""Returns information required to create a build for an experiment."""
project_name = experiment.project.name
experiment_spec = experiment.specification
if experiment_spec.run_exec.git: # We need to fetch the repo first
repo, is_created = ExternalRepo.objects.get_or_create(
project=experiment.project, git_url=experiment_spec.run_exec.git)
if not is_created:
# If the repo already exist, we just need to refetch it
git.fetch(git_url=repo.git_url, repo_path=repo.path)
if not experiment.code_reference.commit:
# Update experiment commit if not set already
code_reference, _ = CodeReference.objects.get_or_create(
repo=repo, commit=repo.last_commit[0])
experiment.code_reference = code_reference
experiment.save()
repo_path = repo.path
repo_name = repo.name
else:
repo_path = experiment.project.repo.path
repo_name = project_name
image_name = '{}/{}'.format(get_registry_host(), repo_name)
image_tag = experiment.code_reference.commit
if not image_tag:
raise Repo.DoesNotExist
return {
'repo_path': repo_path,
'image_name': image_name,
'image_tag': image_tag
}
def build_experiment(experiment):
"""Build necessary code for an experiment to run"""
project_name = experiment.project.name
experiment_spec = experiment.compiled_spec
if experiment_spec.run_exec.git: # We need to fetch the repo first
repo, is_created = ExternalRepo.objects.get_or_create(project=experiment.project,
git_url=experiment_spec.run_exec.git)
if not is_created:
# If the repo already exist, we just need to refetch it
git.fetch(git_url=repo.git_url, repo_path=repo.path)
if not experiment.commit:
# Update experiment commit if not set already
experiment.commit = repo.last_commit[0]
experiment.save()
repo_path = repo.path
repo_name = repo.name
else:
repo_path = experiment.project.repo.path
repo_name = project_name
image_name = '{}/{}'.format(settings.REGISTRY_HOST, repo_name)
image_tag = experiment.commit
if not image_tag:
raise Repo.DoesNotExist(
'Repo was not found for project `{}`.'.format(experiment.unique_name))
# Build the image
docker_builder = ExperimentDockerBuilder(experiment_name=experiment.unique_name,
experiment_uuid=experiment.uuid.hex,
repo_path=repo_path,
from_image=experiment_spec.run_exec.image,
image_name=image_name,
image_tag=image_tag,
steps=experiment_spec.run_exec.steps,
env_vars=experiment_spec.run_exec.env_vars)
docker_builder.login(registry_user=settings.REGISTRY_USER,
registry_password=settings.REGISTRY_PASSWORD,
registry_host=settings.REGISTRY_HOST)
if not docker_builder.build():
docker_builder.clean()
return False
if not docker_builder.push():
docker_builder.clean()
return False
docker_builder.clean()
return True
def build_job(project, job, job_builder):
"""Build necessary code for a job to run"""
project_name = project.name
job_spec = job.compiled_spec
if job_spec.run_exec.git: # We need to fetch the repo first
repo, is_created = ExternalRepo.objects.get_or_create(project=project,
git_url=job_spec.run_exec.git)
if not is_created:
# If the repo already exist, we just need to refetch it
git.fetch(git_url=repo.git_url, repo_path=repo.path)
repo_path = repo.path
repo_name = repo.name
last_commit = repo.last_commit
else:
repo_path = project.repo.path
last_commit = project.repo.last_commit
repo_name = project_name
image_name = '{}/{}'.format(settings.REGISTRY_HOST, repo_name)
if not last_commit:
raise Repo.DoesNotExist(
'Repo was not found for project `{}`.'.format(project.unique_name))
image_tag = last_commit[0]
# Build the image
docker_builder = job_builder(project_id=project.id,
project_name=project.unique_name,
repo_path=repo_path,
from_image=job_spec.run_exec.image,
image_name=image_name,
image_tag=image_tag,
steps=job_spec.run_exec.steps,
env_vars=job_spec.run_exec.env_vars)
docker_builder.login(registry_user=settings.REGISTRY_USER,
registry_password=settings.REGISTRY_PASSWORD,
registry_host=settings.REGISTRY_HOST)
if not docker_builder.build():
docker_builder.clean()
return False
if not docker_builder.push():
docker_builder.clean()
return False
docker_builder.clean()
return True