def build(app_name: str, repository: str, app_config: dict) -> str:
"""Build the docker image of the last version of the app"""
image_tag = git.get_last_tag(repository)
Logger.info(
{"tag": f"{app_name}@{image_tag}"}, "Application name and version",
)
if has_docker_image(app_name, image_tag):
Logger.info({}, "Docker image already built (skipped)")
return image_tag
commit_hash = git.get_commit_hash_from_tag(repository, image_tag)
build_variables = app_config.get("docker", {}).get("build", {}).get("variables", {})
build_variables["COMMIT_HASH"] = commit_hash
# Application build environment variables:
builds_args = []
for key, value in build_variables.items():
builds_args.append(f"--build-arg {key}={value}")
builds_args_str = " ".join(builds_args)
command = f"docker build --tag {app_name}:{image_tag} {builds_args_str} {repository}"
Logger.debug({"command": command}, "Docker build command")
try:
io.execute(command)
except Exception as err:
Logger.error({"err": err}, "Error while building Docker image")
raise err
return image_tag
def branch(repository_dir: str, branch_name: str) -> None:
"""Checkout a branch of a repository"""
Logger.debug({"path": repository_dir}, "[git#branch] Repository path")
exists = is_branch_existing(repository_dir, branch_name)
io.execute(f'git checkout{"" if exists else " -b"} {branch_name}',
repository_dir)
def test_execute_should_raise_if_failure(self, subprocess_run_mock):
subprocess_run_mock.return_value.returncode = 1
subprocess_run_mock.return_value.stderr.decode.return_value = "An error message\n"
with self.assertRaises(RuntimeError) as context:
io.execute("a command with --arg1 arg-value")
self.assertEqual(str(context.exception), "An error message")
def test_execute(mocker):
mocker.patch.object(subprocess, "run")
io.execute("a command with --arg1 arg-value")
subprocess.run.assert_called_with(
["a", "command", "with", "--arg1", "arg-value"],
check=True,
cwd=None,
stdout=-1)
def branch(repository_dir, branch_name):
"""Checkout a branch of a repository"""
Logger.debug({"path": repository_dir}, "[git#branch] Repository path")
stdout = io.execute(f"git branch --list {branch_name}", repository_dir)
exists = len(stdout) > 0
io.execute(f'git checkout{"" if exists else " -b"} {branch_name}',
repository_dir)
def update_repository(repository_dir, git_url, revision="origin/master"):
"""Get the latest version of a revision or clone the repository"""
should_clone = True
if io.exists(repository_dir):
# If the target directory already exists
remote_url = None
try:
remote_url = get_remote_url(repository_dir)
except Exception: # pylint: disable=broad-except
# If the directory is not a git repository, `get_remote_url` will throw
pass
if remote_url == git_url:
# If the remotes are the same, clean and fetch
io.execute("git clean -dfx", repository_dir)
io.execute("git fetch --all", repository_dir)
# No need to clone
should_clone = False
else:
# If the remotes mismatch, remove the old one
io.remove(repository_dir)
if should_clone:
io.execute(f"git clone {git_url} {repository_dir}")
io.execute(f"git reset --hard {revision}", repository_dir)
def change_environment(environment: str,
config_path=Configuration.get_config_path()):
"""Change the environment (branch) of the configuration"""
io.execute("git stash", config_path)
io.execute("git fetch origin", config_path)
io.execute(f"git checkout {environment}", config_path)
io.execute(f"git reset --hard origin/{environment}", config_path)
def tag(repository_dir: str,
tag_name: str,
tag_message: str = "NESTOR_AUTO_TAG") -> str:
"""Add a tag to the repository"""
commit_hash = get_last_commit_hash(repository_dir)
final_tag = f"{tag_name}-sha-{commit_hash}"
if not semver.VersionInfo.isvalid(final_tag):
raise RuntimeError(f'Invalid version tag: "{final_tag}".')
io.execute(f"git tag -a {final_tag} {commit_hash} -m '{tag_message}'",
repository_dir)
return final_tag
def push(app_name: str, image_tag: str, app_config: dict) -> None:
"""Push an image to the configured docker registry"""
if not has_docker_image(app_name, image_tag):
raise RuntimeError("Docker image not available")
# This will need to be done a bit differently to work with other registries (GCP)
# -> the config schema currently expects
# {docker: {registries: {[name: string]: {id: string, organization: string}[]}}}
registry = app_config["docker"]["registries"]["docker.com"][0]
# Create the tag
image = get_registry_image_tag(app_name, image_tag, registry)
io.execute(f"docker tag {app_name}:{image_tag} {image}")
io.execute(f"docker push {image}")
def tag(repository_dir, app_name, tag_name):
"""Add a tag to the repository"""
app_config = config.get_app_config(app_name)
# Move to the corresponding environment branch
branch(repository_dir, app_config.get("workflow")[0])
# Extract the last commit hash:
commit_hash = get_last_commit_hash(repository_dir)
final_tag = f"{tag_name}-sha-{commit_hash}"
if not semver.VersionInfo.isvalid(final_tag):
raise RuntimeError(f'Invalid version tag: "{final_tag}".')
io.execute(f"git tag -a {final_tag} {commit_hash}", repository_dir)
return final_tag
def build(app_name, context):
"""Build the docker image of the last version of the app
Example:
> app_name = "my-app"
> context = {"repository": "/path_to/my-app-doc", "commit_hash": "a2b3c4"}
> build(app_name, context)
"""
repository = context["repository"]
image_tag = git.get_last_tag(repository)
Logger.info(
{"tag": f"{app_name}@{image_tag}"},
"Application name and version",
)
if has_docker_image(app_name, image_tag):
Logger.info({}, "Docker image already built (skipped)")
return image_tag
app_config = config.get_app_config(app_name)
commit_hash = context["commit_hash"]
build_variables = app_config["build"]["variables"]
# Application build environment variables:
builds_args = []
for key, value in build_variables.items():
builds_args.append(f"--build-arg {key}={value}")
builds_args.append(f"--build-arg COMMIT_HASH={commit_hash}")
builds_args_str = " ".join(builds_args)
command = f"docker build --tag {app_name}:{image_tag} {builds_args_str} {repository}"
Logger.debug({"command": command}, "Docker build command")
try:
io.execute(command)
except Exception as err:
Logger.error({"err": err}, "Error while building Docker image")
raise err
return image_tag
def push(app_name, repository):
"""Push an image to the configured docker registry"""
# This will need to be done a bit differently to work with GCP registry
app_config = config.get_app_config(app_name)
image_tag = git.get_last_tag(repository)
if not has_docker_image(app_name, image_tag):
raise RuntimeError("Docker image not available")
registry = app_config["docker"]["registry"]
# Create the tag
image = get_registry_image_tag(app_name, image_tag, registry)
io.execute(f"docker tag {app_name}:{image_tag} {image}")
io.execute(f"docker push {image}")
def get_commits_between_tags(repository_dir: str, tag_old: str,
tag_new: str) -> list:
"""Get the commits between two tags (ordered with the newest first)."""
output = io.execute(
f"git log --oneline --no-decorate refs/tags/{tag_old}..refs/tags/{tag_new}",
repository_dir)
commits = []
for line in output.splitlines():
commit = {}
split_idx = line.find(" ")
commit["hash"] = line[:split_idx]
commit["message"] = line[split_idx + 1:]
commits.append(commit)
return commits
def fetch_resource_configuration(cluster_name: str, namespace: str,
app_name: str,
resources: List[K8sResourceKind]) -> dict:
"""Fetch a resource's configuration using kubectl."""
resources_str = ",".join([str(resource) for resource in resources])
command = ("kubectl "
f"--context {cluster_name} "
f"--namespace {namespace} "
f"get {resources_str} "
"--output=json "
f"--selector app={app_name}")
env = _build_kubectl_env()
stdout = io.execute(command, env=env)
return json.loads(stdout)
def test_execute(self, subprocess_run_mock):
subprocess_run_mock.return_value.returncode = 0
subprocess_run_mock.return_value.stdout.decode.return_value = "some output\n"
output = io.execute("a command with --arg1 arg-value")
self.assertEqual(output, "some output")
subprocess_run_mock.assert_called_with(
"a command with --arg1 arg-value",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=None,
env=None,
shell=True,
check=False,
)
def has_docker_image(app_name, tag):
"""Checks if the docker image already exists for a given app and tag"""
stdout = io.execute(f"docker images {app_name}:{tag} --quiet")
return len(stdout) != 0
def get_remote_url(repository_dir, remote_name="origin"):
"""Retrieves the remote url of a repository"""
return io.execute(f"git remote get-url {remote_name}", repository_dir)
def test_execute_should_raise_if_failure():
with pytest.raises(Exception):
io.execute("a command with --arg1 arg-value")
def push(repository_dir, branch_name="HEAD"):
"""Push to the remote repository"""
io.execute(f"git push origin {branch_name} --tags --follow-tags",
repository_dir)
def get_last_tag(repository_dir):
"""Retrieves the last tag of a repository (locally)"""
return io.execute("git describe --always --abbrev=0", repository_dir)
def apply_config(cluster_name: str, yaml_path: str) -> None:
"""Apply the k8s configuration using kubectl."""
command = "kubectl " f"--context {cluster_name} " f"apply -f {yaml_path}"
env = _build_kubectl_env()
io.execute(command, env=env)
def get_commit_hash_from_tag(repository_dir: str, tag_name: str) -> str:
"""Returns the commit hash associated to the given tag"""
return io.execute(f"git rev-list -1 {tag_name}", repository_dir)
def rebase(repository_dir: str, branch_name: str, *, onto: str = None) -> None:
"""Rebase the current branch on top of the given branch"""
io.execute(
f'git rebase{f" --onto {onto}" if onto else ""} {branch_name} --keep-empty',
repository_dir)
def get_last_commit_hash(repository_dir: str, reference: str = "HEAD") -> str:
"""Retrieves the last commit hash of a repository (locally)"""
return io.execute(f"git rev-parse --short {reference}", repository_dir)
def is_branch_existing(repository_dir: str, branch_name: str) -> bool:
"""Determines if a branch exists on the repository"""
stdout = io.execute(f"git branch --list {branch_name}", repository_dir)
return len(stdout) != 0
def get_last_commit_hash(repository_dir):
"""Retrieves the last commit hash of a repository (locally)"""
return io.execute("git rev-parse --short HEAD", repository_dir)
