def build_app(app_name: str):
"""Build the docker image of an application from the master branch with
a unique tag and uploads it to the configured Docker registry."""
Logger.info({"app": app_name}, "[/api/builds/:app] Building an application image")
Thread(target=_differed_build, args=[app_name]).start()
return "Build processing", HTTPStatus.ACCEPTED
def branch(repository_dir: str, branch_name: str) -> None:
"""Checkout a branch of a repository"""
Logger.debug({"path": repository_dir}, "[git#branch] Repository path")
exists = is_branch_existing(repository_dir, branch_name)
io.execute(f'git checkout{"" if exists else " -b"} {branch_name}',
repository_dir)
def branch(repository_dir, branch_name):
"""Checkout a branch of a repository"""
Logger.debug({"path": repository_dir}, "[git#branch] Repository path")
stdout = io.execute(f"git branch --list {branch_name}", repository_dir)
exists = len(stdout) > 0
io.execute(f'git checkout{"" if exists else " -b"} {branch_name}',
repository_dir)
def non_blocking_clean(path: str, *, message_prefix: str = None):
"""Try to clean directory/file at path but don't
raise an error if it fails (only log it)."""
message = "Error trying to clean temporary directory / file"
if message_prefix is not None:
message = message_prefix.strip() + f" {message}"
try:
io.remove(path)
# pylint: disable=broad-except
except Exception as err:
Logger.warn({"path": path, "err": str(err)}, message)
def create_working_repository(app_name):
"""Checkout a branch of a repository"""
# First, update the pristine repository for this app
pristine_directory = update_pristine_repository(app_name)
repository_dir = io.create_temporary_copy(pristine_directory, app_name)
Logger.debug(
{app_name, repository_dir},
"[git#create_working_directory] Created a working repository",
)
return repository_dir
def update_pristine_repository(app_name):
"""Update the pristine repository of an application"""
app_config = config.get_app_config(app_name)
repository_dir = io.get_pristine_path(app_name)
update_repository(repository_dir, app_config.get("git").get("origin"))
Logger.debug(
{app_name, repository_dir},
"[git#update_pristine_repository] Updated a pristine repository",
)
return repository_dir
def build(app_name: str, repository: str, app_config: dict) -> str:
"""Build the docker image of the last version of the app"""
image_tag = git.get_last_tag(repository)
Logger.info(
{"tag": f"{app_name}@{image_tag}"}, "Application name and version",
)
if has_docker_image(app_name, image_tag):
Logger.info({}, "Docker image already built (skipped)")
return image_tag
commit_hash = git.get_commit_hash_from_tag(repository, image_tag)
build_variables = app_config.get("docker", {}).get("build", {}).get("variables", {})
build_variables["COMMIT_HASH"] = commit_hash
# Application build environment variables:
builds_args = []
for key, value in build_variables.items():
builds_args.append(f"--build-arg {key}={value}")
builds_args_str = " ".join(builds_args)
command = f"docker build --tag {app_name}:{image_tag} {builds_args_str} {repository}"
Logger.debug({"command": command}, "Docker build command")
try:
io.execute(command)
except Exception as err:
Logger.error({"err": err}, "Error while building Docker image")
raise err
return image_tag
def _create_and_protect_branch(
organization: str,
app_name: str,
branch_name: str,
master_sha: str,
user_login: str,
git_provider: AbstractGitProvider,
) -> BranchReport:
"""Try to create and protect a branch on a repository."""
report: BranchReport = {}
branch = None
try:
branch = git_provider.get_branch(organization, app_name, branch_name)
Logger.info({"branch_name": branch_name},
"Branch already exists. Skipped creation.")
report["created"] = CreationStatus(False, True)
except GitResourceNotFoundError as err:
if err.resource == GitResource.BRANCH:
branch = git_provider.create_branch(organization, app_name,
branch_name, master_sha)
Logger.info({"branch_name": branch_name}, "Branch created")
report["created"] = CreationStatus(True, True)
else:
raise
if branch.protected is True:
Logger.info({"branch_name": branch_name},
"Branch is already protected. Skipped protection.")
report["protected"] = ProtectionStatus(False, True)
else:
git_provider.protect_branch(organization, app_name, branch_name,
user_login)
Logger.info({"branch_name": branch_name}, "Branch protected")
report["protected"] = ProtectionStatus(True, True)
return report
def update_pristine_repository(app_name: str, git_url: str) -> str:
"""Update the pristine repository of an application"""
repository_dir = io.get_pristine_path(app_name)
update_repository(repository_dir, git_url)
Logger.debug(
{
"app": app_name,
"repository": repository_dir
},
"[git#update_pristine_repository] Updated a pristine repository",
)
return repository_dir
def create_working_repository(app_name: str, git_url: str) -> str:
"""Create a working copy of an app's repository"""
pristine_directory = update_pristine_repository(app_name, git_url)
repository_dir = io.create_temporary_copy(pristine_directory, app_name)
Logger.debug(
{
"app": app_name,
"repository": repository_dir
},
"[git#create_working_directory] Created a working repository",
)
return repository_dir
def advance_workflow(current_step):
"""Advance the workflow for all applications able to be advanced."""
Logger.info(
{"current_step": current_step},
"[/api/workflow/progress/<current_step>] Workflow advance started",
)
try:
# Creating a copy of the working configuration directory
config_dir = config_lib.create_temporary_config_copy()
config_lib.change_environment(
Configuration.get_config_default_branch(), config_dir)
project_config = config_lib.get_project_config(config_dir)
report_status, report = workflow_lib.advance_workflow(
config_dir, project_config, current_step)
status, message = _get_status_and_message(report_status)
# Clean up
non_blocking_clean(
config_dir,
message_prefix="[/api/workflow/progress/<current_step>]")
# HTTP Response
Logger.info(
{
"current_step": current_step,
"report": report
},
f"[/api/workflow/progress/<current_step>] {message}",
)
return (
{
"current_step": current_step,
"report": report,
"message": message
},
status,
)
# pylint: disable=broad-except
except Exception as err:
Logger.error(
{
"current_step": current_step,
"err": str(err)
},
"[/api/workflow/progress/<current_step>] Workflow advance failed",
)
# HTTP Response
return (
{
"current_step": current_step,
"err": str(err),
"message": "Workflow advance failed",
},
HTTPStatus.INTERNAL_SERVER_ERROR,
)
def build(app_name, context):
"""Build the docker image of the last version of the app
Example:
> app_name = "my-app"
> context = {"repository": "/path_to/my-app-doc", "commit_hash": "a2b3c4"}
> build(app_name, context)
"""
repository = context["repository"]
image_tag = git.get_last_tag(repository)
Logger.info(
{"tag": f"{app_name}@{image_tag}"},
"Application name and version",
)
if has_docker_image(app_name, image_tag):
Logger.info({}, "Docker image already built (skipped)")
return image_tag
app_config = config.get_app_config(app_name)
commit_hash = context["commit_hash"]
build_variables = app_config["build"]["variables"]
# Application build environment variables:
builds_args = []
for key, value in build_variables.items():
builds_args.append(f"--build-arg {key}={value}")
builds_args.append(f"--build-arg COMMIT_HASH={commit_hash}")
builds_args_str = " ".join(builds_args)
command = f"docker build --tag {app_name}:{image_tag} {builds_args_str} {repository}"
Logger.debug({"command": command}, "Docker build command")
try:
io.execute(command)
except Exception as err:
Logger.error({"err": err}, "Error while building Docker image")
raise err
return image_tag
def test_logger_info(self, logging_mock):
Logger.info({"user_id": 1234}, "Found user")
logging_mock.info.assert_called_once_with("%s %s", "Found user",
'{"user_id": 1234}')
def advance_workflow(
config_dir: str, project_config: Dict, current_step: str
) -> Tuple[WorkflowAdvanceStatus, List[AdvanceWorkflowAppReport]]:
"""Advance the application workflow to the next step"""
progress_report: List[AdvanceWorkflowAppReport] = []
next_step = get_next_step(project_config, current_step)
if next_step is None:
raise WorkflowError("Workflow is already in final step.")
status = WorkflowAdvanceStatus.SUCCESS
# List applications
try:
apps = config.list_apps_config(config_dir)
except Exception as err:
raise AppListingError(err)
for (app_name, app_config) in apps.items():
should_app_progress = False
tag = None
app_dir = None
try:
# Determine if app is ready to progress or not
app_dir = git.create_working_repository(
app_name, app_config["git"]["origin"])
should_app_progress, tag = get_app_progress_report(
app_dir, current_step, next_step)
# If app is ready to progress, make it advance to the next step in the workflow
Logger.info(
{
"app": app_name,
"tag": tag,
"current_step": current_step,
"next_step": next_step,
},
"Advancing to the next workflow step" if should_app_progress
else "App is already up-to-date. Skipping.",
)
if should_app_progress:
git.branch(app_dir, next_step)
git.rebase(app_dir, current_step, onto=tag)
git.push(app_dir)
processes = config.get_processes(app_config)
cron_jobs = config.get_cronjobs(app_config)
progress_report.append({
"name": app_name,
"tag": tag,
"step": next_step,
"processes": processes,
"cron_jobs": cron_jobs,
})
# pylint: disable=broad-except
except Exception as err:
Logger.error(
{
"app_name": app_name,
"config_dir": config_dir,
"should_app_progress": should_app_progress,
"tag": tag,
"current_step": current_step,
"err": str(err),
},
"Error while advancing the workflow",
)
status = WorkflowAdvanceStatus.FAIL
if app_dir is not None:
non_blocking_clean(app_dir)
return status, progress_report
def init_workflow(
organization: str, app_name: str, git_provider: AbstractGitProvider
) -> Tuple[WorkflowInitStatus, Report]:
"""Initialize the workflow of an application's repository by creating
all workflow branches. This function is idempotent which means it will not
try to recreate a branch that already exists. However if a branch
already exists but is not protected, it will be set to protected."""
# Create temporary copy for avoiding concurrency problems
config_path = config.create_temporary_config_copy()
# Switch to staging environment in order to get application configuration
config.change_environment("staging", config_path)
# Get application configuration to get the list of workflow branches
app_config = config.get_app_config(app_name, config_path)
master_tag = GitConfiguration.get_master_tag()
workflow_branches = _get_workflow_branches(app_config, master_tag)
branches = {}
status = WorkflowInitStatus.SUCCESS
if len(workflow_branches) != 0:
status = WorkflowInitStatus.FAIL
try:
# Get user_login linked to the GITHUB_TOKEN
user_info = git_provider.get_user_info()
user_login = user_info.login if user_info else None
Logger.info({"user_login": user_login}, "User login retrieved")
# Get the last commit's sha on master branch
branch = git_provider.get_branch(organization, app_name,
master_tag)
master_head_sha = branch and branch.commit and branch.commit.sha
Logger.info({"sha": master_head_sha},
"master last commit sha retrieved")
# Sync all workflow branches with master's head and
# protect them by limiting push rights to user_login
for branch_name in workflow_branches:
branches[branch_name] = _create_and_protect_branch(
organization, app_name, branch_name, master_head_sha,
user_login, git_provider)
except GitResourceNotFoundError as err:
Logger.error(
{
"master_branch_name": master_tag,
"err": err
},
"master last commit sha failed to be retrieved.",
)
except GitProviderError as err:
Logger.error(
{
"organization": organization,
"app_name": app_name,
"err": err
},
"Fail to initialize workflow",
)
else:
status = WorkflowInitStatus.SUCCESS
# Clean temporary copy
non_blocking_clean(config_path)
return status, branches
def _differed_build(app_name: str):
config_dir = None
app_dir = None
try:
# Retrieve app's configuration
config_dir = config.create_temporary_config_copy()
config.change_environment(Configuration.get_config_default_branch(), config_dir)
app_config = config.get_app_config(app_name, config_dir)
Logger.debug(
{"app": app_name, "config_directory": config_dir},
"[/api/builds/:app] Application's configuration retrieved",
)
# Retrieve app's repository
app_dir = git.create_working_repository(app_name, app_config["git"]["origin"])
git.branch(app_dir, app_config["workflow"][0])
Logger.debug(
{"app": app_name, "working_directory": app_dir},
"[/api/builds/:app] Application's repository retrieved",
)
try:
# Create a new tag
version = app.get_version(app_dir)
git_tag = git.tag(app_dir, version)
Logger.debug(
{"app": app_name, "tag": git_tag}, "[/api/builds/:app] New tag created",
)
except Exception as err:
# The tag may already exist
Logger.warn(
{"app": app_name, "err": err}, "[/api/builds/:app] Error while tagging the app"
)
# Build and publish the new docker image
image_tag = docker.build(app_name, app_dir, app_config)
Logger.debug(
{"app": app_name, "image": image_tag}, "[/api/builds/:app] Docker image created"
)
docker.push(app_name, image_tag, app_config)
Logger.debug(
{"app": app_name, "image": image_tag},
"[/api/builds/:app] Docker image published on registry",
)
# Send the new tag to git
git.push(app_dir)
Logger.debug({"app": app_name}, "[/api/builds/:app] Tag pushed to Git")
except Exception as err:
Logger.error(
{"app": app_name, "err": err},
"[/api/builds/:app] Error while tagging and building the app",
)
# Clean up temporary directories
try:
if config_dir is not None:
io.remove(config_dir)
if app_dir is not None:
io.remove(app_dir)
except Exception as err:
Logger.error({"app": app_name, "err": err}, "[/api/builds/:app] Error during cleanup")
def test_logger_error_with_no_context(self, logging_mock):
Logger.error(message="Found user")
logging_mock.error.assert_called_once_with("Found user", exc_info=True)
def test_logger_error(self, logging_mock):
Logger.error({"user_id": 1234}, "Found user")
logging_mock.error.assert_called_once_with("%s %s",
"Found user",
'{"user_id": 1234}',
exc_info=True)
def test_logger_warn_with_no_context(self, logging_mock):
Logger.warn(message="Found user")
logging_mock.warning.assert_called_once_with("Found user")
def test_logger_warn(self, logging_mock):
Logger.warn({"user_id": 1234}, "Found user")
logging_mock.warning.assert_called_once_with("%s %s", "Found user",
'{"user_id": 1234}')
def test_logger_info_with_no_context(self, logging_mock):
Logger.info(message="Found user")
logging_mock.info.assert_called_once_with("Found user")
def test_logger_debug_with_no_context(self, logging_mock):
Logger.debug(message="Found user")
logging_mock.debug.assert_called_once_with("Found user")
def init_workflow(organization, app):
"""Initialize the workflow of an application."""
Logger.info(
{"app": app, "organization": organization},
"[/api/workflow/init/:org/:app] Workflow initialization started",
)
report = None
try:
# Retrieve project configuration
config_dir = config_lib.create_temporary_config_copy()
config_lib.change_environment(Configuration.get_config_default_branch(), config_dir)
project_config = config_lib.get_project_config(config_dir)
git_provider = get_git_provider(project_config)
report_status, report = workflow_lib.init_workflow(organization, app, git_provider)
if report_status == workflow_lib.WorkflowInitStatus.FAIL:
status = HTTPStatus.INTERNAL_SERVER_ERROR
message = "Workflow initialization failed"
elif report_status == workflow_lib.WorkflowInitStatus.SUCCESS:
status = HTTPStatus.OK
message = "Workflow initialization succeeded"
else:
raise ValueError(f"Unexpected status: '{report_status}'")
# Clean the temporary directory
try:
io_lib.remove(config_dir)
# pylint: disable=broad-except
except Exception as err:
Logger.warn(
{"config_dir": config_dir, "err": err}, "Failed to clean temporary config dir"
)
# HTTP Response
Logger.info(
{"organization": organization, "app": app, "report": report},
f"[/api/workflow/init/:org/:app] {message}",
)
return (
{"organization": organization, "app": app, "report": report, "message": message},
status,
)
# pylint: disable=broad-except
except Exception as err:
Logger.error(
{"organization": organization, "app": app, "report": report, "err": str(err)},
"[/api/workflow/init/:org/:app] Workflow initialization failed",
)
# HTTP Response
return (
{
"organization": organization,
"app": app,
"err": str(err),
"message": "Workflow initialization failed",
},
HTTPStatus.INTERNAL_SERVER_ERROR,
)