def __find_apps_config_from_repo(apps_git, root_git):
yaml = YAML()
apps_from_other_repos = [
] # List for all entries in .applications from each config repository
found_app_config_file = None
found_app_config_file_name = None
found_app_config_apps = set()
bootstrap_entries = __get_bootstrap_entries(root_git)
for bootstrap_entry in bootstrap_entries:
if "name" not in bootstrap_entry:
raise GitOpsException(
"Every bootstrap entry must have a 'name' property.")
app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml"
logging.info("Analyzing %s in root repository", app_file_name)
app_config_file = root_git.get_full_file_path(app_file_name)
try:
with open(app_config_file, "r") as stream:
app_config_content = yaml.load(stream)
except FileNotFoundError as ex:
raise GitOpsException(
f"File '{app_file_name}' not found in root repository."
) from ex
if "repository" not in app_config_content:
raise GitOpsException(
f"Cannot find key 'repository' in '{app_file_name}'")
if app_config_content["repository"] == apps_git.get_clone_url():
logging.info("Found apps repository in %s", app_file_name)
found_app_config_file = app_config_file
found_app_config_file_name = app_file_name
found_app_config_apps = __get_applications_from_app_config(
app_config_content)
else:
apps_from_other_repos += __get_applications_from_app_config(
app_config_content)
return found_app_config_file, found_app_config_file_name, found_app_config_apps, apps_from_other_repos
def __get_repo(self):
try:
return self._github.get_repo(f"{self._organisation}/{self._repository_name}")
except BadCredentialsException as ex:
raise GitOpsException("Bad credentials") from ex
except UnknownObjectException as ex:
raise GitOpsException(f"Repository '{self._organisation}/{self._repository_name}' does not exist.") from ex
def __update_values(git, file, values, single_commit, commit_message):
full_file_path = git.get_full_file_path(file)
if not os.path.isfile(full_file_path):
raise GitOpsException(f"No such file: {file}")
updated_values = {}
for key in values:
value = values[key]
try:
updated_value = update_yaml_file(full_file_path, key, value)
except KeyError as ex:
raise GitOpsException(f"Key '{key}' not found in {file}") from ex
if not updated_value:
logging.info("Yaml property %s already up-to-date", key)
continue
logging.info("Updated yaml property %s to %s", key, value)
updated_values[key] = value
if not single_commit and commit_message is None:
git.commit(f"changed '{key}' to '{value}' in {file}")
if updated_values and single_commit and commit_message is None:
if len(updated_values) == 1:
key, value = list(updated_values.items())[0]
git.commit(f"changed '{key}' to '{value}' in {file}")
else:
msg = f"updated {len(updated_values)} value{'s' if len(updated_values) > 1 else ''} in {file}"
msg += f"\n\n{yaml_dump(updated_values)}"
git.commit(msg)
if updated_values and commit_message is not None:
git.commit(commit_message)
return updated_values
def __init__(
self,
git_provider_url: str,
username: Optional[str],
password: Optional[str],
organisation: str,
repository_name: str,
) -> None:
try:
self.__gitlab = gitlab.Gitlab(git_provider_url,
private_token=password)
project = self.__gitlab.projects.get(
f"{organisation}/{repository_name}")
except requests.exceptions.ConnectionError as ex:
raise GitOpsException(
f"Error connecting to '{git_provider_url}''") from ex
except gitlab.exceptions.GitlabAuthenticationError as ex:
raise GitOpsException("Bad Personal Access Token")
except gitlab.exceptions.GitlabGetError as ex:
if ex.response_code == 404:
raise GitOpsException(
f"Repository '{organisation}/{repository_name}' does not exist"
)
raise GitOpsException(
f"Error getting repository: '{ex.error_message}'")
self.__token_name = username
self.__access_token = password
self.__project = project
def clone(self, branch: Optional[str] = None) -> None:
self.__delete_tmp_dir()
self.__tmp_dir = create_tmp_dir()
git_options = []
url = self.get_clone_url()
if branch:
logging.info("Cloning repository: %s (branch: %s)", url, branch)
else:
logging.info("Cloning repository: %s", url)
username = self.__api.get_username()
password = self.__api.get_password()
try:
if username is not None and password is not None:
credentials_file = self.__create_credentials_file(
username, password)
git_options.append(
f"--config credential.helper={credentials_file}")
if branch:
git_options.append(f"--branch {branch}")
self.__repo = Repo.clone_from(url=url,
to_path=f"{self.__tmp_dir}/repo",
multi_options=git_options)
except GitError as ex:
if branch:
raise GitOpsException(
f"Error cloning branch '{branch}' of '{url}'") from ex
raise GitOpsException(f"Error cloning '{url}'") from ex
def get_preview_namespace(self, preview_id: str) -> str:
preview_namespace = self.preview_target_namespace_template
preview_namespace = preview_namespace.replace("${APPLICATION_NAME}", self.application_name)
preview_namespace = preview_namespace.replace("${PREVIEW_ID_HASH}", self.create_preview_id_hash(preview_id))
preview_namespace = preview_namespace.replace(
"${PREVIEW_ID_HASH_SHORT}", self.create_preview_id_hash_short(preview_id)
)
current_length = len(preview_namespace) - len("${PREVIEW_ID}")
remaining_length = self.preview_target_max_namespace_length - current_length
if remaining_length < 1:
preview_namespace = preview_namespace.replace("${PREVIEW_ID}", "")
raise GitOpsException(
f"Preview namespace is too long (max {self.preview_target_max_namespace_length} chars): "
f"{preview_namespace} ({len(preview_namespace)} chars)"
)
sanitized_preview_id = self.__sanitize(preview_id, remaining_length)
preview_namespace = preview_namespace.replace("${PREVIEW_ID}", sanitized_preview_id)
preview_namespace = preview_namespace.lower()
invalid_character = re.search(r"[^a-z0-9-]", preview_namespace)
if invalid_character:
raise GitOpsException(f"Invalid character in preview namespace: '{invalid_character[0]}'")
return preview_namespace
def push(self, branch):
try:
self._repo.git.push("--set-upstream", "origin", branch)
except GitCommandError as ex:
raise GitOpsException(
f"Error pushing branch '{branch}' to origin: {ex}") from ex
except GitError as ex:
raise GitOpsException(
f"Error pushing branch '{branch}' to origin.") from ex
def __update_yaml_file(git_repo: GitRepo, file_path: str, key: str,
value: Any) -> bool:
full_file_path = git_repo.get_full_file_path(file_path)
try:
return update_yaml_file(full_file_path, key, value)
except (FileNotFoundError, IsADirectoryError) as ex:
raise GitOpsException(f"No such file: {file_path}") from ex
except YAMLException as ex:
raise GitOpsException(f"Error loading file: {file_path}") from ex
except KeyError as ex:
raise GitOpsException(
f"Key '{key}' not found in file: {file_path}") from ex
def delete_branch(self, branch):
branches = self._bitbucket.get_branches(self._organisation,
self._repository_name,
filter=branch,
limit=1)
if not branches:
raise GitOpsException(f"Branch '{branch}' not found'")
result = self._bitbucket.delete_branch(self._organisation,
self._repository_name, branch,
branches[0]["latestCommit"])
if result and "errors" in result:
raise GitOpsException(result["errors"][0]["message"])
def __find_apps_config_from_repo(
team_config_git_repo: GitRepo, root_config_git_repo: GitRepo
) -> Tuple[str, str, Set[str], Set[str], str]:
apps_from_other_repos: Set[str] = set(
) # Set for all entries in .applications from each config repository
found_app_config_file = None
found_app_config_file_name = None
found_apps_path = "applications"
found_app_config_apps: Set[str] = set()
bootstrap_entries = __get_bootstrap_entries(root_config_git_repo)
team_config_git_repo_clone_url = team_config_git_repo.get_clone_url()
for bootstrap_entry in bootstrap_entries:
if "name" not in bootstrap_entry:
raise GitOpsException(
"Every bootstrap entry must have a 'name' property.")
app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml"
logging.info("Analyzing %s in root repository", app_file_name)
app_config_file = root_config_git_repo.get_full_file_path(
app_file_name)
try:
app_config_content = yaml_file_load(app_config_file)
except FileNotFoundError as ex:
raise GitOpsException(
f"File '{app_file_name}' not found in root repository."
) from ex
if "config" in app_config_content:
app_config_content = app_config_content["config"]
found_apps_path = "config.applications"
if "repository" not in app_config_content:
raise GitOpsException(
f"Cannot find key 'repository' in '{app_file_name}'")
if app_config_content["repository"] == team_config_git_repo_clone_url:
logging.info("Found apps repository in %s", app_file_name)
found_app_config_file = app_config_file
found_app_config_file_name = app_file_name
found_app_config_apps = __get_applications_from_app_config(
app_config_content)
else:
apps_from_other_repos.update(
__get_applications_from_app_config(app_config_content))
if found_app_config_file is None or found_app_config_file_name is None:
raise GitOpsException(
f"Couldn't find config file for apps repository in root repository's 'apps/' directory"
)
return (
found_app_config_file,
found_app_config_file_name,
found_app_config_apps,
apps_from_other_repos,
found_apps_path,
)
def push(self, branch: Optional[str] = None) -> None:
repo = self.__get_repo()
if not branch:
branch = repo.git.branch("--show-current")
logging.info("Pushing branch: %s", branch)
try:
repo.git.push("--set-upstream", "origin", branch)
except GitCommandError as ex:
raise GitOpsException(
f"Error pushing branch '{branch}' to origin: {ex.stderr}"
) from ex
except GitError as ex:
raise GitOpsException(
f"Error pushing branch '{branch}' to origin.") from ex
def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any:
root_config_git_repo.clone()
bootstrap_values_file = root_config_git_repo.get_full_file_path(
"bootstrap/values.yaml")
try:
bootstrap_yaml = yaml_file_load(bootstrap_values_file)
except FileNotFoundError as ex:
raise GitOpsException(
"File 'bootstrap/values.yaml' not found in root repository."
) from ex
if "bootstrap" not in bootstrap_yaml:
raise GitOpsException(
"Cannot find key 'bootstrap' in 'bootstrap/values.yaml'")
return bootstrap_yaml["bootstrap"]
def __get_bootstrap_entries(root_git):
root_git.checkout("master")
bootstrap_values_file = root_git.get_full_file_path(
"bootstrap/values.yaml")
try:
with open(bootstrap_values_file, "r") as stream:
bootstrap_yaml = YAML().load(stream)
except FileNotFoundError as ex:
raise GitOpsException(
"File 'bootstrap/values.yaml' not found in root repository."
) from ex
if "bootstrap" not in bootstrap_yaml:
raise GitOpsException(
"Cannot find key 'bootstrap' in 'bootstrap/values.yaml'")
return bootstrap_yaml["bootstrap"]
def __check_if_app_already_exists(apps_dirs: Set[str],
apps_from_other_repos: Set[str]) -> None:
for app_key in apps_dirs:
if app_key in apps_from_other_repos:
raise GitOpsException(
f"Application '{app_key}' already exists in a different repository"
)
def delete_branch(self, branch: str) -> None:
branch_hash = self.get_branch_head_hash(branch)
result = self.__bitbucket.delete_branch(self.__organisation,
self.__repository_name, branch,
branch_hash)
if result and "errors" in result:
raise GitOpsException(result["errors"][0]["message"])
def delete_branch(self, branch):
repo = self.__get_repo()
try:
git_ref = repo.get_git_ref(f"heads/{branch}")
except UnknownObjectException as ex:
raise GitOpsException(f"Branch '{branch}' does not exist.") from ex
git_ref.delete()
def get_pull_request_branch(self, pr_id):
pull_request = self._bitbucket.get_pullrequest(self._organisation,
self._repository_name,
pr_id)
if "errors" in pull_request:
raise GitOpsException(pull_request["errors"][0]["message"])
return pull_request["fromRef"]["displayId"]
def __get_pull_request(self, pr_id: int) -> PullRequest.PullRequest:
repo = self.__get_repo()
try:
return repo.get_pull(pr_id)
except UnknownObjectException as ex:
raise GitOpsException(
f"Pull request with ID '{pr_id}' does not exist.") from ex
def __sync_apps(apps_git, root_git):
logging.info("Apps repository: %s", apps_git.get_clone_url())
logging.info("Root repository: %s", root_git.get_clone_url())
repo_apps = __get_repo_apps(apps_git)
logging.info("Found %s app(s) in apps repository: %s", len(repo_apps),
", ".join(repo_apps))
logging.info(
"Searching apps repository in root repository's 'apps/' directory...")
apps_config_file, apps_config_file_name, current_repo_apps, apps_from_other_repos = __find_apps_config_from_repo(
apps_git, root_git)
if apps_config_file is None:
raise GitOpsException(
f"Could't find config file for apps repository in root repository's 'apps/' directory"
)
if current_repo_apps == repo_apps:
logging.info("Root repository already up-to-date. I'm done here.")
return
__check_if_app_already_exists(repo_apps, apps_from_other_repos)
logging.info("Sync applications in root repository's %s.",
apps_config_file_name)
merge_yaml_element(apps_config_file, "applications",
{repo_app: {}
for repo_app in repo_apps}, True)
__commit_and_push(apps_git, root_git, apps_config_file_name)
def test_checkout_error(self):
checkout_exception = GitOpsException("dummy checkout error")
self.git_util_mock.checkout.side_effect = checkout_exception
with pytest.raises(GitOpsException) as ex:
deploy_command(
command="deploy",
file="test/file.yml",
values={
"a.b.c": "foo",
"a.b.d": "bar"
},
username="******",
password="******",
git_user="******",
git_email="GIT_EMAIL",
create_pr=False,
auto_merge=False,
single_commit=False,
organisation="ORGA",
repository_name="REPO",
git_provider="github",
git_provider_url=None,
)
self.assertEqual(ex.value, checkout_exception)
assert self.mock_manager.mock_calls == [
call.create_tmp_dir(),
call.create_git("USERNAME", "PASSWORD", "GIT_USER", "GIT_EMAIL",
"ORGA", "REPO", "github", None,
"/tmp/created-tmp-dir"),
call.git_util.checkout("master"),
call.delete_tmp_dir("/tmp/created-tmp-dir"),
]
def merge_pull_request(
self,
pr_id: int,
merge_method: Literal["squash", "rebase",
"merge"] = "merge") -> None:
merge_request = self.__project.mergerequests.get(pr_id)
max_retries = MAX_MERGE_RETRIES
while max_retries > 0:
try:
if merge_method == "rebase":
merge_request.rebase()
return
merge_request.merge()
return
except gitlab.exceptions.GitlabMRClosedError as ex:
# "Branch cannot be merged" error can occur if the server
# is still processing the merge request internally
max_retries -= 1
logging.warning(
"Retry merging pull request. Attempts: (%s/%s)",
MAX_MERGE_RETRIES - max_retries, MAX_MERGE_RETRIES)
if max_retries == 0:
raise GitOpsException(
"Error merging pull request: 'Branch cannot be merged'"
) from ex
time.sleep(2.5)
def __create_preview_from_template_if_not_existing(
self, template_git_repo: GitRepo, target_git_repo: GitRepo,
gitops_config: GitOpsConfig) -> bool:
preview_namespace = gitops_config.get_preview_namespace(
self.__args.preview_id)
full_preview_folder_path = target_git_repo.get_full_file_path(
preview_namespace)
preview_env_already_exist = os.path.isdir(full_preview_folder_path)
if preview_env_already_exist:
logging.info("Use existing folder for preview: %s",
preview_namespace)
return False
logging.info("Create new folder for preview: %s", preview_namespace)
full_preview_template_folder_path = template_git_repo.get_full_file_path(
gitops_config.preview_template_path)
if not os.path.isdir(full_preview_template_folder_path):
raise GitOpsException(
f"The preview template folder does not exist: {gitops_config.preview_template_path}"
)
logging.info("Using the preview template folder: %s",
gitops_config.preview_template_path)
shutil.copytree(
full_preview_template_folder_path,
full_preview_folder_path,
)
return True
def __create_preview_from_template_if_not_existing(
self, git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool:
preview_namespace = gitops_config.get_preview_namespace(
self.__args.preview_id)
full_preview_folder_path = git_repo.get_full_file_path(
preview_namespace)
preview_env_already_exist = os.path.isdir(full_preview_folder_path)
if preview_env_already_exist:
logging.info("Use existing folder for preview: %s",
preview_namespace)
return False
logging.info("Create new folder for preview: %s", preview_namespace)
preview_template_folder_name = f".preview-templates/{gitops_config.application_name}"
full_preview_template_folder_path = git_repo.get_full_file_path(
preview_template_folder_name)
if not os.path.isdir(full_preview_template_folder_path):
raise GitOpsException(
f"The preview template folder does not exist: {preview_template_folder_name}"
)
logging.info("Using the preview template folder: %s",
preview_template_folder_name)
shutil.copytree(
full_preview_template_folder_path,
full_preview_folder_path,
)
self.__update_yaml_file(git_repo, f"{preview_namespace}/Chart.yaml",
"name", preview_namespace)
return True
def create(config: GitApiConfig, organisation: str,
repository_name: str) -> GitRepoApi:
git_repo_api: Optional[GitRepoApi]
if config.git_provider is GitProvider.GITHUB:
git_repo_api = GithubGitRepoApiAdapter(
username=config.username,
password=config.password,
organisation=organisation,
repository_name=repository_name,
)
elif config.git_provider is GitProvider.BITBUCKET:
if not config.git_provider_url:
raise GitOpsException("Please provide url for Bitbucket!")
git_repo_api = BitbucketGitRepoApiAdapter(
git_provider_url=config.git_provider_url,
username=config.username,
password=config.password,
organisation=organisation,
repository_name=repository_name,
)
elif config.git_provider is GitProvider.GITLAB:
provider_url = config.git_provider_url
if not provider_url:
provider_url = "https://www.gitlab.com"
git_repo_api = GitlabGitRepoApiAdapter(
git_provider_url=provider_url,
username=config.username,
password=config.password,
organisation=organisation,
repository_name=repository_name,
)
return GitRepoApiLoggingProxy(git_repo_api)
def test_clone_error(self):
clone_exception = GitOpsException("dummy clone error")
self.git_repo_mock.clone.side_effect = clone_exception
args = DeployCommand.Args(
file="test/file.yml",
values={
"a.b.c": "foo",
"a.b.d": "bar"
},
username="******",
password="******",
git_user="******",
git_email="GIT_EMAIL",
create_pr=False,
auto_merge=False,
single_commit=False,
organisation="ORGA",
repository_name="REPO",
git_provider=GitProvider.GITHUB,
git_provider_url=None,
commit_message=None,
)
with pytest.raises(GitOpsException) as ex:
DeployCommand(args).execute()
self.assertEqual(ex.value, clone_exception)
assert self.mock_manager.method_calls == [
call.GitRepoApiFactory.create(args, "ORGA", "REPO"),
call.GitRepo(self.git_repo_api_mock),
call.GitRepo.clone(),
]
def execute(self) -> None:
gitops_config = self.__get_gitops_config()
preview_id = self.__args.preview_id
preview_target_git_repo_api = self.__create_preview_target_git_repo_api(
gitops_config)
with GitRepo(preview_target_git_repo_api) as preview_target_git_repo:
preview_target_git_repo.clone(gitops_config.preview_target_branch)
preview_namespace = gitops_config.get_preview_namespace(preview_id)
logging.info("Preview folder name: %s", preview_namespace)
preview_folder_exists = self.__delete_folder_if_exists(
preview_target_git_repo, preview_namespace)
if not preview_folder_exists:
if self.__args.expect_preview_exists:
raise GitOpsException(
f"There was no preview with name: {preview_namespace}")
logging.info(
"No preview environment for '%s' and preview id '%s'. I'm done here.",
gitops_config.application_name,
preview_id,
)
return
self.__commit_and_push(
preview_target_git_repo,
f"Delete preview environment for '{gitops_config.application_name}' and preview id '{preview_id}'.",
)
def get_branch_head_hash(self, branch: str) -> str:
branches = self.__bitbucket.get_branches(self.__organisation,
self.__repository_name,
filter=branch,
limit=1)
if not branches:
raise GitOpsException(f"Branch '{branch}' not found'")
return str(branches[0]["latestCommit"])
def __get_value(self, key: str) -> Any:
keys = key.split(".")
data = self.__yaml
for k in keys:
if not isinstance(data, dict) or k not in data:
raise GitOpsException(f"Key '{key}' not found in GitOps config!")
data = data[k]
return data
def new_branch(self, branch: str) -> None:
logging.info("Creating new branch: %s", branch)
repo = self.__get_repo()
try:
repo.git.checkout("-b", branch)
except GitError as ex:
raise GitOpsException(
f"Error creating new branch '{branch}'.") from ex
def get_clone_url(self) -> str:
try:
repo = self.__bitbucket.get_repo(self.__organisation,
self.__repository_name)
except requests.exceptions.ConnectionError as ex:
raise GitOpsException(
f"Error connecting to '{self.__git_provider_url}''") from ex
if "errors" in repo:
for error in repo["errors"]:
exception = error["exceptionName"]
if exception == "com.atlassian.bitbucket.auth.IncorrectPasswordAuthenticationException":
raise GitOpsException("Bad credentials")
if exception == "com.atlassian.bitbucket.project.NoSuchProjectException":
raise GitOpsException(
f"Organisation '{self.__organisation}' does not exist")
if exception == "com.atlassian.bitbucket.repository.NoSuchRepositoryException":
raise GitOpsException(
f"Repository '{self.__organisation}/{self.__repository_name}' does not exist"
)
raise GitOpsException(error["message"])
if "links" not in repo:
raise GitOpsException(
f"Repository '{self.__organisation}/{self.__repository_name}' does not exist"
)
for clone_link in repo["links"]["clone"]:
if clone_link["name"] == "http":
repo_url = clone_link["href"]
if not repo_url:
raise GitOpsException("Couldn't determine repository URL.")
return str(repo_url)
