def __create_preview_from_template_if_not_existing(
self, template_git_repo: GitRepo, target_git_repo: GitRepo,
gitops_config: GitOpsConfig) -> bool:
preview_namespace = gitops_config.get_preview_namespace(
self.__args.preview_id)
full_preview_folder_path = target_git_repo.get_full_file_path(
preview_namespace)
preview_env_already_exist = os.path.isdir(full_preview_folder_path)
if preview_env_already_exist:
logging.info("Use existing folder for preview: %s",
preview_namespace)
return False
logging.info("Create new folder for preview: %s", preview_namespace)
full_preview_template_folder_path = template_git_repo.get_full_file_path(
gitops_config.preview_template_path)
if not os.path.isdir(full_preview_template_folder_path):
raise GitOpsException(
f"The preview template folder does not exist: {gitops_config.preview_template_path}"
)
logging.info("Using the preview template folder: %s",
gitops_config.preview_template_path)
shutil.copytree(
full_preview_template_folder_path,
full_preview_folder_path,
)
return True
def __create_preview_from_template_if_not_existing(
self, git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool:
preview_namespace = gitops_config.get_preview_namespace(
self.__args.preview_id)
full_preview_folder_path = git_repo.get_full_file_path(
preview_namespace)
preview_env_already_exist = os.path.isdir(full_preview_folder_path)
if preview_env_already_exist:
logging.info("Use existing folder for preview: %s",
preview_namespace)
return False
logging.info("Create new folder for preview: %s", preview_namespace)
preview_template_folder_name = f".preview-templates/{gitops_config.application_name}"
full_preview_template_folder_path = git_repo.get_full_file_path(
preview_template_folder_name)
if not os.path.isdir(full_preview_template_folder_path):
raise GitOpsException(
f"The preview template folder does not exist: {preview_template_folder_name}"
)
logging.info("Using the preview template folder: %s",
preview_template_folder_name)
shutil.copytree(
full_preview_template_folder_path,
full_preview_folder_path,
)
self.__update_yaml_file(git_repo, f"{preview_namespace}/Chart.yaml",
"name", preview_namespace)
return True
def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo,
git_user: str, git_email: str) -> None:
logging.info("Team config repository: %s",
team_config_git_repo.get_clone_url())
logging.info("Root config repository: %s",
root_config_git_repo.get_clone_url())
repo_apps = __get_repo_apps(team_config_git_repo)
logging.info("Found %s app(s) in apps repository: %s", len(repo_apps),
", ".join(repo_apps))
logging.info(
"Searching apps repository in root repository's 'apps/' directory...")
apps_config_file, apps_config_file_name, current_repo_apps, apps_from_other_repos = __find_apps_config_from_repo(
team_config_git_repo, root_config_git_repo)
if current_repo_apps == repo_apps:
logging.info("Root repository already up-to-date. I'm done here.")
return
__check_if_app_already_exists(repo_apps, apps_from_other_repos)
logging.info("Sync applications in root repository's %s.",
apps_config_file_name)
merge_yaml_element(apps_config_file, "applications",
{repo_app: {}
for repo_app in repo_apps})
__commit_and_push(team_config_git_repo, root_config_git_repo, git_user,
git_email, apps_config_file_name)
def __commit_and_push(team_config_git_repo: GitRepo,
root_config_git_repo: GitRepo, git_user: str,
git_email: str, app_file_name: str) -> None:
author = team_config_git_repo.get_author_from_last_commit()
root_config_git_repo.commit(git_user, git_email,
f"{author} updated " + app_file_name)
root_config_git_repo.push()
def _sync_apps_command(args: SyncAppsCommand.Args) -> None:
team_config_git_repo_api = GitRepoApiFactory.create(
args, args.organisation, args.repository_name)
root_config_git_repo_api = GitRepoApiFactory.create(
args, args.root_organisation, args.root_repository_name)
with GitRepo(team_config_git_repo_api) as team_config_git_repo:
with GitRepo(root_config_git_repo_api) as root_config_git_repo:
__sync_apps(team_config_git_repo, root_config_git_repo,
args.git_user, args.git_email)
def __get_repo_apps(team_config_git_repo: GitRepo) -> Set[str]:
team_config_git_repo.clone()
repo_dir = team_config_git_repo.get_full_file_path(".")
return {
name
for name in os.listdir(repo_dir)
if os.path.isdir(os.path.join(repo_dir, name))
and not name.startswith(".")
}
def __find_apps_config_from_repo(
team_config_git_repo: GitRepo, root_config_git_repo: GitRepo
) -> Tuple[str, str, Set[str], Set[str], str]:
apps_from_other_repos: Set[str] = set(
) # Set for all entries in .applications from each config repository
found_app_config_file = None
found_app_config_file_name = None
found_apps_path = "applications"
found_app_config_apps: Set[str] = set()
bootstrap_entries = __get_bootstrap_entries(root_config_git_repo)
team_config_git_repo_clone_url = team_config_git_repo.get_clone_url()
for bootstrap_entry in bootstrap_entries:
if "name" not in bootstrap_entry:
raise GitOpsException(
"Every bootstrap entry must have a 'name' property.")
app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml"
logging.info("Analyzing %s in root repository", app_file_name)
app_config_file = root_config_git_repo.get_full_file_path(
app_file_name)
try:
app_config_content = yaml_file_load(app_config_file)
except FileNotFoundError as ex:
raise GitOpsException(
f"File '{app_file_name}' not found in root repository."
) from ex
if "config" in app_config_content:
app_config_content = app_config_content["config"]
found_apps_path = "config.applications"
if "repository" not in app_config_content:
raise GitOpsException(
f"Cannot find key 'repository' in '{app_file_name}'")
if app_config_content["repository"] == team_config_git_repo_clone_url:
logging.info("Found apps repository in %s", app_file_name)
found_app_config_file = app_config_file
found_app_config_file_name = app_file_name
found_app_config_apps = __get_applications_from_app_config(
app_config_content)
else:
apps_from_other_repos.update(
__get_applications_from_app_config(app_config_content))
if found_app_config_file is None or found_app_config_file_name is None:
raise GitOpsException(
f"Couldn't find config file for apps repository in root repository's 'apps/' directory"
)
return (
found_app_config_file,
found_app_config_file_name,
found_app_config_apps,
apps_from_other_repos,
found_apps_path,
)
def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any:
root_config_git_repo.clone()
bootstrap_values_file = root_config_git_repo.get_full_file_path(
"bootstrap/values.yaml")
try:
bootstrap_yaml = yaml_file_load(bootstrap_values_file)
except FileNotFoundError as ex:
raise GitOpsException(
"File 'bootstrap/values.yaml' not found in root repository."
) from ex
if "bootstrap" not in bootstrap_yaml:
raise GitOpsException(
"Cannot find key 'bootstrap' in 'bootstrap/values.yaml'")
return bootstrap_yaml["bootstrap"]
def test_push_commit_hook_error_reason_is_shown(self, logging_mock):
repo_dir = self.__origin.working_dir
with open(f"{repo_dir}/.git/hooks/pre-receive",
"w") as pre_receive_hook:
pre_receive_hook.write('echo >&2 "we reject this push"; exit 1')
chmod(f"{repo_dir}/.git/hooks/pre-receive",
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
with open(testee.get_full_file_path("foo.md"), "w") as readme:
readme.write("new file")
util_repo = Repo(testee.get_full_file_path("."))
util_repo.git.add("--all")
util_repo.config_writer().set_value("user", "email",
"*****@*****.**").release()
util_repo.git.commit("-m", "new commit")
logging_mock.reset_mock()
with pytest.raises(GitOpsException) as ex:
testee.push("master")
assert "pre-receive" in str(
ex.value) and "we reject this push" in str(ex.value)
logging_mock.info.assert_called_once_with("Pushing branch: %s",
"master")
def execute(self) -> None:
gitops_config = self.__get_gitops_config()
preview_id = self.__args.preview_id
preview_target_git_repo_api = self.__create_preview_target_git_repo_api(
gitops_config)
with GitRepo(preview_target_git_repo_api) as preview_target_git_repo:
preview_target_git_repo.clone(gitops_config.preview_target_branch)
preview_namespace = gitops_config.get_preview_namespace(preview_id)
logging.info("Preview folder name: %s", preview_namespace)
preview_folder_exists = self.__delete_folder_if_exists(
preview_target_git_repo, preview_namespace)
if not preview_folder_exists:
if self.__args.expect_preview_exists:
raise GitOpsException(
f"There was no preview with name: {preview_namespace}")
logging.info(
"No preview environment for '%s' and preview id '%s'. I'm done here.",
gitops_config.application_name,
preview_id,
)
return
self.__commit_and_push(
preview_target_git_repo,
f"Delete preview environment for '{gitops_config.application_name}' and preview id '{preview_id}'.",
)
def execute(self, ) -> None:
gitops_config = self.__get_gitops_config()
self.__create_preview_info_file(gitops_config)
route_host = gitops_config.get_route_host(self.__args.preview_id)
team_config_git_repo_api = self.__create_team_config_git_repo_api(
gitops_config)
with GitRepo(team_config_git_repo_api) as team_config_git_repo:
team_config_git_repo.clone()
created_new_preview = self.__create_preview_from_template_if_not_existing(
team_config_git_repo, gitops_config)
any_values_replaced = self.__replace_values(
team_config_git_repo, gitops_config)
if not created_new_preview and not any_values_replaced:
self.__deployment_already_up_to_date_callback(route_host)
logging.info(
"The preview is already up-to-date. I'm done here.")
return
self.__commit_and_push(
team_config_git_repo,
f"{'Create new' if created_new_preview else 'Update'} preview environment for "
f"'{gitops_config.application_name}' and git hash '{self.__args.git_hash}'.",
)
if created_new_preview:
self.__deployment_created_callback(route_host)
else:
self.__deployment_updated_callback(route_host)
def execute(self) -> None:
git_repo_api = self.__create_git_repo_api()
with GitRepo(git_repo_api) as git_repo:
git_repo.clone()
if self.__args.create_pr:
pr_branch = f"gitopscli-deploy-{str(uuid.uuid4())[:8]}"
git_repo.new_branch(pr_branch)
updated_values = self.__update_values(git_repo)
if not updated_values:
logging.info("All values already up-to-date. I'm done here.")
return
git_repo.push()
if self.__args.create_pr:
title, description = self.__create_pull_request_title_and_description(
updated_values)
pr_id = git_repo_api.create_pull_request_to_default_branch(
pr_branch, title, description).pr_id
if self.__args.auto_merge:
git_repo_api.merge_pull_request(pr_id,
self.__args.merge_method)
git_repo_api.delete_branch(pr_branch)
def test_clone_unknown_url(self, logging_mock):
self.__mock_repo_api.get_clone_url.return_value = "invalid_url"
with GitRepo(self.__mock_repo_api) as testee:
with pytest.raises(GitOpsException) as ex:
testee.clone()
self.assertEqual("Error cloning 'invalid_url'", str(ex.value))
logging_mock.info.assert_called_once_with(
"Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def test_push_no_changes(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
testee.push("master")
logging_mock.info.assert_called_once_with("Pushing branch: %s",
"master")
def test_push_current_branch(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
testee.new_branch("foo")
logging_mock.reset_mock()
testee.push() # current branch
logging_mock.info.assert_called_once_with("Pushing branch: %s", "foo")
def test_enter_and_exit_magic_methods(self):
testee = GitRepo(self.__mock_repo_api)
self.assertEqual(testee, testee.__enter__())
testee.clone()
tmp_dir = testee.get_full_file_path("..")
self.assertTrue(path.exists(tmp_dir))
testee.__exit__(None, None, None)
self.assertFalse(path.exists(tmp_dir))
def test_clone_unknown_branch(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
with pytest.raises(GitOpsException) as ex:
testee.clone("unknown")
self.assertEqual(
f"Error cloning branch 'unknown' of '{self.__mock_repo_api.get_clone_url()}'",
str(ex.value))
logging_mock.info.assert_called_once_with(
"Cloning repository: %s (branch: %s)",
self.__mock_repo_api.get_clone_url(), "unknown")
def test_clone_without_credentials(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
readme = self.__read_file(testee.get_full_file_path("README.md"))
self.assertEqual("master branch readme", readme)
self.assertFalse(
path.exists(testee.get_full_file_path("../credentials.sh")))
logging_mock.info.assert_called_once_with(
"Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def test_new_branch_name_collision(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
with pytest.raises(GitOpsException) as ex:
testee.new_branch("master")
self.assertEqual("Error creating new branch 'master'.",
str(ex.value))
logging_mock.info.assert_called_once_with("Creating new branch: %s",
"master")
def test_push_unknown_branch(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
with pytest.raises(GitOpsException) as ex:
testee.push("unknown")
assert str(ex.value).startswith(
"Error pushing branch 'unknown' to origin")
logging_mock.info.assert_called_once_with("Pushing branch: %s",
"unknown")
def __update_yaml_file(git_repo: GitRepo, file_path: str, key: str,
value: Any) -> bool:
full_file_path = git_repo.get_full_file_path(file_path)
try:
return update_yaml_file(full_file_path, key, value)
except (FileNotFoundError, IsADirectoryError) as ex:
raise GitOpsException(f"No such file: {file_path}") from ex
except YAMLException as ex:
raise GitOpsException(f"Error loading file: {file_path}") from ex
except KeyError as ex:
raise GitOpsException(
f"Key '{key}' not found in file: {file_path}") from ex
def test_new_branch(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
testee.new_branch("foo")
repo = Repo(testee.get_full_file_path("."))
branches = [str(b) for b in repo.branches]
self.assertIn("foo", branches)
logging_mock.info.assert_called_once_with("Creating new branch: %s",
"foo")
def test_finalize(self):
testee = GitRepo(self.__mock_repo_api)
testee.clone()
tmp_dir = testee.get_full_file_path("..")
self.assertTrue(path.exists(tmp_dir))
testee.finalize()
self.assertFalse(path.exists(tmp_dir))
def load_gitops_config(git_api_config: GitApiConfig, organisation: str,
repository_name: str) -> GitOpsConfig:
git_repo_api = GitRepoApiFactory.create(git_api_config, organisation,
repository_name)
with GitRepo(git_repo_api) as git_repo:
git_repo.clone()
gitops_config_file_path = git_repo.get_full_file_path(
".gitops.config.yaml")
try:
gitops_config_yaml = yaml_file_load(gitops_config_file_path)
except FileNotFoundError as ex:
raise GitOpsException("No such file: .gitops.config.yaml") from ex
return GitOpsConfig.from_yaml(gitops_config_yaml)
def test_clone_branch(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone("xyz")
tmp_dir = testee.get_full_file_path("..")
self.assertTrue(path.exists(tmp_dir))
readme = self.__read_file(testee.get_full_file_path("README.md"))
self.assertEqual("xyz branch readme", readme)
self.assertFalse(path.exists(tmp_dir))
logging_mock.info.assert_called_once_with(
"Cloning repository: %s (branch: %s)",
self.__mock_repo_api.get_clone_url(), "xyz")
def test_commit_nothing_to_commit(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
testee.commit(git_user="******",
git_email="*****@*****.**",
message="empty commit")
repo = Repo(testee.get_full_file_path("."))
commits = list(repo.iter_commits("master"))
self.assertEqual(1, len(commits))
self.assertEqual("initial commit\n", commits[0].message)
logging_mock.assert_not_called()
def test_clone_with_credentials(self, logging_mock):
self.__mock_repo_api.get_username.return_value = "User"
self.__mock_repo_api.get_password.return_value = "Pass"
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
credentials_file = self.__read_file(
testee.get_full_file_path("../credentials.sh"))
self.assertEqual(
"""\
#!/bin/sh
echo username=User
echo password=Pass
""",
credentials_file,
)
logging_mock.info.assert_called_once_with(
"Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def __update_values(self, git_repo: GitRepo) -> Dict[str, Any]:
args = self.__args
single_commit = args.single_commit or args.commit_message
full_file_path = git_repo.get_full_file_path(args.file)
updated_values = {}
for key, value in args.values.items():
try:
updated_value = update_yaml_file(full_file_path, key, value)
except (FileNotFoundError, IsADirectoryError) as ex:
raise GitOpsException(f"No such file: {args.file}") from ex
except YAMLException as ex:
raise GitOpsException(
f"Error loading file: {args.file}") from ex
except KeyError as ex:
raise GitOpsException(
f"Key '{key}' not found in file: {args.file}") from ex
if not updated_value:
logging.info("Yaml property %s already up-to-date", key)
continue
logging.info("Updated yaml property %s to %s", key, value)
updated_values[key] = value
if not single_commit:
self.__commit(git_repo,
f"changed '{key}' to '{value}' in {args.file}")
if single_commit and updated_values:
if args.commit_message:
message = args.commit_message
elif len(updated_values) == 1:
key, value = list(updated_values.items())[0]
message = f"changed '{key}' to '{value}' in {args.file}"
else:
updates_count = len(updated_values)
message = f"updated {updates_count} value{'s' if updates_count > 1 else ''} in {args.file}"
message += f"\n\n{yaml_dump(updated_values)}"
self.__commit(git_repo, message)
return updated_values
def test_push(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
with open(testee.get_full_file_path("foo.md"), "w") as readme:
readme.write("new file")
util_repo = Repo(testee.get_full_file_path("."))
util_repo.git.add("--all")
util_repo.config_writer().set_value("user", "email",
"*****@*****.**").release()
util_repo.git.commit("-m", "new commit")
logging_mock.reset_mock()
testee.push("master")
commits = list(self.__origin.iter_commits("master"))
self.assertEqual(2, len(commits))
self.assertEqual("new commit\n", commits[0].message)
logging_mock.info.assert_called_once_with("Pushing branch: %s",
"master")
def test_commit(self, logging_mock):
with GitRepo(self.__mock_repo_api) as testee:
testee.clone()
logging_mock.reset_mock()
with open(testee.get_full_file_path("foo.md"), "w") as outfile:
outfile.write("new file")
with open(testee.get_full_file_path("README.md"), "w") as outfile:
outfile.write("new content")
testee.commit(git_user="******",
git_email="*****@*****.**",
message="new commit")
repo = Repo(testee.get_full_file_path("."))
commits = list(repo.iter_commits("master"))
self.assertEqual(2, len(commits))
self.assertEqual("new commit\n", commits[0].message)
self.assertEqual("john doe", commits[0].author.name)
self.assertEqual("*****@*****.**", commits[0].author.email)
self.assertIn("foo.md", commits[0].stats.files)
self.assertIn("README.md", commits[0].stats.files)
logging_mock.info.assert_called_once_with(
"Creating commit with message: %s", "new commit")
