def __create_preview_from_template_if_not_existing( self, template_git_repo: GitRepo, target_git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool: preview_namespace = gitops_config.get_preview_namespace( self.__args.preview_id) full_preview_folder_path = target_git_repo.get_full_file_path( preview_namespace) preview_env_already_exist = os.path.isdir(full_preview_folder_path) if preview_env_already_exist: logging.info("Use existing folder for preview: %s", preview_namespace) return False logging.info("Create new folder for preview: %s", preview_namespace) full_preview_template_folder_path = template_git_repo.get_full_file_path( gitops_config.preview_template_path) if not os.path.isdir(full_preview_template_folder_path): raise GitOpsException( f"The preview template folder does not exist: {gitops_config.preview_template_path}" ) logging.info("Using the preview template folder: %s", gitops_config.preview_template_path) shutil.copytree( full_preview_template_folder_path, full_preview_folder_path, ) return True
def __create_preview_from_template_if_not_existing( self, git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool: preview_namespace = gitops_config.get_preview_namespace( self.__args.preview_id) full_preview_folder_path = git_repo.get_full_file_path( preview_namespace) preview_env_already_exist = os.path.isdir(full_preview_folder_path) if preview_env_already_exist: logging.info("Use existing folder for preview: %s", preview_namespace) return False logging.info("Create new folder for preview: %s", preview_namespace) preview_template_folder_name = f".preview-templates/{gitops_config.application_name}" full_preview_template_folder_path = git_repo.get_full_file_path( preview_template_folder_name) if not os.path.isdir(full_preview_template_folder_path): raise GitOpsException( f"The preview template folder does not exist: {preview_template_folder_name}" ) logging.info("Using the preview template folder: %s", preview_template_folder_name) shutil.copytree( full_preview_template_folder_path, full_preview_folder_path, ) self.__update_yaml_file(git_repo, f"{preview_namespace}/Chart.yaml", "name", preview_namespace) return True
def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) repo_apps = __get_repo_apps(team_config_git_repo) logging.info("Found %s app(s) in apps repository: %s", len(repo_apps), ", ".join(repo_apps)) logging.info( "Searching apps repository in root repository's 'apps/' directory...") apps_config_file, apps_config_file_name, current_repo_apps, apps_from_other_repos = __find_apps_config_from_repo( team_config_git_repo, root_config_git_repo) if current_repo_apps == repo_apps: logging.info("Root repository already up-to-date. I'm done here.") return __check_if_app_already_exists(repo_apps, apps_from_other_repos) logging.info("Sync applications in root repository's %s.", apps_config_file_name) merge_yaml_element(apps_config_file, "applications", {repo_app: {} for repo_app in repo_apps}) __commit_and_push(team_config_git_repo, root_config_git_repo, git_user, git_email, apps_config_file_name)
def __commit_and_push(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str, app_file_name: str) -> None: author = team_config_git_repo.get_author_from_last_commit() root_config_git_repo.commit(git_user, git_email, f"{author} updated " + app_file_name) root_config_git_repo.push()
def _sync_apps_command(args: SyncAppsCommand.Args) -> None: team_config_git_repo_api = GitRepoApiFactory.create( args, args.organisation, args.repository_name) root_config_git_repo_api = GitRepoApiFactory.create( args, args.root_organisation, args.root_repository_name) with GitRepo(team_config_git_repo_api) as team_config_git_repo: with GitRepo(root_config_git_repo_api) as root_config_git_repo: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email)
def __get_repo_apps(team_config_git_repo: GitRepo) -> Set[str]: team_config_git_repo.clone() repo_dir = team_config_git_repo.get_full_file_path(".") return { name for name in os.listdir(repo_dir) if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") }
def __find_apps_config_from_repo( team_config_git_repo: GitRepo, root_config_git_repo: GitRepo ) -> Tuple[str, str, Set[str], Set[str], str]: apps_from_other_repos: Set[str] = set( ) # Set for all entries in .applications from each config repository found_app_config_file = None found_app_config_file_name = None found_apps_path = "applications" found_app_config_apps: Set[str] = set() bootstrap_entries = __get_bootstrap_entries(root_config_git_repo) team_config_git_repo_clone_url = team_config_git_repo.get_clone_url() for bootstrap_entry in bootstrap_entries: if "name" not in bootstrap_entry: raise GitOpsException( "Every bootstrap entry must have a 'name' property.") app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" logging.info("Analyzing %s in root repository", app_file_name) app_config_file = root_config_git_repo.get_full_file_path( app_file_name) try: app_config_content = yaml_file_load(app_config_file) except FileNotFoundError as ex: raise GitOpsException( f"File '{app_file_name}' not found in root repository." ) from ex if "config" in app_config_content: app_config_content = app_config_content["config"] found_apps_path = "config.applications" if "repository" not in app_config_content: raise GitOpsException( f"Cannot find key 'repository' in '{app_file_name}'") if app_config_content["repository"] == team_config_git_repo_clone_url: logging.info("Found apps repository in %s", app_file_name) found_app_config_file = app_config_file found_app_config_file_name = app_file_name found_app_config_apps = __get_applications_from_app_config( app_config_content) else: apps_from_other_repos.update( __get_applications_from_app_config(app_config_content)) if found_app_config_file is None or found_app_config_file_name is None: raise GitOpsException( f"Couldn't find config file for apps repository in root repository's 'apps/' directory" ) return ( found_app_config_file, found_app_config_file_name, found_app_config_apps, apps_from_other_repos, found_apps_path, )
def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any: root_config_git_repo.clone() bootstrap_values_file = root_config_git_repo.get_full_file_path( "bootstrap/values.yaml") try: bootstrap_yaml = yaml_file_load(bootstrap_values_file) except FileNotFoundError as ex: raise GitOpsException( "File 'bootstrap/values.yaml' not found in root repository." ) from ex if "bootstrap" not in bootstrap_yaml: raise GitOpsException( "Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") return bootstrap_yaml["bootstrap"]
def test_push_commit_hook_error_reason_is_shown(self, logging_mock): repo_dir = self.__origin.working_dir with open(f"{repo_dir}/.git/hooks/pre-receive", "w") as pre_receive_hook: pre_receive_hook.write('echo >&2 "we reject this push"; exit 1') chmod(f"{repo_dir}/.git/hooks/pre-receive", stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) with GitRepo(self.__mock_repo_api) as testee: testee.clone() with open(testee.get_full_file_path("foo.md"), "w") as readme: readme.write("new file") util_repo = Repo(testee.get_full_file_path(".")) util_repo.git.add("--all") util_repo.config_writer().set_value("user", "email", "*****@*****.**").release() util_repo.git.commit("-m", "new commit") logging_mock.reset_mock() with pytest.raises(GitOpsException) as ex: testee.push("master") assert "pre-receive" in str( ex.value) and "we reject this push" in str(ex.value) logging_mock.info.assert_called_once_with("Pushing branch: %s", "master")
def execute(self) -> None: gitops_config = self.__get_gitops_config() preview_id = self.__args.preview_id preview_target_git_repo_api = self.__create_preview_target_git_repo_api( gitops_config) with GitRepo(preview_target_git_repo_api) as preview_target_git_repo: preview_target_git_repo.clone(gitops_config.preview_target_branch) preview_namespace = gitops_config.get_preview_namespace(preview_id) logging.info("Preview folder name: %s", preview_namespace) preview_folder_exists = self.__delete_folder_if_exists( preview_target_git_repo, preview_namespace) if not preview_folder_exists: if self.__args.expect_preview_exists: raise GitOpsException( f"There was no preview with name: {preview_namespace}") logging.info( "No preview environment for '%s' and preview id '%s'. I'm done here.", gitops_config.application_name, preview_id, ) return self.__commit_and_push( preview_target_git_repo, f"Delete preview environment for '{gitops_config.application_name}' and preview id '{preview_id}'.", )
def execute(self, ) -> None: gitops_config = self.__get_gitops_config() self.__create_preview_info_file(gitops_config) route_host = gitops_config.get_route_host(self.__args.preview_id) team_config_git_repo_api = self.__create_team_config_git_repo_api( gitops_config) with GitRepo(team_config_git_repo_api) as team_config_git_repo: team_config_git_repo.clone() created_new_preview = self.__create_preview_from_template_if_not_existing( team_config_git_repo, gitops_config) any_values_replaced = self.__replace_values( team_config_git_repo, gitops_config) if not created_new_preview and not any_values_replaced: self.__deployment_already_up_to_date_callback(route_host) logging.info( "The preview is already up-to-date. I'm done here.") return self.__commit_and_push( team_config_git_repo, f"{'Create new' if created_new_preview else 'Update'} preview environment for " f"'{gitops_config.application_name}' and git hash '{self.__args.git_hash}'.", ) if created_new_preview: self.__deployment_created_callback(route_host) else: self.__deployment_updated_callback(route_host)
def execute(self) -> None: git_repo_api = self.__create_git_repo_api() with GitRepo(git_repo_api) as git_repo: git_repo.clone() if self.__args.create_pr: pr_branch = f"gitopscli-deploy-{str(uuid.uuid4())[:8]}" git_repo.new_branch(pr_branch) updated_values = self.__update_values(git_repo) if not updated_values: logging.info("All values already up-to-date. I'm done here.") return git_repo.push() if self.__args.create_pr: title, description = self.__create_pull_request_title_and_description( updated_values) pr_id = git_repo_api.create_pull_request_to_default_branch( pr_branch, title, description).pr_id if self.__args.auto_merge: git_repo_api.merge_pull_request(pr_id, self.__args.merge_method) git_repo_api.delete_branch(pr_branch)
def test_clone_unknown_url(self, logging_mock): self.__mock_repo_api.get_clone_url.return_value = "invalid_url" with GitRepo(self.__mock_repo_api) as testee: with pytest.raises(GitOpsException) as ex: testee.clone() self.assertEqual("Error cloning 'invalid_url'", str(ex.value)) logging_mock.info.assert_called_once_with( "Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def test_push_no_changes(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() testee.push("master") logging_mock.info.assert_called_once_with("Pushing branch: %s", "master")
def test_push_current_branch(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() testee.new_branch("foo") logging_mock.reset_mock() testee.push() # current branch logging_mock.info.assert_called_once_with("Pushing branch: %s", "foo")
def test_enter_and_exit_magic_methods(self): testee = GitRepo(self.__mock_repo_api) self.assertEqual(testee, testee.__enter__()) testee.clone() tmp_dir = testee.get_full_file_path("..") self.assertTrue(path.exists(tmp_dir)) testee.__exit__(None, None, None) self.assertFalse(path.exists(tmp_dir))
def test_clone_unknown_branch(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: with pytest.raises(GitOpsException) as ex: testee.clone("unknown") self.assertEqual( f"Error cloning branch 'unknown' of '{self.__mock_repo_api.get_clone_url()}'", str(ex.value)) logging_mock.info.assert_called_once_with( "Cloning repository: %s (branch: %s)", self.__mock_repo_api.get_clone_url(), "unknown")
def test_clone_without_credentials(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() readme = self.__read_file(testee.get_full_file_path("README.md")) self.assertEqual("master branch readme", readme) self.assertFalse( path.exists(testee.get_full_file_path("../credentials.sh"))) logging_mock.info.assert_called_once_with( "Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def test_new_branch_name_collision(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() with pytest.raises(GitOpsException) as ex: testee.new_branch("master") self.assertEqual("Error creating new branch 'master'.", str(ex.value)) logging_mock.info.assert_called_once_with("Creating new branch: %s", "master")
def test_push_unknown_branch(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() with pytest.raises(GitOpsException) as ex: testee.push("unknown") assert str(ex.value).startswith( "Error pushing branch 'unknown' to origin") logging_mock.info.assert_called_once_with("Pushing branch: %s", "unknown")
def __update_yaml_file(git_repo: GitRepo, file_path: str, key: str, value: Any) -> bool: full_file_path = git_repo.get_full_file_path(file_path) try: return update_yaml_file(full_file_path, key, value) except (FileNotFoundError, IsADirectoryError) as ex: raise GitOpsException(f"No such file: {file_path}") from ex except YAMLException as ex: raise GitOpsException(f"Error loading file: {file_path}") from ex except KeyError as ex: raise GitOpsException( f"Key '{key}' not found in file: {file_path}") from ex
def test_new_branch(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() testee.new_branch("foo") repo = Repo(testee.get_full_file_path(".")) branches = [str(b) for b in repo.branches] self.assertIn("foo", branches) logging_mock.info.assert_called_once_with("Creating new branch: %s", "foo")
def test_finalize(self): testee = GitRepo(self.__mock_repo_api) testee.clone() tmp_dir = testee.get_full_file_path("..") self.assertTrue(path.exists(tmp_dir)) testee.finalize() self.assertFalse(path.exists(tmp_dir))
def load_gitops_config(git_api_config: GitApiConfig, organisation: str, repository_name: str) -> GitOpsConfig: git_repo_api = GitRepoApiFactory.create(git_api_config, organisation, repository_name) with GitRepo(git_repo_api) as git_repo: git_repo.clone() gitops_config_file_path = git_repo.get_full_file_path( ".gitops.config.yaml") try: gitops_config_yaml = yaml_file_load(gitops_config_file_path) except FileNotFoundError as ex: raise GitOpsException("No such file: .gitops.config.yaml") from ex return GitOpsConfig.from_yaml(gitops_config_yaml)
def test_clone_branch(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone("xyz") tmp_dir = testee.get_full_file_path("..") self.assertTrue(path.exists(tmp_dir)) readme = self.__read_file(testee.get_full_file_path("README.md")) self.assertEqual("xyz branch readme", readme) self.assertFalse(path.exists(tmp_dir)) logging_mock.info.assert_called_once_with( "Cloning repository: %s (branch: %s)", self.__mock_repo_api.get_clone_url(), "xyz")
def test_commit_nothing_to_commit(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() testee.commit(git_user="******", git_email="*****@*****.**", message="empty commit") repo = Repo(testee.get_full_file_path(".")) commits = list(repo.iter_commits("master")) self.assertEqual(1, len(commits)) self.assertEqual("initial commit\n", commits[0].message) logging_mock.assert_not_called()
def test_clone_with_credentials(self, logging_mock): self.__mock_repo_api.get_username.return_value = "User" self.__mock_repo_api.get_password.return_value = "Pass" with GitRepo(self.__mock_repo_api) as testee: testee.clone() credentials_file = self.__read_file( testee.get_full_file_path("../credentials.sh")) self.assertEqual( """\ #!/bin/sh echo username=User echo password=Pass """, credentials_file, ) logging_mock.info.assert_called_once_with( "Cloning repository: %s", self.__mock_repo_api.get_clone_url())
def __update_values(self, git_repo: GitRepo) -> Dict[str, Any]: args = self.__args single_commit = args.single_commit or args.commit_message full_file_path = git_repo.get_full_file_path(args.file) updated_values = {} for key, value in args.values.items(): try: updated_value = update_yaml_file(full_file_path, key, value) except (FileNotFoundError, IsADirectoryError) as ex: raise GitOpsException(f"No such file: {args.file}") from ex except YAMLException as ex: raise GitOpsException( f"Error loading file: {args.file}") from ex except KeyError as ex: raise GitOpsException( f"Key '{key}' not found in file: {args.file}") from ex if not updated_value: logging.info("Yaml property %s already up-to-date", key) continue logging.info("Updated yaml property %s to %s", key, value) updated_values[key] = value if not single_commit: self.__commit(git_repo, f"changed '{key}' to '{value}' in {args.file}") if single_commit and updated_values: if args.commit_message: message = args.commit_message elif len(updated_values) == 1: key, value = list(updated_values.items())[0] message = f"changed '{key}' to '{value}' in {args.file}" else: updates_count = len(updated_values) message = f"updated {updates_count} value{'s' if updates_count > 1 else ''} in {args.file}" message += f"\n\n{yaml_dump(updated_values)}" self.__commit(git_repo, message) return updated_values
def test_push(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() with open(testee.get_full_file_path("foo.md"), "w") as readme: readme.write("new file") util_repo = Repo(testee.get_full_file_path(".")) util_repo.git.add("--all") util_repo.config_writer().set_value("user", "email", "*****@*****.**").release() util_repo.git.commit("-m", "new commit") logging_mock.reset_mock() testee.push("master") commits = list(self.__origin.iter_commits("master")) self.assertEqual(2, len(commits)) self.assertEqual("new commit\n", commits[0].message) logging_mock.info.assert_called_once_with("Pushing branch: %s", "master")
def test_commit(self, logging_mock): with GitRepo(self.__mock_repo_api) as testee: testee.clone() logging_mock.reset_mock() with open(testee.get_full_file_path("foo.md"), "w") as outfile: outfile.write("new file") with open(testee.get_full_file_path("README.md"), "w") as outfile: outfile.write("new content") testee.commit(git_user="******", git_email="*****@*****.**", message="new commit") repo = Repo(testee.get_full_file_path(".")) commits = list(repo.iter_commits("master")) self.assertEqual(2, len(commits)) self.assertEqual("new commit\n", commits[0].message) self.assertEqual("john doe", commits[0].author.name) self.assertEqual("*****@*****.**", commits[0].author.email) self.assertIn("foo.md", commits[0].stats.files) self.assertIn("README.md", commits[0].stats.files) logging_mock.info.assert_called_once_with( "Creating commit with message: %s", "new commit")