def __find_apps_config_from_repo(apps_git, root_git): yaml = YAML() apps_from_other_repos = [ ] # List for all entries in .applications from each config repository found_app_config_file = None found_app_config_file_name = None found_app_config_apps = set() bootstrap_entries = __get_bootstrap_entries(root_git) for bootstrap_entry in bootstrap_entries: if "name" not in bootstrap_entry: raise GitOpsException( "Every bootstrap entry must have a 'name' property.") app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" logging.info("Analyzing %s in root repository", app_file_name) app_config_file = root_git.get_full_file_path(app_file_name) try: with open(app_config_file, "r") as stream: app_config_content = yaml.load(stream) except FileNotFoundError as ex: raise GitOpsException( f"File '{app_file_name}' not found in root repository." ) from ex if "repository" not in app_config_content: raise GitOpsException( f"Cannot find key 'repository' in '{app_file_name}'") if app_config_content["repository"] == apps_git.get_clone_url(): logging.info("Found apps repository in %s", app_file_name) found_app_config_file = app_config_file found_app_config_file_name = app_file_name found_app_config_apps = __get_applications_from_app_config( app_config_content) else: apps_from_other_repos += __get_applications_from_app_config( app_config_content) return found_app_config_file, found_app_config_file_name, found_app_config_apps, apps_from_other_repos
def __get_repo(self): try: return self._github.get_repo(f"{self._organisation}/{self._repository_name}") except BadCredentialsException as ex: raise GitOpsException("Bad credentials") from ex except UnknownObjectException as ex: raise GitOpsException(f"Repository '{self._organisation}/{self._repository_name}' does not exist.") from ex
def __update_values(git, file, values, single_commit, commit_message): full_file_path = git.get_full_file_path(file) if not os.path.isfile(full_file_path): raise GitOpsException(f"No such file: {file}") updated_values = {} for key in values: value = values[key] try: updated_value = update_yaml_file(full_file_path, key, value) except KeyError as ex: raise GitOpsException(f"Key '{key}' not found in {file}") from ex if not updated_value: logging.info("Yaml property %s already up-to-date", key) continue logging.info("Updated yaml property %s to %s", key, value) updated_values[key] = value if not single_commit and commit_message is None: git.commit(f"changed '{key}' to '{value}' in {file}") if updated_values and single_commit and commit_message is None: if len(updated_values) == 1: key, value = list(updated_values.items())[0] git.commit(f"changed '{key}' to '{value}' in {file}") else: msg = f"updated {len(updated_values)} value{'s' if len(updated_values) > 1 else ''} in {file}" msg += f"\n\n{yaml_dump(updated_values)}" git.commit(msg) if updated_values and commit_message is not None: git.commit(commit_message) return updated_values
def __init__( self, git_provider_url: str, username: Optional[str], password: Optional[str], organisation: str, repository_name: str, ) -> None: try: self.__gitlab = gitlab.Gitlab(git_provider_url, private_token=password) project = self.__gitlab.projects.get( f"{organisation}/{repository_name}") except requests.exceptions.ConnectionError as ex: raise GitOpsException( f"Error connecting to '{git_provider_url}''") from ex except gitlab.exceptions.GitlabAuthenticationError as ex: raise GitOpsException("Bad Personal Access Token") except gitlab.exceptions.GitlabGetError as ex: if ex.response_code == 404: raise GitOpsException( f"Repository '{organisation}/{repository_name}' does not exist" ) raise GitOpsException( f"Error getting repository: '{ex.error_message}'") self.__token_name = username self.__access_token = password self.__project = project
def clone(self, branch: Optional[str] = None) -> None: self.__delete_tmp_dir() self.__tmp_dir = create_tmp_dir() git_options = [] url = self.get_clone_url() if branch: logging.info("Cloning repository: %s (branch: %s)", url, branch) else: logging.info("Cloning repository: %s", url) username = self.__api.get_username() password = self.__api.get_password() try: if username is not None and password is not None: credentials_file = self.__create_credentials_file( username, password) git_options.append( f"--config credential.helper={credentials_file}") if branch: git_options.append(f"--branch {branch}") self.__repo = Repo.clone_from(url=url, to_path=f"{self.__tmp_dir}/repo", multi_options=git_options) except GitError as ex: if branch: raise GitOpsException( f"Error cloning branch '{branch}' of '{url}'") from ex raise GitOpsException(f"Error cloning '{url}'") from ex
def get_preview_namespace(self, preview_id: str) -> str: preview_namespace = self.preview_target_namespace_template preview_namespace = preview_namespace.replace("${APPLICATION_NAME}", self.application_name) preview_namespace = preview_namespace.replace("${PREVIEW_ID_HASH}", self.create_preview_id_hash(preview_id)) preview_namespace = preview_namespace.replace( "${PREVIEW_ID_HASH_SHORT}", self.create_preview_id_hash_short(preview_id) ) current_length = len(preview_namespace) - len("${PREVIEW_ID}") remaining_length = self.preview_target_max_namespace_length - current_length if remaining_length < 1: preview_namespace = preview_namespace.replace("${PREVIEW_ID}", "") raise GitOpsException( f"Preview namespace is too long (max {self.preview_target_max_namespace_length} chars): " f"{preview_namespace} ({len(preview_namespace)} chars)" ) sanitized_preview_id = self.__sanitize(preview_id, remaining_length) preview_namespace = preview_namespace.replace("${PREVIEW_ID}", sanitized_preview_id) preview_namespace = preview_namespace.lower() invalid_character = re.search(r"[^a-z0-9-]", preview_namespace) if invalid_character: raise GitOpsException(f"Invalid character in preview namespace: '{invalid_character[0]}'") return preview_namespace
def push(self, branch): try: self._repo.git.push("--set-upstream", "origin", branch) except GitCommandError as ex: raise GitOpsException( f"Error pushing branch '{branch}' to origin: {ex}") from ex except GitError as ex: raise GitOpsException( f"Error pushing branch '{branch}' to origin.") from ex
def __update_yaml_file(git_repo: GitRepo, file_path: str, key: str, value: Any) -> bool: full_file_path = git_repo.get_full_file_path(file_path) try: return update_yaml_file(full_file_path, key, value) except (FileNotFoundError, IsADirectoryError) as ex: raise GitOpsException(f"No such file: {file_path}") from ex except YAMLException as ex: raise GitOpsException(f"Error loading file: {file_path}") from ex except KeyError as ex: raise GitOpsException( f"Key '{key}' not found in file: {file_path}") from ex
def delete_branch(self, branch): branches = self._bitbucket.get_branches(self._organisation, self._repository_name, filter=branch, limit=1) if not branches: raise GitOpsException(f"Branch '{branch}' not found'") result = self._bitbucket.delete_branch(self._organisation, self._repository_name, branch, branches[0]["latestCommit"]) if result and "errors" in result: raise GitOpsException(result["errors"][0]["message"])
def __find_apps_config_from_repo( team_config_git_repo: GitRepo, root_config_git_repo: GitRepo ) -> Tuple[str, str, Set[str], Set[str], str]: apps_from_other_repos: Set[str] = set( ) # Set for all entries in .applications from each config repository found_app_config_file = None found_app_config_file_name = None found_apps_path = "applications" found_app_config_apps: Set[str] = set() bootstrap_entries = __get_bootstrap_entries(root_config_git_repo) team_config_git_repo_clone_url = team_config_git_repo.get_clone_url() for bootstrap_entry in bootstrap_entries: if "name" not in bootstrap_entry: raise GitOpsException( "Every bootstrap entry must have a 'name' property.") app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" logging.info("Analyzing %s in root repository", app_file_name) app_config_file = root_config_git_repo.get_full_file_path( app_file_name) try: app_config_content = yaml_file_load(app_config_file) except FileNotFoundError as ex: raise GitOpsException( f"File '{app_file_name}' not found in root repository." ) from ex if "config" in app_config_content: app_config_content = app_config_content["config"] found_apps_path = "config.applications" if "repository" not in app_config_content: raise GitOpsException( f"Cannot find key 'repository' in '{app_file_name}'") if app_config_content["repository"] == team_config_git_repo_clone_url: logging.info("Found apps repository in %s", app_file_name) found_app_config_file = app_config_file found_app_config_file_name = app_file_name found_app_config_apps = __get_applications_from_app_config( app_config_content) else: apps_from_other_repos.update( __get_applications_from_app_config(app_config_content)) if found_app_config_file is None or found_app_config_file_name is None: raise GitOpsException( f"Couldn't find config file for apps repository in root repository's 'apps/' directory" ) return ( found_app_config_file, found_app_config_file_name, found_app_config_apps, apps_from_other_repos, found_apps_path, )
def push(self, branch: Optional[str] = None) -> None: repo = self.__get_repo() if not branch: branch = repo.git.branch("--show-current") logging.info("Pushing branch: %s", branch) try: repo.git.push("--set-upstream", "origin", branch) except GitCommandError as ex: raise GitOpsException( f"Error pushing branch '{branch}' to origin: {ex.stderr}" ) from ex except GitError as ex: raise GitOpsException( f"Error pushing branch '{branch}' to origin.") from ex
def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any: root_config_git_repo.clone() bootstrap_values_file = root_config_git_repo.get_full_file_path( "bootstrap/values.yaml") try: bootstrap_yaml = yaml_file_load(bootstrap_values_file) except FileNotFoundError as ex: raise GitOpsException( "File 'bootstrap/values.yaml' not found in root repository." ) from ex if "bootstrap" not in bootstrap_yaml: raise GitOpsException( "Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") return bootstrap_yaml["bootstrap"]
def __get_bootstrap_entries(root_git): root_git.checkout("master") bootstrap_values_file = root_git.get_full_file_path( "bootstrap/values.yaml") try: with open(bootstrap_values_file, "r") as stream: bootstrap_yaml = YAML().load(stream) except FileNotFoundError as ex: raise GitOpsException( "File 'bootstrap/values.yaml' not found in root repository." ) from ex if "bootstrap" not in bootstrap_yaml: raise GitOpsException( "Cannot find key 'bootstrap' in 'bootstrap/values.yaml'") return bootstrap_yaml["bootstrap"]
def __check_if_app_already_exists(apps_dirs: Set[str], apps_from_other_repos: Set[str]) -> None: for app_key in apps_dirs: if app_key in apps_from_other_repos: raise GitOpsException( f"Application '{app_key}' already exists in a different repository" )
def delete_branch(self, branch: str) -> None: branch_hash = self.get_branch_head_hash(branch) result = self.__bitbucket.delete_branch(self.__organisation, self.__repository_name, branch, branch_hash) if result and "errors" in result: raise GitOpsException(result["errors"][0]["message"])
def delete_branch(self, branch): repo = self.__get_repo() try: git_ref = repo.get_git_ref(f"heads/{branch}") except UnknownObjectException as ex: raise GitOpsException(f"Branch '{branch}' does not exist.") from ex git_ref.delete()
def get_pull_request_branch(self, pr_id): pull_request = self._bitbucket.get_pullrequest(self._organisation, self._repository_name, pr_id) if "errors" in pull_request: raise GitOpsException(pull_request["errors"][0]["message"]) return pull_request["fromRef"]["displayId"]
def __get_pull_request(self, pr_id: int) -> PullRequest.PullRequest: repo = self.__get_repo() try: return repo.get_pull(pr_id) except UnknownObjectException as ex: raise GitOpsException( f"Pull request with ID '{pr_id}' does not exist.") from ex
def __sync_apps(apps_git, root_git): logging.info("Apps repository: %s", apps_git.get_clone_url()) logging.info("Root repository: %s", root_git.get_clone_url()) repo_apps = __get_repo_apps(apps_git) logging.info("Found %s app(s) in apps repository: %s", len(repo_apps), ", ".join(repo_apps)) logging.info( "Searching apps repository in root repository's 'apps/' directory...") apps_config_file, apps_config_file_name, current_repo_apps, apps_from_other_repos = __find_apps_config_from_repo( apps_git, root_git) if apps_config_file is None: raise GitOpsException( f"Could't find config file for apps repository in root repository's 'apps/' directory" ) if current_repo_apps == repo_apps: logging.info("Root repository already up-to-date. I'm done here.") return __check_if_app_already_exists(repo_apps, apps_from_other_repos) logging.info("Sync applications in root repository's %s.", apps_config_file_name) merge_yaml_element(apps_config_file, "applications", {repo_app: {} for repo_app in repo_apps}, True) __commit_and_push(apps_git, root_git, apps_config_file_name)
def test_checkout_error(self): checkout_exception = GitOpsException("dummy checkout error") self.git_util_mock.checkout.side_effect = checkout_exception with pytest.raises(GitOpsException) as ex: deploy_command( command="deploy", file="test/file.yml", values={ "a.b.c": "foo", "a.b.d": "bar" }, username="******", password="******", git_user="******", git_email="GIT_EMAIL", create_pr=False, auto_merge=False, single_commit=False, organisation="ORGA", repository_name="REPO", git_provider="github", git_provider_url=None, ) self.assertEqual(ex.value, checkout_exception) assert self.mock_manager.mock_calls == [ call.create_tmp_dir(), call.create_git("USERNAME", "PASSWORD", "GIT_USER", "GIT_EMAIL", "ORGA", "REPO", "github", None, "/tmp/created-tmp-dir"), call.git_util.checkout("master"), call.delete_tmp_dir("/tmp/created-tmp-dir"), ]
def merge_pull_request( self, pr_id: int, merge_method: Literal["squash", "rebase", "merge"] = "merge") -> None: merge_request = self.__project.mergerequests.get(pr_id) max_retries = MAX_MERGE_RETRIES while max_retries > 0: try: if merge_method == "rebase": merge_request.rebase() return merge_request.merge() return except gitlab.exceptions.GitlabMRClosedError as ex: # "Branch cannot be merged" error can occur if the server # is still processing the merge request internally max_retries -= 1 logging.warning( "Retry merging pull request. Attempts: (%s/%s)", MAX_MERGE_RETRIES - max_retries, MAX_MERGE_RETRIES) if max_retries == 0: raise GitOpsException( "Error merging pull request: 'Branch cannot be merged'" ) from ex time.sleep(2.5)
def __create_preview_from_template_if_not_existing( self, template_git_repo: GitRepo, target_git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool: preview_namespace = gitops_config.get_preview_namespace( self.__args.preview_id) full_preview_folder_path = target_git_repo.get_full_file_path( preview_namespace) preview_env_already_exist = os.path.isdir(full_preview_folder_path) if preview_env_already_exist: logging.info("Use existing folder for preview: %s", preview_namespace) return False logging.info("Create new folder for preview: %s", preview_namespace) full_preview_template_folder_path = template_git_repo.get_full_file_path( gitops_config.preview_template_path) if not os.path.isdir(full_preview_template_folder_path): raise GitOpsException( f"The preview template folder does not exist: {gitops_config.preview_template_path}" ) logging.info("Using the preview template folder: %s", gitops_config.preview_template_path) shutil.copytree( full_preview_template_folder_path, full_preview_folder_path, ) return True
def __create_preview_from_template_if_not_existing( self, git_repo: GitRepo, gitops_config: GitOpsConfig) -> bool: preview_namespace = gitops_config.get_preview_namespace( self.__args.preview_id) full_preview_folder_path = git_repo.get_full_file_path( preview_namespace) preview_env_already_exist = os.path.isdir(full_preview_folder_path) if preview_env_already_exist: logging.info("Use existing folder for preview: %s", preview_namespace) return False logging.info("Create new folder for preview: %s", preview_namespace) preview_template_folder_name = f".preview-templates/{gitops_config.application_name}" full_preview_template_folder_path = git_repo.get_full_file_path( preview_template_folder_name) if not os.path.isdir(full_preview_template_folder_path): raise GitOpsException( f"The preview template folder does not exist: {preview_template_folder_name}" ) logging.info("Using the preview template folder: %s", preview_template_folder_name) shutil.copytree( full_preview_template_folder_path, full_preview_folder_path, ) self.__update_yaml_file(git_repo, f"{preview_namespace}/Chart.yaml", "name", preview_namespace) return True
def create(config: GitApiConfig, organisation: str, repository_name: str) -> GitRepoApi: git_repo_api: Optional[GitRepoApi] if config.git_provider is GitProvider.GITHUB: git_repo_api = GithubGitRepoApiAdapter( username=config.username, password=config.password, organisation=organisation, repository_name=repository_name, ) elif config.git_provider is GitProvider.BITBUCKET: if not config.git_provider_url: raise GitOpsException("Please provide url for Bitbucket!") git_repo_api = BitbucketGitRepoApiAdapter( git_provider_url=config.git_provider_url, username=config.username, password=config.password, organisation=organisation, repository_name=repository_name, ) elif config.git_provider is GitProvider.GITLAB: provider_url = config.git_provider_url if not provider_url: provider_url = "https://www.gitlab.com" git_repo_api = GitlabGitRepoApiAdapter( git_provider_url=provider_url, username=config.username, password=config.password, organisation=organisation, repository_name=repository_name, ) return GitRepoApiLoggingProxy(git_repo_api)
def test_clone_error(self): clone_exception = GitOpsException("dummy clone error") self.git_repo_mock.clone.side_effect = clone_exception args = DeployCommand.Args( file="test/file.yml", values={ "a.b.c": "foo", "a.b.d": "bar" }, username="******", password="******", git_user="******", git_email="GIT_EMAIL", create_pr=False, auto_merge=False, single_commit=False, organisation="ORGA", repository_name="REPO", git_provider=GitProvider.GITHUB, git_provider_url=None, commit_message=None, ) with pytest.raises(GitOpsException) as ex: DeployCommand(args).execute() self.assertEqual(ex.value, clone_exception) assert self.mock_manager.method_calls == [ call.GitRepoApiFactory.create(args, "ORGA", "REPO"), call.GitRepo(self.git_repo_api_mock), call.GitRepo.clone(), ]
def execute(self) -> None: gitops_config = self.__get_gitops_config() preview_id = self.__args.preview_id preview_target_git_repo_api = self.__create_preview_target_git_repo_api( gitops_config) with GitRepo(preview_target_git_repo_api) as preview_target_git_repo: preview_target_git_repo.clone(gitops_config.preview_target_branch) preview_namespace = gitops_config.get_preview_namespace(preview_id) logging.info("Preview folder name: %s", preview_namespace) preview_folder_exists = self.__delete_folder_if_exists( preview_target_git_repo, preview_namespace) if not preview_folder_exists: if self.__args.expect_preview_exists: raise GitOpsException( f"There was no preview with name: {preview_namespace}") logging.info( "No preview environment for '%s' and preview id '%s'. I'm done here.", gitops_config.application_name, preview_id, ) return self.__commit_and_push( preview_target_git_repo, f"Delete preview environment for '{gitops_config.application_name}' and preview id '{preview_id}'.", )
def get_branch_head_hash(self, branch: str) -> str: branches = self.__bitbucket.get_branches(self.__organisation, self.__repository_name, filter=branch, limit=1) if not branches: raise GitOpsException(f"Branch '{branch}' not found'") return str(branches[0]["latestCommit"])
def __get_value(self, key: str) -> Any: keys = key.split(".") data = self.__yaml for k in keys: if not isinstance(data, dict) or k not in data: raise GitOpsException(f"Key '{key}' not found in GitOps config!") data = data[k] return data
def new_branch(self, branch: str) -> None: logging.info("Creating new branch: %s", branch) repo = self.__get_repo() try: repo.git.checkout("-b", branch) except GitError as ex: raise GitOpsException( f"Error creating new branch '{branch}'.") from ex
def get_clone_url(self) -> str: try: repo = self.__bitbucket.get_repo(self.__organisation, self.__repository_name) except requests.exceptions.ConnectionError as ex: raise GitOpsException( f"Error connecting to '{self.__git_provider_url}''") from ex if "errors" in repo: for error in repo["errors"]: exception = error["exceptionName"] if exception == "com.atlassian.bitbucket.auth.IncorrectPasswordAuthenticationException": raise GitOpsException("Bad credentials") if exception == "com.atlassian.bitbucket.project.NoSuchProjectException": raise GitOpsException( f"Organisation '{self.__organisation}' does not exist") if exception == "com.atlassian.bitbucket.repository.NoSuchRepositoryException": raise GitOpsException( f"Repository '{self.__organisation}/{self.__repository_name}' does not exist" ) raise GitOpsException(error["message"]) if "links" not in repo: raise GitOpsException( f"Repository '{self.__organisation}/{self.__repository_name}' does not exist" ) for clone_link in repo["links"]["clone"]: if clone_link["name"] == "http": repo_url = clone_link["href"] if not repo_url: raise GitOpsException("Couldn't determine repository URL.") return str(repo_url)