def __init__(self, config_path=None, config_string=None): self.configuration = Configuration(config_path, config_string) self.url = self.configuration.get("gitlab|url", os.getenv("GITLAB_URL")) self.token = self.configuration.get("gitlab|token", os.getenv("GITLAB_TOKEN")) self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True) self.timeout = self.configuration.get("gitlab|timeout", 10) self.session = requests.Session() retries = Retry(total=3, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504]) self.session.mount("http://", HTTPAdapter(max_retries=retries)) self.session.mount("https://", HTTPAdapter(max_retries=retries)) self.session.verify = self.ssl_verify if not self.ssl_verify: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) try: version = self._make_requests_to_api("version") cli_ui.debug( f"Connected to GitLab version: {version['version']} ({version['revision']})" ) self.version = version["version"] except Exception as e: raise TestRequestFailedException(e)
def run_git_captured(working_path: Path, *cmd: str, check: bool = True) -> Tuple[int, str]: """Run git `cmd` in given `working_path`, capturing the output. Return a tuple (returncode, output). Raise GitCommandError if return code is non-zero and check is True. """ assert_working_path(working_path) git_cmd = list(cmd) git_cmd.insert(0, "git") options: Dict[str, Any] = {} options["stdout"] = subprocess.PIPE options["stderr"] = subprocess.STDOUT ui.debug(ui.lightgray, working_path, "$", ui.reset, *git_cmd) process = subprocess.Popen(git_cmd, cwd=working_path, **options) out, _ = process.communicate() out = out.decode("utf-8") if out.endswith("\n"): out = out.strip("\n") returncode = process.returncode ui.debug(ui.lightgray, "[", returncode, "]", ui.reset, out) if check and returncode != 0: raise GitCommandError(working_path, cmd, output=out) return returncode, out
def _process_configuration(self, group: str, configuration: dict, do_apply: bool = True): group_settings = configuration["group_settings"] logging.debug("Group settings BEFORE: %s", self.gitlab.get_group_settings(group)) cli_ui.debug(f"Setting group settings: {group_settings}") self.gitlab.put_group_settings(group, group_settings) logging.debug("Group settings AFTER: %s", self.gitlab.get_group_settings(group))
def run(working_path: Path, *cmd: str, check: bool = True) -> None: """ Run git `cmd` in given `working_path` Raise GitCommandError if return code is non-zero and `check` is True. """ git_cmd = list(cmd) git_cmd.insert(0, "git") ui.debug(ui.lightgray, working_path, "$", ui.reset, *git_cmd) returncode = subprocess.call(git_cmd, cwd=working_path) if returncode != 0 and check: raise CommandError(working_path, cmd)
def _process_configuration(self, project_and_group: str, configuration: dict): logging.debug( "Deploy keys BEFORE: %s", self.gitlab.get_deploy_keys(project_and_group) ) for deploy_key in sorted(configuration["deploy_keys"]): cli_ui.debug(f"Setting deploy key: {deploy_key}") self.gitlab.post_deploy_key( project_and_group, configuration["deploy_keys"][deploy_key] ) logging.debug( "Deploy keys AFTER: %s", self.gitlab.get_deploy_keys(project_and_group) )
def _process_configuration(self, project_and_group: str, configuration: dict, do_apply: bool = True): project = configuration["project"] if project: if "archive" in project: if project["archive"]: cli_ui.debug("Archiving project...") self.gitlab.archive(project_and_group) else: cli_ui.debug("Unarchiving project...") self.gitlab.unarchive(project_and_group)
def _process_configuration(self, project_and_group: str, configuration: dict): project_settings = configuration["project_settings"] logging.debug( "Project settings BEFORE: %s", self.gitlab.get_project_settings(project_and_group), ) cli_ui.debug(f"Setting project settings: {project_settings}") self.gitlab.put_project_settings(project_and_group, project_settings) logging.debug( "Project settings AFTER: %s", self.gitlab.get_project_settings(project_and_group), )
def _process_configuration(self, project_and_group: str, configuration: dict): push_rules = configuration["project_push_rules"] old_project_push_rules = self.gitlab.get_project_push_rules(project_and_group) logging.debug("Project push rules settings BEFORE: %s", old_project_push_rules) if old_project_push_rules: cli_ui.debug(f"Updating project push rules: {push_rules}") self.gitlab.put_project_push_rules(project_and_group, push_rules) else: cli_ui.debug(f"Creating project push rules: {push_rules}") self.gitlab.post_project_push_rules(project_and_group, push_rules) logging.debug( "Project push rules AFTER: %s", self.gitlab.get_project_push_rules(project_and_group), )
def log_diff( subject, current_config, config_to_apply, only_changed=False, hide_entries=None, test=False, ): # Compose values in list of `[key, from_config, from_server]`` changes = [[ k, json.dumps( current_config.get(k, "???") if type(current_config) == dict else "???"), json.dumps(v), ] for k, v in config_to_apply.items()] # Remove unchanged if needed if only_changed: changes = filter(lambda i: i[1] != i[2], changes) # Hide secrets if hide_entries: changes = list( map( lambda i: [i[0], hide(i[1]), hide(i[2])] if i[0] in hide_entries else i, changes, )) # calculate field size for nice formatting max_key_len = str(max(map(lambda i: len(i[0]), changes))) max_val_1 = str(max(map(lambda i: len(i[1]), changes))) max_val_2 = str(max(map(lambda i: len(i[2]), changes))) # generate placeholders for output pattern: ` value: before => after ` pattern = ("{:>" + max_key_len + "}: {:<" + max_val_1 + "} => {:<" + max_val_2 + "}") # create string text = "{subject}:\n{diff}".format(subject=subject, diff="\n".join( starmap(pattern.format, changes))) if test: return text else: cli_ui.debug(text)
def _process_configuration(self, project_and_group: str, configuration: dict): if ( self.gitlab.get_project_settings(project_and_group)["builds_access_level"] == "disabled" ): cli_ui.warning( "Builds disabled in this project so I can't set secret variables here." ) return logging.debug( "Secret variables BEFORE: %s", self.gitlab.get_secret_variables(project_and_group), ) for secret_variable in sorted(configuration["secret_variables"]): if "delete" in configuration["secret_variables"][secret_variable]: key = configuration["secret_variables"][secret_variable]["key"] if configuration["secret_variables"][secret_variable]["delete"]: cli_ui.debug( f"Deleting {secret_variable}: {key} in project {project_and_group}" ) try: self.gitlab.delete_secret_variable(project_and_group, key) except: logging.warn( f"Could not delete variable {key} in group {project_and_group}" ) continue cli_ui.debug(f"Setting secret variable: {secret_variable}") try: self.gitlab.put_secret_variable( project_and_group, configuration["secret_variables"][secret_variable], ) except NotFoundException: self.gitlab.post_secret_variable( project_and_group, configuration["secret_variables"][secret_variable], ) logging.debug( "Secret variables AFTER: %s", self.gitlab.get_secret_variables(project_and_group), )
def process( self, project_or_project_and_group: str, configuration: dict, dry_run: bool, output_file: Optional[TextIO], ): if self.__configuration_name in configuration: if configuration.get(f"{self.__configuration_name}|skip"): cli_ui.debug( f"Skipping {self.__configuration_name} - explicitly configured to do so." ) return elif (configuration.get("project|archive") and self.__configuration_name != "project"): cli_ui.debug( f"Skipping {self.__configuration_name} - it is configured to be archived." ) return if dry_run: cli_ui.debug( f"Processing {self.__configuration_name} in dry-run mode.") self._print_diff( project_or_project_and_group, configuration.get(self.__configuration_name), ) else: cli_ui.debug(f"Processing {self.__configuration_name}") self._process_configuration(project_or_project_and_group, configuration) if output_file: cli_ui.debug( f"Writing effective configuration for {self.__configuration_name} to the output file." ) self._write_to_file( configuration.get(self.__configuration_name), output_file, ) else: logging.debug("Skipping %s - not in config." % self.__configuration_name)
def _process_configuration(self, group: str, configuration: dict, do_apply: bool = True): logging.debug( "Group secret variables BEFORE: %s", self.gitlab.get_group_secret_variables(group), ) for secret_variable in sorted(configuration["group_secret_variables"]): if "delete" in configuration["group_secret_variables"][ secret_variable]: key = configuration["group_secret_variables"][secret_variable][ "key"] if configuration["group_secret_variables"][secret_variable][ "delete"]: cli_ui.debug( f"Deleting {secret_variable}: {key} in group {group}") try: self.gitlab.delete_group_secret_variable(group, key) except: cli_ui.info( f"Could not delete variable {key} in group {group}" ) continue cli_ui.debug(f"Setting group secret variable: {secret_variable}") try: self.gitlab.put_group_secret_variable( group, configuration["group_secret_variables"][secret_variable]) except NotFoundException: self.gitlab.post_group_secret_variable( group, configuration["group_secret_variables"][secret_variable]) logging.debug( "Groups secret variables AFTER: %s", self.gitlab.get_group_secret_variables(group), )
def _process_configuration(self, project_and_group: str, configuration: dict): for service in sorted(configuration["services"]): if configuration.get("services|" + service + "|delete"): cli_ui.debug(f"Deleting service: {service}") self.gitlab.delete_service(project_and_group, service) else: if ("recreate" in configuration["services"][service] and configuration["services"][service]["recreate"]): # support from this configuration key has been added in v1.13.4 # we will remove it here to avoid passing it to the GitLab API cli_ui.warning( f"Ignoring deprecated 'recreate' field in the '{service}' service config. " "Please remove it from the config file permanently as this workaround is not " "needed anymore.") del configuration["services"][service]["recreate"] cli_ui.debug(f"Setting service: {service}") self.gitlab.set_service(project_and_group, service, configuration["services"][service])
def main(self): if self.project_or_group == "ALL": cli_ui.info(">>> Processing ALL groups and projects") elif self.project_or_group == "ALL_DEFINED": cli_ui.info( ">>> Processing ALL groups and projects defined in config") groups = self.get_groups(self.project_or_group) projects = self.get_projects(self.project_or_group, groups) if len(groups) == 0 and len(projects) == 0: cli_ui.error( f"Entity {self.project_or_group} cannot be found in GitLab!") sys.exit(EXIT_INVALID_INPUT) else: cli_ui.debug(f"groups: {groups}") cli_ui.debug(f"projects: {projects}") cli_ui.info_1(f"# of groups to process: {len(groups)}") cli_ui.info_1(f"# of projects to process: {len(projects)}") self.process_all(projects, groups)
def _print_diff(self, project_and_group: str, configuration): try: current_secret_variables = self.gitlab.get_secret_variables( project_and_group ) for secret_variable in current_secret_variables: secret_variable["value"] = hide(secret_variable["value"]) cli_ui.debug(f"Secret variables for {project_and_group} in GitLab:") cli_ui.debug( textwrap.indent( yaml.dump(current_secret_variables, default_flow_style=False), " ", ) ) except: cli_ui.debug( f"Secret variables for {project_and_group} in GitLab cannot be checked." ) cli_ui.debug(f"Secret variables in {project_and_group} in configuration:") configured_secret_variables = copy.deepcopy(configuration) for key in configured_secret_variables.keys(): configured_secret_variables[key]["value"] = hide( configured_secret_variables[key]["value"] ) cli_ui.debug( textwrap.indent( yaml.dump(configured_secret_variables, default_flow_style=False), " ", ) )
def _process_configuration(self, project_and_group: str, configuration: dict): for file in sorted(configuration["files"]): logging.debug("Processing file '%s'...", file) if configuration.get("files|" + file + "|skip"): logging.debug("Skipping file '%s'", file) continue all_branches = self.gitlab.get_branches(project_and_group) if configuration["files"][file]["branches"] == "all": branches = sorted(all_branches) elif configuration["files"][file]["branches"] == "protected": protected_branches = self.gitlab.get_protected_branches( project_and_group) branches = sorted(protected_branches) else: branches = [] for branch in configuration["files"][file]["branches"]: if branch in all_branches: branches.append(branch) else: message = f"! Branch '{branch}' not found, not processing file '{file}' in it" if self.strict: cli_ui.error(message) sys.exit(EXIT_INVALID_INPUT) else: cli_ui.warning(message) for branch in branches: cli_ui.debug(f"Processing file '{file}' in branch '{branch}'") # unprotect protected branch temporarily for operations below if configuration.get("branches|" + branch + "|protected"): logging.debug( "> Temporarily unprotecting the branch for managing files in it..." ) self.gitlab.unprotect_branch(project_and_group, branch) if configuration.get("files|" + file + "|delete"): try: self.gitlab.get_file(project_and_group, branch, file) logging.debug("Deleting file '%s' in branch '%s'", file, branch) self.gitlab.delete_file( project_and_group, branch, file, self.get_commit_message_for_file_change( "delete", configuration.get("files|" + file + "|skip_ci"), ), ) except NotFoundException: logging.debug( "Not deleting file '%s' in branch '%s' (already doesn't exist)", file, branch, ) else: # change or create file if configuration.get("files|" + file + "|content") and configuration.get( "files|" + file + "|file"): cli_ui.error( f"File '{file}' in '{project_and_group}' has both `content` and `file` set - " "use only one of these keys.") sys.exit(EXIT_INVALID_INPUT) elif configuration.get("files|" + file + "|content"): new_content = configuration.get("files|" + file + "|content") else: path_in_config = Path( configuration.get("files|" + file + "|file")) if path_in_config.is_absolute(): path = path_in_config.read_text() else: # relative paths are relative to config file location path = Path( os.path.join(self.config.config_dir, str(path_in_config))) new_content = path.read_text() if configuration.get("files|" + file + "|template", True): new_content = self.get_file_content_as_template( new_content, project_and_group, **configuration.get("files|" + file + "|jinja_env", dict()), ) try: current_content = self.gitlab.get_file( project_and_group, branch, file) if current_content != new_content: if configuration.get("files|" + file + "|overwrite"): logging.debug( "Changing file '%s' in branch '%s'", file, branch) self.gitlab.set_file( project_and_group, branch, file, new_content, self.get_commit_message_for_file_change( "change", configuration.get("files|" + file + "|skip_ci"), ), ) else: logging.debug( "Not changing file '%s' in branch '%s' " "(overwrite flag not set)", file, branch, ) else: logging.debug( "Not changing file '%s' in branch '%s' (it's content is already" " as provided)", file, branch, ) except NotFoundException: logging.debug("Creating file '%s' in branch '%s'", file, branch) self.gitlab.add_file( project_and_group, branch, file, new_content, self.get_commit_message_for_file_change( "add", configuration.get("files|" + file + "|skip_ci")), ) # protect branch back after above operations if configuration.get("branches|" + branch + "|protected"): logging.debug("> Protecting the branch again.") self.branch_protector.protect_branch( project_and_group, configuration, branch) if configuration.get("files|" + file + "|only_first_branch"): cli_ui.debug( "Skipping other branches for this file, as configured." ) break
def _print_diff(self, project_or_project_and_group: str, configuration_to_process): cli_ui.debug( f"Diffing for {self.__configuration_name} section is not supported yet" )
def _process_configuration(self, project_and_group: str, configuration: dict): approvals = configuration.get("merge_requests|approvals") if approvals: cli_ui.debug(f"Setting approvals settings: {approvals}") self.gitlab.post_approvals_settings(project_and_group, approvals) approvers = configuration.get("merge_requests|approvers") approver_groups = configuration.get("merge_requests|approver_groups") remove_other_approval_rules = configuration.get( "merge_requests|remove_other_approval_rules" ) # checking if "is not None" allows configs with empty array to work if ( approvers is not None or approver_groups is not None and approvals and "approvals_before_merge" in approvals ): # in pre-12.3 API approvers (users and groups) were configured under the same endpoint as approvals settings approvals_settings = self.gitlab.get_approvals_settings(project_and_group) if ( "approvers" in approvals_settings or "approver_groups" in approvals_settings ): # /approvers endpoint has been removed in 13.11.x GitLab version if LooseVersion(self.gitlab.version) < LooseVersion("13.11"): logging.debug("Deleting legacy approvers setup") self.gitlab.delete_legacy_approvers(project_and_group) approval_rule_name = "Approvers (configured using GitLabForm)" # is a rule already configured and just needs updating? approval_rule_id = None rules = self.gitlab.get_approvals_rules(project_and_group) for rule in rules: if rule["name"] == approval_rule_name: approval_rule_id = rule["id"] else: if remove_other_approval_rules: logging.debug( "Deleting extra approval rule '%s'" % rule["name"] ) self.gitlab.delete_approvals_rule(project_and_group, rule["id"]) if not approvers: approvers = [] if not approver_groups: approver_groups = [] if approval_rule_id: # the rule exists, needs an update cli_ui.debug( f"Updating approvers rule to users {approvers} and groups {approver_groups}" ) self.gitlab.update_approval_rule( project_and_group, approval_rule_id, approval_rule_name, approvals["approvals_before_merge"], approvers, approver_groups, ) else: # the rule does not exist yet, let's create it cli_ui.debug( f"Creating approvers rule to users {approvers} and groups {approver_groups}" ) self.gitlab.create_approval_rule( project_and_group, approval_rule_name, approvals["approvals_before_merge"], approvers, approver_groups, )
def __init__(self, config_path=None, config_string=None): if config_path and config_string: cli_ui.fatal( "Please initialize with either config_path or config_string, not both." ) sys.exit(EXIT_INVALID_INPUT) try: if config_string: cli_ui.debug("Reading config from provided string.") self.config = yaml.safe_load(textwrap.dedent(config_string)) self.config_dir = "." else: # maybe config_path if "APP_HOME" in os.environ: # using this env var should be considered unofficial, we need this temporarily # for backwards compatibility. support for it may be removed without notice, do not use it! config_path = os.path.join(os.environ["APP_HOME"], "config.yml") elif not config_path: # this case is only meant for using gitlabform as a library config_path = os.path.join( str(Path.home()), ".gitlabform", "config.yml" ) elif config_path in [os.path.join(".", "config.yml"), "config.yml"]: # provided points to config.yml in the app current working dir config_path = os.path.join(os.getcwd(), "config.yml") cli_ui.debug(f"Reading config from file: {config_path}") with open(config_path, "r") as ymlfile: self.config = yaml.safe_load(ymlfile) logging.debug("Config parsed successfully as YAML.") # we need config path for accessing files for relative paths self.config_dir = os.path.dirname(config_path) if self.config.get("example_config"): cli_ui.fatal( "Example config detected, aborting.\n" "Haven't you forgotten to use `-c <config_file>` parameter?\n" "If you created your config based on the example config.yml," " then please remove 'example_config' key." ) sys.exit(EXIT_INVALID_INPUT) if self.config.get("config_version", 1) != 2: cli_ui.fatal( "This version of GitLabForm requires 'config_version: 2' entry in the config.\n" "This ensures that when the application behavior changes in a backward incompatible way," " you won't apply unexpected configuration to your GitLab instance.\n" "Please read the upgrading guide here: https://bit.ly/3ub1g5C\n" ) sys.exit(EXIT_INVALID_INPUT) try: self.config.get("projects_and_groups") except KeyNotFoundException: cli_ui.fatal("'projects_and_groups' key in the config is required.") sys.exit(EXIT_INVALID_INPUT) except (FileNotFoundError, IOError): raise ConfigFileNotFoundException(config_path) except Exception: if config_path: raise ConfigInvalidException(config_path) else: raise ConfigInvalidException(config_string)