def _parse_yaml(source: str, config_string: bool): logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False) log = ConsolePrinter(logging_args) yaml = Parsers.get_yaml_editor() # for better backward compatibility with PyYAML (that supports only YAML 1.1) used in the previous # GitLabForm versions, let's force ruamel.yaml to use YAML version 1.1 by default too yaml.version = (1, 1) if config_string: config_string = textwrap.dedent(source) verbose("Reading config from the provided string.") (yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, config_string, literal=True) else: config_path = source verbose(f"Reading config from file: {config_path}") (yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, config_path) if doc_loaded: debug("Config parsed successfully as YAML.") else: # an error message has already been printed via ConsolePrinter exit(EXIT_INVALID_INPUT) return yaml_data
def _print_diff(self, project_and_group: str, configuration): try: current_secret_variables = self.gitlab.get_secret_variables( project_and_group) for secret_variable in current_secret_variables: secret_variable["value"] = hide(secret_variable["value"]) verbose(f"Secret variables for {project_and_group} in GitLab:") verbose( textwrap.indent( ez_yaml.to_string(current_secret_variables), " ", )) except: verbose( f"Secret variables for {project_and_group} in GitLab cannot be checked." ) verbose(f"Secret variables in {project_and_group} in configuration:") configured_secret_variables = copy.deepcopy(configuration) for key in configured_secret_variables.keys(): configured_secret_variables[key]["value"] = hide( configured_secret_variables[key]["value"]) verbose( textwrap.indent( ez_yaml.to_string(configured_secret_variables), " ", ))
def _process_configuration(self, project_and_group: str, configuration: dict): if configuration["project"].get("archive"): verbose("Archiving project...") self.gitlab.archive(project_and_group) else: verbose("Unarchiving project...") self.gitlab.unarchive(project_and_group)
def _process_configuration(self, project_and_group: str, configuration: dict): debug("Deploy keys BEFORE: %s", self.gitlab.get_deploy_keys(project_and_group)) for deploy_key in sorted(configuration["deploy_keys"]): verbose(f"Setting deploy key: {deploy_key}") self.gitlab.post_deploy_key( project_and_group, configuration["deploy_keys"][deploy_key]) debug("Deploy keys AFTER: %s", self.gitlab.get_deploy_keys(project_and_group))
def _process_groups(self, project_and_group: str, groups: dict, enforce_members: bool): if groups: verbose("Processing groups as members...") current_groups = self.gitlab.get_groups_from_project( project_and_group) for group in groups: expires_at = (groups[group]["expires_at"].strftime("%Y-%m-%d") if "expires_at" in groups[group] else None) access_level = (groups[group]["group_access"] if "group_access" in groups[group] else None) # we only add the group if it doesn't have the correct settings if (group in current_groups and expires_at == current_groups[group]["expires_at"] and access_level == current_groups[group]["group_access_level"]): debug("Ignoring group '%s' as it is already a member", group) debug("Current settings for '%s' are: %s" % (group, current_groups[group])) else: debug("Setting group '%s' as a member", group) access = access_level expiry = expires_at # we will remove group access first and then re-add them, # to ensure that the groups have the expected access level self.gitlab.unshare_with_group(project_and_group, group) self.gitlab.share_with_group(project_and_group, group, access, expiry) if enforce_members: current_groups = self.gitlab.get_groups_from_project( project_and_group) groups_in_config = groups.keys() groups_in_gitlab = current_groups.keys() groups_not_in_config = set(groups_in_gitlab) - set( groups_in_config) for group_not_in_config in groups_not_in_config: debug( f"Removing group '{group_not_in_config}' that is not configured to be a member." ) self.gitlab.unshare_with_group(project_and_group, group_not_in_config) else: debug("Not enforcing group members.")
def log_diff( subject, current_config, config_to_apply, only_changed=False, hide_entries=None, test=False, ): # Compose values in list of `[key, from_config, from_server]`` changes = [[ k, json.dumps( current_config.get(k, "???") if type(current_config) == dict else "???"), json.dumps(v), ] for k, v in config_to_apply.items()] # Remove unchanged if needed if only_changed: changes = filter(lambda i: i[1] != i[2], changes) # Hide secrets if hide_entries: changes = list( map( lambda i: [i[0], hide(i[1]), hide(i[2])] if i[0] in hide_entries else i, changes, )) # calculate field size for nice formatting max_key_len = str(max(map(lambda i: len(i[0]), changes))) max_val_1 = str(max(map(lambda i: len(i[1]), changes))) max_val_2 = str(max(map(lambda i: len(i[2]), changes))) # generate placeholders for output pattern: ` value: before => after ` pattern = ("{:>" + max_key_len + "}: {:<" + max_val_1 + "} => {:<" + max_val_2 + "}") # create string text = "{subject}:\n{diff}".format(subject=subject, diff="\n".join( starmap(pattern.format, changes))) if test: return text else: verbose(text)
def _process_configuration(self, project_and_group: str, configuration: dict): if (self.gitlab.get_project_settings(project_and_group) ["builds_access_level"] == "disabled"): warning( "Builds disabled in this project so I can't set secret variables here." ) return debug( "Secret variables BEFORE: %s", self.gitlab.get_secret_variables(project_and_group), ) for secret_variable in sorted(configuration["secret_variables"]): if "delete" in configuration["secret_variables"][secret_variable]: key = configuration["secret_variables"][secret_variable]["key"] if configuration["secret_variables"][secret_variable][ "delete"]: verbose( f"Deleting {secret_variable}: {key} in project {project_and_group}" ) try: self.gitlab.delete_secret_variable( project_and_group, key) except: warning( f"Could not delete variable {key} in group {project_and_group}" ) continue verbose(f"Setting secret variable: {secret_variable}") try: self.gitlab.put_secret_variable( project_and_group, configuration["secret_variables"][secret_variable], ) except NotFoundException: self.gitlab.post_secret_variable( project_and_group, configuration["secret_variables"][secret_variable], ) debug( "Secret variables AFTER: %s", self.gitlab.get_secret_variables(project_and_group), )
def process_entity( self, entity_reference: str, configuration: dict, dry_run: bool, effective_configuration: EffectiveConfiguration, only_sections: List[str], ): for processor in self.processors: if only_sections == "all" or processor.configuration_name in only_sections: processor.process(entity_reference, configuration, dry_run, effective_configuration) else: verbose( f"Skipping section '{processor.configuration_name}' - not in --only-sections list." )
def _process_users(self, project_and_group: str, users: dict, enforce_members: bool): if users: verbose("Processing users as members...") current_members = self.gitlab.get_members_from_project( project_and_group) for user in users: expires_at = (users[user]["expires_at"].strftime("%Y-%m-%d") if "expires_at" in users[user] else None) access_level = (users[user]["access_level"] if "access_level" in users[user] else None) # we only add the user if it doesn't have the correct settings if (user in current_members and expires_at == current_members[user]["expires_at"] and access_level == current_members[user]["access_level"]): debug("Ignoring user '%s' as it is already a member", user) debug("Current settings for '%s' are: %s" % (user, current_members[user])) else: debug("Setting user '%s' as a member", user) access = access_level expiry = expires_at self.gitlab.remove_member_from_project( project_and_group, user) self.gitlab.add_member_to_project(project_and_group, user, access, expiry) if enforce_members: current_members = self.gitlab.get_members_from_project( project_and_group) users_in_config = users.keys() users_in_gitlab = current_members.keys() users_not_in_config = set(users_in_gitlab) - set(users_in_config) for user_not_in_config in users_not_in_config: debug( f"Removing user '{user_not_in_config}' that is not configured to be a member." ) self.gitlab.remove_member_from_project(project_and_group, user_not_in_config) else: debug("Not enforcing user members.")
def process( self, project_or_project_and_group: str, configuration: dict, dry_run: bool, effective_configuration: EffectiveConfiguration, ): if self._section_is_in_config(configuration): if configuration.get(f"{self.configuration_name}|skip"): verbose( f"Skipping section '{self.configuration_name}' - explicitly configured to do so." ) return elif ( configuration.get("project|archive") and self.configuration_name != "project" ): verbose( f"Skipping section '{self.configuration_name}' - it is configured to be archived." ) return if dry_run: verbose( f"Processing section '{self.configuration_name}' in dry-run mode." ) self._print_diff( project_or_project_and_group, configuration.get(self.configuration_name), ) else: verbose(f"Processing section '{self.configuration_name}'") self._process_configuration(project_or_project_and_group, configuration) effective_configuration.add_configuration( project_or_project_and_group, self.configuration_name, configuration.get(self.configuration_name), ) else: verbose(f"Skipping section '{self.configuration_name}' - not in config.")
def show_input_entities(entities: Entities): info_1(f"# of {entities.name} to process: {len(entities.get_effective())}") entities_omitted = "" entities_verbose = f"{entities.name}: {entities.get_effective()}" if entities.any_omitted(): entities_omitted += f"(# of omitted {entities.name} -" first = True for reason in entities.omitted: if len(entities.omitted[reason]) > 0: if not first: entities_omitted += "," entities_omitted += f" {reason}: {len(entities.omitted[reason])}" entities_verbose += f"\nomitted {entities.name} - {reason}: {entities.get_omitted(reason)}" first = False entities_omitted += ")" if entities_omitted: info_1(entities_omitted) verbose(entities_verbose)
def __init__(self, config_path=None, config_string=None): self.configuration = Configuration(config_path, config_string) self.url = self.configuration.get("gitlab|url", os.getenv("GITLAB_URL")) self.token = self.configuration.get("gitlab|token", os.getenv("GITLAB_TOKEN")) self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True) self.timeout = self.configuration.get("gitlab|timeout", 10) self.session = requests.Session() retries = Retry( total=3, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504] ) self.session.mount("http://", HTTPAdapter(max_retries=retries)) self.session.mount("https://", HTTPAdapter(max_retries=retries)) self.session.verify = self.ssl_verify if not self.ssl_verify: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) self.gitlabform_version = pkg_resources.get_distribution("gitlabform").version self.requests_version = pkg_resources.get_distribution("requests").version self.session.headers.update( { "private-token": self.token, "authorization": f"Bearer {self.token}", "user-agent": f"GitLabForm/{self.gitlabform_version} (python-requests/{self.requests_version})", } ) try: version = self._make_requests_to_api("version") verbose( f"Connected to GitLab version: {version['version']} ({version['revision']})" ) self.version = version["version"] except Exception as e: raise TestRequestFailedException(e)
def _process_configuration(self, group: str, configuration: dict): debug( "Group secret variables BEFORE: %s", self.gitlab.get_group_secret_variables(group), ) for secret_variable in sorted(configuration["group_secret_variables"]): if "delete" in configuration["group_secret_variables"][ secret_variable]: key = configuration["group_secret_variables"][secret_variable][ "key"] if configuration["group_secret_variables"][secret_variable][ "delete"]: verbose( f"Deleting {secret_variable}: {key} in group {group}") try: self.gitlab.delete_group_secret_variable(group, key) except: warning( f"Could not delete variable {key} in group {group}" ) continue verbose(f"Setting group secret variable: {secret_variable}") try: self.gitlab.put_group_secret_variable( group, configuration["group_secret_variables"][secret_variable]) except NotFoundException: self.gitlab.post_group_secret_variable( group, configuration["group_secret_variables"][secret_variable]) debug( "Groups secret variables AFTER: %s", self.gitlab.get_group_secret_variables(group), )
def _process_configuration(self, project_and_group: str, configuration: dict): for service in sorted(configuration["services"]): if configuration.get("services|" + service + "|delete"): verbose(f"Deleting service: {service}") self.gitlab.delete_service(project_and_group, service) else: if ( "recreate" in configuration["services"][service] and configuration["services"][service]["recreate"] ): # support from this configuration key has been added in v1.13.4 # we will remove it here to avoid passing it to the GitLab API warning( f"Ignoring deprecated 'recreate' field in the '{service}' service config. " "Please remove it from the config file permanently as this workaround is not " "needed anymore." ) del configuration["services"][service]["recreate"] verbose(f"Setting service: {service}") self.gitlab.set_service( project_and_group, service, configuration["services"][service] )
def _process_configuration(self, project_or_group: str, configuration: dict): entity_config = configuration[self.configuration_name] entity_in_gitlab = self.get_method(project_or_group) debug(f"{self.configuration_name} BEFORE: ^^^") if entity_in_gitlab: if self._needs_update(entity_in_gitlab, entity_config): verbose(f"Editing {self.configuration_name} in {project_or_group}") self.edit_method(project_or_group, entity_config) debug(f"{self.configuration_name} AFTER: ^^^") else: verbose( f"{self.configuration_name} in {project_or_group} doesn't need an update." ) else: verbose(f"Adding {self.configuration_name} in {project_or_group}") self.add_method(project_or_group, entity_config) debug(f"{self.configuration_name} AFTER: ^^^")
def add_configuration( self, project_or_group: str, configuration_name: str, configuration: dict ): if self.output_file: verbose(f"Adding effective configuration for {configuration_name}.") self.config[project_or_group][configuration_name] = configuration
def _process_configuration(self, project_or_group: str, configuration: dict): entities_in_configuration = configuration[self.configuration_name] self._find_duplicates(project_or_group, entities_in_configuration) entities_in_gitlab = self.list_method(project_or_group) debug(f"{self.configuration_name} BEFORE: ^^^") for entity_name, entity_config in entities_in_configuration.items(): entity_in_gitlab = self._is_in_gitlab(entity_config, entities_in_gitlab) if entity_in_gitlab: if "delete" in entity_config and entity_config["delete"]: self._validate_required_to_delete(project_or_group, entity_name, entity_config) verbose( f"Deleting {entity_name} of {self.configuration_name} in {project_or_group}" ) self.delete_method(project_or_group, entity_in_gitlab) elif self._needs_update(entity_in_gitlab, entity_config): self._validate_required_to_create_or_update( project_or_group, entity_name, entity_config) if self.edit_method: verbose( f"Editing {entity_name} of {self.configuration_name} in {project_or_group}" ) self.edit_method(project_or_group, entity_in_gitlab, entity_config) debug(f"{self.configuration_name} AFTER: ^^^") else: verbose( f"Recreating {entity_name} of {self.configuration_name} in {project_or_group}" ) self.delete_method(project_or_group, entity_in_gitlab) self.add_method(project_or_group, entity_config) debug(f"{self.configuration_name} AFTER: ^^^") else: verbose( f"{entity_name} of {self.configuration_name} in {project_or_group} doesn't need an update." ) else: if "delete" in entity_config and entity_config["delete"]: verbose( f"{entity_name} of {self.configuration_name} in {project_or_group} already doesn't exist." ) else: self._validate_required_to_create_or_update( project_or_group, entity_name, entity_config) verbose( f"Adding {entity_name} of {self.configuration_name} in {project_or_group}" ) self.add_method(project_or_group, entity_config) debug(f"{self.configuration_name} AFTER: ^^^")
def _process_configuration(self, project_and_group: str, configuration: dict): for file in sorted(configuration["files"]): debug("Processing file '%s'...", file) if configuration.get("files|" + file + "|skip"): debug("Skipping file '%s'", file) continue if configuration["files"][file]["branches"] == "all": all_branches = self.gitlab.get_branches(project_and_group) branches = sorted(all_branches) elif configuration["files"][file]["branches"] == "protected": protected_branches = self.gitlab.get_protected_branches( project_and_group ) branches = sorted(protected_branches) else: all_branches = self.gitlab.get_branches(project_and_group) branches = [] for branch in configuration["files"][file]["branches"]: if branch in all_branches: branches.append(branch) else: message = f"! Branch '{branch}' not found, not processing file '{file}' in it" if self.strict: fatal( message, exit_code=EXIT_INVALID_INPUT, ) else: warning(message) for branch in branches: verbose(f"Processing file '{file}' in branch '{branch}'") if configuration.get( "files|" + file + "|content" ) and configuration.get("files|" + file + "|file"): fatal( f"File '{file}' in '{project_and_group}' has both `content` and `file` set - " "use only one of these keys.", exit_code=EXIT_INVALID_INPUT, ) if configuration.get("files|" + file + "|delete"): try: self.gitlab.get_file(project_and_group, branch, file) debug("Deleting file '%s' in branch '%s'", file, branch) self.modify_file_dealing_with_branch_protection( project_and_group, branch, file, "delete", configuration, ) except NotFoundException: debug( "Not deleting file '%s' in branch '%s' (already doesn't exist)", file, branch, ) else: # change or create file if configuration.get("files|" + file + "|content"): new_content = configuration.get("files|" + file + "|content") else: path_in_config = Path( configuration.get("files|" + file + "|file") ) if path_in_config.is_absolute(): # TODO: does this work? we are reading the content twice in this case... path = path_in_config.read_text() else: # relative paths are relative to config file location path = Path( os.path.join( self.config.config_dir, str(path_in_config) ) ) new_content = path.read_text() if configuration.get("files|" + file + "|template", True): new_content = self.get_file_content_as_template( new_content, project_and_group, **configuration.get("files|" + file + "|jinja_env", dict()), ) try: current_content = self.gitlab.get_file( project_and_group, branch, file ) if current_content != new_content: if configuration.get("files|" + file + "|overwrite"): debug("Changing file '%s' in branch '%s'", file, branch) self.modify_file_dealing_with_branch_protection( project_and_group, branch, file, "modify", configuration, new_content, ) else: debug( "Not changing file '%s' in branch '%s' - overwrite flag not set.", file, branch, ) else: debug( "Not changing file '%s' in branch '%s' - it's content is already" " as provided)", file, branch, ) except NotFoundException: debug("Creating file '%s' in branch '%s'", file, branch) self.modify_file_dealing_with_branch_protection( project_and_group, branch, file, "add", configuration, new_content, ) if configuration.get("files|" + file + "|only_first_branch"): verbose("Skipping other branches for this file, as configured.") break
def _process_configuration(self, project_and_group: str, configuration: dict): approvals = configuration.get("merge_requests|approvals") if approvals: verbose(f"Setting approvals settings: {approvals}") self.gitlab.post_approvals_settings(project_and_group, approvals) approvers = configuration.get("merge_requests|approvers") approver_groups = configuration.get("merge_requests|approver_groups") remove_other_approval_rules = configuration.get( "merge_requests|remove_other_approval_rules") # checking if "is not None" allows configs with empty array to work if (approvers is not None or approver_groups is not None and approvals and "approvals_before_merge" in approvals): verbose(f"Setting approvers...") # in pre-12.3 API approvers (users and groups) were configured under the same endpoint as approvals settings approvals_settings = self.gitlab.get_approvals_settings( project_and_group) if ("approvers" in approvals_settings or "approver_groups" in approvals_settings): # /approvers endpoint has been removed in 13.11.x GitLab version if LooseVersion(self.gitlab.version) < LooseVersion("13.11"): debug("Deleting legacy approvers setup") self.gitlab.delete_legacy_approvers(project_and_group) approval_rule_name = "Approvers (configured using GitLabForm)" # is a rule already configured and just needs updating? approval_rule_id = None rules = self.gitlab.get_approvals_rules(project_and_group) for rule in rules: if rule["name"] == approval_rule_name: approval_rule_id = rule["id"] else: if remove_other_approval_rules: debug("Deleting extra approval rule '%s'" % rule["name"]) self.gitlab.delete_approvals_rule( project_and_group, rule["id"]) if not approvers: approvers = [] if not approver_groups: approver_groups = [] if approval_rule_id: # the rule exists, needs an update verbose( f"Updating approvers rule to users {approvers} and groups {approver_groups}" ) self.gitlab.update_approval_rule( project_and_group, approval_rule_id, approval_rule_name, approvals["approvals_before_merge"], approvers, approver_groups, ) else: # the rule does not exist yet, let's create it verbose( f"Creating approvers rule to users {approvers} and groups {approver_groups}" ) self.gitlab.create_approval_rule( project_and_group, approval_rule_name, approvals["approvals_before_merge"], approvers, approver_groups, )
def _print_diff(self, project_or_project_and_group: str, entity_config): verbose(f"Diffing for section '{self.configuration_name}' is not supported yet")