def _parse_yaml(source: str, config_string: bool):
logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False)
log = ConsolePrinter(logging_args)
yaml = Parsers.get_yaml_editor()
# for better backward compatibility with PyYAML (that supports only YAML 1.1) used in the previous
# GitLabForm versions, let's force ruamel.yaml to use YAML version 1.1 by default too
yaml.version = (1, 1)
if config_string:
config_string = textwrap.dedent(source)
verbose("Reading config from the provided string.")
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml,
log,
config_string,
literal=True)
else:
config_path = source
verbose(f"Reading config from file: {config_path}")
(yaml_data,
doc_loaded) = Parsers.get_yaml_data(yaml, log, config_path)
if doc_loaded:
debug("Config parsed successfully as YAML.")
else:
# an error message has already been printed via ConsolePrinter
exit(EXIT_INVALID_INPUT)
return yaml_data
def _print_diff(self, project_and_group: str, configuration):
try:
current_secret_variables = self.gitlab.get_secret_variables(
project_and_group)
for secret_variable in current_secret_variables:
secret_variable["value"] = hide(secret_variable["value"])
verbose(f"Secret variables for {project_and_group} in GitLab:")
verbose(
textwrap.indent(
ez_yaml.to_string(current_secret_variables),
" ",
))
except:
verbose(
f"Secret variables for {project_and_group} in GitLab cannot be checked."
)
verbose(f"Secret variables in {project_and_group} in configuration:")
configured_secret_variables = copy.deepcopy(configuration)
for key in configured_secret_variables.keys():
configured_secret_variables[key]["value"] = hide(
configured_secret_variables[key]["value"])
verbose(
textwrap.indent(
ez_yaml.to_string(configured_secret_variables),
" ",
))
def _process_configuration(self, project_and_group: str, configuration: dict):
if configuration["project"].get("archive"):
verbose("Archiving project...")
self.gitlab.archive(project_and_group)
else:
verbose("Unarchiving project...")
self.gitlab.unarchive(project_and_group)
def _process_configuration(self, project_and_group: str,
configuration: dict):
debug("Deploy keys BEFORE: %s",
self.gitlab.get_deploy_keys(project_and_group))
for deploy_key in sorted(configuration["deploy_keys"]):
verbose(f"Setting deploy key: {deploy_key}")
self.gitlab.post_deploy_key(
project_and_group, configuration["deploy_keys"][deploy_key])
debug("Deploy keys AFTER: %s",
self.gitlab.get_deploy_keys(project_and_group))
def _process_groups(self, project_and_group: str, groups: dict,
enforce_members: bool):
if groups:
verbose("Processing groups as members...")
current_groups = self.gitlab.get_groups_from_project(
project_and_group)
for group in groups:
expires_at = (groups[group]["expires_at"].strftime("%Y-%m-%d")
if "expires_at" in groups[group] else None)
access_level = (groups[group]["group_access"]
if "group_access" in groups[group] else None)
# we only add the group if it doesn't have the correct settings
if (group in current_groups
and expires_at == current_groups[group]["expires_at"]
and access_level
== current_groups[group]["group_access_level"]):
debug("Ignoring group '%s' as it is already a member",
group)
debug("Current settings for '%s' are: %s" %
(group, current_groups[group]))
else:
debug("Setting group '%s' as a member", group)
access = access_level
expiry = expires_at
# we will remove group access first and then re-add them,
# to ensure that the groups have the expected access level
self.gitlab.unshare_with_group(project_and_group, group)
self.gitlab.share_with_group(project_and_group, group,
access, expiry)
if enforce_members:
current_groups = self.gitlab.get_groups_from_project(
project_and_group)
groups_in_config = groups.keys()
groups_in_gitlab = current_groups.keys()
groups_not_in_config = set(groups_in_gitlab) - set(
groups_in_config)
for group_not_in_config in groups_not_in_config:
debug(
f"Removing group '{group_not_in_config}' that is not configured to be a member."
)
self.gitlab.unshare_with_group(project_and_group,
group_not_in_config)
else:
debug("Not enforcing group members.")
def log_diff(
subject,
current_config,
config_to_apply,
only_changed=False,
hide_entries=None,
test=False,
):
# Compose values in list of `[key, from_config, from_server]``
changes = [[
k,
json.dumps(
current_config.get(k, "???") if type(current_config) ==
dict else "???"),
json.dumps(v),
] for k, v in config_to_apply.items()]
# Remove unchanged if needed
if only_changed:
changes = filter(lambda i: i[1] != i[2], changes)
# Hide secrets
if hide_entries:
changes = list(
map(
lambda i: [i[0], hide(i[1]), hide(i[2])]
if i[0] in hide_entries else i,
changes,
))
# calculate field size for nice formatting
max_key_len = str(max(map(lambda i: len(i[0]), changes)))
max_val_1 = str(max(map(lambda i: len(i[1]), changes)))
max_val_2 = str(max(map(lambda i: len(i[2]), changes)))
# generate placeholders for output pattern: ` value: before => after `
pattern = ("{:>" + max_key_len + "}: {:<" + max_val_1 + "} => {:<" +
max_val_2 + "}")
# create string
text = "{subject}:\n{diff}".format(subject=subject,
diff="\n".join(
starmap(pattern.format,
changes)))
if test:
return text
else:
verbose(text)
def _process_configuration(self, project_and_group: str,
configuration: dict):
if (self.gitlab.get_project_settings(project_and_group)
["builds_access_level"] == "disabled"):
warning(
"Builds disabled in this project so I can't set secret variables here."
)
return
debug(
"Secret variables BEFORE: %s",
self.gitlab.get_secret_variables(project_and_group),
)
for secret_variable in sorted(configuration["secret_variables"]):
if "delete" in configuration["secret_variables"][secret_variable]:
key = configuration["secret_variables"][secret_variable]["key"]
if configuration["secret_variables"][secret_variable][
"delete"]:
verbose(
f"Deleting {secret_variable}: {key} in project {project_and_group}"
)
try:
self.gitlab.delete_secret_variable(
project_and_group, key)
except:
warning(
f"Could not delete variable {key} in group {project_and_group}"
)
continue
verbose(f"Setting secret variable: {secret_variable}")
try:
self.gitlab.put_secret_variable(
project_and_group,
configuration["secret_variables"][secret_variable],
)
except NotFoundException:
self.gitlab.post_secret_variable(
project_and_group,
configuration["secret_variables"][secret_variable],
)
debug(
"Secret variables AFTER: %s",
self.gitlab.get_secret_variables(project_and_group),
)
def process_entity(
self,
entity_reference: str,
configuration: dict,
dry_run: bool,
effective_configuration: EffectiveConfiguration,
only_sections: List[str],
):
for processor in self.processors:
if only_sections == "all" or processor.configuration_name in only_sections:
processor.process(entity_reference, configuration, dry_run,
effective_configuration)
else:
verbose(
f"Skipping section '{processor.configuration_name}' - not in --only-sections list."
)
def _process_users(self, project_and_group: str, users: dict,
enforce_members: bool):
if users:
verbose("Processing users as members...")
current_members = self.gitlab.get_members_from_project(
project_and_group)
for user in users:
expires_at = (users[user]["expires_at"].strftime("%Y-%m-%d")
if "expires_at" in users[user] else None)
access_level = (users[user]["access_level"]
if "access_level" in users[user] else None)
# we only add the user if it doesn't have the correct settings
if (user in current_members
and expires_at == current_members[user]["expires_at"]
and access_level
== current_members[user]["access_level"]):
debug("Ignoring user '%s' as it is already a member", user)
debug("Current settings for '%s' are: %s" %
(user, current_members[user]))
else:
debug("Setting user '%s' as a member", user)
access = access_level
expiry = expires_at
self.gitlab.remove_member_from_project(
project_and_group, user)
self.gitlab.add_member_to_project(project_and_group, user,
access, expiry)
if enforce_members:
current_members = self.gitlab.get_members_from_project(
project_and_group)
users_in_config = users.keys()
users_in_gitlab = current_members.keys()
users_not_in_config = set(users_in_gitlab) - set(users_in_config)
for user_not_in_config in users_not_in_config:
debug(
f"Removing user '{user_not_in_config}' that is not configured to be a member."
)
self.gitlab.remove_member_from_project(project_and_group,
user_not_in_config)
else:
debug("Not enforcing user members.")
def process(
self,
project_or_project_and_group: str,
configuration: dict,
dry_run: bool,
effective_configuration: EffectiveConfiguration,
):
if self._section_is_in_config(configuration):
if configuration.get(f"{self.configuration_name}|skip"):
verbose(
f"Skipping section '{self.configuration_name}' - explicitly configured to do so."
)
return
elif (
configuration.get("project|archive")
and self.configuration_name != "project"
):
verbose(
f"Skipping section '{self.configuration_name}' - it is configured to be archived."
)
return
if dry_run:
verbose(
f"Processing section '{self.configuration_name}' in dry-run mode."
)
self._print_diff(
project_or_project_and_group,
configuration.get(self.configuration_name),
)
else:
verbose(f"Processing section '{self.configuration_name}'")
self._process_configuration(project_or_project_and_group, configuration)
effective_configuration.add_configuration(
project_or_project_and_group,
self.configuration_name,
configuration.get(self.configuration_name),
)
else:
verbose(f"Skipping section '{self.configuration_name}' - not in config.")
def show_input_entities(entities: Entities):
info_1(f"# of {entities.name} to process: {len(entities.get_effective())}")
entities_omitted = ""
entities_verbose = f"{entities.name}: {entities.get_effective()}"
if entities.any_omitted():
entities_omitted += f"(# of omitted {entities.name} -"
first = True
for reason in entities.omitted:
if len(entities.omitted[reason]) > 0:
if not first:
entities_omitted += ","
entities_omitted += f" {reason}: {len(entities.omitted[reason])}"
entities_verbose += f"\nomitted {entities.name} - {reason}: {entities.get_omitted(reason)}"
first = False
entities_omitted += ")"
if entities_omitted:
info_1(entities_omitted)
verbose(entities_verbose)
def __init__(self, config_path=None, config_string=None):
self.configuration = Configuration(config_path, config_string)
self.url = self.configuration.get("gitlab|url", os.getenv("GITLAB_URL"))
self.token = self.configuration.get("gitlab|token", os.getenv("GITLAB_TOKEN"))
self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True)
self.timeout = self.configuration.get("gitlab|timeout", 10)
self.session = requests.Session()
retries = Retry(
total=3, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504]
)
self.session.mount("http://", HTTPAdapter(max_retries=retries))
self.session.mount("https://", HTTPAdapter(max_retries=retries))
self.session.verify = self.ssl_verify
if not self.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.gitlabform_version = pkg_resources.get_distribution("gitlabform").version
self.requests_version = pkg_resources.get_distribution("requests").version
self.session.headers.update(
{
"private-token": self.token,
"authorization": f"Bearer {self.token}",
"user-agent": f"GitLabForm/{self.gitlabform_version} (python-requests/{self.requests_version})",
}
)
try:
version = self._make_requests_to_api("version")
verbose(
f"Connected to GitLab version: {version['version']} ({version['revision']})"
)
self.version = version["version"]
except Exception as e:
raise TestRequestFailedException(e)
def _process_configuration(self, group: str, configuration: dict):
debug(
"Group secret variables BEFORE: %s",
self.gitlab.get_group_secret_variables(group),
)
for secret_variable in sorted(configuration["group_secret_variables"]):
if "delete" in configuration["group_secret_variables"][
secret_variable]:
key = configuration["group_secret_variables"][secret_variable][
"key"]
if configuration["group_secret_variables"][secret_variable][
"delete"]:
verbose(
f"Deleting {secret_variable}: {key} in group {group}")
try:
self.gitlab.delete_group_secret_variable(group, key)
except:
warning(
f"Could not delete variable {key} in group {group}"
)
continue
verbose(f"Setting group secret variable: {secret_variable}")
try:
self.gitlab.put_group_secret_variable(
group,
configuration["group_secret_variables"][secret_variable])
except NotFoundException:
self.gitlab.post_group_secret_variable(
group,
configuration["group_secret_variables"][secret_variable])
debug(
"Groups secret variables AFTER: %s",
self.gitlab.get_group_secret_variables(group),
)
def _process_configuration(self, project_and_group: str, configuration: dict):
for service in sorted(configuration["services"]):
if configuration.get("services|" + service + "|delete"):
verbose(f"Deleting service: {service}")
self.gitlab.delete_service(project_and_group, service)
else:
if (
"recreate" in configuration["services"][service]
and configuration["services"][service]["recreate"]
):
# support from this configuration key has been added in v1.13.4
# we will remove it here to avoid passing it to the GitLab API
warning(
f"Ignoring deprecated 'recreate' field in the '{service}' service config. "
"Please remove it from the config file permanently as this workaround is not "
"needed anymore."
)
del configuration["services"][service]["recreate"]
verbose(f"Setting service: {service}")
self.gitlab.set_service(
project_and_group, service, configuration["services"][service]
)
def _process_configuration(self, project_or_group: str, configuration: dict):
entity_config = configuration[self.configuration_name]
entity_in_gitlab = self.get_method(project_or_group)
debug(f"{self.configuration_name} BEFORE: ^^^")
if entity_in_gitlab:
if self._needs_update(entity_in_gitlab, entity_config):
verbose(f"Editing {self.configuration_name} in {project_or_group}")
self.edit_method(project_or_group, entity_config)
debug(f"{self.configuration_name} AFTER: ^^^")
else:
verbose(
f"{self.configuration_name} in {project_or_group} doesn't need an update."
)
else:
verbose(f"Adding {self.configuration_name} in {project_or_group}")
self.add_method(project_or_group, entity_config)
debug(f"{self.configuration_name} AFTER: ^^^")
def add_configuration(
self, project_or_group: str, configuration_name: str, configuration: dict
):
if self.output_file:
verbose(f"Adding effective configuration for {configuration_name}.")
self.config[project_or_group][configuration_name] = configuration
def _process_configuration(self, project_or_group: str,
configuration: dict):
entities_in_configuration = configuration[self.configuration_name]
self._find_duplicates(project_or_group, entities_in_configuration)
entities_in_gitlab = self.list_method(project_or_group)
debug(f"{self.configuration_name} BEFORE: ^^^")
for entity_name, entity_config in entities_in_configuration.items():
entity_in_gitlab = self._is_in_gitlab(entity_config,
entities_in_gitlab)
if entity_in_gitlab:
if "delete" in entity_config and entity_config["delete"]:
self._validate_required_to_delete(project_or_group,
entity_name,
entity_config)
verbose(
f"Deleting {entity_name} of {self.configuration_name} in {project_or_group}"
)
self.delete_method(project_or_group, entity_in_gitlab)
elif self._needs_update(entity_in_gitlab, entity_config):
self._validate_required_to_create_or_update(
project_or_group, entity_name, entity_config)
if self.edit_method:
verbose(
f"Editing {entity_name} of {self.configuration_name} in {project_or_group}"
)
self.edit_method(project_or_group, entity_in_gitlab,
entity_config)
debug(f"{self.configuration_name} AFTER: ^^^")
else:
verbose(
f"Recreating {entity_name} of {self.configuration_name} in {project_or_group}"
)
self.delete_method(project_or_group, entity_in_gitlab)
self.add_method(project_or_group, entity_config)
debug(f"{self.configuration_name} AFTER: ^^^")
else:
verbose(
f"{entity_name} of {self.configuration_name} in {project_or_group} doesn't need an update."
)
else:
if "delete" in entity_config and entity_config["delete"]:
verbose(
f"{entity_name} of {self.configuration_name} in {project_or_group} already doesn't exist."
)
else:
self._validate_required_to_create_or_update(
project_or_group, entity_name, entity_config)
verbose(
f"Adding {entity_name} of {self.configuration_name} in {project_or_group}"
)
self.add_method(project_or_group, entity_config)
debug(f"{self.configuration_name} AFTER: ^^^")
def _process_configuration(self, project_and_group: str, configuration: dict):
for file in sorted(configuration["files"]):
debug("Processing file '%s'...", file)
if configuration.get("files|" + file + "|skip"):
debug("Skipping file '%s'", file)
continue
if configuration["files"][file]["branches"] == "all":
all_branches = self.gitlab.get_branches(project_and_group)
branches = sorted(all_branches)
elif configuration["files"][file]["branches"] == "protected":
protected_branches = self.gitlab.get_protected_branches(
project_and_group
)
branches = sorted(protected_branches)
else:
all_branches = self.gitlab.get_branches(project_and_group)
branches = []
for branch in configuration["files"][file]["branches"]:
if branch in all_branches:
branches.append(branch)
else:
message = f"! Branch '{branch}' not found, not processing file '{file}' in it"
if self.strict:
fatal(
message,
exit_code=EXIT_INVALID_INPUT,
)
else:
warning(message)
for branch in branches:
verbose(f"Processing file '{file}' in branch '{branch}'")
if configuration.get(
"files|" + file + "|content"
) and configuration.get("files|" + file + "|file"):
fatal(
f"File '{file}' in '{project_and_group}' has both `content` and `file` set - "
"use only one of these keys.",
exit_code=EXIT_INVALID_INPUT,
)
if configuration.get("files|" + file + "|delete"):
try:
self.gitlab.get_file(project_and_group, branch, file)
debug("Deleting file '%s' in branch '%s'", file, branch)
self.modify_file_dealing_with_branch_protection(
project_and_group,
branch,
file,
"delete",
configuration,
)
except NotFoundException:
debug(
"Not deleting file '%s' in branch '%s' (already doesn't exist)",
file,
branch,
)
else:
# change or create file
if configuration.get("files|" + file + "|content"):
new_content = configuration.get("files|" + file + "|content")
else:
path_in_config = Path(
configuration.get("files|" + file + "|file")
)
if path_in_config.is_absolute():
# TODO: does this work? we are reading the content twice in this case...
path = path_in_config.read_text()
else:
# relative paths are relative to config file location
path = Path(
os.path.join(
self.config.config_dir, str(path_in_config)
)
)
new_content = path.read_text()
if configuration.get("files|" + file + "|template", True):
new_content = self.get_file_content_as_template(
new_content,
project_and_group,
**configuration.get("files|" + file + "|jinja_env", dict()),
)
try:
current_content = self.gitlab.get_file(
project_and_group, branch, file
)
if current_content != new_content:
if configuration.get("files|" + file + "|overwrite"):
debug("Changing file '%s' in branch '%s'", file, branch)
self.modify_file_dealing_with_branch_protection(
project_and_group,
branch,
file,
"modify",
configuration,
new_content,
)
else:
debug(
"Not changing file '%s' in branch '%s' - overwrite flag not set.",
file,
branch,
)
else:
debug(
"Not changing file '%s' in branch '%s' - it's content is already"
" as provided)",
file,
branch,
)
except NotFoundException:
debug("Creating file '%s' in branch '%s'", file, branch)
self.modify_file_dealing_with_branch_protection(
project_and_group,
branch,
file,
"add",
configuration,
new_content,
)
if configuration.get("files|" + file + "|only_first_branch"):
verbose("Skipping other branches for this file, as configured.")
break
def _process_configuration(self, project_and_group: str,
configuration: dict):
approvals = configuration.get("merge_requests|approvals")
if approvals:
verbose(f"Setting approvals settings: {approvals}")
self.gitlab.post_approvals_settings(project_and_group, approvals)
approvers = configuration.get("merge_requests|approvers")
approver_groups = configuration.get("merge_requests|approver_groups")
remove_other_approval_rules = configuration.get(
"merge_requests|remove_other_approval_rules")
# checking if "is not None" allows configs with empty array to work
if (approvers is not None or approver_groups is not None and approvals
and "approvals_before_merge" in approvals):
verbose(f"Setting approvers...")
# in pre-12.3 API approvers (users and groups) were configured under the same endpoint as approvals settings
approvals_settings = self.gitlab.get_approvals_settings(
project_and_group)
if ("approvers" in approvals_settings
or "approver_groups" in approvals_settings):
# /approvers endpoint has been removed in 13.11.x GitLab version
if LooseVersion(self.gitlab.version) < LooseVersion("13.11"):
debug("Deleting legacy approvers setup")
self.gitlab.delete_legacy_approvers(project_and_group)
approval_rule_name = "Approvers (configured using GitLabForm)"
# is a rule already configured and just needs updating?
approval_rule_id = None
rules = self.gitlab.get_approvals_rules(project_and_group)
for rule in rules:
if rule["name"] == approval_rule_name:
approval_rule_id = rule["id"]
else:
if remove_other_approval_rules:
debug("Deleting extra approval rule '%s'" %
rule["name"])
self.gitlab.delete_approvals_rule(
project_and_group, rule["id"])
if not approvers:
approvers = []
if not approver_groups:
approver_groups = []
if approval_rule_id:
# the rule exists, needs an update
verbose(
f"Updating approvers rule to users {approvers} and groups {approver_groups}"
)
self.gitlab.update_approval_rule(
project_and_group,
approval_rule_id,
approval_rule_name,
approvals["approvals_before_merge"],
approvers,
approver_groups,
)
else:
# the rule does not exist yet, let's create it
verbose(
f"Creating approvers rule to users {approvers} and groups {approver_groups}"
)
self.gitlab.create_approval_rule(
project_and_group,
approval_rule_name,
approvals["approvals_before_merge"],
approvers,
approver_groups,
)
def _print_diff(self, project_or_project_and_group: str, entity_config):
verbose(f"Diffing for section '{self.configuration_name}' is not supported yet")