def __init__(self, config_path=None, config_string=None):
self.configuration = Configuration(config_path, config_string)
self.url = self.configuration.get("gitlab|url",
os.getenv("GITLAB_URL"))
self.token = self.configuration.get("gitlab|token",
os.getenv("GITLAB_TOKEN"))
self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True)
self.timeout = self.configuration.get("gitlab|timeout", 10)
self.session = requests.Session()
retries = Retry(total=3,
backoff_factor=0.25,
status_forcelist=[500, 502, 503, 504])
self.session.mount("http://", HTTPAdapter(max_retries=retries))
self.session.mount("https://", HTTPAdapter(max_retries=retries))
self.session.verify = self.ssl_verify
if not self.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
try:
version = self._make_requests_to_api("version")
cli_ui.debug(
f"Connected to GitLab version: {version['version']} ({version['revision']})"
)
self.version = version["version"]
except Exception as e:
raise TestRequestFailedException(e)
def run_git_captured(working_path: Path,
*cmd: str,
check: bool = True) -> Tuple[int, str]:
"""Run git `cmd` in given `working_path`, capturing the output.
Return a tuple (returncode, output).
Raise GitCommandError if return code is non-zero and check is True.
"""
assert_working_path(working_path)
git_cmd = list(cmd)
git_cmd.insert(0, "git")
options: Dict[str, Any] = {}
options["stdout"] = subprocess.PIPE
options["stderr"] = subprocess.STDOUT
ui.debug(ui.lightgray, working_path, "$", ui.reset, *git_cmd)
process = subprocess.Popen(git_cmd, cwd=working_path, **options)
out, _ = process.communicate()
out = out.decode("utf-8")
if out.endswith("\n"):
out = out.strip("\n")
returncode = process.returncode
ui.debug(ui.lightgray, "[", returncode, "]", ui.reset, out)
if check and returncode != 0:
raise GitCommandError(working_path, cmd, output=out)
return returncode, out
def _process_configuration(self,
group: str,
configuration: dict,
do_apply: bool = True):
group_settings = configuration["group_settings"]
logging.debug("Group settings BEFORE: %s",
self.gitlab.get_group_settings(group))
cli_ui.debug(f"Setting group settings: {group_settings}")
self.gitlab.put_group_settings(group, group_settings)
logging.debug("Group settings AFTER: %s",
self.gitlab.get_group_settings(group))
def run(working_path: Path, *cmd: str, check: bool = True) -> None:
""" Run git `cmd` in given `working_path`
Raise GitCommandError if return code is non-zero and `check` is True.
"""
git_cmd = list(cmd)
git_cmd.insert(0, "git")
ui.debug(ui.lightgray, working_path, "$", ui.reset, *git_cmd)
returncode = subprocess.call(git_cmd, cwd=working_path)
if returncode != 0 and check:
raise CommandError(working_path, cmd)
def _process_configuration(self, project_and_group: str, configuration: dict):
logging.debug(
"Deploy keys BEFORE: %s", self.gitlab.get_deploy_keys(project_and_group)
)
for deploy_key in sorted(configuration["deploy_keys"]):
cli_ui.debug(f"Setting deploy key: {deploy_key}")
self.gitlab.post_deploy_key(
project_and_group, configuration["deploy_keys"][deploy_key]
)
logging.debug(
"Deploy keys AFTER: %s", self.gitlab.get_deploy_keys(project_and_group)
)
def _process_configuration(self,
project_and_group: str,
configuration: dict,
do_apply: bool = True):
project = configuration["project"]
if project:
if "archive" in project:
if project["archive"]:
cli_ui.debug("Archiving project...")
self.gitlab.archive(project_and_group)
else:
cli_ui.debug("Unarchiving project...")
self.gitlab.unarchive(project_and_group)
def _process_configuration(self, project_and_group: str,
configuration: dict):
project_settings = configuration["project_settings"]
logging.debug(
"Project settings BEFORE: %s",
self.gitlab.get_project_settings(project_and_group),
)
cli_ui.debug(f"Setting project settings: {project_settings}")
self.gitlab.put_project_settings(project_and_group, project_settings)
logging.debug(
"Project settings AFTER: %s",
self.gitlab.get_project_settings(project_and_group),
)
def _process_configuration(self, project_and_group: str, configuration: dict):
push_rules = configuration["project_push_rules"]
old_project_push_rules = self.gitlab.get_project_push_rules(project_and_group)
logging.debug("Project push rules settings BEFORE: %s", old_project_push_rules)
if old_project_push_rules:
cli_ui.debug(f"Updating project push rules: {push_rules}")
self.gitlab.put_project_push_rules(project_and_group, push_rules)
else:
cli_ui.debug(f"Creating project push rules: {push_rules}")
self.gitlab.post_project_push_rules(project_and_group, push_rules)
logging.debug(
"Project push rules AFTER: %s",
self.gitlab.get_project_push_rules(project_and_group),
)
def log_diff(
subject,
current_config,
config_to_apply,
only_changed=False,
hide_entries=None,
test=False,
):
# Compose values in list of `[key, from_config, from_server]``
changes = [[
k,
json.dumps(
current_config.get(k, "???") if type(current_config) ==
dict else "???"),
json.dumps(v),
] for k, v in config_to_apply.items()]
# Remove unchanged if needed
if only_changed:
changes = filter(lambda i: i[1] != i[2], changes)
# Hide secrets
if hide_entries:
changes = list(
map(
lambda i: [i[0], hide(i[1]), hide(i[2])]
if i[0] in hide_entries else i,
changes,
))
# calculate field size for nice formatting
max_key_len = str(max(map(lambda i: len(i[0]), changes)))
max_val_1 = str(max(map(lambda i: len(i[1]), changes)))
max_val_2 = str(max(map(lambda i: len(i[2]), changes)))
# generate placeholders for output pattern: ` value: before => after `
pattern = ("{:>" + max_key_len + "}: {:<" + max_val_1 + "} => {:<" +
max_val_2 + "}")
# create string
text = "{subject}:\n{diff}".format(subject=subject,
diff="\n".join(
starmap(pattern.format,
changes)))
if test:
return text
else:
cli_ui.debug(text)
def _process_configuration(self, project_and_group: str, configuration: dict):
if (
self.gitlab.get_project_settings(project_and_group)["builds_access_level"]
== "disabled"
):
cli_ui.warning(
"Builds disabled in this project so I can't set secret variables here."
)
return
logging.debug(
"Secret variables BEFORE: %s",
self.gitlab.get_secret_variables(project_and_group),
)
for secret_variable in sorted(configuration["secret_variables"]):
if "delete" in configuration["secret_variables"][secret_variable]:
key = configuration["secret_variables"][secret_variable]["key"]
if configuration["secret_variables"][secret_variable]["delete"]:
cli_ui.debug(
f"Deleting {secret_variable}: {key} in project {project_and_group}"
)
try:
self.gitlab.delete_secret_variable(project_and_group, key)
except:
logging.warn(
f"Could not delete variable {key} in group {project_and_group}"
)
continue
cli_ui.debug(f"Setting secret variable: {secret_variable}")
try:
self.gitlab.put_secret_variable(
project_and_group,
configuration["secret_variables"][secret_variable],
)
except NotFoundException:
self.gitlab.post_secret_variable(
project_and_group,
configuration["secret_variables"][secret_variable],
)
logging.debug(
"Secret variables AFTER: %s",
self.gitlab.get_secret_variables(project_and_group),
)
def process(
self,
project_or_project_and_group: str,
configuration: dict,
dry_run: bool,
output_file: Optional[TextIO],
):
if self.__configuration_name in configuration:
if configuration.get(f"{self.__configuration_name}|skip"):
cli_ui.debug(
f"Skipping {self.__configuration_name} - explicitly configured to do so."
)
return
elif (configuration.get("project|archive")
and self.__configuration_name != "project"):
cli_ui.debug(
f"Skipping {self.__configuration_name} - it is configured to be archived."
)
return
if dry_run:
cli_ui.debug(
f"Processing {self.__configuration_name} in dry-run mode.")
self._print_diff(
project_or_project_and_group,
configuration.get(self.__configuration_name),
)
else:
cli_ui.debug(f"Processing {self.__configuration_name}")
self._process_configuration(project_or_project_and_group,
configuration)
if output_file:
cli_ui.debug(
f"Writing effective configuration for {self.__configuration_name} to the output file."
)
self._write_to_file(
configuration.get(self.__configuration_name),
output_file,
)
else:
logging.debug("Skipping %s - not in config." %
self.__configuration_name)
def _process_configuration(self,
group: str,
configuration: dict,
do_apply: bool = True):
logging.debug(
"Group secret variables BEFORE: %s",
self.gitlab.get_group_secret_variables(group),
)
for secret_variable in sorted(configuration["group_secret_variables"]):
if "delete" in configuration["group_secret_variables"][
secret_variable]:
key = configuration["group_secret_variables"][secret_variable][
"key"]
if configuration["group_secret_variables"][secret_variable][
"delete"]:
cli_ui.debug(
f"Deleting {secret_variable}: {key} in group {group}")
try:
self.gitlab.delete_group_secret_variable(group, key)
except:
cli_ui.info(
f"Could not delete variable {key} in group {group}"
)
continue
cli_ui.debug(f"Setting group secret variable: {secret_variable}")
try:
self.gitlab.put_group_secret_variable(
group,
configuration["group_secret_variables"][secret_variable])
except NotFoundException:
self.gitlab.post_group_secret_variable(
group,
configuration["group_secret_variables"][secret_variable])
logging.debug(
"Groups secret variables AFTER: %s",
self.gitlab.get_group_secret_variables(group),
)
def _process_configuration(self, project_and_group: str,
configuration: dict):
for service in sorted(configuration["services"]):
if configuration.get("services|" + service + "|delete"):
cli_ui.debug(f"Deleting service: {service}")
self.gitlab.delete_service(project_and_group, service)
else:
if ("recreate" in configuration["services"][service]
and configuration["services"][service]["recreate"]):
# support from this configuration key has been added in v1.13.4
# we will remove it here to avoid passing it to the GitLab API
cli_ui.warning(
f"Ignoring deprecated 'recreate' field in the '{service}' service config. "
"Please remove it from the config file permanently as this workaround is not "
"needed anymore.")
del configuration["services"][service]["recreate"]
cli_ui.debug(f"Setting service: {service}")
self.gitlab.set_service(project_and_group, service,
configuration["services"][service])
def main(self):
if self.project_or_group == "ALL":
cli_ui.info(">>> Processing ALL groups and projects")
elif self.project_or_group == "ALL_DEFINED":
cli_ui.info(
">>> Processing ALL groups and projects defined in config")
groups = self.get_groups(self.project_or_group)
projects = self.get_projects(self.project_or_group, groups)
if len(groups) == 0 and len(projects) == 0:
cli_ui.error(
f"Entity {self.project_or_group} cannot be found in GitLab!")
sys.exit(EXIT_INVALID_INPUT)
else:
cli_ui.debug(f"groups: {groups}")
cli_ui.debug(f"projects: {projects}")
cli_ui.info_1(f"# of groups to process: {len(groups)}")
cli_ui.info_1(f"# of projects to process: {len(projects)}")
self.process_all(projects, groups)
def _print_diff(self, project_and_group: str, configuration):
try:
current_secret_variables = self.gitlab.get_secret_variables(
project_and_group
)
for secret_variable in current_secret_variables:
secret_variable["value"] = hide(secret_variable["value"])
cli_ui.debug(f"Secret variables for {project_and_group} in GitLab:")
cli_ui.debug(
textwrap.indent(
yaml.dump(current_secret_variables, default_flow_style=False),
" ",
)
)
except:
cli_ui.debug(
f"Secret variables for {project_and_group} in GitLab cannot be checked."
)
cli_ui.debug(f"Secret variables in {project_and_group} in configuration:")
configured_secret_variables = copy.deepcopy(configuration)
for key in configured_secret_variables.keys():
configured_secret_variables[key]["value"] = hide(
configured_secret_variables[key]["value"]
)
cli_ui.debug(
textwrap.indent(
yaml.dump(configured_secret_variables, default_flow_style=False),
" ",
)
)
def _process_configuration(self, project_and_group: str,
configuration: dict):
for file in sorted(configuration["files"]):
logging.debug("Processing file '%s'...", file)
if configuration.get("files|" + file + "|skip"):
logging.debug("Skipping file '%s'", file)
continue
all_branches = self.gitlab.get_branches(project_and_group)
if configuration["files"][file]["branches"] == "all":
branches = sorted(all_branches)
elif configuration["files"][file]["branches"] == "protected":
protected_branches = self.gitlab.get_protected_branches(
project_and_group)
branches = sorted(protected_branches)
else:
branches = []
for branch in configuration["files"][file]["branches"]:
if branch in all_branches:
branches.append(branch)
else:
message = f"! Branch '{branch}' not found, not processing file '{file}' in it"
if self.strict:
cli_ui.error(message)
sys.exit(EXIT_INVALID_INPUT)
else:
cli_ui.warning(message)
for branch in branches:
cli_ui.debug(f"Processing file '{file}' in branch '{branch}'")
# unprotect protected branch temporarily for operations below
if configuration.get("branches|" + branch + "|protected"):
logging.debug(
"> Temporarily unprotecting the branch for managing files in it..."
)
self.gitlab.unprotect_branch(project_and_group, branch)
if configuration.get("files|" + file + "|delete"):
try:
self.gitlab.get_file(project_and_group, branch, file)
logging.debug("Deleting file '%s' in branch '%s'",
file, branch)
self.gitlab.delete_file(
project_and_group,
branch,
file,
self.get_commit_message_for_file_change(
"delete",
configuration.get("files|" + file +
"|skip_ci"),
),
)
except NotFoundException:
logging.debug(
"Not deleting file '%s' in branch '%s' (already doesn't exist)",
file,
branch,
)
else:
# change or create file
if configuration.get("files|" + file +
"|content") and configuration.get(
"files|" + file + "|file"):
cli_ui.error(
f"File '{file}' in '{project_and_group}' has both `content` and `file` set - "
"use only one of these keys.")
sys.exit(EXIT_INVALID_INPUT)
elif configuration.get("files|" + file + "|content"):
new_content = configuration.get("files|" + file +
"|content")
else:
path_in_config = Path(
configuration.get("files|" + file + "|file"))
if path_in_config.is_absolute():
path = path_in_config.read_text()
else:
# relative paths are relative to config file location
path = Path(
os.path.join(self.config.config_dir,
str(path_in_config)))
new_content = path.read_text()
if configuration.get("files|" + file + "|template", True):
new_content = self.get_file_content_as_template(
new_content,
project_and_group,
**configuration.get("files|" + file + "|jinja_env",
dict()),
)
try:
current_content = self.gitlab.get_file(
project_and_group, branch, file)
if current_content != new_content:
if configuration.get("files|" + file +
"|overwrite"):
logging.debug(
"Changing file '%s' in branch '%s'", file,
branch)
self.gitlab.set_file(
project_and_group,
branch,
file,
new_content,
self.get_commit_message_for_file_change(
"change",
configuration.get("files|" + file +
"|skip_ci"),
),
)
else:
logging.debug(
"Not changing file '%s' in branch '%s' "
"(overwrite flag not set)",
file,
branch,
)
else:
logging.debug(
"Not changing file '%s' in branch '%s' (it's content is already"
" as provided)",
file,
branch,
)
except NotFoundException:
logging.debug("Creating file '%s' in branch '%s'",
file, branch)
self.gitlab.add_file(
project_and_group,
branch,
file,
new_content,
self.get_commit_message_for_file_change(
"add",
configuration.get("files|" + file +
"|skip_ci")),
)
# protect branch back after above operations
if configuration.get("branches|" + branch + "|protected"):
logging.debug("> Protecting the branch again.")
self.branch_protector.protect_branch(
project_and_group, configuration, branch)
if configuration.get("files|" + file + "|only_first_branch"):
cli_ui.debug(
"Skipping other branches for this file, as configured."
)
break
def _print_diff(self, project_or_project_and_group: str,
configuration_to_process):
cli_ui.debug(
f"Diffing for {self.__configuration_name} section is not supported yet"
)
def _process_configuration(self, project_and_group: str, configuration: dict):
approvals = configuration.get("merge_requests|approvals")
if approvals:
cli_ui.debug(f"Setting approvals settings: {approvals}")
self.gitlab.post_approvals_settings(project_and_group, approvals)
approvers = configuration.get("merge_requests|approvers")
approver_groups = configuration.get("merge_requests|approver_groups")
remove_other_approval_rules = configuration.get(
"merge_requests|remove_other_approval_rules"
)
# checking if "is not None" allows configs with empty array to work
if (
approvers is not None
or approver_groups is not None
and approvals
and "approvals_before_merge" in approvals
):
# in pre-12.3 API approvers (users and groups) were configured under the same endpoint as approvals settings
approvals_settings = self.gitlab.get_approvals_settings(project_and_group)
if (
"approvers" in approvals_settings
or "approver_groups" in approvals_settings
):
# /approvers endpoint has been removed in 13.11.x GitLab version
if LooseVersion(self.gitlab.version) < LooseVersion("13.11"):
logging.debug("Deleting legacy approvers setup")
self.gitlab.delete_legacy_approvers(project_and_group)
approval_rule_name = "Approvers (configured using GitLabForm)"
# is a rule already configured and just needs updating?
approval_rule_id = None
rules = self.gitlab.get_approvals_rules(project_and_group)
for rule in rules:
if rule["name"] == approval_rule_name:
approval_rule_id = rule["id"]
else:
if remove_other_approval_rules:
logging.debug(
"Deleting extra approval rule '%s'" % rule["name"]
)
self.gitlab.delete_approvals_rule(project_and_group, rule["id"])
if not approvers:
approvers = []
if not approver_groups:
approver_groups = []
if approval_rule_id:
# the rule exists, needs an update
cli_ui.debug(
f"Updating approvers rule to users {approvers} and groups {approver_groups}"
)
self.gitlab.update_approval_rule(
project_and_group,
approval_rule_id,
approval_rule_name,
approvals["approvals_before_merge"],
approvers,
approver_groups,
)
else:
# the rule does not exist yet, let's create it
cli_ui.debug(
f"Creating approvers rule to users {approvers} and groups {approver_groups}"
)
self.gitlab.create_approval_rule(
project_and_group,
approval_rule_name,
approvals["approvals_before_merge"],
approvers,
approver_groups,
)
def __init__(self, config_path=None, config_string=None):
if config_path and config_string:
cli_ui.fatal(
"Please initialize with either config_path or config_string, not both."
)
sys.exit(EXIT_INVALID_INPUT)
try:
if config_string:
cli_ui.debug("Reading config from provided string.")
self.config = yaml.safe_load(textwrap.dedent(config_string))
self.config_dir = "."
else: # maybe config_path
if "APP_HOME" in os.environ:
# using this env var should be considered unofficial, we need this temporarily
# for backwards compatibility. support for it may be removed without notice, do not use it!
config_path = os.path.join(os.environ["APP_HOME"], "config.yml")
elif not config_path:
# this case is only meant for using gitlabform as a library
config_path = os.path.join(
str(Path.home()), ".gitlabform", "config.yml"
)
elif config_path in [os.path.join(".", "config.yml"), "config.yml"]:
# provided points to config.yml in the app current working dir
config_path = os.path.join(os.getcwd(), "config.yml")
cli_ui.debug(f"Reading config from file: {config_path}")
with open(config_path, "r") as ymlfile:
self.config = yaml.safe_load(ymlfile)
logging.debug("Config parsed successfully as YAML.")
# we need config path for accessing files for relative paths
self.config_dir = os.path.dirname(config_path)
if self.config.get("example_config"):
cli_ui.fatal(
"Example config detected, aborting.\n"
"Haven't you forgotten to use `-c <config_file>` parameter?\n"
"If you created your config based on the example config.yml,"
" then please remove 'example_config' key."
)
sys.exit(EXIT_INVALID_INPUT)
if self.config.get("config_version", 1) != 2:
cli_ui.fatal(
"This version of GitLabForm requires 'config_version: 2' entry in the config.\n"
"This ensures that when the application behavior changes in a backward incompatible way,"
" you won't apply unexpected configuration to your GitLab instance.\n"
"Please read the upgrading guide here: https://bit.ly/3ub1g5C\n"
)
sys.exit(EXIT_INVALID_INPUT)
try:
self.config.get("projects_and_groups")
except KeyNotFoundException:
cli_ui.fatal("'projects_and_groups' key in the config is required.")
sys.exit(EXIT_INVALID_INPUT)
except (FileNotFoundError, IOError):
raise ConfigFileNotFoundException(config_path)
except Exception:
if config_path:
raise ConfigInvalidException(config_path)
else:
raise ConfigInvalidException(config_string)
