def __init__(self, config_path=None, config_string=None):
self.configuration = Configuration(config_path, config_string)
self.url = self.configuration.get("gitlab|url",
os.getenv("GITLAB_URL"))
self.token = self.configuration.get("gitlab|token",
os.getenv("GITLAB_TOKEN"))
self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True)
self.timeout = self.configuration.get("gitlab|timeout", 10)
self.session = requests.Session()
retries = Retry(total=3,
backoff_factor=0.25,
status_forcelist=[500, 502, 503, 504])
self.session.mount("http://", HTTPAdapter(max_retries=retries))
self.session.mount("https://", HTTPAdapter(max_retries=retries))
self.session.verify = self.ssl_verify
if not self.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
try:
version = self._make_requests_to_api("version")
cli_ui.debug(
f"Connected to GitLab version: {version['version']} ({version['revision']})"
)
self.version = version["version"]
except Exception as e:
raise TestRequestFailedException(e)
def configuration_with_multiple_levels_and_lists():
config_yaml = """
---
projects_and_groups:
"*":
merge_requests:
approvals:
approvals_before_merge: 3
approvers:
- common_approvers
"some_group/*":
merge_requests:
approvals:
reset_approvals_on_push: true
approvers:
- group_approvers
"some_group/my_project":
merge_requests:
approvals:
reset_approvals_on_push: false
disable_overriding_approvers_per_merge_request: true
approvers:
- project_approvers
"""
return Configuration(config_string=config_yaml)
def initialize_configuration_and_gitlab(self):
try:
if hasattr(self, 'config_string'):
gl = GitLab(config_string=self.config_string)
c = Configuration(config_string=self.config_string)
else:
gl = GitLab(self.config.strip())
c = Configuration(self.config.strip())
return gl, c
except ConfigFileNotFoundException as e:
logging.fatal('Aborting - config file not found at: %s', e)
sys.exit(1)
except TestRequestFailedException as e:
logging.fatal("Aborting - GitLab test request failed, details: '%s'", e)
sys.exit(2)
def configuration_for_skip_groups_skip_projects():
config_yaml = """
---
projects_and_groups:
"*":
group_settings:
one: two
project_settings:
three: four
skip_groups:
- group-skip
- group-skip-wildcard/*
- group-not-skip/subgroup-skip
- group-not-skip/subgroup-skip-wildcard/*
skip_projects:
- project-skip
- group-skip/project-skip
- group-skip-wildcard/*
- group-not-skip/project-skip
- group-not-skip/subgroup-skip-wildcard/*
"""
return Configuration(config_string=config_yaml)
def test__config_with_different_case_group():
group_name_with_varying_case = "GROUPnameWITHvaryingCASE"
config_yaml = f"""
projects_and_groups:
{group_name_with_varying_case}/*:
project_settings:
visibility: internal
"""
configuration = Configuration(config_string=config_yaml)
group_name_with_other_case = group_name_with_varying_case.lower()
effective_configuration = configuration.get_effective_config_for_group(
group_name_with_other_case)
assert effective_configuration["project_settings"] == {
"visibility": "internal"
}
def test__config_with_different_case_duplicate_skip_projects():
config_yaml = """
skip_projects:
- GroupNameWithVaryingCase/projectwithvaryingcase
- GroupNameWithVaryingCase/ProjectWithVaryingCase
"""
with pytest.raises(SystemExit):
Configuration(config_string=config_yaml)
def test__config_with_different_case_duplicate_skip_groups():
config_yaml = """
skip_groups:
- groupnamewithvaryingcase
- GROUPnameWITHvaryingCASE
"""
with pytest.raises(SystemExit):
Configuration(config_string=config_yaml)
def configuration_with_yes():
config_yaml = """
---
projects_and_groups:
some_group/*:
project_settings:
foo: yes # in YAML 1.1 this should be interpreted as boolean true
"""
return Configuration(config_string=config_yaml)
def test__config_with_different_case_duplicate_skip_groups():
config_yaml = """
skip_groups:
- groupnamewithvaryingcase
- GROUPnameWITHvaryingCASE
"""
with pytest.raises(SystemExit) as e:
Configuration(config_string=config_yaml)
assert e.value.code == EXIT_INVALID_INPUT
def test__config_with_different_case_duplicate_skip_projects():
config_yaml = """
skip_projects:
- GroupNameWithVaryingCase/projectwithvaryingcase
- GroupNameWithVaryingCase/ProjectWithVaryingCase
"""
with pytest.raises(SystemExit) as e:
Configuration(config_string=config_yaml)
assert e.value.code == EXIT_INVALID_INPUT
def test__config__with_access_level_names__branches_premium_syntax():
config_yaml = f"""
projects_and_groups:
foobar/*:
branches:
special:
protected: true
allowed_to_push:
- user: jsmith # you can use usernames...
- user: bdoe
- group: another-group # ...or group names (paths)...
allowed_to_merge:
- user_id: 15 # ...or user ids, if you know them...
- access_level: developer # ...or the whole access levels, like in the other syntax
- group_id: 456 # ...or group ids, if you know them...
allowed_to_unprotect:
- access_level: maintainer # ...or the whole access levels, like in the other syntax
"""
configuration = Configuration(config_string=config_yaml)
AccessLevelsTransformer.transform(configuration)
config_with_numbers = f"""
projects_and_groups:
foobar/*:
branches:
special:
protected: true
allowed_to_push:
- user: jsmith # you can use usernames...
- user: bdoe
- group: another-group # ...or group names (paths)...
allowed_to_merge:
- user_id: 15 # ...or user ids, if you know them...
- access_level: 30 # ...or the whole access levels, like in the other syntax
- group_id: 456 # ...or group ids, if you know them...
allowed_to_unprotect:
- access_level: 40 # ...or the whole access levels, like in the other syntax
"""
configuration_with_numbers = Configuration(config_string=config_with_numbers)
ddiff = DeepDiff(configuration.config, configuration_with_numbers.config)
assert not ddiff
def test__config_with_different_case_project():
group_and_project_name_with_varying_case = (
"GroupNameWithVaryingCase/projectwithvaryingcase")
config_yaml = f"""
projects_and_groups:
{group_and_project_name_with_varying_case}:
project_settings:
visibility: public
"""
configuration = Configuration(config_string=config_yaml)
group_and_project_name_with_other_case = (
group_and_project_name_with_varying_case.upper())
effective_configuration = configuration.get_effective_config_for_project(
group_and_project_name_with_other_case)
assert effective_configuration["project_settings"] == {
"visibility": "public"
}
def __init__(self, config_path=None, config_string=None):
self.configuration = Configuration(config_path, config_string)
self.url = self.configuration.get("gitlab|url", os.getenv("GITLAB_URL"))
self.token = self.configuration.get("gitlab|token", os.getenv("GITLAB_TOKEN"))
self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True)
self.timeout = self.configuration.get("gitlab|timeout", 10)
self.session = requests.Session()
retries = Retry(
total=3, backoff_factor=0.25, status_forcelist=[500, 502, 503, 504]
)
self.session.mount("http://", HTTPAdapter(max_retries=retries))
self.session.mount("https://", HTTPAdapter(max_retries=retries))
self.session.verify = self.ssl_verify
if not self.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.gitlabform_version = pkg_resources.get_distribution("gitlabform").version
self.requests_version = pkg_resources.get_distribution("requests").version
self.session.headers.update(
{
"private-token": self.token,
"authorization": f"Bearer {self.token}",
"user-agent": f"GitLabForm/{self.gitlabform_version} (python-requests/{self.requests_version})",
}
)
try:
version = self._make_requests_to_api("version")
verbose(
f"Connected to GitLab version: {version['version']} ({version['revision']})"
)
self.version = version["version"]
except Exception as e:
raise TestRequestFailedException(e)
def test__config_with_different_case_duplicate_projects():
config_yaml = """
projects_and_groups:
GroupNameWithVaryingCase/projectwithvaryingcase:
project_settings:
visibility: internal
GroupNameWithVaryingCase/ProjectWithVaryingCase:
project_settings:
visibility: public
"""
with pytest.raises(SystemExit):
Configuration(config_string=config_yaml)
def initialize_configuration_and_gitlab(self):
try:
gl = GitLab(self.args.config)
c = Configuration(self.args.config)
return gl, c
except ConfigFileNotFoundException as e:
logging.fatal('Aborting - config file not found at: %s', e)
sys.exit(1)
except TestRequestFailedException as e:
logging.fatal(
"Aborting - GitLab test request failed, details: '%s'", e)
sys.exit(2)
def test__config__with_access_level_names__group_ldap_links():
config_yaml = f"""
projects_and_groups:
foobar/*:
group_ldap_links:
# "provider" field should contain a value that you can find in the GitLab web UI,
# see https://github.com/gdubicki/gitlabform/issues/261
devops_are_maintainers:
provider: "AD"
cn: "devops"
group_access: maintainer
developers_are_developers:
provider: "AD"
filter: "(employeeType=developer)"
group_access: developer
"""
configuration = Configuration(config_string=config_yaml)
AccessLevelsTransformer.transform(configuration)
config_with_numbers = f"""
projects_and_groups:
foobar/*:
group_ldap_links:
# "provider" field should contain a value that you can find in the GitLab web UI,
# see https://github.com/gdubicki/gitlabform/issues/261
devops_are_maintainers:
provider: "AD"
cn: "devops"
group_access: 40
developers_are_developers:
provider: "AD"
filter: "(employeeType=developer)"
group_access: 30
"""
configuration_with_numbers = Configuration(config_string=config_with_numbers)
ddiff = DeepDiff(configuration.config, configuration_with_numbers.config)
assert not ddiff
def test__config_with_different_case_duplicate_groups():
config_yaml = """
projects_and_groups:
groupnamewithvaryingcase/*:
project_settings:
visibility: internal
GROUPnameWITHvaryingCASE/*: # different case than defined above
project_settings:
visibility: public
"""
with pytest.raises(SystemExit):
Configuration(config_string=config_yaml)
def configuration_with_subgroups_and_projects():
config_yaml = """
---
projects_and_groups:
some_group/*:
project_settings:
foo: bar
hooks:
a:
foo: bar
some_group/subgroup_level_1/*:
project_settings:
foo: bar2
hooks:
a:
foo: bar2
some_group/subgroup_level_1/subgroup_level_2/*:
project_settings:
foo: bar3
hooks:
a:
foo: bar3
some_group/some_project:
project_settings:
bar: something_else
hooks:
b:
bar: something_else
some_group/subgroup_level_1/some_project:
project_settings:
bar: something_else2
hooks:
b:
bar: something_else2
some_group/subgroup_level_1/subgroup_level_2/some_project:
project_settings:
bar: something_else3
hooks:
b:
bar: something_else3
"""
return Configuration(config_string=config_yaml)
def configuration_for_other_project():
config_yaml = """
---
projects_and_groups:
"some_group/*":
secret_variables:
first:
key: foo
value: bar
"some_group/my_project":
secret_variables:
second:
key: foo
value: bar
"""
return Configuration(config_string=config_yaml)
def configuration_with_subgroups():
config_yaml = """
---
projects_and_groups:
some_group/*:
group_members:
my-user:
access_level: 10
enforce_group_members: true
some_group/subgroup/*:
group_members:
my-user2:
access_level: 20
enforce_group_members: true
"""
return Configuration(config_string=config_yaml)
def configuration_with_only_group_and_project():
config_yaml = """
---
projects_and_groups:
some_group/*:
project_settings:
foo: bar
hooks:
a:
foo: bar
some_group/some_project:
project_settings:
bar: foo
hooks:
b:
bar: foo
"""
return Configuration(config_string=config_yaml)
def test__config__with_access_level_names__invalid_name():
config_yaml = f"""
projects_and_groups:
foobar/*:
branches:
special:
protected: true
allowed_to_push:
- user: jsmith # you can use usernames...
- user: bdoe
- group: another-group # ...or group names (paths)...
allowed_to_merge:
- user_id: 15 # ...or user ids, if you know them...
- access_level: developers # <-------------------------- this is invalid, it's plural
- group_id: 456 # ...or group ids, if you know them...
allowed_to_unprotect:
- access_level: maintainer # ...or the whole access levels, like in the other syntax
"""
configuration = Configuration(config_string=config_yaml)
with pytest.raises(SystemExit) as e:
AccessLevelsTransformer.transform(configuration)
assert e.value.code == EXIT_INVALID_INPUT
class GitLabCore:
def __init__(self, config_path=None, config_string=None):
self.configuration = Configuration(config_path, config_string)
self.url = self.configuration.get("gitlab|url",
os.getenv("GITLAB_URL"))
self.token = self.configuration.get("gitlab|token",
os.getenv("GITLAB_TOKEN"))
self.ssl_verify = self.configuration.get("gitlab|ssl_verify", True)
self.timeout = self.configuration.get("gitlab|timeout", 10)
self.session = requests.Session()
retries = Retry(total=3,
backoff_factor=0.25,
status_forcelist=[500, 502, 503, 504])
self.session.mount("http://", HTTPAdapter(max_retries=retries))
self.session.mount("https://", HTTPAdapter(max_retries=retries))
self.session.verify = self.ssl_verify
if not self.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
try:
version = self._make_requests_to_api("version")
cli_ui.debug(
f"Connected to GitLab version: {version['version']} ({version['revision']})"
)
self.version = version["version"]
except Exception as e:
raise TestRequestFailedException(e)
def get_configuration(self):
return self.configuration
def get_project(self, project_and_group_or_id):
return self._make_requests_to_api("projects/%s",
project_and_group_or_id)
def _get_user_id(self, username):
users = self._make_requests_to_api("users?username=%s", username,
"GET")
# this API endpoint is for lookup, not search, so 'username' has to be full and exact username
# also it's not possible to get more than 1 user as a result
if len(users) == 0:
raise NotFoundException(
"No users found when searching for username '%s'" % username)
return users[0]["id"]
def _get_user(self, user_id):
return self._make_requests_to_api("users/%s", str(user_id), "GET")
def _get_group_id(self, path):
group = self._make_requests_to_api("groups/%s", path, "GET")
# TODO: add tests for all that uses this and then stop converting these ints to strings here
return str(group["id"])
def _get_project_id(self, project_and_group):
# This is a NEW workaround for https://github.com/gitlabhq/gitlabhq/issues/8290
result = self.get_project(project_and_group)
return str(result["id"])
def _make_requests_to_api(
self,
path_as_format_string,
args=None,
method="GET",
data=None,
expected_codes=200,
paginated=False,
json=None,
):
"""
Makes a HTTP request (or requests) to the GitLab API endpoint. It takes case of making as many requests as
needed in case we are using a paginated endpoint. (See underlying method for authentication, retries,
timeout etc.)
:param path_as_format_string: path with parts to be replaced by values from `args` replaced by '%s'
(aka the old-style Python string formatting, see:
https://docs.python.org/2/library/stdtypes.html#string-formatting )
:param args: single element or a tuple of values to put under '%s's in `path_as_format_string`
:param method: uppercase string of a HTTP method name, like 'GET' or 'PUT'
:param data: dict with data to be 'PUT'ted or 'POST'ed
:param expected_codes: a single HTTP code (like: 200) or a list of accepted HTTP codes
- if the call to the API will return other code an exception will be thrown
:param paginated: if given API is paginated (see https://docs.gitlab.com/ee/api/#pagination )
:param json: alternatively to `dict` you can set this to a string that can be parsed as JSON that will
be used as data to be 'PUT'ted or 'POST'ed
:return: data returned by the endpoint, as a JSON object. If the API is paginated the it returns JSONs with
arrays of objects and then this method returns JSON with a single array that contains all of those
objects.
"""
if not paginated:
response = self._make_request_to_api(path_as_format_string, args,
method, data, expected_codes,
json)
return response.json()
else:
if "?" in path_as_format_string:
path_as_format_string += "&per_page=100"
else:
path_as_format_string += "?per_page=100"
first_response = self._make_request_to_api(path_as_format_string,
args, method, data,
expected_codes, json)
results = first_response.json()
# In newer versions of GitLab the 'X-Total-Pages' may not be available
# anymore, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43159
# so let's switch to thew newer style.
response = first_response
while True:
if ("x-next-page" in response.headers
and response.headers["x-next-page"]):
next_page = response.headers["x-next-page"]
response = self._make_request_to_api(
path_as_format_string + "&page=" + str(next_page),
args,
method,
data,
expected_codes,
json,
)
results += response.json()
else:
break
return results
def _make_request_to_api(self, path_as_format_string, args, method,
dict_data, expected_codes, json_data):
"""
Makes a single request to the GitLab API. Takes care of the authentication, basic error processing,
retries, timeout etc.
:param for the params description please see `_make_requests_to_api()`
:return: data returned by the endpoint, as a JSON object.
"""
expected_codes = self._listify(expected_codes)
if dict_data and json_data:
raise Exception(
"You need to pass the either as dict (dict_data) or JSON (json_data), not both!"
)
url = (self.url + "/api/v4/" +
self._format_with_url_encoding(path_as_format_string, args))
logging.debug(
"url = %s , method = %s , data = %s, json = %s",
url,
method,
json.dumps(dict_data, sort_keys=True),
json.dumps(json_data, sort_keys=True),
)
headers = {
"PRIVATE-TOKEN": self.token,
"Authorization": "Bearer " + self.token,
}
if dict_data:
response = self.session.request(method,
url,
headers=headers,
data=dict_data,
timeout=self.timeout)
elif json_data:
response = self.session.request(method,
url,
headers=headers,
json=json_data,
timeout=self.timeout)
else:
response = self.session.request(method,
url,
headers=headers,
timeout=self.timeout)
logging.debug("response code=%s" % response.status_code)
if response.status_code in expected_codes:
# if we accept error responses then they will likely not contain a JSON body
# so fake it to fix further calls to response.json()
if response.status_code == 204 or (400 <= response.status_code <=
499):
logging.debug("faking response body to be {}")
response.json = lambda: {}
else:
if response.status_code == 404:
raise NotFoundException("Resource path='%s' not found!" % url)
else:
raise UnexpectedResponseException(
"Request url='%s', method=%s, data='%s' failed - expected code(s) %s, got code %s & body: '%s'"
% (
url,
method,
dict_data,
str(expected_codes),
response.status_code,
response.content,
),
response.status_code,
)
logging.debug("response json=%s" %
json.dumps(response.json(), sort_keys=True))
return response
@staticmethod
def _format_with_url_encoding(format_string, single_arg_or_args_tuple):
# we want to URL-encode all the args, but not the path itself which looks like "/foo/%s/bar"
# because '/'s here are NOT to be URL-encoded
if not single_arg_or_args_tuple:
# there are no params, so the format_string is the URL
return format_string
else:
if type(single_arg_or_args_tuple) == tuple:
# URL-encode each arg in the tuple and return it as tuple too
url_encoded_args = ()
for arg in single_arg_or_args_tuple:
url_encoded_args += (parse.quote_plus(str(arg)), )
else:
# URL-encode single arg
url_encoded_args = parse.quote_plus(single_arg_or_args_tuple)
return format_string % url_encoded_args
@staticmethod
def _listify(expected_codes):
if isinstance(expected_codes, int):
return [expected_codes]
else:
return expected_codes
def test__config__with_access_level_names__branches():
config_yaml = f"""
projects_and_groups:
foobar/*:
branches:
develop:
protected: false
main:
protected: true
push_access_level: no access
merge_access_level: developer
unprotect_access_level: maintainer
code_owner_approval_required: true
branch_protected_from_changes:
protected: true
push_access_level: no access
merge_access_level: no access
unprotect_access_level: maintainer
"*":
branches:
'*-something':
protected: true
push_access_level: no access
merge_access_level: developer
unprotect_access_level: maintainer
allow_to_force_push:
protected: true
push_access_level: developer
merge_access_level: developer
unprotect_access_level: maintainer
allow_force_push: true
"""
configuration = Configuration(config_string=config_yaml)
AccessLevelsTransformer.transform(configuration)
config_with_numbers = f"""
projects_and_groups:
foobar/*:
branches:
develop:
protected: false
main:
protected: true
push_access_level: 0
merge_access_level: 30
unprotect_access_level: 40
code_owner_approval_required: true
branch_protected_from_changes:
protected: true
push_access_level: 0
merge_access_level: 0
unprotect_access_level: 40
"*":
branches:
'*-something':
protected: true
push_access_level: 0
merge_access_level: 30
unprotect_access_level: 40
allow_to_force_push:
protected: true
push_access_level: 30
merge_access_level: 30
unprotect_access_level: 40
allow_force_push: true
"""
configuration_with_numbers = Configuration(config_string=config_with_numbers)
ddiff = DeepDiff(configuration.config, configuration_with_numbers.config)
assert not ddiff
