def test_match_url_regex():
rules = ({
'schemes': ['bad_scheme'],
}, {
'schemes': ['https'],
'netlocs': ['bad_netloc'],
}, {
'schemes': ['https'],
'netlocs': ['hg.mozilla.org'],
'path_regexes': [
"^bad_regex$",
],
}, {
'schemes': ['https'],
'netlocs': ['hg.mozilla.org'],
'path_regexes': [
"^.*$",
"^/(?P<path>mozilla-(central|unified))(/|$)",
],
})
def cb(m):
import logging
log = logging.getLogger()
log.error(m)
path_info = m.groupdict()
if 'path' in path_info:
return path_info['path']
assert utils.match_url_regex(rules,
"https://hg.mozilla.org/mozilla-central",
cb) == "mozilla-central"
assert utils.match_url_regex(
(), "https://hg.mozilla.org/mozilla-central", cb) is None
def test_match_url_regex():
rules = ({
'schemes': ['bad_scheme'],
}, {
'schemes': ['https'],
'netlocs': ['bad_netloc'],
}, {
'schemes': ['https'],
'netlocs': ['hg.mozilla.org'],
'path_regexes': [
"^bad_regex$",
],
}, {
'schemes': ['https'],
'netlocs': ['hg.mozilla.org'],
'path_regexes': [
"^.*$",
"^/(?P<path>mozilla-(central|unified))(/|$)",
],
})
def cb(m):
import logging
log = logging.getLogger()
log.error(m)
path_info = m.groupdict()
if 'path' in path_info:
return path_info['path']
assert utils.match_url_regex(rules, "https://hg.mozilla.org/mozilla-central", cb) == "mozilla-central"
assert utils.match_url_regex((), "https://hg.mozilla.org/mozilla-central", cb) is None
def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
"""Ensure a URL fits in given scheme, netloc, and path restrictions.
If we fail any checks, raise a ScriptWorkerTaskException with
``malformed-payload``.
Args:
valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``,
and ``path_regexes``.
valid_artifact_task_ids (list): the list of valid task IDs to download from.
url (str): the url of the artifact.
Returns:
str: the ``filepath`` of the path regex.
Raises:
ScriptWorkerTaskException: on failure to validate.
"""
def callback(match):
path_info = match.groupdict()
# make sure we're pointing at a valid task ID
if 'taskId' in path_info and \
path_info['taskId'] not in valid_artifact_task_ids:
return
if 'filepath' not in path_info:
return
return path_info['filepath']
filepath = match_url_regex(valid_artifact_rules, url, callback)
if filepath is None:
raise ScriptWorkerTaskException(
"Can't validate url {}".format(url),
exit_code=STATUSES['malformed-payload'])
return unquote(filepath).lstrip('/')
def test_match_url_regex():
rules = (
{"schemes": ["bad_scheme"]},
{"schemes": ["https"], "netlocs": ["bad_netloc"]},
{"schemes": ["https"], "netlocs": ["hg.mozilla.org"], "path_regexes": ["^bad_regex$"]},
{"schemes": ["https"], "netlocs": ["hg.mozilla.org"], "path_regexes": ["^.*$", "^/(?P<path>mozilla-(central|unified))(/|$)"]},
)
def cb(m):
import logging
log = logging.getLogger()
log.error(m)
path_info = m.groupdict()
if "path" in path_info:
return path_info["path"]
assert utils.match_url_regex(rules, "https://hg.mozilla.org/mozilla-central", cb) == "mozilla-central"
assert utils.match_url_regex((), "https://hg.mozilla.org/mozilla-central", cb) is None
def get_and_check_project(valid_vcs_rules, source_url):
"""Given vcs rules and a source_url, return the project.
The project is in the path, but is the repo name.
`releases/mozilla-beta` is the path; `mozilla-beta` is the project.
Args:
valid_vcs_rules (tuple of frozendicts): the valid vcs rules, per
``match_url_regex``.
source_url (str): the source url to find the project for.
Raises:
RuntimeError: on failure to find the project.
Returns:
str: the project.
"""
project_path = match_url_regex(valid_vcs_rules, source_url, match_url_path_callback)
if project_path is None:
raise ValueError("Unknown repo for source url {}!".format(source_url))
project = project_path.split('/')[-1]
return project
def get_and_check_project(valid_vcs_rules, source_url):
"""Given vcs rules and a source_url, return the project.
The project is in the path, but is the repo name.
`releases/mozilla-beta` is the path; `mozilla-beta` is the project.
Args:
valid_vcs_rules (tuple of frozendicts): the valid vcs rules, per
``match_url_regex``.
source_url (str): the source url to find the project for.
Raises:
RuntimeError: on failure to find the project.
Returns:
str: the project.
"""
project_path = match_url_regex(valid_vcs_rules, source_url,
match_url_path_callback)
if project_path is None:
raise ValueError("Unknown repo for source url {}!".format(source_url))
project = project_path.split('/')[-1]
return project
def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
"""Ensure a URL fits in given scheme, netloc, and path restrictions.
If we fail any checks, raise a ScriptWorkerTaskException with
``malformed-payload``.
Args:
valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``,
and ``path_regexes``.
valid_artifact_task_ids (list): the list of valid task IDs to download from.
url (str): the url of the artifact.
Returns:
str: the ``filepath`` of the path regex.
Raises:
ScriptWorkerTaskException: on failure to validate.
"""
def callback(match):
path_info = match.groupdict()
# make sure we're pointing at a valid task ID
if 'taskId' in path_info and \
path_info['taskId'] not in valid_artifact_task_ids:
return
if 'filepath' not in path_info:
return
return path_info['filepath']
filepath = match_url_regex(valid_artifact_rules, url, callback)
if filepath is None:
raise ScriptWorkerTaskException(
"Can't validate url {}".format(url),
exit_code=STATUSES['malformed-payload']
)
return unquote(filepath).lstrip('/')
async def trace_back_to_firefox_tree(chain):
"""Trace the chain back to the firefox tree.
task.metadata.source: "https://hg.mozilla.org/projects/date//file/a80373508881bfbff67a2a49297c328ff8052572/taskcluster/ci/build"
task.payload.env.GECKO_HEAD_REPOSITORY "https://hg.mozilla.org/projects/date/"
Args:
chain (ChainOfTrust): the chain we're operating on
Raises:
CoTError: on error.
"""
errors = []
repos = {}
restricted_privs = None
rules = {}
cot_product = chain.context.config['cot_product']
for my_key, config_key in {
'scopes': 'cot_restricted_scopes',
'trees': 'cot_restricted_trees'
}.items():
rules[my_key] = chain.context.config[config_key].get(cot_product)
if not isinstance(rules[my_key], (dict, frozendict)):
raise_on_errors([
"{} invalid for {}: {}!".format(config_key, cot_product,
rules[my_key])
])
def callback(match):
path_info = match.groupdict()
return path_info['path']
# a repo_path of None means we have no restricted privs.
# a string repo_path may mean we have higher privs
for obj in [chain] + chain.links:
source_url = get_firefox_source_url(obj)
repo_path = match_url_regex(chain.context.config['valid_vcs_rules'],
source_url, callback)
repos[obj] = repo_path
# check for restricted scopes.
my_repo = repos[chain]
for scope in chain.task['scopes']:
if scope in rules['scopes']:
log.info("Found privileged scope {}".format(scope))
restricted_privs = True
level = rules['scopes'][scope]
if my_repo not in rules['trees'][level]:
errors.append(
"{} {}: repo {} not allowlisted for scope {}!".format(
chain.name, chain.task_id, my_repo, scope))
# verify all tasks w/ same decision_task_id have the same source repo.
if len(set(repos.values())) > 1:
for obj, repo in repos.items():
if obj.decision_task_id == chain.decision_task_id:
if repo != my_repo:
errors.append(
"{} {} repo {} doesn't match my repo {}!".format(
obj.name, obj.task_id, repo, my_repo))
# if we have restricted privs, the non-sibling tasks must at least be in
# a known repo.
# (Not currently requiring that all tasks have the same privilege level,
# in case a docker-image build is run on mozilla-central and that image
# is used for a release-priv task, for example.)
elif restricted_privs and repo is None:
errors.append(
"{} {} has no privileged repo on an restricted privilege scope!"
.format(obj.name, obj.task_id))
# Disallow restricted privs on is_try. This may be a redundant check.
if restricted_privs and chain.is_try():
errors.append(
"{} {} has restricted privilege scope, and is_try()!".format(
chain.name, chain.task_id))
raise_on_errors(errors)
