def test_match_url_regex(): rules = ({ 'schemes': ['bad_scheme'], }, { 'schemes': ['https'], 'netlocs': ['bad_netloc'], }, { 'schemes': ['https'], 'netlocs': ['hg.mozilla.org'], 'path_regexes': [ "^bad_regex$", ], }, { 'schemes': ['https'], 'netlocs': ['hg.mozilla.org'], 'path_regexes': [ "^.*$", "^/(?P<path>mozilla-(central|unified))(/|$)", ], }) def cb(m): import logging log = logging.getLogger() log.error(m) path_info = m.groupdict() if 'path' in path_info: return path_info['path'] assert utils.match_url_regex(rules, "https://hg.mozilla.org/mozilla-central", cb) == "mozilla-central" assert utils.match_url_regex( (), "https://hg.mozilla.org/mozilla-central", cb) is None
def test_match_url_regex(): rules = ({ 'schemes': ['bad_scheme'], }, { 'schemes': ['https'], 'netlocs': ['bad_netloc'], }, { 'schemes': ['https'], 'netlocs': ['hg.mozilla.org'], 'path_regexes': [ "^bad_regex$", ], }, { 'schemes': ['https'], 'netlocs': ['hg.mozilla.org'], 'path_regexes': [ "^.*$", "^/(?P<path>mozilla-(central|unified))(/|$)", ], }) def cb(m): import logging log = logging.getLogger() log.error(m) path_info = m.groupdict() if 'path' in path_info: return path_info['path'] assert utils.match_url_regex(rules, "https://hg.mozilla.org/mozilla-central", cb) == "mozilla-central" assert utils.match_url_regex((), "https://hg.mozilla.org/mozilla-central", cb) is None
def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url): """Ensure a URL fits in given scheme, netloc, and path restrictions. If we fail any checks, raise a ScriptWorkerTaskException with ``malformed-payload``. Args: valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``, and ``path_regexes``. valid_artifact_task_ids (list): the list of valid task IDs to download from. url (str): the url of the artifact. Returns: str: the ``filepath`` of the path regex. Raises: ScriptWorkerTaskException: on failure to validate. """ def callback(match): path_info = match.groupdict() # make sure we're pointing at a valid task ID if 'taskId' in path_info and \ path_info['taskId'] not in valid_artifact_task_ids: return if 'filepath' not in path_info: return return path_info['filepath'] filepath = match_url_regex(valid_artifact_rules, url, callback) if filepath is None: raise ScriptWorkerTaskException( "Can't validate url {}".format(url), exit_code=STATUSES['malformed-payload']) return unquote(filepath).lstrip('/')
def test_match_url_regex(): rules = ( {"schemes": ["bad_scheme"]}, {"schemes": ["https"], "netlocs": ["bad_netloc"]}, {"schemes": ["https"], "netlocs": ["hg.mozilla.org"], "path_regexes": ["^bad_regex$"]}, {"schemes": ["https"], "netlocs": ["hg.mozilla.org"], "path_regexes": ["^.*$", "^/(?P<path>mozilla-(central|unified))(/|$)"]}, ) def cb(m): import logging log = logging.getLogger() log.error(m) path_info = m.groupdict() if "path" in path_info: return path_info["path"] assert utils.match_url_regex(rules, "https://hg.mozilla.org/mozilla-central", cb) == "mozilla-central" assert utils.match_url_regex((), "https://hg.mozilla.org/mozilla-central", cb) is None
def get_and_check_project(valid_vcs_rules, source_url): """Given vcs rules and a source_url, return the project. The project is in the path, but is the repo name. `releases/mozilla-beta` is the path; `mozilla-beta` is the project. Args: valid_vcs_rules (tuple of frozendicts): the valid vcs rules, per ``match_url_regex``. source_url (str): the source url to find the project for. Raises: RuntimeError: on failure to find the project. Returns: str: the project. """ project_path = match_url_regex(valid_vcs_rules, source_url, match_url_path_callback) if project_path is None: raise ValueError("Unknown repo for source url {}!".format(source_url)) project = project_path.split('/')[-1] return project
def get_and_check_project(valid_vcs_rules, source_url): """Given vcs rules and a source_url, return the project. The project is in the path, but is the repo name. `releases/mozilla-beta` is the path; `mozilla-beta` is the project. Args: valid_vcs_rules (tuple of frozendicts): the valid vcs rules, per ``match_url_regex``. source_url (str): the source url to find the project for. Raises: RuntimeError: on failure to find the project. Returns: str: the project. """ project_path = match_url_regex(valid_vcs_rules, source_url, match_url_path_callback) if project_path is None: raise ValueError("Unknown repo for source url {}!".format(source_url)) project = project_path.split('/')[-1] return project
def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url): """Ensure a URL fits in given scheme, netloc, and path restrictions. If we fail any checks, raise a ScriptWorkerTaskException with ``malformed-payload``. Args: valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``, and ``path_regexes``. valid_artifact_task_ids (list): the list of valid task IDs to download from. url (str): the url of the artifact. Returns: str: the ``filepath`` of the path regex. Raises: ScriptWorkerTaskException: on failure to validate. """ def callback(match): path_info = match.groupdict() # make sure we're pointing at a valid task ID if 'taskId' in path_info and \ path_info['taskId'] not in valid_artifact_task_ids: return if 'filepath' not in path_info: return return path_info['filepath'] filepath = match_url_regex(valid_artifact_rules, url, callback) if filepath is None: raise ScriptWorkerTaskException( "Can't validate url {}".format(url), exit_code=STATUSES['malformed-payload'] ) return unquote(filepath).lstrip('/')
async def trace_back_to_firefox_tree(chain): """Trace the chain back to the firefox tree. task.metadata.source: "https://hg.mozilla.org/projects/date//file/a80373508881bfbff67a2a49297c328ff8052572/taskcluster/ci/build" task.payload.env.GECKO_HEAD_REPOSITORY "https://hg.mozilla.org/projects/date/" Args: chain (ChainOfTrust): the chain we're operating on Raises: CoTError: on error. """ errors = [] repos = {} restricted_privs = None rules = {} cot_product = chain.context.config['cot_product'] for my_key, config_key in { 'scopes': 'cot_restricted_scopes', 'trees': 'cot_restricted_trees' }.items(): rules[my_key] = chain.context.config[config_key].get(cot_product) if not isinstance(rules[my_key], (dict, frozendict)): raise_on_errors([ "{} invalid for {}: {}!".format(config_key, cot_product, rules[my_key]) ]) def callback(match): path_info = match.groupdict() return path_info['path'] # a repo_path of None means we have no restricted privs. # a string repo_path may mean we have higher privs for obj in [chain] + chain.links: source_url = get_firefox_source_url(obj) repo_path = match_url_regex(chain.context.config['valid_vcs_rules'], source_url, callback) repos[obj] = repo_path # check for restricted scopes. my_repo = repos[chain] for scope in chain.task['scopes']: if scope in rules['scopes']: log.info("Found privileged scope {}".format(scope)) restricted_privs = True level = rules['scopes'][scope] if my_repo not in rules['trees'][level]: errors.append( "{} {}: repo {} not allowlisted for scope {}!".format( chain.name, chain.task_id, my_repo, scope)) # verify all tasks w/ same decision_task_id have the same source repo. if len(set(repos.values())) > 1: for obj, repo in repos.items(): if obj.decision_task_id == chain.decision_task_id: if repo != my_repo: errors.append( "{} {} repo {} doesn't match my repo {}!".format( obj.name, obj.task_id, repo, my_repo)) # if we have restricted privs, the non-sibling tasks must at least be in # a known repo. # (Not currently requiring that all tasks have the same privilege level, # in case a docker-image build is run on mozilla-central and that image # is used for a release-priv task, for example.) elif restricted_privs and repo is None: errors.append( "{} {} has no privileged repo on an restricted privilege scope!" .format(obj.name, obj.task_id)) # Disallow restricted privs on is_try. This may be a redundant check. if restricted_privs and chain.is_try(): errors.append( "{} {} has restricted privilege scope, and is_try()!".format( chain.name, chain.task_id)) raise_on_errors(errors)