async def has_commit_landed_on_repository(self, context, revision): """Tell if a commit was landed on the repository or if it just comes from a pull request. Args: context (scriptworker.context.Context): the scriptworker context. revision (str): the commit hash or the tag name. Returns: bool: True if the commit is present in one of the branches of the main repository """ # Revision may be a tag name. `branch_commits` doesn't work on tags if not _is_git_full_hash(revision): revision = self.get_tag_hash(tag_name=revision) repo = self._github_repository.html_url url = '/'.join([repo.rstrip('/'), 'branch_commits', revision]) from scriptworker.task import get_decision_task_id cache_key = '{}-{}'.format(get_decision_task_id(context.task), url) async with _branch_commits_cache_lock: if cache_key in _branch_commits_cache: html_text = _branch_commits_cache[cache_key] else: html_data = await retry_request(context, url) html_text = html_data.strip() _branch_commits_cache[cache_key] = html_text # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it # represents the branches on which the given revision is present. return html_text != ''
async def download_artifacts(context, file_urls, parent_dir=None, session=None, download_func=download_file, valid_artifact_task_ids=None): """Download artifacts in parallel after validating their URLs. Valid ``taskId``s for download include the task's dependencies and the ``taskGroupId``, which by convention is the ``taskId`` of the decision task. Args: context (scriptworker.context.Context): the scriptworker context. file_urls (list): the list of artifact urls to download. parent_dir (str, optional): the path of the directory to download the artifacts into. If None, defaults to ``work_dir``. Default is None. session (aiohttp.ClientSession, optional): the session to use to download. If None, defaults to context.session. Default is None. download_func (function, optional): the function to call to download the files. default is ``download_file``. valid_artifact_task_ids (list, optional): the list of task ids that are valid to download from. If None, defaults to all task dependencies plus the decision taskId. Defaults to None. Returns: list: the full paths to the files downloaded Raises: scriptworker.exceptions.BaseDownloadError: on download failure after any applicable retries. """ parent_dir = parent_dir or context.config["work_dir"] session = session or context.session tasks = [] files = [] valid_artifact_rules = context.config["valid_artifact_rules"] # XXX when chain of trust is on everywhere, hardcode the chain of trust task list valid_artifact_task_ids = valid_artifact_task_ids or list( context.task["dependencies"] + [get_decision_task_id(context.task)]) for file_url in file_urls: rel_path = validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, file_url) abs_file_path = os.path.join(parent_dir, rel_path) assert_is_parent(abs_file_path, parent_dir) files.append(abs_file_path) tasks.append( asyncio.ensure_future( retry_async( download_func, args=(context, file_url, abs_file_path), retry_exceptions=(DownloadError, aiohttp.ClientError, asyncio.TimeoutError), kwargs={"session": session}, ))) await raise_future_exceptions(tasks) return files
def __init__(self, context, name, task_id=None): """Initialize ChainOfTrust. Args: context (scriptworker.context.Context): the scriptworker context name (str): the name of the task (e.g., signing) task_id (str, optional): the task_id of the task. If None, use ``get_task_id(context.claim_task)``. Defaults to None. """ self.name = name self.task_type = guess_task_type(name) self.context = context self.task_id = task_id or get_task_id(context.claim_task) self.task = context.task self.worker_impl = guess_worker_impl( self) # this should be scriptworker self.decision_task_id = get_decision_task_id(self.task) self.links = []
def find_sorted_task_dependencies(task, task_name, task_id): """Find the taskIds of the chain of trust dependencies of a given task. Args: task (dict): the task definition to inspect. task_name (str): the name of the task, for logging and naming children. task_id (str): the taskId of the task. Returns: list: tuples associating dependent task ``name`` to dependent task ``taskId``. """ log.info("find_sorted_task_dependencies {} {}".format(task_name, task_id)) cot_input_dependencies = [ _craft_dependency_tuple(task_name, task_type, task_id) for task_type, task_id in task['extra'].get('chainOfTrust', {}).get( 'inputs', {}).items() ] upstream_artifacts_dependencies = [ _craft_dependency_tuple(task_name, artifact_dict['taskType'], artifact_dict['taskId']) for artifact_dict in task.get('payload', {}).get( 'upstreamArtifacts', []) ] dependencies = [*cot_input_dependencies, *upstream_artifacts_dependencies] dependencies = _sort_dependencies_by_name_then_task_id(dependencies) decision_task_id = get_decision_task_id(task) task_type = guess_task_type(task_name) if decision_task_id != task_id and task_type != 'decision': # make sure we deal with the decision task first, or we may populate # signing:build0:decision before signing:decision dependencies.insert( 0, _craft_dependency_tuple(task_name, 'decision', decision_task_id)) log.info('found dependencies: {}'.format(dependencies)) return dependencies
async def has_commit_landed_on_repository(self, context, revision): """Tell if a commit was landed on the repository or if it just comes from a pull request. Args: context (scriptworker.context.Context): the scriptworker context. revision (str): the commit hash or the tag name. Returns: bool: True if the commit is present in one of the branches of the main repository """ # Revision may be a tag name. `branch_commits` doesn't work on tags if not _is_git_full_hash(revision): revision = self.get_tag_hash(tag_name=revision) repo = self._github_repository.html_url if any( vcs_rule.get('require_secret') for vcs_rule in context.config['trusted_vcs_rules']): # This check uses unofficial API on github, which we can't easily # check for private repos, assume its true in the private case. log.info( "has_commit_landed_on_repository() not implemented for private" "repositories, assume True") return True url = '/'.join([repo.rstrip('/'), 'branch_commits', revision]) from scriptworker.task import get_decision_task_id cache_key = '{}-{}'.format(get_decision_task_id(context.task), url) if cache_key in _branch_commits_cache: html_text = _branch_commits_cache[cache_key] else: html_data = await retry_request(context, url) html_text = html_data.strip() _branch_commits_cache[cache_key] = html_text # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it # represents the branches on which the given revision is present. return html_text != ''
def test_get_decision_task_id(defn, result): assert task.get_decision_task_id(defn) == result
def test_get_decision_task_id(task, result): assert swtask.get_decision_task_id(task) == result
async def download_artifacts(context, file_urls, parent_dir=None, session=None, download_func=download_file, valid_artifact_task_ids=None): """Download artifacts in parallel after validating their URLs. Valid ``taskId``s for download include the task's dependencies and the ``taskGroupId``, which by convention is the ``taskId`` of the decision task. Args: context (scriptworker.context.Context): the scriptworker context. file_urls (list): the list of artifact urls to download. parent_dir (str, optional): the path of the directory to download the artifacts into. If None, defaults to ``work_dir``. Default is None. session (aiohttp.ClientSession, optional): the session to use to download. If None, defaults to context.session. Default is None. download_func (function, optional): the function to call to download the files. default is ``download_file``. valid_artifact_task_ids (list, optional): the list of task ids that are valid to download from. If None, defaults to all task dependencies plus the decision taskId. Defaults to None. Returns: list: the full paths to the files downloaded Raises: scriptworker.exceptions.BaseDownloadError: on download failure after any applicable retries. """ parent_dir = parent_dir or context.config['work_dir'] session = session or context.session tasks = [] files = [] valid_artifact_rules = context.config['valid_artifact_rules'] # XXX when chain of trust is on everywhere, hardcode the chain of trust task list valid_artifact_task_ids = valid_artifact_task_ids or list(context.task['dependencies'] + [get_decision_task_id(context.task)]) for file_url in file_urls: rel_path = validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, file_url) abs_file_path = os.path.join(parent_dir, rel_path) files.append(abs_file_path) tasks.append( asyncio.ensure_future( retry_async( download_func, args=(context, file_url, abs_file_path), retry_exceptions=(DownloadError, aiohttp.ClientError, asyncio.TimeoutError), kwargs={'session': session}, ) ) ) await raise_future_exceptions(tasks) return files
def task(self, task): self._set('_task', task) self.decision_task_id = get_decision_task_id(self.task) self.worker_impl = guess_worker_impl(self) self.is_try = is_try(self.task)