async def has_commit_landed_on_repository(self, context, revision):
"""Tell if a commit was landed on the repository or if it just comes from a pull request.
Args:
context (scriptworker.context.Context): the scriptworker context.
revision (str): the commit hash or the tag name.
Returns:
bool: True if the commit is present in one of the branches of the main repository
"""
# Revision may be a tag name. `branch_commits` doesn't work on tags
if not _is_git_full_hash(revision):
revision = self.get_tag_hash(tag_name=revision)
repo = self._github_repository.html_url
url = '/'.join([repo.rstrip('/'), 'branch_commits', revision])
from scriptworker.task import get_decision_task_id
cache_key = '{}-{}'.format(get_decision_task_id(context.task), url)
async with _branch_commits_cache_lock:
if cache_key in _branch_commits_cache:
html_text = _branch_commits_cache[cache_key]
else:
html_data = await retry_request(context, url)
html_text = html_data.strip()
_branch_commits_cache[cache_key] = html_text
# https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
# when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it
# represents the branches on which the given revision is present.
return html_text != ''
async def download_artifacts(context,
file_urls,
parent_dir=None,
session=None,
download_func=download_file,
valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config["work_dir"]
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config["valid_artifact_rules"]
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(
context.task["dependencies"] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules,
valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
assert_is_parent(abs_file_path, parent_dir)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func,
args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError,
asyncio.TimeoutError),
kwargs={"session": session},
)))
await raise_future_exceptions(tasks)
return files
def __init__(self, context, name, task_id=None):
"""Initialize ChainOfTrust.
Args:
context (scriptworker.context.Context): the scriptworker context
name (str): the name of the task (e.g., signing)
task_id (str, optional): the task_id of the task. If None, use
``get_task_id(context.claim_task)``. Defaults to None.
"""
self.name = name
self.task_type = guess_task_type(name)
self.context = context
self.task_id = task_id or get_task_id(context.claim_task)
self.task = context.task
self.worker_impl = guess_worker_impl(
self) # this should be scriptworker
self.decision_task_id = get_decision_task_id(self.task)
self.links = []
def find_sorted_task_dependencies(task, task_name, task_id):
"""Find the taskIds of the chain of trust dependencies of a given task.
Args:
task (dict): the task definition to inspect.
task_name (str): the name of the task, for logging and naming children.
task_id (str): the taskId of the task.
Returns:
list: tuples associating dependent task ``name`` to dependent task ``taskId``.
"""
log.info("find_sorted_task_dependencies {} {}".format(task_name, task_id))
cot_input_dependencies = [
_craft_dependency_tuple(task_name, task_type, task_id)
for task_type, task_id in task['extra'].get('chainOfTrust', {}).get(
'inputs', {}).items()
]
upstream_artifacts_dependencies = [
_craft_dependency_tuple(task_name, artifact_dict['taskType'],
artifact_dict['taskId'])
for artifact_dict in task.get('payload', {}).get(
'upstreamArtifacts', [])
]
dependencies = [*cot_input_dependencies, *upstream_artifacts_dependencies]
dependencies = _sort_dependencies_by_name_then_task_id(dependencies)
decision_task_id = get_decision_task_id(task)
task_type = guess_task_type(task_name)
if decision_task_id != task_id and task_type != 'decision':
# make sure we deal with the decision task first, or we may populate
# signing:build0:decision before signing:decision
dependencies.insert(
0, _craft_dependency_tuple(task_name, 'decision',
decision_task_id))
log.info('found dependencies: {}'.format(dependencies))
return dependencies
async def has_commit_landed_on_repository(self, context, revision):
"""Tell if a commit was landed on the repository or if it just comes from a pull request.
Args:
context (scriptworker.context.Context): the scriptworker context.
revision (str): the commit hash or the tag name.
Returns:
bool: True if the commit is present in one of the branches of the main repository
"""
# Revision may be a tag name. `branch_commits` doesn't work on tags
if not _is_git_full_hash(revision):
revision = self.get_tag_hash(tag_name=revision)
repo = self._github_repository.html_url
if any(
vcs_rule.get('require_secret')
for vcs_rule in context.config['trusted_vcs_rules']):
# This check uses unofficial API on github, which we can't easily
# check for private repos, assume its true in the private case.
log.info(
"has_commit_landed_on_repository() not implemented for private"
"repositories, assume True")
return True
url = '/'.join([repo.rstrip('/'), 'branch_commits', revision])
from scriptworker.task import get_decision_task_id
cache_key = '{}-{}'.format(get_decision_task_id(context.task), url)
if cache_key in _branch_commits_cache:
html_text = _branch_commits_cache[cache_key]
else:
html_data = await retry_request(context, url)
html_text = html_data.strip()
_branch_commits_cache[cache_key] = html_text
# https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
# when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it
# represents the branches on which the given revision is present.
return html_text != ''
def test_get_decision_task_id(defn, result):
assert task.get_decision_task_id(defn) == result
def test_get_decision_task_id(task, result):
assert swtask.get_decision_task_id(task) == result
def test_get_decision_task_id(task, result):
assert swtask.get_decision_task_id(task) == result
async def download_artifacts(context, file_urls, parent_dir=None, session=None,
download_func=download_file, valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config['valid_artifact_rules']
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(context.task['dependencies'] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func, args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError, asyncio.TimeoutError),
kwargs={'session': session},
)
)
)
await raise_future_exceptions(tasks)
return files
def task(self, task):
self._set('_task', task)
self.decision_task_id = get_decision_task_id(self.task)
self.worker_impl = guess_worker_impl(self)
self.is_try = is_try(self.task)
