async def get_props(context):
source = get_manifest_url(context.task['payload'])
# extra validation check is useful for the url scheme, netloc and path
# restrictions
beet_config = deepcopy(context.config)
beet_config.setdefault('valid_artifact_task_ids', context.task['dependencies'])
validate_artifact_url(beet_config, source)
return (await retry_request(context, source, method='get',
return_type='json'))['properties']
async def download_artifacts(context,
file_urls,
parent_dir=None,
session=None,
download_func=download_file,
valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config["work_dir"]
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config["valid_artifact_rules"]
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(
context.task["dependencies"] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules,
valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
assert_is_parent(abs_file_path, parent_dir)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func,
args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError,
asyncio.TimeoutError),
kwargs={"session": session},
)))
await raise_future_exceptions(tasks)
return files
async def download_artifacts(context, file_urls, parent_dir=None, session=None,
download_func=download_file, valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config['valid_artifact_rules']
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(context.task['dependencies'] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func, args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError, asyncio.TimeoutError),
kwargs={'session': session},
)
)
)
await raise_future_exceptions(tasks)
return files
async def move_beet(context, source, destinations, locale, update_balrog_manifest):
beet_config = deepcopy(context.config)
beet_config.setdefault('valid_artifact_task_ids', context.task['dependencies'])
rel_path = validate_artifact_url(beet_config, source)
abs_file_path = os.path.join(context.config['work_dir'], rel_path)
await retry_download(context=context, url=source, path=abs_file_path)
await retry_upload(context=context, destinations=destinations, path=abs_file_path)
if update_balrog_manifest:
context.balrog_manifest.append(
enrich_balrog_manifest(context, abs_file_path, locale, destinations)
)
async def download_artifacts(context,
file_urls,
parent_dir=None,
session=None,
download_func=download_file):
"""Download artifacts in parallel after validating their URLs.
Valid `taskId`s for download include the task's dependencies and the
`taskGroupId`, which by convention is the `taskId` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to `work_dir`. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is `download_file`.
Returns:
list: the relative paths to the files downloaded, relative to
`parent_dir`.
Raises:
scriptworker.exceptions.DownloadError: on download failure after
max retries.
"""
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
download_config = deepcopy(context.config)
download_config.setdefault(
'valid_artifact_task_ids',
context.task['dependencies'] + [context.task['taskGroupId']])
for file_url in file_urls:
rel_path = validate_artifact_url(download_config, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(rel_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func,
args=(context, file_url, abs_file_path),
kwargs={'session': session},
)))
await raise_future_exceptions(tasks)
return files
def test_bad_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
with pytest.raises(ScriptWorkerTaskException):
client.validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url)
def test_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url, expected):
value = client.validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url)
assert value == expected
def test_bad_artifact_url(params):
with pytest.raises(ScriptWorkerTaskException):
client.validate_artifact_url(params[0], params[1])
def test_artifact_url(params):
value = client.validate_artifact_url(params[0], params[1])
assert value == params[2]
def test_bad_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
with pytest.raises(ScriptWorkerTaskException):
client.validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url)
def test_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url, expected):
value = client.validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url)
assert value == expected
