async def async_main(args, signing_certs):
tasks = []
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
for definition in task["extra"]["funsize"]["partials"]:
workenv = WorkEnv(
mar=definition.get('mar_binary'),
mbsdiff=definition.get('mbsdiff_binary')
)
await workenv.setup()
tasks.append(asyncio.ensure_future(retry_async(
manage_partial,
retry_exceptions=(
aiohttp.ClientError,
asyncio.TimeoutError
),
kwargs=dict(
partial_def=definition,
filename_template=args.filename_template,
artifacts_dir=args.artifacts_dir,
work_env=workenv,
signing_certs=signing_certs
))))
manifest = await asyncio.gather(*tasks)
return manifest
def test_retry_async_fail_first(event_loop):
global retry_count
retry_count['fail_first'] = 0
status = event_loop.run_until_complete(
utils.retry_async(fail_first, sleeptime_kwargs={'delay_factor': 0}))
assert status == "yay"
assert retry_count['fail_first'] == 2
async def async_main(args, signing_certs):
tasks = []
allowed_url_prefixes = list(ALLOWED_URL_PREFIXES)
if args.allow_staging_prefixes:
allowed_url_prefixes += STAGING_URL_PREFIXES
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
for definition in task["extra"]["funsize"]["partials"]:
tasks.append(asyncio.ensure_future(retry_async(
manage_partial,
retry_exceptions=(
aiohttp.ClientError,
asyncio.TimeoutError
),
kwargs=dict(
partial_def=definition,
filename_template=args.filename_template,
artifacts_dir=args.artifacts_dir,
allowed_url_prefixes=allowed_url_prefixes,
signing_certs=signing_certs,
arch=args.arch
))))
manifest = await asyncio.gather(*tasks)
return manifest
async def download_artifacts(context,
file_urls,
parent_dir=None,
session=None,
download_func=download_file,
valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config["work_dir"]
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config["valid_artifact_rules"]
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(
context.task["dependencies"] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules,
valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
assert_is_parent(abs_file_path, parent_dir)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func,
args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError,
asyncio.TimeoutError),
kwargs={"session": session},
)))
await raise_future_exceptions(tasks)
return files
def get_temp_creds_from_file(config):
"""Retry _get_temp_creds_from_file
"""
loop = asyncio.get_event_loop()
return loop.run_until_complete(retry_async(
_get_temp_creds_from_file, retry_exceptions=(ScriptWorkerTaskException,),
args=(config, ),
))
def test_retry_async_fail_first(event_loop):
global retry_count
retry_count['fail_first'] = 0
status = event_loop.run_until_complete(
utils.retry_async(fail_first)
)
assert status == "yay"
assert retry_count['fail_first'] == 2
def test_retry_async_always_fail(event_loop):
global retry_count
retry_count['always_fail'] = 0
with mock.patch('asyncio.sleep', new=fake_sleep):
with pytest.raises(ScriptWorkerException):
status = event_loop.run_until_complete(
utils.retry_async(always_fail,
sleeptime_kwargs={'delay_factor': 0}))
assert status is None
assert retry_count['always_fail'] == 5
def get_temp_creds_from_file(config):
"""Retry _get_temp_creds_from_file
"""
loop = asyncio.get_event_loop()
return loop.run_until_complete(
retry_async(
_get_temp_creds_from_file,
retry_exceptions=(ScriptWorkerTaskException, ),
args=(config, ),
))
async def download_artifacts(context, file_urls, parent_dir=None, session=None,
download_func=download_file, valid_artifact_task_ids=None):
"""Download artifacts in parallel after validating their URLs.
Valid ``taskId``s for download include the task's dependencies and the
``taskGroupId``, which by convention is the ``taskId`` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to ``work_dir``. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is ``download_file``.
valid_artifact_task_ids (list, optional): the list of task ids that are
valid to download from. If None, defaults to all task dependencies
plus the decision taskId. Defaults to None.
Returns:
list: the full paths to the files downloaded
Raises:
scriptworker.exceptions.BaseDownloadError: on download failure after
any applicable retries.
"""
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
valid_artifact_rules = context.config['valid_artifact_rules']
# XXX when chain of trust is on everywhere, hardcode the chain of trust task list
valid_artifact_task_ids = valid_artifact_task_ids or list(context.task['dependencies'] + [get_decision_task_id(context.task)])
for file_url in file_urls:
rel_path = validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func, args=(context, file_url, abs_file_path),
retry_exceptions=(DownloadError, aiohttp.ClientError, asyncio.TimeoutError),
kwargs={'session': session},
)
)
)
await raise_future_exceptions(tasks)
return files
async def download_artifacts(context,
file_urls,
parent_dir=None,
session=None,
download_func=download_file):
"""Download artifacts in parallel after validating their URLs.
Valid `taskId`s for download include the task's dependencies and the
`taskGroupId`, which by convention is the `taskId` of the decision task.
Args:
context (scriptworker.context.Context): the scriptworker context.
file_urls (list): the list of artifact urls to download.
parent_dir (str, optional): the path of the directory to download the
artifacts into. If None, defaults to `work_dir`. Default is None.
session (aiohttp.ClientSession, optional): the session to use to download.
If None, defaults to context.session. Default is None.
download_func (function, optional): the function to call to download the files.
default is `download_file`.
Returns:
list: the relative paths to the files downloaded, relative to
`parent_dir`.
Raises:
scriptworker.exceptions.DownloadError: on download failure after
max retries.
"""
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
download_config = deepcopy(context.config)
download_config.setdefault(
'valid_artifact_task_ids',
context.task['dependencies'] + [context.task['taskGroupId']])
for file_url in file_urls:
rel_path = validate_artifact_url(download_config, file_url)
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(rel_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func,
args=(context, file_url, abs_file_path),
kwargs={'session': session},
)))
await raise_future_exceptions(tasks)
return files
async def async_main(args, signing_cert):
tasks = []
allowed_url_prefixes = list(ALLOWED_URL_PREFIXES)
if args.allow_staging_prefixes:
allowed_url_prefixes += STAGING_URL_PREFIXES
task = json.load(args.task_definition)
downloads = await download_and_verify_mars(
task["extra"]["funsize"]["partials"], allowed_url_prefixes, signing_cert
)
tools_dir = Path(tempfile.mkdtemp())
await download_buildsystem_bits(
partials_config=task["extra"]["funsize"]["partials"],
downloads=downloads,
tools_dir=tools_dir,
)
# May want to consider os.cpu_count() if we ever run on osx/win.
# sched_getaffinity is the list of cores we can run on, not the total.
semaphore = asyncio.Semaphore(len(os.sched_getaffinity(0)))
for definition in task["extra"]["funsize"]["partials"]:
tasks.append(
asyncio.ensure_future(
retry_async(
manage_partial,
retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError),
kwargs=dict(
partial_def=definition,
artifacts_dir=args.artifacts_dir,
tools_dir=tools_dir,
arch=args.arch,
downloads=downloads,
semaphore=semaphore,
),
)
)
)
manifest = await asyncio.gather(*tasks)
for url in downloads:
downloads[url]["download_path"].unlink()
shutil.rmtree(downloads[url]["extracted_path"])
shutil.rmtree(tools_dir)
return manifest
async def download_all_zip_artifacts(upstream_zip_definitions, session):
async_tasks = tuple([
asyncio.ensure_future(
retry_async(download_file,
kwargs={
'context': None,
'url': definition['url'],
'abs_filename': definition['abs_filename'],
'session': session
},
retry_exceptions=(DownloadError, aiohttp.ClientError,
asyncio.TimeoutError)))
for definition in upstream_zip_definitions
])
await raise_future_exceptions(async_tasks)
def download_artifacts(context, file_urls, parent_dir=None, session=None,
download_func=download_file):
parent_dir = parent_dir or context.config['work_dir']
session = session or context.session
tasks = []
files = []
for file_url in file_urls:
rel_path = file_url.rsplit('/', 1)[-1]
abs_file_path = os.path.join(parent_dir, rel_path)
files.append(abs_file_path)
tasks.append(
asyncio.ensure_future(
retry_async(
download_func, args=(context, file_url, abs_file_path),
kwargs={'session': session},
)
)
)
return tasks, files
