def get_optional_artifacts_per_task_id(upstream_artifacts):
"""Return every optional artifact defined in ``upstream_artifacts``, ordered by taskId.
Args:
upstream_artifacts: the list of upstream artifact definitions
Returns:
dict: list of paths to downloaded artifacts ordered by taskId
"""
# A given taskId might be defined many times in upstreamArtifacts. Thus, we can't
# use a dict comprehension
optional_artifacts_per_task_id = {}
for artifact_definition in upstream_artifacts:
if artifact_definition.get('optional', False) is True:
task_id = artifact_definition['taskId']
artifacts_paths = artifact_definition['paths']
add_enumerable_item_to_dict(
dict_=optional_artifacts_per_task_id,
key=task_id, item=artifacts_paths
)
return optional_artifacts_per_task_id
def get_optional_artifacts_per_task_id(upstream_artifacts):
"""Return every optional artifact defined in ``upstream_artifacts``, ordered by taskId.
Args:
upstream_artifacts: the list of upstream artifact definitions
Returns:
dict: list of paths to downloaded artifacts ordered by taskId
"""
# A given taskId might be defined many times in upstreamArtifacts. Thus, we can't
# use a dict comprehension
optional_artifacts_per_task_id = {}
for artifact_definition in upstream_artifacts:
if artifact_definition.get('optional', False) is True:
task_id = artifact_definition['taskId']
artifacts_paths = artifact_definition['paths']
add_enumerable_item_to_dict(
dict_=optional_artifacts_per_task_id,
key=task_id, item=artifacts_paths
)
return optional_artifacts_per_task_id
def get_upstream_artifacts_full_paths_per_task_id(context):
"""List the downloaded upstream artifacts.
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
dict, dict: lists of the paths to upstream artifacts, sorted by task_id.
First dict represents the existing upstream artifacts. The second one
maps the optional artifacts that couldn't be downloaded
Raises:
scriptworker.exceptions.ScriptWorkerTaskException: when an artifact doesn't exist.
"""
upstream_artifacts = context.task['payload']['upstreamArtifacts']
task_ids_and_relative_paths = [
(artifact_definition['taskId'], artifact_definition['paths'])
for artifact_definition in upstream_artifacts
]
optional_artifacts_per_task_id = get_optional_artifacts_per_task_id(
upstream_artifacts)
upstream_artifacts_full_paths_per_task_id = {}
failed_paths_per_task_id = {}
for task_id, paths in task_ids_and_relative_paths:
for path in paths:
try:
path_to_add = get_and_check_single_upstream_artifact_full_path(
context, task_id, path)
add_enumerable_item_to_dict(
dict_=upstream_artifacts_full_paths_per_task_id,
key=task_id,
item=path_to_add)
except ScriptWorkerTaskException:
if path in optional_artifacts_per_task_id.get(task_id, []):
log.warning(
'Optional artifact "{}" of task "{}" not found'.format(
path, task_id))
add_enumerable_item_to_dict(dict_=failed_paths_per_task_id,
key=task_id,
item=path)
else:
raise
return upstream_artifacts_full_paths_per_task_id, failed_paths_per_task_id
def get_upstream_artifacts_full_paths_per_task_id(context):
"""List the downloaded upstream artifacts.
Args:
context (scriptworker.context.Context): the scriptworker context.
Returns:
dict, dict: lists of the paths to upstream artifacts, sorted by task_id.
First dict represents the existing upstream artifacts. The second one
maps the optional artifacts that couldn't be downloaded
Raises:
scriptworker.exceptions.ScriptWorkerTaskException: when an artifact doesn't exist.
"""
upstream_artifacts = context.task['payload']['upstreamArtifacts']
task_ids_and_relative_paths = [
(artifact_definition['taskId'], artifact_definition['paths'])
for artifact_definition in upstream_artifacts
]
optional_artifacts_per_task_id = get_optional_artifacts_per_task_id(upstream_artifacts)
upstream_artifacts_full_paths_per_task_id = {}
failed_paths_per_task_id = {}
for task_id, paths in task_ids_and_relative_paths:
for path in paths:
try:
path_to_add = get_and_check_single_upstream_artifact_full_path(context, task_id, path)
add_enumerable_item_to_dict(
dict_=upstream_artifacts_full_paths_per_task_id,
key=task_id, item=path_to_add
)
except ScriptWorkerTaskException:
if path in optional_artifacts_per_task_id.get(task_id, []):
log.warning('Optional artifact "{}" of task "{}" not found'.format(path, task_id))
add_enumerable_item_to_dict(
dict_=failed_paths_per_task_id,
key=task_id, item=path
)
else:
raise
return upstream_artifacts_full_paths_per_task_id, failed_paths_per_task_id
def test_add_enumerable_item_to_dict(dict_, key, item, expected):
utils.add_enumerable_item_to_dict(dict_, key, item)
assert dict_ == expected
def test_add_enumerable_item_to_dict(dict_, key, item, expected):
utils.add_enumerable_item_to_dict(dict_, key, item)
assert dict_ == expected