def get_updated_buildhub_artifact(path,
installer_artifact,
installer_path,
context,
locale,
manifest=None,
artifact_map=None):
"""
Read the file into a dict, alter the fields below, and return the updated dict
buildhub.json fields that should be changed: download.size, download.date, download.url
"""
contents = utils.load_json(path)
url_prefix = context.config["bucket_config"][context.bucket]["url_prefix"]
if artifact_map:
task_id = get_taskId_from_full_path(installer_path)
cfg = utils.extract_file_config_from_artifact_map(
artifact_map, installer_artifact, task_id, locale)
path = urllib.parse.quote(cfg["destinations"][0])
else:
dest = manifest["mapping"][locale][installer_artifact]["destinations"][
0]
path = urllib.parse.quote(
urllib.parse.urljoin(manifest["s3_bucket_path"], dest))
url = urllib.parse.urljoin(url_prefix, path)
# Update fields
contents["download"]["size"] = utils.get_size(installer_path)
contents["download"]["date"] = str(arrow.utcnow())
contents["download"]["url"] = url
return contents
def test_extract_file_config_from_artifact_map():
task_def = get_fake_valid_task(taskjson="task_artifact_map.json")
task_id = "eSzfNqMZT_mSiQQXu8hyqg"
locale = "en-US"
filename = "target.txt"
expected = {
"destinations": [
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
],
"checksums_path": "",
"from_buildid": 19991231235959,
"update_balrog_manifest": True,
}
assert extract_file_config_from_artifact_map(task_def["payload"]["artifactMap"], filename, task_id, locale) == expected
async def move_beets(context,
artifacts_to_beetmove,
manifest=None,
artifact_map=None):
beets = []
for locale in artifacts_to_beetmove:
installer_artifact = ""
buildhub_artifact_exists = False
# get path of installer beet
for artifact in artifacts_to_beetmove[locale]:
if exists_or_endswith(artifact, INSTALLER_ARTIFACTS):
installer_artifact = artifact
if exists_or_endswith(artifact, BUILDHUB_ARTIFACT):
buildhub_artifact_exists = True
# throws error if buildhub.json is present and installer isn't
if not installer_artifact and buildhub_artifact_exists:
raise ScriptWorkerTaskException(
"could not determine installer path from task payload")
# move beets
for artifact in artifacts_to_beetmove[locale]:
source = artifacts_to_beetmove[locale][artifact]
# update buildhub.json file
# if there is no installer then there will be no buildhub.json artifact
# in logical coding terms, this means that if installer_path is an empty
# string, then this if-block is never reached
if exists_or_endswith(artifact, BUILDHUB_ARTIFACT):
write_json(
source,
get_updated_buildhub_artifact(
path=source,
installer_artifact=installer_artifact,
installer_path=artifacts_to_beetmove[locale]
[installer_artifact],
context=context,
locale=locale,
manifest=manifest,
artifact_map=artifact_map,
),
)
if artifact_map:
task_id = get_taskId_from_full_path(source)
# Should only ever be one (taskId, locale) match.
map_entry = extract_file_config_from_artifact_map(
artifact_map, artifact, task_id, locale)
artifact_pretty_name = map_entry["checksums_path"]
destinations = map_entry["destinations"]
update_balrog_manifest = map_entry.get(
"update_balrog_manifest", False)
balrog_format = map_entry.get("balrog_format", "")
from_buildid = map_entry.get("from_buildid")
else:
artifact_pretty_name = manifest["mapping"][locale][artifact][
"s3_key"]
destinations = [
os.path.join(manifest["s3_bucket_path"], dest) for dest in
manifest["mapping"][locale][artifact]["destinations"]
]
update_balrog_manifest = manifest["mapping"][locale][
artifact].get("update_balrog_manifest")
# Adjust old template format.
# artifact map specifies these separately.
# templates say "update_balrog_manifest": true
# or "update_balrog_manifest": {"format": mozinfo}
if isinstance(update_balrog_manifest, dict):
balrog_format = update_balrog_manifest.get("format")
update_balrog_manifest = True
else:
balrog_format = ""
from_buildid = manifest["mapping"][locale][artifact].get(
"from_buildid")
beets.append(
asyncio.ensure_future(
move_beet(
context,
source,
destinations,
locale=locale,
update_balrog_manifest=update_balrog_manifest,
balrog_format=balrog_format,
from_buildid=from_buildid,
artifact_pretty_name=artifact_pretty_name,
)))
await raise_future_exceptions(beets)
# Fix up balrog manifest. We need an entry with both completes and
# partials, which is why we store up the data from each moved beet
# and collate it now.
for locale, info in context.raw_balrog_manifest.items():
for format in info["completeInfo"]:
balrog_entry = enrich_balrog_manifest(context, locale)
balrog_entry["completeInfo"] = [info["completeInfo"][format]]
if "partialInfo" in info:
balrog_entry["partialInfo"] = info["partialInfo"]
if format:
balrog_entry["blob_suffix"] = "-{}".format(format)
context.balrog_manifest.append(balrog_entry)
def test_extract_file_config_from_artifact_map_raises(task_id, locale, filename):
task_def = get_fake_valid_task(taskjson='task_artifact_map.json')
with pytest.raises(TaskVerificationError):
extract_file_config_from_artifact_map(
task_def['payload']['artifactMap'], filename, task_id, locale)
