def test_write_json():
sample_data = get_fake_balrog_props()
with tempfile.NamedTemporaryFile(delete=True) as fp:
write_json(fp.name, sample_data)
with open(fp.name, "r") as fread:
retrieved_data = json.load(fread)
assert sample_data == retrieved_data
def test_write_json():
sample_data = get_fake_valid_task()['payload']['releaseProperties']
with tempfile.NamedTemporaryFile(delete=True) as fp:
write_json(fp.name, sample_data)
with open(fp.name, "r") as fread:
retrieved_data = json.load(fread)
assert sample_data == retrieved_data
async def move_beets(context,
artifacts_to_beetmove,
manifest=None,
artifact_map=None):
beets = []
for locale in artifacts_to_beetmove:
installer_artifact = ""
buildhub_artifact_exists = False
# get path of installer beet
for artifact in artifacts_to_beetmove[locale]:
if exists_or_endswith(artifact, INSTALLER_ARTIFACTS):
installer_artifact = artifact
if exists_or_endswith(artifact, BUILDHUB_ARTIFACT):
buildhub_artifact_exists = True
# throws error if buildhub.json is present and installer isn't
if not installer_artifact and buildhub_artifact_exists:
raise ScriptWorkerTaskException(
"could not determine installer path from task payload")
# move beets
for artifact in artifacts_to_beetmove[locale]:
source = artifacts_to_beetmove[locale][artifact]
# update buildhub.json file
# if there is no installer then there will be no buildhub.json artifact
# in logical coding terms, this means that if installer_path is an empty
# string, then this if-block is never reached
if exists_or_endswith(artifact, BUILDHUB_ARTIFACT):
write_json(
source,
get_updated_buildhub_artifact(
path=source,
installer_artifact=installer_artifact,
installer_path=artifacts_to_beetmove[locale]
[installer_artifact],
context=context,
locale=locale,
manifest=manifest,
artifact_map=artifact_map,
),
)
if artifact_map:
task_id = get_taskId_from_full_path(source)
# Should only ever be one (taskId, locale) match.
map_entry = extract_file_config_from_artifact_map(
artifact_map, artifact, task_id, locale)
artifact_pretty_name = map_entry["checksums_path"]
destinations = map_entry["destinations"]
update_balrog_manifest = map_entry.get(
"update_balrog_manifest", False)
balrog_format = map_entry.get("balrog_format", "")
from_buildid = map_entry.get("from_buildid")
else:
artifact_pretty_name = manifest["mapping"][locale][artifact][
"s3_key"]
destinations = [
os.path.join(manifest["s3_bucket_path"], dest) for dest in
manifest["mapping"][locale][artifact]["destinations"]
]
update_balrog_manifest = manifest["mapping"][locale][
artifact].get("update_balrog_manifest")
# Adjust old template format.
# artifact map specifies these separately.
# templates say "update_balrog_manifest": true
# or "update_balrog_manifest": {"format": mozinfo}
if isinstance(update_balrog_manifest, dict):
balrog_format = update_balrog_manifest.get("format")
update_balrog_manifest = True
else:
balrog_format = ""
from_buildid = manifest["mapping"][locale][artifact].get(
"from_buildid")
beets.append(
asyncio.ensure_future(
move_beet(
context,
source,
destinations,
locale=locale,
update_balrog_manifest=update_balrog_manifest,
balrog_format=balrog_format,
from_buildid=from_buildid,
artifact_pretty_name=artifact_pretty_name,
)))
await raise_future_exceptions(beets)
# Fix up balrog manifest. We need an entry with both completes and
# partials, which is why we store up the data from each moved beet
# and collate it now.
for locale, info in context.raw_balrog_manifest.items():
for format in info["completeInfo"]:
balrog_entry = enrich_balrog_manifest(context, locale)
balrog_entry["completeInfo"] = [info["completeInfo"][format]]
if "partialInfo" in info:
balrog_entry["partialInfo"] = info["partialInfo"]
if format:
balrog_entry["blob_suffix"] = "-{}".format(format)
context.balrog_manifest.append(balrog_entry)
def add_balrog_manifest_to_artifacts(context):
abs_file_path = os.path.join(context.config['artifact_dir'],
'public/manifest.json')
utils.write_json(abs_file_path, context.balrog_manifest)
async def move_beets(context, artifacts_to_beetmove, manifest):
beets = []
for locale in artifacts_to_beetmove:
installer_path = ''
buildhub_artifact_exists = False
# get path of installer beet
for artifact in artifacts_to_beetmove[locale]:
if artifact in INSTALLER_ARTIFACTS:
installer_path = artifacts_to_beetmove[locale][artifact]
if artifact == BUILDHUB_ARTIFACT:
buildhub_artifact_exists = True
# throws error if buildhub.json is present and installer isn't
if not installer_path and buildhub_artifact_exists:
raise ScriptWorkerTaskException(
"could not determine installer path from task payload")
# move beets
for artifact in artifacts_to_beetmove[locale]:
source = artifacts_to_beetmove[locale][artifact]
artifact_pretty_name = manifest['mapping'][locale][artifact][
's3_key']
# update buildhub.json file
# if there is no installer then there will be no buildhub.json artifact
# in logical coding terms, this means that if installer_path is an empty
# string, then this if-block is never reached
if artifact == BUILDHUB_ARTIFACT:
write_json(
source,
get_updated_buildhub_artifact(source, installer_path,
context, manifest, locale))
destinations = [
os.path.join(manifest["s3_bucket_path"], dest) for dest in
manifest['mapping'][locale][artifact]['destinations']
]
balrog_manifest = manifest['mapping'][locale][artifact].get(
'update_balrog_manifest')
# For partials
from_buildid = manifest['mapping'][locale][artifact].get(
'from_buildid')
beets.append(
asyncio.ensure_future(
move_beet(context,
source,
destinations,
locale=locale,
update_balrog_manifest=balrog_manifest,
from_buildid=from_buildid,
artifact_pretty_name=artifact_pretty_name)))
await raise_future_exceptions(beets)
# Fix up balrog manifest. We need an entry with both completes and
# partials, which is why we store up the data from each moved beet
# and collate it now.
for locale in context.raw_balrog_manifest:
balrog_entry = enrich_balrog_manifest(context, locale)
balrog_entry['completeInfo'] = context.raw_balrog_manifest[locale][
'completeInfo']
if 'partialInfo' in context.raw_balrog_manifest[locale]:
balrog_entry['partialInfo'] = context.raw_balrog_manifest[locale][
'partialInfo']
context.balrog_manifest.append(balrog_entry)
