async def move_partner_beets(context, manifest):
artifacts_to_beetmove = context.artifacts_to_beetmove
beets = []
for locale in artifacts_to_beetmove:
for full_path_artifact in artifacts_to_beetmove[locale]:
source = artifacts_to_beetmove[locale][full_path_artifact]
destination = get_destination_for_partner_repack_path(
context, manifest, full_path_artifact, locale)
beets.append(
asyncio.ensure_future(
upload_to_s3(context=context,
s3_key=destination,
path=source)))
if is_partner_public_task(context):
# we trim the full destination to the part after
# candidates/{version}-candidates/build{build_number}/
artifact_pretty_name = destination[destination.find(locale):]
if context.checksums.get(artifact_pretty_name) is None:
context.checksums[artifact_pretty_name] = {
algo: get_hash(source, algo)
for algo in context.config["checksums_digests"]
}
context.checksums[artifact_pretty_name]["size"] = get_size(
source)
await raise_future_exceptions(beets)
async def move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format, from_buildid,
artifact_pretty_name):
await retry_upload(context=context, destinations=destinations, path=source)
if context.checksums.get(artifact_pretty_name) is None:
context.checksums[artifact_pretty_name] = {
algo: get_hash(source, algo)
for algo in context.config['checksums_digests']
}
context.checksums[artifact_pretty_name]['size'] = get_size(source)
if update_balrog_manifest:
context.raw_balrog_manifest.setdefault(locale, {})
balrog_info = generate_balrog_info(
context,
artifact_pretty_name,
locale,
destinations,
from_buildid,
)
if from_buildid:
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(balrog_info)
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = balrog_info
def get_updated_buildhub_artifact(path,
installer_artifact,
installer_path,
context,
locale,
manifest=None,
artifact_map=None):
"""
Read the file into a dict, alter the fields below, and return the updated dict
buildhub.json fields that should be changed: download.size, download.date, download.url
"""
contents = utils.load_json(path)
url_prefix = context.config["bucket_config"][context.bucket]["url_prefix"]
if artifact_map:
task_id = get_taskId_from_full_path(installer_path)
cfg = utils.extract_file_config_from_artifact_map(
artifact_map, installer_artifact, task_id, locale)
path = urllib.parse.quote(cfg["destinations"][0])
else:
dest = manifest["mapping"][locale][installer_artifact]["destinations"][
0]
path = urllib.parse.quote(
urllib.parse.urljoin(manifest["s3_bucket_path"], dest))
url = urllib.parse.urljoin(url_prefix, path)
# Update fields
contents["download"]["size"] = utils.get_size(installer_path)
contents["download"]["date"] = str(arrow.utcnow())
contents["download"]["url"] = url
return contents
async def move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
await retry_upload(context=context, destinations=destinations, path=source)
if context.checksums.get(artifact_pretty_name) is None:
context.checksums[artifact_pretty_name] = {
algo: get_hash(source, algo) for algo in context.config['checksums_digests']
}
context.checksums[artifact_pretty_name]['size'] = get_size(source)
if update_balrog_manifest:
context.balrog_manifest.append(
enrich_balrog_manifest(context, artifact_pretty_name, locale, destinations)
)
def get_updated_buildhub_artifact(path, installer_path, context, manifest, locale):
"""
Read the file into a dict, alter the fields below, and return the updated dict
buildhub.json fields that should be changed: download.size, download.date, download.url
"""
contents = utils.load_json(path)
installer_name = os.path.basename(installer_path)
dest = manifest['mapping'][locale][installer_name]['destinations']
url_prefix = context.config["bucket_config"][context.bucket]["url_prefix"]
# assume url_prefix is ASCII safe
url = urllib.parse.quote(urllib.parse.urljoin(manifest["s3_bucket_path"], dest[0]))
# Update fields
contents['download']['size'] = utils.get_size(installer_path)
contents['download']['date'] = str(arrow.utcnow())
contents['download']['url'] = urllib.parse.urljoin(url_prefix, url)
return contents
