async def move_partner_beets(context, manifest):
artifacts_to_beetmove = context.artifacts_to_beetmove
beets = []
for locale in artifacts_to_beetmove:
for full_path_artifact in artifacts_to_beetmove[locale]:
source = artifacts_to_beetmove[locale][full_path_artifact]
destination = get_destination_for_partner_repack_path(
context, manifest, full_path_artifact, locale)
beets.append(
asyncio.ensure_future(
upload_to_s3(context=context,
s3_key=destination,
path=source)))
if is_partner_public_task(context):
# we trim the full destination to the part after
# candidates/{version}-candidates/build{build_number}/
artifact_pretty_name = destination[destination.find(locale):]
if context.checksums.get(artifact_pretty_name) is None:
context.checksums[artifact_pretty_name] = {
algo: get_hash(source, algo)
for algo in context.config["checksums_digests"]
}
context.checksums[artifact_pretty_name]["size"] = get_size(
source)
await raise_future_exceptions(beets)
def get_destination_for_partner_repack_path(context, manifest, full_path, locale):
"""Function to process the final destination path, relative to the root of
the S3 bucket. Depending on whether it's a private or public destination, it
performs several string manipulations.
Input: 'releng/partner/ghost/ghost-var/v1/linux-i686/ro/target.tar.bz2'
Possible ouput(s):
-> ghost/59.0b20-2/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
-> pub/firefox/candidates/59.0b20-candidates/build2/partner-repacks/ghost/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
"""
# make sure we're calling this function from private-partner context
if not is_partner_action(context.action):
raise ScriptWorkerRetryException("Outside of private-partner context!")
# pretty name the `target` part to the actual filename
pretty_full_path = os.path.join(locale, manifest["mapping"][locale][os.path.basename(full_path)])
build_number = context.task["payload"]["build_number"]
version = context.task["payload"]["version"]
if is_partner_private_task(context):
sanity_check_partner_path(locale, {"version": version, "build_number": build_number}, PARTNER_REPACK_PRIVATE_REGEXES)
return pretty_full_path
elif is_partner_public_task(context):
sanity_check_partner_path(locale, {"version": version, "build_number": build_number}, PARTNER_REPACK_PUBLIC_REGEXES)
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(version=version, build_number=build_number)
return os.path.join(prefix, pretty_full_path)
def test_is_partner_private_public_task(context, action, bucket,
expected_private, expected_public):
context.action = action
context.bucket = bucket
assert is_partner_private_task(context) == expected_private
assert is_partner_public_task(context) == expected_public
def get_destination_for_private_repack_path(context, manifest, full_path,
locale):
"""Function to process the final destination path, relative to the root of
the S3 bucket. Depending on whether it's a private or public destination, it
performs several string manipulations.
Input: 'releng/partner/ghost/ghost-var/v1/linux-i686/ro/target.tar.bz2'
Possible ouput(s):
-> ghost/59.0b20-2/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
-> pub/firefox/candidates/59.0b20-candidates/build2/partner-repacks/ghost/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
"""
# make sure we're calling this function from private-partner context
if not is_partner_action(context.action):
raise ScriptWorkerRetryException("Outside of private-partner context!")
# pretty name the `target` part to the actual filename
pretty_full_path = os.path.join(
os.path.dirname(full_path),
manifest['mapping'][locale][os.path.basename(full_path)])
# get rid of leading "releng/partner"
if pretty_full_path.startswith(PARTNER_LEADING_STRING):
pretty_full_path = pretty_full_path[len(PARTNER_LEADING_STRING):]
build_number = context.task['payload']['build_number']
version = context.task['payload']['version']
if is_partner_private_task(context):
elements = pretty_full_path.split('/')
identifier = '{version}-{build_number}'.format(
version=version, build_number=build_number)
# we need need to manually insert the "version-buildno" identifier in
# between `partner` # and `partner-variant` to be consistent
elements.insert(1, identifier)
# TODO: potentially need to remove the `v1` from the path?
path = '/'.join(elements)
# XXX: temp hack until bug 1447673 is solved
if context.bucket == "dep":
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(
version=version, build_number=build_number)
path = os.path.join(prefix, path)
return path
elif is_partner_public_task(context):
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(
version=version, build_number=build_number)
return os.path.join(prefix, pretty_full_path)
def test_is_partner_public_task(context, action, payload_id, expected):
context.action = action
if payload_id:
context.task['payload'][PARTNER_REPACK_PUBLIC_PAYLOAD_ID] = True
assert is_partner_public_task(context) == expected
