def test_generate_manifest(context, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
def test_validate_scopes():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context.config, context.task,
context.properties)
context.task['scopes'] = []
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:mightly"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:dep"]
manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
def test_get_destination_for_partner_repack_path(context, full_path, expected,
bucket, raises, locale):
context.bucket = bucket
context.action = 'push-to-partner'
context.task['payload']['build_number'] = 99
context.task['payload']['version'] = '9999.0'
context.task['payload']['releaseProperties'] = {
"appName": "Firefox",
"buildid": "20180328233904",
"appVersion": "9999.0",
"hashType": "sha512",
"platform": "linux",
"branch": "maple"
}
# hack in locale
for artifact_dict in context.task['payload']['upstreamArtifacts']:
artifact_dict['locale'] = locale
context.artifacts_to_beetmove = get_upstream_artifacts(
context, preserve_full_paths=True)
context.release_props = get_release_props(context)
mapping_manifest = generate_beetmover_manifest(context)
if raises:
context.action = 'push-to-dummy'
with pytest.raises(ScriptWorkerRetryException):
get_destination_for_partner_repack_path(context, mapping_manifest,
full_path, locale)
else:
assert expected == get_destination_for_partner_repack_path(
context, mapping_manifest, full_path, locale)
async def push_to_nightly(context):
"""Push artifacts to a certain location (e.g. nightly/ or candidates/).
Determine the list of artifacts to be transferred, generate the
mapping manifest, run some data validations, and upload the bits.
Upon successful transfer, generate checksums files and manifests to be
consumed downstream by balrogworkers."""
context.release_props = get_release_props(context)
# balrog_manifest is written and uploaded as an artifact which is used by
# a subsequent balrogworker task in the release graph. Balrogworker uses
# this manifest to submit release blob info (e.g. mar filename, size, etc)
context.balrog_manifest = list()
# Used as a staging area to generate balrog_manifest, so that all the
# completes and partials for a release end up in the same data structure
context.raw_balrog_manifest = dict()
# the checksums manifest is written and uploaded as an artifact which is
# used by a subsequent signing task and again by another beetmover task to
# upload it to S3
context.checksums = dict()
# TODO: if artifactMap passes schema validation
if context.task['payload'].get('artifactMap'):
# determine artifacts to beetmove
context.artifacts_to_beetmove = get_upstream_artifacts(
context, preserve_full_paths=True)
await move_beets(context,
context.artifacts_to_beetmove,
artifact_map=context.task['payload']['artifactMap'])
else:
# determine artifacts to beetmove
context.artifacts_to_beetmove = get_upstream_artifacts(context)
# generate beetmover mapping manifest
mapping_manifest = generate_beetmover_manifest(context)
# perform another validation check against the bucket path
validate_bucket_paths(context.bucket,
mapping_manifest['s3_bucket_path'])
# some files to-be-determined via script configs need to have their
# contents pretty named, so doing it here before even beetmoving begins
blobs = context.config.get('blobs_needing_prettynaming_contents', [])
alter_unpretty_contents(context, blobs, mapping_manifest)
# for each artifact in manifest
# a. map each upstream artifact to pretty name release bucket format
# b. upload to corresponding S3 location
await move_beets(context,
context.artifacts_to_beetmove,
manifest=mapping_manifest)
# write balrog_manifest to a file and add it to list of artifacts
add_balrog_manifest_to_artifacts(context)
# determine the correct checksum filename and generate it, adding it to
# the list of artifacts afterwards
add_checksums_to_artifacts(context)
async def push_to_maven(context):
"""Push artifacts to locations expected by maven clients (like mvn or gradle)"""
artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(context)
context.release_props = get_release_props(context)
context.checksums = dict() # Needed by downstream calls
context.raw_balrog_manifest = dict() # Needed by downstream calls
if context.task['payload'].get('artifactMap'):
context.artifacts_to_beetmove = _extract_and_check_maven_artifacts_to_beetmove(
artifacts_to_beetmove,
context.config.get('zip_max_file_size_in_mb', DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB),
artifact_map=context.task['payload'].get('artifactMap')
)
await move_beets(context, context.artifacts_to_beetmove, artifact_map=context.task['payload']['artifactMap'])
else:
mapping_manifest = generate_beetmover_manifest(context)
validate_bucket_paths(context.bucket, mapping_manifest['s3_bucket_path'])
context.artifacts_to_beetmove = _extract_and_check_maven_artifacts_to_beetmove(
artifacts_to_beetmove,
context.config.get('zip_max_file_size_in_mb', DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB),
mapping_manifest=mapping_manifest
)
await move_beets(context, context.artifacts_to_beetmove, manifest=mapping_manifest)
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
async def test_move_partner_beets(context, mocker):
context.artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=True)
context.release_props = get_release_props(context)
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
mapping_manifest = generate_beetmover_manifest(context)
mocker.patch.object(beetmoverscript.script, "get_destination_for_partner_repack_path", new=noop_sync)
mocker.patch.object(beetmoverscript.script, "upload_to_s3", new=noop_async)
await move_partner_beets(context, mapping_manifest)
def test_generate_manifest_maven(context, mocker, version, build_id, artifact_id, expected):
context.bucket = 'maven'
context.action = 'push-to-maven'
context.task['payload']['version'] = version
context.task['payload']['artifact_id'] = artifact_id
context.release_props['branch'] = 'mozilla-central'
context.release_props['buildid'] = build_id
context.release_props['appName'] = 'geckoview'
assert generate_beetmover_manifest(context) == expected
async def push_to_partner(context):
"""Push private repack artifacts to a certain location. They can be either
private (to private S3 buckets) or public (going under regular firefox
bucket).
Determine the list of artifacts to be transferred, generate the
mapping manifest and upload the bits."""
context.artifacts_to_beetmove = get_upstream_artifacts(
context, preserve_full_paths=True)
context.release_props, release_props_file = get_release_props(context)
mapping_manifest = generate_beetmover_manifest(context)
await move_partner_beets(context, mapping_manifest)
async def push_to_maven(context):
"""Push artifacts to locations expected by maven clients (like mvn or gradle)"""
context.release_props = get_release_props(context)
context.checksums = dict() # Needed by downstream calls
context.raw_balrog_manifest = dict() # Needed by downstream calls
# XXX this is needed in order to avoid the need to land the in-tree
# corresponding patches across all trees altogether. Similarly for github
# projects. This allows us a gradual rollout of this across all projects
is_zip_archive = any([
d.get('zipExtract')
for d in context.task['payload']['upstreamArtifacts']
])
if context.task['payload'].get('artifactMap'):
if is_zip_archive:
artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(
context)
context.artifacts_to_beetmove = _extract_and_check_maven_artifacts_to_beetmove(
artifacts_to_beetmove,
context.config.get('zip_max_file_size_in_mb',
DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB),
artifact_map=context.task['payload'].get('artifactMap'))
await move_beets(
context,
context.artifacts_to_beetmove,
artifact_map=context.task['payload']['artifactMap'])
else:
# overwrite artifacts_to_beetmove with the declarative artifacts ones
context.artifacts_to_beetmove = task.get_upstream_artifacts(
context, preserve_full_paths=True)
await move_beets(
context,
context.artifacts_to_beetmove,
artifact_map=context.task['payload']['artifactMap'])
else:
# TODO: remove this once we're done with migrating from maven.zip
artifacts_to_beetmove = task.get_upstream_artifacts_with_zip_extract_param(
context)
mapping_manifest = generate_beetmover_manifest(context)
validate_bucket_paths(context.bucket,
mapping_manifest['s3_bucket_path'])
context.artifacts_to_beetmove = _extract_and_check_maven_artifacts_to_beetmove(
artifacts_to_beetmove,
context.config.get('zip_max_file_size_in_mb',
DEFAULT_ZIP_MAX_FILE_SIZE_IN_MB),
mapping_manifest=mapping_manifest)
await move_beets(context,
context.artifacts_to_beetmove,
manifest=mapping_manifest)
async def async_main(context):
# balrog_manifest is written and uploaded as an artifact which is used by a subsequent
# balrogworker task in the release graph. Balrogworker uses this manifest to submit
# release blob info with things like mar filename, size, etc
context.balrog_manifest = list()
# the checksums manifest is written and uploaded as an artifact which is used
# by a subsequent signing task and again by another beetmover task to
# upload it along with the other artifacts
context.checksums = dict()
# determine and validate the task schema along with its scopes
context.task = get_task(context.config) # e.g. $cfg['work_dir']/task.json
validate_task_schema(context)
# determine artifacts to beetmove
context.artifacts_to_beetmove = get_upstream_artifacts(context)
# determine the release properties and make a copy in the artifacts
# directory
release_props_file = get_initial_release_props_file(context)
context.release_props = get_release_props(release_props_file)
# generate beetmover mapping manifest
mapping_manifest = generate_beetmover_manifest(context.config,
context.task,
context.release_props)
# validate scopes to prevent beetmoving in the wrong place
validate_task_scopes(context, mapping_manifest)
# some files to-be-determined via script configs need to have their
# contents pretty named, so doing it here before even beetmoving begins
blobs = context.config.get('blobs_needing_prettynaming_contents', [])
alter_unpretty_contents(context, blobs, mapping_manifest)
# for each artifact in manifest
# a. map each upstream artifact to pretty name release bucket format
# b. upload to candidates/dated location
await move_beets(context, context.artifacts_to_beetmove, mapping_manifest)
# write balrog_manifest to a file and add it to list of artifacts
add_balrog_manifest_to_artifacts(context)
# determine the correct checksum filename and generate it, adding it to
# the list of artifacts afterwards
add_checksums_to_artifacts(context)
# add release props file to later be used by beetmover jobs than upload
# the checksums file
add_release_props_to_artifacts(context, release_props_file)
log.info('Success!')
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_beetmover_manifest(context.config, context.task, context.properties)
mapping = manifest['mapping']
s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt']
assert (
manifest.get('s3_prefix_dated') ==
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and
manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/'
)
async def push_to_nightly(context):
# determine artifacts to beetmove
context.artifacts_to_beetmove = get_upstream_artifacts(context)
# find release properties and make a copy in the artifacts directory
release_props_file = get_initial_release_props_file(context)
context.release_props = get_release_props(release_props_file)
# generate beetmover mapping manifest
mapping_manifest = generate_beetmover_manifest(context)
# perform another validation check against the bucket path
validate_bucket_paths(context.bucket, mapping_manifest['s3_bucket_path'])
# some files to-be-determined via script configs need to have their
# contents pretty named, so doing it here before even beetmoving begins
blobs = context.config.get('blobs_needing_prettynaming_contents', [])
alter_unpretty_contents(context, blobs, mapping_manifest)
# balrog_manifest is written and uploaded as an artifact which is used by
# a subsequent balrogworker task in the release graph. Balrogworker uses
# this manifest to submit release blob info (e.g. mar filename, size, etc)
context.balrog_manifest = list()
# the checksums manifest is written and uploaded as an artifact which is
# used by a subsequent signing task and again by another beetmover task to
# upload it to S3
context.checksums = dict()
# for each artifact in manifest
# a. map each upstream artifact to pretty name release bucket format
# b. upload to corresponding S3 location
await move_beets(context, context.artifacts_to_beetmove, mapping_manifest)
# write balrog_manifest to a file and add it to list of artifacts
add_balrog_manifest_to_artifacts(context)
# determine the correct checksum filename and generate it, adding it to
# the list of artifacts afterwards
add_checksums_to_artifacts(context)
# add release props file to later be used by beetmover jobs than upload
# the checksums file
add_release_props_to_artifacts(context, release_props_file)
def test_generate_manifest(context, mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
manifest = generate_beetmover_manifest(context)
mapping = manifest["mapping"]
s3_keys = [mapping[m].get("target_info.txt", {}).get("s3_key") for m in mapping]
assert sorted(mapping.keys()) == ["en-US", "multi"]
assert sorted(s3_keys) == ["fake-99.0a1.en-US.target_info.txt", "fake-99.0a1.multi.target_info.txt"]
expected_destinations = {
"en-US": [
"2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
"latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
],
"multi": [
"2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt",
"latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt",
],
}
actual_destinations = {k: mapping[k]["target_info.txt"]["destinations"] for k in sorted(mapping.keys())}
assert expected_destinations == actual_destinations
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
async def test_move_beets(task_filename, partials, mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task["payload"]["releaseProperties"]
context.release_props["stage_platform"] = context.release_props["platform"]
context.bucket = "nightly"
context.action = "push-to-nightly"
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task["payload"].get("artifactMap"):
artifact_map = context.task["payload"].get("artifactMap")
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk"),
]
expected_destinations = [
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
],
]
expected_balrog_manifest = []
for complete_info in [
{
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
}
]
},
{
"blob_suffix": "-mozinfo",
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
}
],
},
]:
entry = {
"tc_nightly": True,
"appName": "Fake",
"appVersion": "99.0a1",
"branch": "mozilla-central",
"buildid": "20990205110000",
"extVersion": "99.0a1",
"hashType": "sha512",
"locale": "en-US",
"platform": "android-api-15",
"url_replacements": [["http://archive.mozilla.org/pub", "http://download.cdn.mozilla.net/pub"]],
}
entry.update(complete_info)
if partials:
entry["partialInfo"] = [
{
"from_buildid": 19991231235959,
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
}
]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get("blob_suffix", ""))
async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {"hash": "dummyhash", "size": 123456, "url": destinations[0]}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault("partialInfo", []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault("completeInfo", {})[balrog_format] = data
with mock.patch("beetmoverscript.script.move_beet", fake_move_beet):
await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
