async def test_push_to_maven_with_map(context, mocker, artifact_map,
extract_zip_output, ErrorRaised):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context.task['payload']['upstreamArtifacts'] = []
context.task['payload']['artifactMap'] = artifact_map
mocker.patch(
'beetmoverscript.task.get_upstream_artifacts_with_zip_extract_param',
new=lambda _: None)
mocker.patch(
'beetmoverscript.maven_utils.get_maven_expected_files_per_archive_per_task_id',
new=lambda _, **kwargs: ('', {}))
mocker.patch('beetmoverscript.zip.check_and_extract_zip_archives',
new=lambda _, __, ___, ____: extract_zip_output)
if ErrorRaised is None:
async def assert_artifacts_to_beetmove(_, artifacts_to_beetmove,
**kwargs):
assert artifacts_to_beetmove == {
'en-US': {
'org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar':
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar', # noqa E501
'org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5':
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.md5', # noqa E501
'org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1':
'/work_dir/cot/someTaskId/public/build/target.maven.zip.out/org/mozilla/geckoview-beta-x86/62.0b3/geckoview-beta-x86-62.0b3.aar.sha1', # noqa E501
},
}
mocker.patch('beetmoverscript.script.move_beets',
new=assert_artifacts_to_beetmove)
await push_to_maven(context)
else:
with pytest.raises(ErrorRaised):
await push_to_maven(context)
def test_generate_manifest(context, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
async def test_move_partner_beets(context, mocker):
context.artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=True)
context.release_props = get_release_props(context)
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
mapping_manifest = generate_beetmover_manifest(context)
mocker.patch.object(beetmoverscript.script, 'get_destination_for_partner_repack_path', new=noop_sync)
mocker.patch.object(beetmoverscript.script, 'upload_to_s3', new=noop_async)
await move_partner_beets(context, mapping_manifest)
async def test_async_main(context, mocker, action, raises):
context.action = action
def fake_action(*args):
return action
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
mocker.patch('beetmoverscript.script.move_beets', new=noop_async)
mocker.patch.object(beetmoverscript.script, 'get_task_action', new=fake_action)
if raises:
with pytest.raises(SystemExit):
await async_main(context)
else:
await async_main(context)
for module in ("botocore", "boto3", "chardet"):
assert logging.getLogger(module).level == logging.INFO
async def test_move_beets_raises(mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson='task_missing_installer.json')
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
with pytest.raises(ScriptWorkerTaskException):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=None,
artifact_map=None)
async def test_push_to_partner(context, mocker):
mocker.patch('beetmoverscript.script.move_partner_beets', new=noop_async)
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
await push_to_partner(context)
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
