def test_beetmover_template_args_generation(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' expected_template_args = { 'branch': 'mozilla-central', 'platform': 'android-api-15', 'product': 'Fake', 'stage_platform': 'android-api-15', 'template_key': 'fennec_nightly', 'upload_date': '2016/09/2016-09-01-16-26-14', 'version': '99.0a1' } template_args = generate_beetmover_template_args(context) assert template_args == expected_template_args context.task['payload']['locale'] = 'ro' template_args = generate_beetmover_template_args(context) assert template_args['template_key'] == 'fake_nightly_repacks'
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def context(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = get_fake_balrog_props()['properties'] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' yield context
def context(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = context.task["payload"]["releaseProperties"] context.release_props["stage_platform"] = context.release_props["platform"] context.bucket = "nightly" context.action = "push-to-nightly" yield context
async def test_move_beets_raises(mocker): mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson="task_missing_installer.json") context.release_props = context.task["payload"]["releaseProperties"] context.release_props["stage_platform"] = context.release_props["platform"] context.bucket = "nightly" context.action = "push-to-nightly" context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) with pytest.raises(ScriptWorkerTaskException): await move_beets(context, context.artifacts_to_beetmove, manifest=None, artifact_map=None)
def test_generate_manifest(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' manifest = generate_beetmover_manifest(context) mapping = manifest['mapping'] s3_keys = [ mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping ] assert sorted(mapping.keys()) == ['en-US', 'multi'] assert sorted(s3_keys) == [ 'fake-99.0a1.en-US.target_info.txt', 'fake-99.0a1.multi.target_info.txt' ] expected_destinations = { 'en-US': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], 'multi': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt', 'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt' ] } actual_destinations = { k: mapping[k]['target_info.txt']['destinations'] for k in sorted(mapping.keys()) } assert expected_destinations == actual_destinations
async def test_move_beet(update_manifest, action): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.task['extra'] = dict() context.task['extra']['partials'] = [{ "artifact_name": "target-98.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "19991231235959", "previousVersion": "98.0b96", "previousBuildNumber": "1" }, { "artifact_name": "target-97.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "22423423402984", "previousVersion": "97.0b96", "previousBuildNumber": "1" }] context.action = action context.bucket = 'nightly' context.checksums = dict() context.balrog_manifest = list() context.raw_balrog_manifest = dict() context.release_props = context.task['payload']['releaseProperties'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): await move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format='', artifact_pretty_name=pretty_name, from_buildid=None) assert expected_upload_args == actual_upload_args if update_manifest: for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['completeInfo'][''][k] == expected_balrog_manifest[k]) expected_balrog_manifest['from_buildid'] = '19991231235959' with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): await move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format='', artifact_pretty_name=pretty_name, from_buildid='19991231235959') if update_manifest: if is_promotion_action(context.action): expected_balrog_manifest['previousBuildNumber'] = '1' expected_balrog_manifest['previousVersion'] = '98.0b96' for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] == expected_balrog_manifest[k])
async def test_move_beets(task_filename, partials, mocker): mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson=task_filename) context.release_props = context.task['payload']['releaseProperties'] context.release_props['stage_platform'] = context.release_props['platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) if context.task['payload'].get('artifactMap'): artifact_map = context.task['payload'].get('artifactMap') manifest = None else: artifact_map = None manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk' ) ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk' ] ] expected_balrog_manifest = [] for complete_info in [ { 'completeInfo': [{ 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' }], }, { 'blob_suffix': '-mozinfo', 'completeInfo': [ { 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' }, ], }, ]: entry = { 'tc_nightly': True, 'appName': 'Fake', 'appVersion': '99.0a1', 'branch': 'mozilla-central', 'buildid': '20990205110000', 'extVersion': '99.0a1', 'hashType': 'sha512', 'locale': 'en-US', 'platform': 'android-api-15', 'url_replacements': [[ 'http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub' ]], } entry.update(complete_info) if partials: entry['partialInfo'] = [{ 'from_buildid': 19991231235959, 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' }] expected_balrog_manifest.append(entry) actual_sources = [] actual_destinations = [] def sort_manifest(manifest): manifest.sort(key=lambda entry: entry.get('blob_suffix', '')) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = { "hash": 'dummyhash', "size": 123456, "url": destinations[0] } context.raw_balrog_manifest.setdefault(locale, {}) if from_buildid: if partials: data["from_buildid"] = from_buildid context.raw_balrog_manifest[locale].setdefault( 'partialInfo', []).append(data) else: return else: context.raw_balrog_manifest[locale].setdefault( 'completeInfo', {})[balrog_format] = data with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_manifest(context.balrog_manifest) sort_manifest(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest
async def test_move_beet(update_manifest, action): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.task["extra"] = dict() context.task["extra"]["partials"] = [ { "artifact_name": "target-98.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "19991231235959", "previousVersion": "98.0b96", "previousBuildNumber": "1", }, { "artifact_name": "target-97.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "22423423402984", "previousVersion": "97.0b96", "previousBuildNumber": "1", }, ] context.action = action context.bucket = "nightly" context.checksums = dict() context.balrog_manifest = list() context.raw_balrog_manifest = dict() context.release_props = context.task["payload"]["releaseProperties"] locale = "sample-locale" target_source = "tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt" pretty_name = "fake-99.0a1.en-US.target.txt" target_destinations = ( "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", ) expected_upload_args = [ ( "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", ), "tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt", ] expected_balrog_manifest = { "hash": "73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b", "size": 18, "url": "https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch("beetmoverscript.script.retry_upload", fake_retry_upload): await move_beet( context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format="", artifact_pretty_name=pretty_name, from_buildid=None, ) assert expected_upload_args == actual_upload_args if update_manifest: for k in expected_balrog_manifest.keys(): assert context.raw_balrog_manifest[locale]["completeInfo"][""][k] == expected_balrog_manifest[k] expected_balrog_manifest["from_buildid"] = "19991231235959" with mock.patch("beetmoverscript.script.retry_upload", fake_retry_upload): await move_beet( context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format="", artifact_pretty_name=pretty_name, from_buildid="19991231235959", ) if update_manifest: if is_promotion_action(context.action): expected_balrog_manifest["previousBuildNumber"] = "1" expected_balrog_manifest["previousVersion"] = "98.0b96" for k in expected_balrog_manifest.keys(): assert context.raw_balrog_manifest[locale]["partialInfo"][0][k] == expected_balrog_manifest[k]
async def test_move_beets(task_filename, partials, mocker): mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson=task_filename) context.release_props = context.task["payload"]["releaseProperties"] context.release_props["stage_platform"] = context.release_props["platform"] context.bucket = "nightly" context.action = "push-to-nightly" context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) if context.task["payload"].get("artifactMap"): artifact_map = context.task["payload"].get("artifactMap") manifest = None else: artifact_map = None manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk"), ] expected_destinations = [ [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk", ], ] expected_balrog_manifest = [] for complete_info in [ { "completeInfo": [ { "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", } ] }, { "blob_suffix": "-mozinfo", "completeInfo": [ { "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", } ], }, ]: entry = { "tc_nightly": True, "appName": "Fake", "appVersion": "99.0a1", "branch": "mozilla-central", "buildid": "20990205110000", "extVersion": "99.0a1", "hashType": "sha512", "locale": "en-US", "platform": "android-api-15", "url_replacements": [["http://archive.mozilla.org/pub", "http://download.cdn.mozilla.net/pub"]], } entry.update(complete_info) if partials: entry["partialInfo"] = [ { "from_buildid": 19991231235959, "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", } ] expected_balrog_manifest.append(entry) actual_sources = [] actual_destinations = [] def sort_manifest(manifest): manifest.sort(key=lambda entry: entry.get("blob_suffix", "")) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = {"hash": "dummyhash", "size": 123456, "url": destinations[0]} context.raw_balrog_manifest.setdefault(locale, {}) if from_buildid: if partials: data["from_buildid"] = from_buildid context.raw_balrog_manifest[locale].setdefault("partialInfo", []).append(data) else: return else: context.raw_balrog_manifest[locale].setdefault("completeInfo", {})[balrog_format] = data with mock.patch("beetmoverscript.script.move_beet", fake_move_beet): await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_manifest(context.balrog_manifest) sort_manifest(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest