def test_validate_scopes(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context.config, context.task, context.properties) context.task['scopes'] = [] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:mightly"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:dep"] manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.." manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.." with pytest.raises(SystemExit): validate_task_scopes(context, manifest)
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_exception_get_upstream_artifacts(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = context.task["payload"]["releaseProperties"] context.task["payload"]["upstreamArtifacts"][0]["paths"].append( "fake_file") with pytest.raises(ScriptWorkerTaskException): context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_exception_get_upstream_artifacts(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file') with pytest.raises(ScriptWorkerTaskException): context.artifacts_to_beetmove = get_upstream_artifacts(context)
async def test_move_beets_raises(mocker): mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson="task_missing_installer.json") context.release_props = context.task["payload"]["releaseProperties"] context.release_props["stage_platform"] = context.release_props["platform"] context.bucket = "nightly" context.action = "push-to-nightly" context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) with pytest.raises(ScriptWorkerTaskException): await move_beets(context, context.artifacts_to_beetmove, manifest=None, artifact_map=None)
async def test_move_beets(task_filename, partials, mocker): mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson=task_filename) context.release_props = context.task['payload']['releaseProperties'] context.release_props['stage_platform'] = context.release_props['platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) if context.task['payload'].get('artifactMap'): artifact_map = context.task['payload'].get('artifactMap') manifest = None else: artifact_map = None manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk' ) ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk' ] ] expected_balrog_manifest = [] for complete_info in [ { 'completeInfo': [{ 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' }], }, { 'blob_suffix': '-mozinfo', 'completeInfo': [ { 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' }, ], }, ]: entry = { 'tc_nightly': True, 'appName': 'Fake', 'appVersion': '99.0a1', 'branch': 'mozilla-central', 'buildid': '20990205110000', 'extVersion': '99.0a1', 'hashType': 'sha512', 'locale': 'en-US', 'platform': 'android-api-15', 'url_replacements': [[ 'http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub' ]], } entry.update(complete_info) if partials: entry['partialInfo'] = [{ 'from_buildid': 19991231235959, 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' }] expected_balrog_manifest.append(entry) actual_sources = [] actual_destinations = [] def sort_manifest(manifest): manifest.sort(key=lambda entry: entry.get('blob_suffix', '')) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = { "hash": 'dummyhash', "size": 123456, "url": destinations[0] } context.raw_balrog_manifest.setdefault(locale, {}) if from_buildid: if partials: data["from_buildid"] = from_buildid context.raw_balrog_manifest[locale].setdefault( 'partialInfo', []).append(data) else: return else: context.raw_balrog_manifest[locale].setdefault( 'completeInfo', {})[balrog_format] = data with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_manifest(context.balrog_manifest) sort_manifest(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest
async def test_move_beets(task_filename, partials, mocker): mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson=task_filename) context.release_props = context.task["payload"]["releaseProperties"] context.release_props["stage_platform"] = context.release_props["platform"] context.bucket = "nightly" context.action = "push-to-nightly" context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) if context.task["payload"].get("artifactMap"): artifact_map = context.task["payload"].get("artifactMap") manifest = None else: artifact_map = None manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json"), os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk"), ] expected_destinations = [ [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json", ], [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk", ], ] expected_balrog_manifest = [] for complete_info in [ { "completeInfo": [ { "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt", } ] }, { "blob_suffix": "-mozinfo", "completeInfo": [ { "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json", } ], }, ]: entry = { "tc_nightly": True, "appName": "Fake", "appVersion": "99.0a1", "branch": "mozilla-central", "buildid": "20990205110000", "extVersion": "99.0a1", "hashType": "sha512", "locale": "en-US", "platform": "android-api-15", "url_replacements": [["http://archive.mozilla.org/pub", "http://download.cdn.mozilla.net/pub"]], } entry.update(complete_info) if partials: entry["partialInfo"] = [ { "from_buildid": 19991231235959, "hash": "dummyhash", "size": 123456, "url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", } ] expected_balrog_manifest.append(entry) actual_sources = [] actual_destinations = [] def sort_manifest(manifest): manifest.sort(key=lambda entry: entry.get("blob_suffix", "")) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = {"hash": "dummyhash", "size": 123456, "url": destinations[0]} context.raw_balrog_manifest.setdefault(locale, {}) if from_buildid: if partials: data["from_buildid"] = from_buildid context.raw_balrog_manifest[locale].setdefault("partialInfo", []).append(data) else: return else: context.raw_balrog_manifest[locale].setdefault("completeInfo", {})[balrog_format] = data with mock.patch("beetmoverscript.script.move_beet", fake_move_beet): await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_manifest(context.balrog_manifest) sort_manifest(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest