def test_validate_scopes(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context.config, context.task, context.properties) context.task['scopes'] = [] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:mightly"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:dep"] manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.." manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.." with pytest.raises(SystemExit): validate_task_scopes(context, manifest)
def test_beetmover_template_args_generation(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] expected_template_args = { 'branch': 'mozilla-central', 'platform': 'android-api-15', 'product': 'Fake', 'stage_platform': 'android-api-15', 'template_key': 'fennec_nightly', 'upload_date': '2016/09/2016-09-01-16-26-14', 'version': '99.0a1' } template_args = generate_beetmover_template_args(context.task, context.properties) assert template_args == expected_template_args context.task['payload']['locale'] = 'ro' template_args = generate_beetmover_template_args(context.task, context.properties) assert template_args['template_key'] == 'fake_nightly_repacks'
def test_beetmover_template_args_fennec_nightly(context): """Ensure that fennec which is en-US and multi don't get the repack template""" context.task = get_fake_valid_task('task_fennec.json') template_args = generate_beetmover_template_args(context) assert 'locale' not in template_args assert template_args['locales'] == ['en-US', 'multi'] assert template_args['template_key'] == 'fake_nightly'
def test_beetmover_template_args_generation(context, taskjson, partials): context.task = get_fake_valid_task(taskjson) expected_template_args = { 'branch': 'mozilla-central', 'filename_platform': 'android-arm', 'product': 'Fake', 'stage_platform': 'android-api-15', 'platform': 'android-api-15', 'template_key': 'fake_nightly', 'upload_date': '2016/09/2016-09-01-16-26-14', 'version': '99.0a1', 'buildid': '20990205110000', 'partials': partials, 'locales': ['en-US'], } template_args = generate_beetmover_template_args(context) assert template_args == expected_template_args context.task['payload']['locale'] = 'en-US' context.task['payload']['upstreamArtifacts'][0]['locale'] = 'en-US' expected_template_args['template_key'] = 'fake_nightly' expected_template_args['locales'] = ['en-US'] template_args = generate_beetmover_template_args(context) assert template_args == expected_template_args
def test_get_release_props(context, mocker, taskjson, locale, relprops, expected): context.task = get_fake_valid_task(taskjson) if locale: context.task['payload']['locale'] = 'lang' context.task['payload']['releaseProperties'] = relprops assert get_release_props(context) == (expected, None) # also check balrog_props method works with same data # TODO remove the end of this function when method is not supported anymore del context.task['payload']['releaseProperties'] context.task['payload']['upstreamArtifacts'] = [{ "locale": "lang", "paths": [ "public/build/lang/balrog_props.json" ], "taskId": "buildTaskId", "taskType": "build" }] balrog_props_path = os.path.abspath(os.path.join(context.config['work_dir'], 'cot', 'buildTaskId', 'public/build/lang/balrog_props.json')) makedirs(os.path.dirname(balrog_props_path)) with open(balrog_props_path, 'w') as f: json.dump({ 'properties': relprops }, f) assert get_release_props(context) == (expected, balrog_props_path)
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] manifest = generate_candidates_manifest(context) expected_sources = [ 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package', 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package' ] expected_destinations = [ ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ), ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package' ) ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete(move_beets(context, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_balrog_manifest_to_artifacts(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() fake_balrog_manifest = context.task['payload']['releaseProperties'] context.balrog_manifest = fake_balrog_manifest # fake the path to to able to check the contents written later on with tempfile.TemporaryDirectory() as tmpdirname: context.config['artifact_dir'] = tmpdirname file_path = os.path.join(context.config['artifact_dir'], 'public/manifest.json') # <temp-dir>/public doesn't exist yet and it's not automatically # being created so we need to ensure it exists public_tmpdirname = os.path.join(tmpdirname, 'public') if not os.path.exists(public_tmpdirname): os.makedirs(public_tmpdirname) add_balrog_manifest_to_artifacts(context) with open(file_path, "r") as fread: retrieved_data = json.load(fread) assert fake_balrog_manifest == retrieved_data
def test_get_schema_key_by_action(scopes, expected): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.task["scopes"] = scopes assert expected == get_schema_key_by_action(context)
def test_get_initial_release_props_file(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.task['payload']['upstreamArtifacts'] = [{'paths': []}] with pytest.raises(ScriptWorkerTaskException): get_initial_release_props_file(context)
def test_get_release_props(context, mocker, taskjson, locale, relprops, expected): context.task = get_fake_valid_task(taskjson) if locale: context.task['payload']['locale'] = 'lang' context.task['payload']['releaseProperties'] = relprops assert get_release_props(context) == expected
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_upstream_artifacts(expected, preserve): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=preserve) assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_get_upstream_artifacts(expected, preserve): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = context.task['payload']['releaseProperties'] artifacts_to_beetmove = get_upstream_artifacts( context, preserve_full_paths=preserve) assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_write_json(): sample_data = get_fake_valid_task()['payload']['releaseProperties'] with tempfile.NamedTemporaryFile(delete=True) as fp: write_json(fp.name, sample_data) with open(fp.name, "r") as fread: retrieved_data = json.load(fread) assert sample_data == retrieved_data
def test_exception_get_upstream_artifacts(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file') with pytest.raises(ScriptWorkerTaskException): context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_exception_get_upstream_artifacts(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = context.task['payload']['releaseProperties'] context.task['payload']['upstreamArtifacts'][0]['paths'].append( 'fake_file') with pytest.raises(ScriptWorkerTaskException): context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_beetmover_template_args_locales(context, payload, expected_locales): context.task = get_fake_valid_task('task_partials.json') context.task['payload'] = payload context.task['payload']['upload_date'] = '2018/04/2018-04-09-15-30-00' template_args = generate_beetmover_template_args(context) if expected_locales: assert 'locale' not in template_args # locale used to be the old way of filling locale assert template_args['locales'] == expected_locales else: assert 'locale' not in template_args assert 'locales' not in template_args
def test_get_release_props(context, mocker, taskjson, locale, relprops, expected): context.task = get_fake_valid_task(taskjson) if locale: context.task['payload']['locale'] = 'lang' context.task['payload']['releaseProperties'] = relprops assert get_release_props(context) == expected context.task['payload']['releaseProperties'] = None with pytest.raises(ScriptWorkerTaskException): get_release_props(context)
def test_extract_file_config_from_artifact_map(): task_def = get_fake_valid_task(taskjson='task_artifact_map.json') task_id = "eSzfNqMZT_mSiQQXu8hyqg" locale = "en-US" filename = "target.txt" expected = { "destinations": [ "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt", "pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt" ], "checksums_path": "", "from_buildid": 19991231235959, "update_balrog_manifest": True } assert extract_file_config_from_artifact_map( task_def['payload']['artifactMap'], filename, task_id, locale) == expected
def test_checksums_manifest_generation(): checksums = { "firefox-53.0a1.en-US.linux-i686.complete.mar": { "sha512": "14f2d1cb999a8b42a3b6b671f7376c3e246daa65d108e2b8fe880f069601dc2b26afa155b52001235db059", "size": 618149, "sha256": "293975734953874539475" } } context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.checksums = checksums expected_checksums_manifest_dump = get_fake_checksums_manifest() checksums_manifest_dump = generate_checksums_manifest(context) assert checksums_manifest_dump == expected_checksums_manifest_dump
def test_generate_manifest(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] manifest = generate_beetmover_manifest(context.config, context.task, context.properties) mapping = manifest['mapping'] s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping] assert sorted(mapping.keys()) == ['en-US', 'multi'] assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt', 'fake-99.0a1.multi.target_info.txt'] assert ( manifest.get('s3_prefix_dated') == 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/' )
def test_move_beet(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.checksums = dict() context.balrog_manifest = list() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=True, artifact_pretty_name=pretty_name)) assert expected_upload_args == actual_upload_args for k in expected_balrog_manifest.keys(): assert (context.balrog_manifest[0]['completeInfo'][0][k] == expected_balrog_manifest[k])
async def test_move_beets_raises(mocker): mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson='task_missing_installer.json') context.release_props = context.task['payload']['releaseProperties'] context.release_props['stage_platform'] = context.release_props['platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) with pytest.raises(ScriptWorkerTaskException): await move_beets(context, context.artifacts_to_beetmove, manifest=None, artifact_map=None)
async def test_async_main(context, mocker, action, raises, task_filename): context.action = action context.task = get_fake_valid_task(taskjson=task_filename) def fake_action(*args): return action mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) mocker.patch('beetmoverscript.script.move_beets', new=noop_async) mocker.patch.object(beetmoverscript.script, 'get_task_action', new=fake_action) if raises: with pytest.raises(SystemExit): await async_main(context) else: await async_main(context) for module in ("botocore", "boto3", "chardet"): assert logging.getLogger(module).level == logging.INFO
def test_move_beet(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() locale = "sample-locale" target_source = 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ) expected_download_args = [ 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package', 'beetmoverscript/test/test_work_dir/public/build/target.package' ] expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ), 'beetmoverscript/test/test_work_dir/public/build/target.package'] actual_download_args = [] actual_upload_args = [] async def fake_retry_download(context, url, path): actual_download_args.extend([url, path]) async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_download', fake_retry_download): with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=False)) assert sorted(expected_download_args) == sorted(actual_download_args) assert expected_upload_args == actual_upload_args
def test_generate_manifest(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' manifest = generate_beetmover_manifest(context) mapping = manifest['mapping'] s3_keys = [ mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping ] assert sorted(mapping.keys()) == ['en-US', 'multi'] assert sorted(s3_keys) == [ 'fake-99.0a1.en-US.target_info.txt', 'fake-99.0a1.multi.target_info.txt' ] expected_destinations = { 'en-US': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], 'multi': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt', 'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt' ] } actual_destinations = { k: mapping[k]['target_info.txt']['destinations'] for k in sorted(mapping.keys()) } assert expected_destinations == actual_destinations
def test_extract_file_config_from_artifact_map_raises(task_id, locale, filename): task_def = get_fake_valid_task(taskjson='task_artifact_map.json') with pytest.raises(TaskVerificationError): extract_file_config_from_artifact_map( task_def['payload']['artifactMap'], filename, task_id, locale)
async def test_move_beet(update_manifest, action): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.task['extra'] = dict() context.task['extra']['partials'] = [{ "artifact_name": "target-98.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "19991231235959", "previousVersion": "98.0b96", "previousBuildNumber": "1" }, { "artifact_name": "target-97.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "22423423402984", "previousVersion": "97.0b96", "previousBuildNumber": "1" }] context.action = action context.bucket = 'nightly' context.checksums = dict() context.balrog_manifest = list() context.raw_balrog_manifest = dict() context.release_props = context.task['payload']['releaseProperties'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): await move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format='', artifact_pretty_name=pretty_name, from_buildid=None) assert expected_upload_args == actual_upload_args if update_manifest: for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['completeInfo'][''][k] == expected_balrog_manifest[k]) expected_balrog_manifest['from_buildid'] = '19991231235959' with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): await move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, balrog_format='', artifact_pretty_name=pretty_name, from_buildid='19991231235959') if update_manifest: if is_promotion_action(context.action): expected_balrog_manifest['previousBuildNumber'] = '1' expected_balrog_manifest['previousVersion'] = '98.0b96' for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] == expected_balrog_manifest[k])
def test_extract_full_artifact_map_path(path, locale, found): task_def = get_fake_valid_task(taskjson='task_artifact_map.json') assert extract_full_artifact_map_path(task_def['payload']['artifactMap'], path, locale) == found
async def test_move_beets(task_filename, partials, mocker): mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task(taskjson=task_filename) context.release_props = context.task['payload']['releaseProperties'] context.release_props['stage_platform'] = context.release_props['platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) if context.task['payload'].get('artifactMap'): artifact_map = context.task['payload'].get('artifactMap') manifest = None else: artifact_map = None manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk' ) ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk' ] ] expected_balrog_manifest = [] for complete_info in [ { 'completeInfo': [{ 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' }], }, { 'blob_suffix': '-mozinfo', 'completeInfo': [ { 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' }, ], }, ]: entry = { 'tc_nightly': True, 'appName': 'Fake', 'appVersion': '99.0a1', 'branch': 'mozilla-central', 'buildid': '20990205110000', 'extVersion': '99.0a1', 'hashType': 'sha512', 'locale': 'en-US', 'platform': 'android-api-15', 'url_replacements': [[ 'http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub' ]], } entry.update(complete_info) if partials: entry['partialInfo'] = [{ 'from_buildid': 19991231235959, 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' }] expected_balrog_manifest.append(entry) actual_sources = [] actual_destinations = [] def sort_manifest(manifest): manifest.sort(key=lambda entry: entry.get('blob_suffix', '')) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = { "hash": 'dummyhash', "size": 123456, "url": destinations[0] } context.raw_balrog_manifest.setdefault(locale, {}) if from_buildid: if partials: data["from_buildid"] = from_buildid context.raw_balrog_manifest[locale].setdefault( 'partialInfo', []).append(data) else: return else: context.raw_balrog_manifest[locale].setdefault( 'completeInfo', {})[balrog_format] = data with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_manifest(context.balrog_manifest) sort_manifest(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest