def test_balrog_manifest_to_artifacts(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() fake_balrog_manifest = get_fake_balrog_props() context.balrog_manifest = fake_balrog_manifest # fake the path to to able to check the contents written later on with tempfile.TemporaryDirectory() as tmpdirname: context.config['artifact_dir'] = tmpdirname file_path = os.path.join(context.config['artifact_dir'], 'public/manifest.json') # <temp-dir>/public doesn't exist yet and it's not automatically # being created so we need to ensure it exists public_tmpdirname = os.path.join(tmpdirname, 'public') if not os.path.exists(public_tmpdirname): os.makedirs(public_tmpdirname) add_balrog_manifest_to_artifacts(context) with open(file_path, "r") as fread: retrieved_data = json.load(fread) assert fake_balrog_manifest == retrieved_data
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] manifest = generate_candidates_manifest(context) expected_sources = [ 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package', 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package' ] expected_destinations = [ ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ), ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package' ) ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete(move_beets(context, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_beetmover_template_args_generation(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] expected_template_args = { 'branch': 'mozilla-central', 'platform': 'android-api-15', 'product': 'Fake', 'stage_platform': 'android-api-15', 'template_key': 'fennec_nightly', 'upload_date': '2016/09/2016-09-01-16-26-14', 'version': '99.0a1' } template_args = generate_beetmover_template_args(context.task, context.properties) assert template_args == expected_template_args context.task['payload']['locale'] = 'ro' template_args = generate_beetmover_template_args(context.task, context.properties) assert template_args['template_key'] == 'fake_nightly_repacks'
def test_validate_scopes(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context.config, context.task, context.properties) context.task['scopes'] = [] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:mightly"] with pytest.raises(SystemExit): validate_task_scopes(context, manifest) context.task['scopes'] = ["project:releng:beetmover:dep"] manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.." manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.." with pytest.raises(SystemExit): validate_task_scopes(context, manifest)
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_upstream_artifacts(expected, preserve): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=preserve) assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_write_json(): sample_data = get_fake_balrog_props() with tempfile.NamedTemporaryFile(delete=True) as fp: write_json(fp.name, sample_data) with open(fp.name, "r") as fread: retrieved_data = json.load(fread) assert sample_data == retrieved_data
def test_exception_get_upstream_artifacts(): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file') with pytest.raises(ScriptWorkerTaskException): context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_move_beet(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.checksums = dict() context.balrog_manifest = list() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=True, artifact_pretty_name=pretty_name)) assert expected_upload_args == actual_upload_args for k in expected_balrog_manifest.keys(): assert (context.balrog_manifest[0]['completeInfo'][0][k] == expected_balrog_manifest[k])
def test_generate_manifest(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] manifest = generate_beetmover_manifest(context.config, context.task, context.properties) mapping = manifest['mapping'] s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping] assert sorted(mapping.keys()) == ['en-US', 'multi'] assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt', 'fake-99.0a1.multi.target_info.txt'] assert ( manifest.get('s3_prefix_dated') == 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/' )
def test_generate_manifest(): context = Context() context.task = get_fake_valid_task() context.config = get_fake_valid_config() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' manifest = generate_beetmover_manifest(context) mapping = manifest['mapping'] s3_keys = [ mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping ] assert sorted(mapping.keys()) == ['en-US', 'multi'] assert sorted(s3_keys) == [ 'fake-99.0a1.en-US.target_info.txt', 'fake-99.0a1.multi.target_info.txt' ] expected_destinations = { 'en-US': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], 'multi': [ '2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt', 'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt' ] } actual_destinations = { k: mapping[k]['target_info.txt']['destinations'] for k in sorted(mapping.keys()) } assert expected_destinations == actual_destinations
async def get_fake_props(context): return get_fake_balrog_props()['properties']