def test_balrog_manifest_to_artifacts():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
fake_balrog_manifest = get_fake_balrog_props()
context.balrog_manifest = fake_balrog_manifest
# fake the path to to able to check the contents written later on
with tempfile.TemporaryDirectory() as tmpdirname:
context.config['artifact_dir'] = tmpdirname
file_path = os.path.join(context.config['artifact_dir'],
'public/manifest.json')
# <temp-dir>/public doesn't exist yet and it's not automatically
# being created so we need to ensure it exists
public_tmpdirname = os.path.join(tmpdirname, 'public')
if not os.path.exists(public_tmpdirname):
os.makedirs(public_tmpdirname)
add_balrog_manifest_to_artifacts(context)
with open(file_path, "r") as fread:
retrieved_data = json.load(fread)
assert fake_balrog_manifest == retrieved_data
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_candidates_manifest(context)
expected_sources = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package'
]
expected_destinations = [
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
),
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package'
)
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(move_beets(context, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_beetmover_template_args_generation():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
expected_template_args = {
'branch': 'mozilla-central',
'platform': 'android-api-15',
'product': 'Fake',
'stage_platform': 'android-api-15',
'template_key': 'fennec_nightly',
'upload_date': '2016/09/2016-09-01-16-26-14',
'version': '99.0a1'
}
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args == expected_template_args
context.task['payload']['locale'] = 'ro'
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args['template_key'] == 'fake_nightly_repacks'
def test_validate_scopes():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context.config, context.task,
context.properties)
context.task['scopes'] = []
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:mightly"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:dep"]
manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_upstream_artifacts(expected, preserve):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=preserve)
assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_write_json():
sample_data = get_fake_balrog_props()
with tempfile.NamedTemporaryFile(delete=True) as fp:
write_json(fp.name, sample_data)
with open(fp.name, "r") as fread:
retrieved_data = json.load(fread)
assert sample_data == retrieved_data
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.checksums = dict()
context.balrog_manifest = list()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=True,
artifact_pretty_name=pretty_name))
assert expected_upload_args == actual_upload_args
for k in expected_balrog_manifest.keys():
assert (context.balrog_manifest[0]['completeInfo'][0][k] ==
expected_balrog_manifest[k])
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_beetmover_manifest(context.config, context.task, context.properties)
mapping = manifest['mapping']
s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt']
assert (
manifest.get('s3_prefix_dated') ==
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and
manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/'
)
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
async def get_fake_props(context):
return get_fake_balrog_props()['properties']
