def test_validate_scopes():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context.config, context.task,
context.properties)
context.task['scopes'] = []
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:mightly"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:dep"]
manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
def test_beetmover_template_args_generation():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
expected_template_args = {
'branch': 'mozilla-central',
'platform': 'android-api-15',
'product': 'Fake',
'stage_platform': 'android-api-15',
'template_key': 'fennec_nightly',
'upload_date': '2016/09/2016-09-01-16-26-14',
'version': '99.0a1'
}
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args == expected_template_args
context.task['payload']['locale'] = 'ro'
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args['template_key'] == 'fake_nightly_repacks'
def test_beetmover_template_args_fennec_nightly(context):
"""Ensure that fennec which is en-US and multi don't get the repack template"""
context.task = get_fake_valid_task('task_fennec.json')
template_args = generate_beetmover_template_args(context)
assert 'locale' not in template_args
assert template_args['locales'] == ['en-US', 'multi']
assert template_args['template_key'] == 'fake_nightly'
def test_beetmover_template_args_generation(context, taskjson, partials):
context.task = get_fake_valid_task(taskjson)
expected_template_args = {
'branch': 'mozilla-central',
'filename_platform': 'android-arm',
'product': 'Fake',
'stage_platform': 'android-api-15',
'platform': 'android-api-15',
'template_key': 'fake_nightly',
'upload_date': '2016/09/2016-09-01-16-26-14',
'version': '99.0a1',
'buildid': '20990205110000',
'partials': partials,
'locales': ['en-US'],
}
template_args = generate_beetmover_template_args(context)
assert template_args == expected_template_args
context.task['payload']['locale'] = 'en-US'
context.task['payload']['upstreamArtifacts'][0]['locale'] = 'en-US'
expected_template_args['template_key'] = 'fake_nightly'
expected_template_args['locales'] = ['en-US']
template_args = generate_beetmover_template_args(context)
assert template_args == expected_template_args
def test_get_release_props(context, mocker, taskjson, locale, relprops, expected):
context.task = get_fake_valid_task(taskjson)
if locale:
context.task['payload']['locale'] = 'lang'
context.task['payload']['releaseProperties'] = relprops
assert get_release_props(context) == (expected, None)
# also check balrog_props method works with same data
# TODO remove the end of this function when method is not supported anymore
del context.task['payload']['releaseProperties']
context.task['payload']['upstreamArtifacts'] = [{
"locale": "lang",
"paths": [
"public/build/lang/balrog_props.json"
],
"taskId": "buildTaskId",
"taskType": "build"
}]
balrog_props_path = os.path.abspath(os.path.join(context.config['work_dir'], 'cot', 'buildTaskId', 'public/build/lang/balrog_props.json'))
makedirs(os.path.dirname(balrog_props_path))
with open(balrog_props_path, 'w') as f:
json.dump({
'properties': relprops
}, f)
assert get_release_props(context) == (expected, balrog_props_path)
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_candidates_manifest(context)
expected_sources = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package'
]
expected_destinations = [
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
),
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package'
)
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(move_beets(context, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_balrog_manifest_to_artifacts():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
fake_balrog_manifest = context.task['payload']['releaseProperties']
context.balrog_manifest = fake_balrog_manifest
# fake the path to to able to check the contents written later on
with tempfile.TemporaryDirectory() as tmpdirname:
context.config['artifact_dir'] = tmpdirname
file_path = os.path.join(context.config['artifact_dir'],
'public/manifest.json')
# <temp-dir>/public doesn't exist yet and it's not automatically
# being created so we need to ensure it exists
public_tmpdirname = os.path.join(tmpdirname, 'public')
if not os.path.exists(public_tmpdirname):
os.makedirs(public_tmpdirname)
add_balrog_manifest_to_artifacts(context)
with open(file_path, "r") as fread:
retrieved_data = json.load(fread)
assert fake_balrog_manifest == retrieved_data
def test_get_schema_key_by_action(scopes, expected):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task["scopes"] = scopes
assert expected == get_schema_key_by_action(context)
def test_get_initial_release_props_file():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.task['payload']['upstreamArtifacts'] = [{'paths': []}]
with pytest.raises(ScriptWorkerTaskException):
get_initial_release_props_file(context)
def test_get_release_props(context, mocker, taskjson, locale, relprops,
expected):
context.task = get_fake_valid_task(taskjson)
if locale:
context.task['payload']['locale'] = 'lang'
context.task['payload']['releaseProperties'] = relprops
assert get_release_props(context) == expected
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_upstream_artifacts(expected, preserve):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=preserve)
assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_get_upstream_artifacts(expected, preserve):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task['payload']['releaseProperties']
artifacts_to_beetmove = get_upstream_artifacts(
context, preserve_full_paths=preserve)
assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_write_json():
sample_data = get_fake_valid_task()['payload']['releaseProperties']
with tempfile.NamedTemporaryFile(delete=True) as fp:
write_json(fp.name, sample_data)
with open(fp.name, "r") as fread:
retrieved_data = json.load(fread)
assert sample_data == retrieved_data
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task['payload']['releaseProperties']
context.task['payload']['upstreamArtifacts'][0]['paths'].append(
'fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_beetmover_template_args_locales(context, payload, expected_locales):
context.task = get_fake_valid_task('task_partials.json')
context.task['payload'] = payload
context.task['payload']['upload_date'] = '2018/04/2018-04-09-15-30-00'
template_args = generate_beetmover_template_args(context)
if expected_locales:
assert 'locale' not in template_args # locale used to be the old way of filling locale
assert template_args['locales'] == expected_locales
else:
assert 'locale' not in template_args
assert 'locales' not in template_args
def test_get_release_props(context, mocker, taskjson, locale, relprops,
expected):
context.task = get_fake_valid_task(taskjson)
if locale:
context.task['payload']['locale'] = 'lang'
context.task['payload']['releaseProperties'] = relprops
assert get_release_props(context) == expected
context.task['payload']['releaseProperties'] = None
with pytest.raises(ScriptWorkerTaskException):
get_release_props(context)
def test_extract_file_config_from_artifact_map():
task_def = get_fake_valid_task(taskjson='task_artifact_map.json')
task_id = "eSzfNqMZT_mSiQQXu8hyqg"
locale = "en-US"
filename = "target.txt"
expected = {
"destinations": [
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt"
],
"checksums_path": "",
"from_buildid": 19991231235959,
"update_balrog_manifest": True
}
assert extract_file_config_from_artifact_map(
task_def['payload']['artifactMap'], filename, task_id, locale) == expected
def test_checksums_manifest_generation():
checksums = {
"firefox-53.0a1.en-US.linux-i686.complete.mar": {
"sha512": "14f2d1cb999a8b42a3b6b671f7376c3e246daa65d108e2b8fe880f069601dc2b26afa155b52001235db059",
"size": 618149,
"sha256": "293975734953874539475"
}
}
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.checksums = checksums
expected_checksums_manifest_dump = get_fake_checksums_manifest()
checksums_manifest_dump = generate_checksums_manifest(context)
assert checksums_manifest_dump == expected_checksums_manifest_dump
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_beetmover_manifest(context.config, context.task, context.properties)
mapping = manifest['mapping']
s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt']
assert (
manifest.get('s3_prefix_dated') ==
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and
manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/'
)
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.checksums = dict()
context.balrog_manifest = list()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=True,
artifact_pretty_name=pretty_name))
assert expected_upload_args == actual_upload_args
for k in expected_balrog_manifest.keys():
assert (context.balrog_manifest[0]['completeInfo'][0][k] ==
expected_balrog_manifest[k])
async def test_move_beets_raises(mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson='task_missing_installer.json')
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
with pytest.raises(ScriptWorkerTaskException):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=None,
artifact_map=None)
async def test_async_main(context, mocker, action, raises, task_filename):
context.action = action
context.task = get_fake_valid_task(taskjson=task_filename)
def fake_action(*args):
return action
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
mocker.patch('beetmoverscript.script.move_beets', new=noop_async)
mocker.patch.object(beetmoverscript.script,
'get_task_action',
new=fake_action)
if raises:
with pytest.raises(SystemExit):
await async_main(context)
else:
await async_main(context)
for module in ("botocore", "boto3", "chardet"):
assert logging.getLogger(module).level == logging.INFO
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
locale = "sample-locale"
target_source = 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
)
expected_download_args = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'beetmoverscript/test/test_work_dir/public/build/target.package'
]
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
), 'beetmoverscript/test/test_work_dir/public/build/target.package']
actual_download_args = []
actual_upload_args = []
async def fake_retry_download(context, url, path):
actual_download_args.extend([url, path])
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_download',
fake_retry_download):
with mock.patch('beetmoverscript.script.retry_upload',
fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=False))
assert sorted(expected_download_args) == sorted(actual_download_args)
assert expected_upload_args == actual_upload_args
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
def test_extract_file_config_from_artifact_map_raises(task_id, locale, filename):
task_def = get_fake_valid_task(taskjson='task_artifact_map.json')
with pytest.raises(TaskVerificationError):
extract_file_config_from_artifact_map(
task_def['payload']['artifactMap'], filename, task_id, locale)
async def test_move_beet(update_manifest, action):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task['extra'] = dict()
context.task['extra']['partials'] = [{
"artifact_name": "target-98.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "19991231235959",
"previousVersion": "98.0b96",
"previousBuildNumber": "1"
}, {
"artifact_name": "target-97.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "22423423402984",
"previousVersion": "97.0b96",
"previousBuildNumber": "1"
}]
context.action = action
context.bucket = 'nightly'
context.checksums = dict()
context.balrog_manifest = list()
context.raw_balrog_manifest = dict()
context.release_props = context.task['payload']['releaseProperties']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid=None)
assert expected_upload_args == actual_upload_args
if update_manifest:
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['completeInfo'][''][k]
== expected_balrog_manifest[k])
expected_balrog_manifest['from_buildid'] = '19991231235959'
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid='19991231235959')
if update_manifest:
if is_promotion_action(context.action):
expected_balrog_manifest['previousBuildNumber'] = '1'
expected_balrog_manifest['previousVersion'] = '98.0b96'
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] ==
expected_balrog_manifest[k])
def test_extract_full_artifact_map_path(path, locale, found):
task_def = get_fake_valid_task(taskjson='task_artifact_map.json')
assert extract_full_artifact_map_path(task_def['payload']['artifactMap'], path, locale) == found
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
