def test_validate_scopes():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context.config, context.task,
context.properties)
context.task['scopes'] = []
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:mightly"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:dep"]
manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task["payload"]["releaseProperties"]
context.task["payload"]["upstreamArtifacts"][0]["paths"].append(
"fake_file")
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
async def test_move_beets_raises(mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson="task_missing_installer.json")
context.release_props = context.task["payload"]["releaseProperties"]
context.release_props["stage_platform"] = context.release_props["platform"]
context.bucket = "nightly"
context.action = "push-to-nightly"
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
with pytest.raises(ScriptWorkerTaskException):
await move_beets(context, context.artifacts_to_beetmove, manifest=None, artifact_map=None)
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
async def test_move_beets(task_filename, partials, mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task["payload"]["releaseProperties"]
context.release_props["stage_platform"] = context.release_props["platform"]
context.bucket = "nightly"
context.action = "push-to-nightly"
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task["payload"].get("artifactMap"):
artifact_map = context.task["payload"].get("artifactMap")
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk"),
]
expected_destinations = [
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
],
]
expected_balrog_manifest = []
for complete_info in [
{
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
}
]
},
{
"blob_suffix": "-mozinfo",
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
}
],
},
]:
entry = {
"tc_nightly": True,
"appName": "Fake",
"appVersion": "99.0a1",
"branch": "mozilla-central",
"buildid": "20990205110000",
"extVersion": "99.0a1",
"hashType": "sha512",
"locale": "en-US",
"platform": "android-api-15",
"url_replacements": [["http://archive.mozilla.org/pub", "http://download.cdn.mozilla.net/pub"]],
}
entry.update(complete_info)
if partials:
entry["partialInfo"] = [
{
"from_buildid": 19991231235959,
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
}
]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get("blob_suffix", ""))
async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {"hash": "dummyhash", "size": 123456, "url": destinations[0]}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault("partialInfo", []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault("completeInfo", {})[balrog_format] = data
with mock.patch("beetmoverscript.script.move_beet", fake_move_beet):
await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
