def test_balrog_manifest_to_artifacts():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
fake_balrog_manifest = context.task["payload"]["releaseProperties"]
context.balrog_manifest = fake_balrog_manifest
# fake the path to to able to check the contents written later on
with tempfile.TemporaryDirectory() as tmpdirname:
context.config["artifact_dir"] = tmpdirname
file_path = os.path.join(context.config["artifact_dir"],
"public/manifest.json")
# <temp-dir>/public doesn't exist yet and it's not automatically
# being created so we need to ensure it exists
public_tmpdirname = os.path.join(tmpdirname, "public")
if not os.path.exists(public_tmpdirname):
os.makedirs(public_tmpdirname)
add_balrog_manifest_to_artifacts(context)
with open(file_path, "r") as fread:
retrieved_data = json.load(fread)
assert fake_balrog_manifest == retrieved_data
async def test_move_beets_raises(mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson="task_missing_installer.json")
context.release_props = context.task["payload"]["releaseProperties"]
context.release_props["stage_platform"] = context.release_props["platform"]
context.bucket = "nightly"
context.action = "push-to-nightly"
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
with pytest.raises(ScriptWorkerTaskException):
await move_beets(context, context.artifacts_to_beetmove, manifest=None, artifact_map=None)
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.checksums = dict()
context.balrog_manifest = list()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=True,
artifact_pretty_name=pretty_name))
assert expected_upload_args == actual_upload_args
for k in expected_balrog_manifest.keys():
assert (context.balrog_manifest[0]['completeInfo'][0][k] ==
expected_balrog_manifest[k])
async def test_move_beet(update_manifest, action):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task['extra'] = dict()
context.task['extra']['partials'] = [{
"artifact_name": "target-98.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "19991231235959",
"previousVersion": "98.0b96",
"previousBuildNumber": "1"
}, {
"artifact_name": "target-97.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "22423423402984",
"previousVersion": "97.0b96",
"previousBuildNumber": "1"
}]
context.action = action
context.bucket = 'nightly'
context.checksums = dict()
context.balrog_manifest = list()
context.raw_balrog_manifest = dict()
context.release_props = context.task['payload']['releaseProperties']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid=None)
assert expected_upload_args == actual_upload_args
if update_manifest:
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['completeInfo'][''][k]
== expected_balrog_manifest[k])
expected_balrog_manifest['from_buildid'] = '19991231235959'
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid='19991231235959')
if update_manifest:
if is_promotion_action(context.action):
expected_balrog_manifest['previousBuildNumber'] = '1'
expected_balrog_manifest['previousVersion'] = '98.0b96'
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] ==
expected_balrog_manifest[k])
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
async def test_move_beet(update_manifest, action):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task["extra"] = dict()
context.task["extra"]["partials"] = [
{
"artifact_name": "target-98.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "19991231235959",
"previousVersion": "98.0b96",
"previousBuildNumber": "1",
},
{
"artifact_name": "target-97.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "22423423402984",
"previousVersion": "97.0b96",
"previousBuildNumber": "1",
},
]
context.action = action
context.bucket = "nightly"
context.checksums = dict()
context.balrog_manifest = list()
context.raw_balrog_manifest = dict()
context.release_props = context.task["payload"]["releaseProperties"]
locale = "sample-locale"
target_source = "tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"
pretty_name = "fake-99.0a1.en-US.target.txt"
target_destinations = (
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
)
expected_upload_args = [
(
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
),
"tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt",
]
expected_balrog_manifest = {
"hash": "73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b",
"size": 18,
"url": "https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch("beetmoverscript.script.retry_upload", fake_retry_upload):
await move_beet(
context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format="",
artifact_pretty_name=pretty_name,
from_buildid=None,
)
assert expected_upload_args == actual_upload_args
if update_manifest:
for k in expected_balrog_manifest.keys():
assert context.raw_balrog_manifest[locale]["completeInfo"][""][k] == expected_balrog_manifest[k]
expected_balrog_manifest["from_buildid"] = "19991231235959"
with mock.patch("beetmoverscript.script.retry_upload", fake_retry_upload):
await move_beet(
context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format="",
artifact_pretty_name=pretty_name,
from_buildid="19991231235959",
)
if update_manifest:
if is_promotion_action(context.action):
expected_balrog_manifest["previousBuildNumber"] = "1"
expected_balrog_manifest["previousVersion"] = "98.0b96"
for k in expected_balrog_manifest.keys():
assert context.raw_balrog_manifest[locale]["partialInfo"][0][k] == expected_balrog_manifest[k]
async def test_move_beets(task_filename, partials, mocker):
mocker.patch("beetmoverscript.utils.JINJA_ENV", get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task["payload"]["releaseProperties"]
context.release_props["stage_platform"] = context.release_props["platform"]
context.bucket = "nightly"
context.action = "push-to-nightly"
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task["payload"].get("artifactMap"):
artifact_map = context.task["payload"].get("artifactMap")
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json"),
os.path.abspath("tests/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk"),
]
expected_destinations = [
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json",
],
[
"pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
"pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk",
],
]
expected_balrog_manifest = []
for complete_info in [
{
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt",
}
]
},
{
"blob_suffix": "-mozinfo",
"completeInfo": [
{
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json",
}
],
},
]:
entry = {
"tc_nightly": True,
"appName": "Fake",
"appVersion": "99.0a1",
"branch": "mozilla-central",
"buildid": "20990205110000",
"extVersion": "99.0a1",
"hashType": "sha512",
"locale": "en-US",
"platform": "android-api-15",
"url_replacements": [["http://archive.mozilla.org/pub", "http://download.cdn.mozilla.net/pub"]],
}
entry.update(complete_info)
if partials:
entry["partialInfo"] = [
{
"from_buildid": 19991231235959,
"hash": "dummyhash",
"size": 123456,
"url": "pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt",
}
]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get("blob_suffix", ""))
async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, balrog_format, artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {"hash": "dummyhash", "size": 123456, "url": destinations[0]}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault("partialInfo", []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault("completeInfo", {})[balrog_format] = data
with mock.patch("beetmoverscript.script.move_beet", fake_move_beet):
await move_beets(context, context.artifacts_to_beetmove, manifest=manifest, artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
