def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_candidates_manifest(context)
expected_sources = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package'
]
expected_destinations = [
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
),
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package'
)
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(move_beets(context, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_do_failed_with_TimeoutError_api_call(
context, mocker, event_loop, fake_TimeoutError_throwing_session):
context.server = get_task_server(context.task, context.config)
context.action = get_task_action(context.task, context.config)
context.session = fake_TimeoutError_throwing_session
with pytest.raises(aiohttp.ServerTimeoutError):
event_loop.run_until_complete(_do_api_call(context, 'dummy', {}))
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_put_failure(event_loop, fake_session_500):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session_500
with pytest.raises(ScriptWorkerRetryException):
event_loop.run_until_complete(
put(context, url=URL('https://foo.com/packages/fake.package'), headers={},
abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session_500)
)
def test_async_main(event_loop):
context = Context()
context.config = get_fake_valid_config()
async def fake_move_beets(context, artifacts_to_beetmove, manifest):
pass
with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets):
event_loop.run_until_complete(async_main(context))
def test_async_main(event_loop):
context = Context()
context.config = get_fake_valid_config()
async def fake_move_beets(context, manifest):
pass
async def get_fake_props(context):
return get_fake_balrog_props()['properties']
with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets):
with mock.patch('beetmoverscript.script.get_props',
new=get_fake_props):
event_loop.run_until_complete(async_main(context))
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.checksums = dict()
context.balrog_manifest = list()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=True,
artifact_pretty_name=pretty_name))
assert expected_upload_args == actual_upload_args
for k in expected_balrog_manifest.keys():
assert (context.balrog_manifest[0]['completeInfo'][0][k] ==
expected_balrog_manifest[k])
def test_do_failed_api_call(context, mocker, event_loop, fake_session_500):
context.server = get_task_server(context.task, context.config)
context.action = get_task_action(context.task, context.config)
context.session = fake_session_500
response = event_loop.run_until_complete(_do_api_call(
context, 'dummy', {}))
assert response == '{}'
def test_put_success(event_loop, fake_session):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session
response = event_loop.run_until_complete(
put(context, url=URL('https://foo.com/packages/fake.package'), headers={},
abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session)
)
assert response.status == 200
assert response.resp == [b'asdf', b'asdf']
assert response.content.url == "https://foo.com/packages/fake.package"
def test_do_successful_api_call(context, mocker, event_loop, fake_session,
data, credentials):
context.server = get_task_server(context.task, context.config)
context.action = get_task_action(context.task, context.config)
context.session = fake_session
if not credentials:
del context.config["bouncer_config"][context.server]["username"]
del context.config["bouncer_config"][context.server]["password"]
with pytest.raises(KeyError):
response = event_loop.run_until_complete(
_do_api_call(context, 'dummy', data))
return
response = event_loop.run_until_complete(
_do_api_call(context, 'dummy', data))
assert response == '{}'
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
locale = "sample-locale"
target_source = 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
)
expected_download_args = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'beetmoverscript/test/test_work_dir/public/build/target.package'
]
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
), 'beetmoverscript/test/test_work_dir/public/build/target.package']
actual_download_args = []
actual_upload_args = []
async def fake_retry_download(context, url, path):
actual_download_args.extend([url, path])
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_download',
fake_retry_download):
with mock.patch('beetmoverscript.script.retry_upload',
fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=False))
assert sorted(expected_download_args) == sorted(actual_download_args)
assert expected_upload_args == actual_upload_args
def test_download(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session
url = 'https://fake.com'
path = '/fake/path'
async def fake_download(context, url, path, session):
return context, url, path, session
# just make sure retry_download ends up calling scriptworker's download_file and passes the
# right args, kwargs
with mock.patch('beetmoverscript.script.download_file', fake_download):
result = event_loop.run_until_complete(
retry_download(context, url, path))
assert result == (context, url, path, context.session)
def test_move_beet(event_loop, update_manifest, action):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task['extra'] = dict()
context.task['extra']['partials'] = [{
"artifact_name": "target-98.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "19991231235959",
"previousVersion": "98.0b96",
"previousBuildNumber": "1"
}, {
"artifact_name": "target-97.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "22423423402984",
"previousVersion": "97.0b96",
"previousBuildNumber": "1"
}]
context.action = action
context.bucket = 'nightly'
context.checksums = dict()
context.balrog_manifest = list()
context.raw_balrog_manifest = dict()
context.release_props = context.task['payload']['releaseProperties']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
artifact_pretty_name=pretty_name,
from_buildid=None))
assert expected_upload_args == actual_upload_args
if update_manifest:
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['completeInfo'][0][k]
== expected_balrog_manifest[k])
expected_balrog_manifest['from_buildid'] = '19991231235959'
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
artifact_pretty_name=pretty_name,
from_buildid='19991231235959'))
if update_manifest:
if is_promotion_action(context.action):
expected_balrog_manifest['previousBuildNumber'] = '1'
expected_balrog_manifest['previousVersion'] = '98.0b96'
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] ==
expected_balrog_manifest[k])
def test_move_beets(event_loop, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake 99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = [{
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
}, {
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
}]
if partials:
expected_balrog_manifest[0]['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
actual_sources = []
actual_destinations = []
def sort_completeInfo(manifest):
completeInfo = manifest[0]['completeInfo']
manifest[0]['completeInfo'] = sorted(completeInfo,
key=lambda ci: ci['url'])
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name,
from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
component = 'partialInfo'
else:
return
else:
component = 'completeInfo'
context.raw_balrog_manifest.setdefault(locale, {})
context.raw_balrog_manifest[locale].setdefault(component, [])
context.raw_balrog_manifest[locale][component].append(data)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_completeInfo(context.balrog_manifest)
sort_completeInfo(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
