def test_beetmover_template_args_generation():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
expected_template_args = {
'branch': 'mozilla-central',
'platform': 'android-api-15',
'product': 'Fake',
'stage_platform': 'android-api-15',
'template_key': 'fennec_nightly',
'upload_date': '2016/09/2016-09-01-16-26-14',
'version': '99.0a1'
}
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args == expected_template_args
context.task['payload']['locale'] = 'ro'
template_args = generate_beetmover_template_args(context.task,
context.properties)
assert template_args['template_key'] == 'fake_nightly_repacks'
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_candidates_manifest(context)
expected_sources = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package'
]
expected_destinations = [
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
),
('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package'
)
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(move_beets(context, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_schema_key_by_action(scopes, expected):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task["scopes"] = scopes
assert expected == get_schema_key_by_action(context)
def test_validate_scopes():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context.config, context.task,
context.properties)
context.task['scopes'] = []
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:!@#nightly_(@#$"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:mightly"]
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
context.task['scopes'] = ["project:releng:beetmover:dep"]
manifest['s3_prefix_dated'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
manifest['s3_prefix_latest'] = "pub/mobile/nightly/2017/01/2017-01-dep.."
with pytest.raises(SystemExit):
validate_task_scopes(context, manifest)
def test_balrog_manifest_to_artifacts():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
fake_balrog_manifest = context.task['payload']['releaseProperties']
context.balrog_manifest = fake_balrog_manifest
# fake the path to to able to check the contents written later on
with tempfile.TemporaryDirectory() as tmpdirname:
context.config['artifact_dir'] = tmpdirname
file_path = os.path.join(context.config['artifact_dir'],
'public/manifest.json')
# <temp-dir>/public doesn't exist yet and it's not automatically
# being created so we need to ensure it exists
public_tmpdirname = os.path.join(tmpdirname, 'public')
if not os.path.exists(public_tmpdirname):
os.makedirs(public_tmpdirname)
add_balrog_manifest_to_artifacts(context)
with open(file_path, "r") as fread:
retrieved_data = json.load(fread)
assert fake_balrog_manifest == retrieved_data
def test_get_initial_release_props_file():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.task['payload']['upstreamArtifacts'] = [{'paths': []}]
with pytest.raises(ScriptWorkerTaskException):
get_initial_release_props_file(context)
def test_move_beets(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.artifacts_to_beetmove = get_upstream_artifacts(context)
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
]
actual_sources = []
actual_destinations = []
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, artifact_pretty_name):
actual_sources.append(source)
actual_destinations.append(destinations)
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
event_loop.run_until_complete(
move_beets(context, context.artifacts_to_beetmove, manifest))
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
def test_get_upstream_artifacts(expected, preserve):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
artifacts_to_beetmove = get_upstream_artifacts(context, preserve_full_paths=preserve)
assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_async_main(event_loop):
context = Context()
context.config = get_fake_valid_config()
async def fake_move_beets(context, artifacts_to_beetmove, manifest):
pass
with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets):
event_loop.run_until_complete(async_main(context))
async def test_put_failure(fake_session_500):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session_500
with pytest.raises(ScriptWorkerRetryException):
await put(
context, url=URL('https://foo.com/packages/fake.package'), headers={},
abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session_500
)
def test_get_upstream_artifacts(expected, preserve):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task['payload']['releaseProperties']
artifacts_to_beetmove = get_upstream_artifacts(
context, preserve_full_paths=preserve)
assert sorted(list(artifacts_to_beetmove['en-US'])) == sorted(expected)
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task['payload']['releaseProperties']
context.task['payload']['upstreamArtifacts'][0]['paths'].append(
'fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
async def test_put_success(fake_session):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session
response = await put(
context, url=URL('https://foo.com/packages/fake.package'), headers={},
abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session
)
assert response.status == 200
assert response.resp == [b'asdf', b'asdf']
def test_exception_get_upstream_artifacts():
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
context.task['payload']['upstreamArtifacts'][0]['paths'].append('fake_file')
with pytest.raises(ScriptWorkerTaskException):
context.artifacts_to_beetmove = get_upstream_artifacts(context)
def test_put_failure(event_loop, fake_session_500):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session_500
with pytest.raises(ScriptWorkerRetryException):
event_loop.run_until_complete(
put(context,
url='https://foo.com/packages/fake.package',
headers={},
abs_filename='beetmoverscript/test/fake_artifact.json',
session=fake_session_500))
def test_put_success(event_loop, fake_session):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session
response = event_loop.run_until_complete(
put(context, url=URL('https://foo.com/packages/fake.package'), headers={},
abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session)
)
assert response.status == 200
assert response.resp == [b'asdf', b'asdf']
assert response.content.url == "https://foo.com/packages/fake.package"
def test_setup_config():
expected_context = Context()
expected_context.config = get_fake_valid_config()
with pytest.raises(SystemExit):
setup_config(None)
actual_context = setup_config("beetmoverscript/test/fake_config.json")
assert expected_context.config == actual_context.config
args = ['beetmoverscript', "beetmoverscript/test/fake_config.json"]
with mock.patch.object(sys, 'argv', args):
actual_context = setup_config(None)
assert expected_context.config == actual_context.config
def test_async_main(event_loop):
context = Context()
context.config = get_fake_valid_config()
async def fake_move_beets(context, manifest):
pass
async def get_fake_props(context):
return get_fake_balrog_props()['properties']
with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets):
with mock.patch('beetmoverscript.script.get_props',
new=get_fake_props):
event_loop.run_until_complete(async_main(context))
def test_download(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.session = fake_session
url = 'https://fake.com'
path = '/fake/path'
async def fake_download(context, url, path, session):
return context, url, path, session
# just make sure retry_download ends up calling scriptworker's download_file and passes the
# right args, kwargs
with mock.patch('beetmoverscript.script.download_file', fake_download):
result = event_loop.run_until_complete(
retry_download(context, url, path))
assert result == (context, url, path, context.session)
def test_checksums_manifest_generation():
checksums = {
"firefox-53.0a1.en-US.linux-i686.complete.mar": {
"sha512": "14f2d1cb999a8b42a3b6b671f7376c3e246daa65d108e2b8fe880f069601dc2b26afa155b52001235db059",
"size": 618149,
"sha256": "293975734953874539475"
}
}
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.checksums = checksums
expected_checksums_manifest_dump = get_fake_checksums_manifest()
checksums_manifest_dump = generate_checksums_manifest(context)
assert checksums_manifest_dump == expected_checksums_manifest_dump
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.properties = get_fake_balrog_props()["properties"]
context.properties['platform'] = context.properties['stage_platform']
manifest = generate_beetmover_manifest(context.config, context.task, context.properties)
mapping = manifest['mapping']
s3_keys = [mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == ['en-US/fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt']
assert (
manifest.get('s3_prefix_dated') ==
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/' and
manifest.get('s3_prefix_latest') == 'pub/mobile/nightly/latest-mozilla-central-fake/'
)
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.checksums = dict()
context.balrog_manifest = list()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=True,
artifact_pretty_name=pretty_name))
assert expected_upload_args == actual_upload_args
for k in expected_balrog_manifest.keys():
assert (context.balrog_manifest[0]['completeInfo'][0][k] ==
expected_balrog_manifest[k])
def test_main(fake_session):
context = Context()
context.config = get_fake_valid_config()
async def fake_async_main(context):
pass
async def fake_async_main_with_exception(context):
raise ScriptWorkerTaskException("This is wrong, the answer is 42")
with mock.patch('beetmoverscript.script.async_main', new=fake_async_main):
main(config_path='beetmoverscript/test/fake_config.json')
with mock.patch('beetmoverscript.script.async_main', new=fake_async_main_with_exception):
try:
main(config_path='beetmoverscript/test/fake_config.json')
except SystemExit as exc:
assert exc.code == 1
async def test_move_beets_raises(mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson='task_missing_installer.json')
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
with pytest.raises(ScriptWorkerTaskException):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=None,
artifact_map=None)
def test_move_beet(event_loop):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
locale = "sample-locale"
target_source = 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
)
expected_download_args = [
'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package',
'beetmoverscript/test/test_work_dir/public/build/target.package'
]
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package',
'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package'
), 'beetmoverscript/test/test_work_dir/public/build/target.package']
actual_download_args = []
actual_upload_args = []
async def fake_retry_download(context, url, path):
actual_download_args.extend([url, path])
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_download',
fake_retry_download):
with mock.patch('beetmoverscript.script.retry_upload',
fake_retry_upload):
event_loop.run_until_complete(
move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=False))
assert sorted(expected_download_args) == sorted(actual_download_args)
assert expected_upload_args == actual_upload_args
def test_generate_manifest():
context = Context()
context.task = get_fake_valid_task()
context.config = get_fake_valid_config()
context.release_props = get_fake_balrog_props()["properties"]
context.release_props['platform'] = context.release_props['stage_platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
manifest = generate_beetmover_manifest(context)
mapping = manifest['mapping']
s3_keys = [
mapping[m].get('target_info.txt', {}).get('s3_key') for m in mapping
]
assert sorted(mapping.keys()) == ['en-US', 'multi']
assert sorted(s3_keys) == [
'fake-99.0a1.en-US.target_info.txt',
'fake-99.0a1.multi.target_info.txt'
]
expected_destinations = {
'en-US': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
'multi': [
'2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.target_info.txt',
'latest-mozilla-central-fake/fake-99.0a1.multi.target_info.txt'
]
}
actual_destinations = {
k: mapping[k]['target_info.txt']['destinations']
for k in sorted(mapping.keys())
}
assert expected_destinations == actual_destinations
def test_setup_config():
expected_context = Context()
expected_context.config = get_fake_valid_config()
actual_context = setup_config("beetmoverscript/test/fake_config.json")
assert expected_context.config == actual_context.config
async def test_move_beet(update_manifest, action):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.task['extra'] = dict()
context.task['extra']['partials'] = [{
"artifact_name": "target-98.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "19991231235959",
"previousVersion": "98.0b96",
"previousBuildNumber": "1"
}, {
"artifact_name": "target-97.0b96.partial.mar",
"platform": "linux",
"locale": "de",
"buildid": "22423423402984",
"previousVersion": "97.0b96",
"previousBuildNumber": "1"
}]
context.action = action
context.bucket = 'nightly'
context.checksums = dict()
context.balrog_manifest = list()
context.raw_balrog_manifest = dict()
context.release_props = context.task['payload']['releaseProperties']
locale = "sample-locale"
target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
pretty_name = 'fake-99.0a1.en-US.target.txt'
target_destinations = (
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
)
expected_upload_args = [(
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt'
]
expected_balrog_manifest = {
'hash':
'73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b',
'size':
18,
'url':
'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
}
actual_upload_args = []
async def fake_retry_upload(context, destinations, path):
actual_upload_args.extend([destinations, path])
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid=None)
assert expected_upload_args == actual_upload_args
if update_manifest:
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['completeInfo'][''][k]
== expected_balrog_manifest[k])
expected_balrog_manifest['from_buildid'] = '19991231235959'
with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload):
await move_beet(context,
target_source,
target_destinations,
locale,
update_balrog_manifest=update_manifest,
balrog_format='',
artifact_pretty_name=pretty_name,
from_buildid='19991231235959')
if update_manifest:
if is_promotion_action(context.action):
expected_balrog_manifest['previousBuildNumber'] = '1'
expected_balrog_manifest['previousVersion'] = '98.0b96'
for k in expected_balrog_manifest.keys():
assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] ==
expected_balrog_manifest[k])
async def test_move_beets(task_filename, partials, mocker):
mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env())
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task(taskjson=task_filename)
context.release_props = context.task['payload']['releaseProperties']
context.release_props['stage_platform'] = context.release_props['platform']
context.bucket = 'nightly'
context.action = 'push-to-nightly'
context.raw_balrog_manifest = dict()
context.balrog_manifest = list()
context.artifacts_to_beetmove = get_upstream_artifacts(context)
if context.task['payload'].get('artifactMap'):
artifact_map = context.task['payload'].get('artifactMap')
manifest = None
else:
artifact_map = None
manifest = generate_beetmover_manifest(context)
expected_sources = [
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt',
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json'
),
os.path.abspath(
'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk'
)
]
expected_destinations = [
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json'
],
[
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk',
'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk'
]
]
expected_balrog_manifest = []
for complete_info in [
{
'completeInfo': [{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt'
}],
},
{
'blob_suffix':
'-mozinfo',
'completeInfo': [
{
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json'
},
],
},
]:
entry = {
'tc_nightly':
True,
'appName':
'Fake',
'appVersion':
'99.0a1',
'branch':
'mozilla-central',
'buildid':
'20990205110000',
'extVersion':
'99.0a1',
'hashType':
'sha512',
'locale':
'en-US',
'platform':
'android-api-15',
'url_replacements': [[
'http://archive.mozilla.org/pub',
'http://download.cdn.mozilla.net/pub'
]],
}
entry.update(complete_info)
if partials:
entry['partialInfo'] = [{
'from_buildid':
19991231235959,
'hash':
'dummyhash',
'size':
123456,
'url':
'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt'
}]
expected_balrog_manifest.append(entry)
actual_sources = []
actual_destinations = []
def sort_manifest(manifest):
manifest.sort(key=lambda entry: entry.get('blob_suffix', ''))
async def fake_move_beet(context, source, destinations, locale,
update_balrog_manifest, balrog_format,
artifact_pretty_name, from_buildid):
actual_sources.append(source)
actual_destinations.append(destinations)
if update_balrog_manifest:
data = {
"hash": 'dummyhash',
"size": 123456,
"url": destinations[0]
}
context.raw_balrog_manifest.setdefault(locale, {})
if from_buildid:
if partials:
data["from_buildid"] = from_buildid
context.raw_balrog_manifest[locale].setdefault(
'partialInfo', []).append(data)
else:
return
else:
context.raw_balrog_manifest[locale].setdefault(
'completeInfo', {})[balrog_format] = data
with mock.patch('beetmoverscript.script.move_beet', fake_move_beet):
await move_beets(context,
context.artifacts_to_beetmove,
manifest=manifest,
artifact_map=artifact_map)
assert sorted(expected_sources) == sorted(actual_sources)
assert sorted(expected_destinations) == sorted(actual_destinations)
# Deal with different-sorted completeInfo
sort_manifest(context.balrog_manifest)
sort_manifest(expected_balrog_manifest)
assert context.balrog_manifest == expected_balrog_manifest
def test_get_upstream_artifacts_with_zip_extract_param(monkeypatch):
context = Context()
context.config = get_fake_valid_config()
context.task = get_fake_valid_task()
context.properties = context.task['payload']['releaseProperties']
context.task['payload']['upstreamArtifacts'] = [{
'paths': ['a/non/archive', 'another/non/archive'],
'taskId':
'firstTaskId',
'taskType':
'someType1',
'zipExtract':
False,
}, {
'paths': ['archive1.zip', 'subfolder/archive2.zip'],
'taskId':
'firstTaskId',
'taskType':
'someType1',
'zipExtract':
True,
}, {
'paths': ['just/another/regular/file'],
'taskId':
'secondTaskId',
'taskType':
'someType2',
'zipExtract':
False,
}, {
'paths': ['archive1.zip'],
'taskId': 'thirdTaskId',
'taskType': 'someType3',
'zipExtract': True,
}]
def mock_upstream_artifact_full_path(context, task_id, path):
# doesn't check whether the file exists on disk
return os.path.join(context.config['work_dir'], 'cot', task_id, path)
monkeypatch.setattr(
'scriptworker.artifacts.get_and_check_single_upstream_artifact_full_path',
mock_upstream_artifact_full_path)
assert get_upstream_artifacts_with_zip_extract_param(context) == {
'firstTaskId': [{
'paths': [
os.path.join(context.config['work_dir'], 'cot', 'firstTaskId',
'a/non/archive'),
os.path.join(context.config['work_dir'], 'cot', 'firstTaskId',
'another/non/archive'),
],
'zip_extract':
False,
}, {
'paths': [
os.path.join(context.config['work_dir'], 'cot', 'firstTaskId',
'archive1.zip'),
os.path.join(context.config['work_dir'], 'cot', 'firstTaskId',
'subfolder/archive2.zip'),
],
'zip_extract':
True,
}],
'secondTaskId': [{
'paths': [
os.path.join(context.config['work_dir'], 'cot', 'secondTaskId',
'just/another/regular/file'),
],
'zip_extract':
False,
}],
'thirdTaskId': [{
'paths': [
os.path.join(context.config['work_dir'], 'cot', 'thirdTaskId',
'archive1.zip'),
],
'zip_extract':
True,
}],
}
