def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.properties = get_fake_balrog_props()["properties"] context.properties['platform'] = context.properties['stage_platform'] manifest = generate_candidates_manifest(context) expected_sources = [ 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package', 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/en-US/target.package' ] expected_destinations = [ ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ), ('pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.fake.package' ) ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete(move_beets(context, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_do_failed_with_TimeoutError_api_call( context, mocker, event_loop, fake_TimeoutError_throwing_session): context.server = get_task_server(context.task, context.config) context.action = get_task_action(context.task, context.config) context.session = fake_TimeoutError_throwing_session with pytest.raises(aiohttp.ServerTimeoutError): event_loop.run_until_complete(_do_api_call(context, 'dummy', {}))
def test_move_beets(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], ] actual_sources = [] actual_destinations = [] async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name): actual_sources.append(source) actual_destinations.append(destinations) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations)
def test_put_failure(event_loop, fake_session_500): context = Context() context.config = get_fake_valid_config() context.session = fake_session_500 with pytest.raises(ScriptWorkerRetryException): event_loop.run_until_complete( put(context, url=URL('https://foo.com/packages/fake.package'), headers={}, abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session_500) )
def test_async_main(event_loop): context = Context() context.config = get_fake_valid_config() async def fake_move_beets(context, artifacts_to_beetmove, manifest): pass with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets): event_loop.run_until_complete(async_main(context))
def test_async_main(event_loop): context = Context() context.config = get_fake_valid_config() async def fake_move_beets(context, manifest): pass async def get_fake_props(context): return get_fake_balrog_props()['properties'] with mock.patch('beetmoverscript.script.move_beets', new=fake_move_beets): with mock.patch('beetmoverscript.script.get_props', new=get_fake_props): event_loop.run_until_complete(async_main(context))
def test_move_beet(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.checksums = dict() context.balrog_manifest = list() context.release_props = get_fake_balrog_props()["properties"] context.release_props['platform'] = context.release_props['stage_platform'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.mozilla.org/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=True, artifact_pretty_name=pretty_name)) assert expected_upload_args == actual_upload_args for k in expected_balrog_manifest.keys(): assert (context.balrog_manifest[0]['completeInfo'][0][k] == expected_balrog_manifest[k])
def test_do_failed_api_call(context, mocker, event_loop, fake_session_500): context.server = get_task_server(context.task, context.config) context.action = get_task_action(context.task, context.config) context.session = fake_session_500 response = event_loop.run_until_complete(_do_api_call( context, 'dummy', {})) assert response == '{}'
def test_put_success(event_loop, fake_session): context = Context() context.config = get_fake_valid_config() context.session = fake_session response = event_loop.run_until_complete( put(context, url=URL('https://foo.com/packages/fake.package'), headers={}, abs_filename='beetmoverscript/test/fake_artifact.json', session=fake_session) ) assert response.status == 200 assert response.resp == [b'asdf', b'asdf'] assert response.content.url == "https://foo.com/packages/fake.package"
def test_do_successful_api_call(context, mocker, event_loop, fake_session, data, credentials): context.server = get_task_server(context.task, context.config) context.action = get_task_action(context.task, context.config) context.session = fake_session if not credentials: del context.config["bouncer_config"][context.server]["username"] del context.config["bouncer_config"][context.server]["password"] with pytest.raises(KeyError): response = event_loop.run_until_complete( _do_api_call(context, 'dummy', data)) return response = event_loop.run_until_complete( _do_api_call(context, 'dummy', data)) assert response == '{}'
def test_move_beet(event_loop): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() locale = "sample-locale" target_source = 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ) expected_download_args = [ 'https://queue.taskcluster.net/v1/task/VALID_TASK_ID/artifacts/public/build/target.package', 'beetmoverscript/test/test_work_dir/public/build/target.package' ] expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/fake-99.0a1.multi.fake.package', 'pub/mobile/nightly/latest-mozilla-central-fake/fake-99.0a1.multi.fake.package' ), 'beetmoverscript/test/test_work_dir/public/build/target.package'] actual_download_args = [] actual_upload_args = [] async def fake_retry_download(context, url, path): actual_download_args.extend([url, path]) async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_download', fake_retry_download): with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=False)) assert sorted(expected_download_args) == sorted(actual_download_args) assert expected_upload_args == actual_upload_args
def test_download(event_loop): context = Context() context.config = get_fake_valid_config() context.session = fake_session url = 'https://fake.com' path = '/fake/path' async def fake_download(context, url, path, session): return context, url, path, session # just make sure retry_download ends up calling scriptworker's download_file and passes the # right args, kwargs with mock.patch('beetmoverscript.script.download_file', fake_download): result = event_loop.run_until_complete( retry_download(context, url, path)) assert result == (context, url, path, context.session)
def test_move_beet(event_loop, update_manifest, action): context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.task['extra'] = dict() context.task['extra']['partials'] = [{ "artifact_name": "target-98.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "19991231235959", "previousVersion": "98.0b96", "previousBuildNumber": "1" }, { "artifact_name": "target-97.0b96.partial.mar", "platform": "linux", "locale": "de", "buildid": "22423423402984", "previousVersion": "97.0b96", "previousBuildNumber": "1" }] context.action = action context.bucket = 'nightly' context.checksums = dict() context.balrog_manifest = list() context.raw_balrog_manifest = dict() context.release_props = context.task['payload']['releaseProperties'] locale = "sample-locale" target_source = 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' pretty_name = 'fake-99.0a1.en-US.target.txt' target_destinations = ( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ) expected_upload_args = [( 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ), 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt' ] expected_balrog_manifest = { 'hash': '73b91c3625d70e9ba1992f119bdfd3fba85041e6f804a985a18efe06ebb1d4147fb044ac06b28773130b4887dd8b5b3bc63958e1bd74003077d8bc2a3909416b', 'size': 18, 'url': 'https://archive.test/pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', } actual_upload_args = [] async def fake_retry_upload(context, destinations, path): actual_upload_args.extend([destinations, path]) with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, artifact_pretty_name=pretty_name, from_buildid=None)) assert expected_upload_args == actual_upload_args if update_manifest: for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['completeInfo'][0][k] == expected_balrog_manifest[k]) expected_balrog_manifest['from_buildid'] = '19991231235959' with mock.patch('beetmoverscript.script.retry_upload', fake_retry_upload): event_loop.run_until_complete( move_beet(context, target_source, target_destinations, locale, update_balrog_manifest=update_manifest, artifact_pretty_name=pretty_name, from_buildid='19991231235959')) if update_manifest: if is_promotion_action(context.action): expected_balrog_manifest['previousBuildNumber'] = '1' expected_balrog_manifest['previousVersion'] = '98.0b96' for k in expected_balrog_manifest.keys(): assert (context.raw_balrog_manifest[locale]['partialInfo'][0][k] == expected_balrog_manifest[k])
def test_move_beets(event_loop, partials, mocker): mocker.patch('beetmoverscript.utils.JINJA_ENV', get_test_jinja_env()) context = Context() context.config = get_fake_valid_config() context.task = get_fake_valid_task() context.release_props = context.task['payload']['releaseProperties'] context.release_props['stage_platform'] = context.release_props['platform'] context.bucket = 'nightly' context.action = 'push-to-nightly' context.raw_balrog_manifest = dict() context.balrog_manifest = list() context.artifacts_to_beetmove = get_upstream_artifacts(context) manifest = generate_beetmover_manifest(context) expected_sources = [ os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.mozinfo.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.txt', ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target_info.txt' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.test_packages.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/buildhub.json' ), os.path.abspath( 'beetmoverscript/test/test_work_dir/cot/eSzfNqMZT_mSiQQXu8hyqg/public/build/target.apk' ) ] expected_destinations = [ [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.test_packages.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.buildhub.json' ], [ 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake 99.0a1.en-US.target.apk', 'pub/mobile/nightly/latest-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.apk' ] ] expected_balrog_manifest = [{ 'tc_nightly': True, 'appName': 'Fake', 'appVersion': '99.0a1', 'branch': 'mozilla-central', 'buildid': '20990205110000', 'extVersion': '99.0a1', 'hashType': 'sha512', 'locale': 'en-US', 'platform': 'android-api-15', 'url_replacements': [[ 'http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub' ]], 'completeInfo': [{ 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.mozinfo.json' }, { 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target_info.txt' }], }] if partials: expected_balrog_manifest[0]['partialInfo'] = [{ 'from_buildid': 19991231235959, 'hash': 'dummyhash', 'size': 123456, 'url': 'pub/mobile/nightly/2016/09/2016-09-01-16-26-14-mozilla-central-fake/en-US/fake-99.0a1.en-US.target.txt' }] actual_sources = [] actual_destinations = [] def sort_completeInfo(manifest): completeInfo = manifest[0]['completeInfo'] manifest[0]['completeInfo'] = sorted(completeInfo, key=lambda ci: ci['url']) async def fake_move_beet(context, source, destinations, locale, update_balrog_manifest, artifact_pretty_name, from_buildid): actual_sources.append(source) actual_destinations.append(destinations) if update_balrog_manifest: data = { "hash": 'dummyhash', "size": 123456, "url": destinations[0] } if from_buildid: if partials: data["from_buildid"] = from_buildid component = 'partialInfo' else: return else: component = 'completeInfo' context.raw_balrog_manifest.setdefault(locale, {}) context.raw_balrog_manifest[locale].setdefault(component, []) context.raw_balrog_manifest[locale][component].append(data) with mock.patch('beetmoverscript.script.move_beet', fake_move_beet): event_loop.run_until_complete( move_beets(context, context.artifacts_to_beetmove, manifest)) assert sorted(expected_sources) == sorted(actual_sources) assert sorted(expected_destinations) == sorted(actual_destinations) # Deal with different-sorted completeInfo sort_completeInfo(context.balrog_manifest) sort_completeInfo(expected_balrog_manifest) assert context.balrog_manifest == expected_balrog_manifest