def test_influx_service(on_try): if ON_TRY and sys.platform == "darwin": # macos slave in the CI are restricted return mocks() with running_on_try(on_try), temp_dir() as output: args = { "verbose": True, "output": output, "perfboard-influx-password": "******", "perfboard-grafana-key": "xxx", "perfboard-grafana-host": "grafana", "perfboard-influx-port": 0, "perfboard-influx-host": "influxdb", "tests": [EXAMPLE_TEST], } mach_cmd, metadata, env = get_running_env(**args) metadata.add_result({"results": str(BT_DATA), "name": "browsertime"}) layer = Influx(env, mach_cmd) layer.setup() try: metadata = layer.run(metadata) finally: layer.teardown() index = on_try and 2 or 1 sent_data = responses.calls[index].request.body.split(b"\n") fields = [line.split(b",")[0].strip() for line in sent_data] assert b"rumspeedindex" in fields responses.reset()
def test_xpcshell_perfherder(*mocked): mach_cmd, metadata, env = get_running_env(flavor="xpcshell", perfherder=True, xpcshell_cycles=10) sys = env.layers[SYSTEM] xpcshell = env.layers[TEST] env.set_arg("tests", [str(EXAMPLE_XPCSHELL_TEST)]) metrics = env.layers[METRICS] with temp_file() as output: env.set_arg("output", output) try: with sys as s, xpcshell as x, metrics as m: m(x(s(metadata))) finally: shutil.rmtree(mach_cmd._mach_context.state_dir) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" assert output["framework"]["name"] == "xpcshell" # Check some numbers in our data assert len(output["suites"]) == 1 assert len(output["suites"][0]["subtests"]) == 3 assert output["suites"][0]["value"] > 0 for subtest in output["suites"][0]["subtests"]: assert subtest["name"].startswith("metrics")
def ptetls(files): resources, dirs, output = files["resources"], files["dirs"], files[ "output"] _, metadata, _ = get_running_env() config = {"output": output} file_group_list = {"group_1": list(resources.values())} file_group_str = {"group_1": dirs["resources"].resolve().as_posix()} yield { "ptetl_list": PerftestETL( file_groups=file_group_list, config=config, prefix="PerftestETL", logger=metadata, sort_files=True, ), "ptetl_str": PerftestETL( file_groups=file_group_str, config=config, prefix="PerftestETL", logger=metadata, sort_files=True, ), }
def test_proxy(install_mozproxy): mach_cmd, metadata, env = get_running_env(proxy=True) system = env.layers[SYSTEM] # XXX this will run for real, we need to mock HTTP calls with system as proxy, silence(): proxy(metadata)
def test_metrics_last(): mach, metadata, env = get_running_env() system = create_mock() browser = create_mock() # Check that the metrics layer is entered after # other have finished and that the other layers # were only called once class M: def __enter__(self): system.setup.assert_called_once() browser.setup.assert_called_once() system.teardown.assert_called_once() browser.teardown.assert_called_once() return self def __exit__(self, *args, **kw): return def __call__(self, metadata): return env.layers = [system, browser, M()] with env: env.run(metadata)
def test_browser_desktop(*mocked): mach_cmd, metadata, env = get_running_env( browsertime_iterations=1, browsertime_extra_options="one=1,two=2", tests=[EXAMPLE_TEST], browsertime_no_window_recorder=False, browsertime_viewport_size="1234x567", ) browser = env.layers[TEST] sys = env.layers[SYSTEM] try: with sys as s, browser as b, silence(): # just checking that the setup_helper property gets # correctly initialized browsertime = browser.layers[-1] assert browsertime.setup_helper is not None helper = browsertime.setup_helper assert browsertime.setup_helper is helper b(s(metadata)) finally: shutil.rmtree(mach_cmd._mach_context.state_dir) assert mach_cmd.run_process.call_count == 1 cmd = " ".join(mach_cmd.run_process.call_args[0][0]) # check that --firefox.binaryPath is set automatically assert "--firefox.binaryPath" in cmd
def test_browser(*mocked): mach_cmd, metadata, env = get_running_env( android=True, android_app_name="something", browsertime_geckodriver="GECKODRIVER", browsertime_iterations=1, browsertime_extra_options="one=1,two=2", ) browser = env.layers[BROWSER] env.set_arg("tests", [EXAMPLE_TEST]) try: with browser as b, silence(): b(metadata) finally: shutil.rmtree(mach_cmd._mach_context.state_dir) assert mach_cmd.run_process.call_count == 1 # Make sure all arguments are of type str for option in mach_cmd.run_process.call_args[0][0]: assert isinstance(option, str) cmd = " ".join(mach_cmd.run_process.call_args[0][0]) assert EXAMPLE_TEST in cmd assert "--firefox.geckodriverPath GECKODRIVER" in cmd assert "--one 1" in cmd assert "--two 2" in cmd results = metadata.get_results() assert len(results) == 1 assert set(list(results[0].keys())) - set(["name", "results"]) == set() assert results[0]["name"] == "Example"
def test_browsertime_should_reinstall(): _, _, env = get_running_env( android=True, android_app_name="something", browsertime_geckodriver="GECKODRIVER", browsertime_iterations=1, browsertime_extra_options="one=1,two=2", tests=[EXAMPLE_TEST], ) with mock.patch( "mozperftest.test.browsertime.runner.pathlib.Path.open", build_mock_open([ { "devDependencies": { "browsertime": "89771a1d6be54114db190427dbc281582cba3d47" } }, { "_from": ("browsertime@https://github.com/sitespeedio/browsertime" "/tarball/98747854be54114db190427dbc281582cba3d47") }, ]), ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): browser = env.layers[TEST] btime_layer = browser.layers[0] assert btime_layer._should_install()
def test_visual_metrics(device): os.environ["VISUALMETRICS_PY"] = "" mach_cmd, metadata, env = get_running_env( visualmetrics=True, perfherder=True, verbose=True, tests=[EXAMPLE_TEST], ) metrics = env.layers[METRICS] metadata.add_result({ "results": str(BT_DATA_VIDEO.parent), "name": "browsertime" }) with temp_file() as output: env.set_arg("output", output) with metrics as m: metadata = m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" visual_metrics = [i["name"] for i in output["suites"][1]["subtests"]] assert "VisualProgress96" in visual_metrics
def test_android_log(*mocked): with temp_file() as logcat, temp_dir() as output: args = { "flavor": "mobile-browser", "android-install-apk": ["this.apk"], "android": True, "console": True, "android-timeout": 30, "android-capture-adb": "stdout", "android-capture-logcat": logcat, "android-app-name": "org.mozilla.fenix", "androidlog": True, "output": output, } mach_cmd, metadata, env = get_running_env(**args) env.set_arg("tests", [EXAMPLE_TEST]) with env.layers[SYSTEM] as sys, env.layers[TEST] as andro: metadata = andro(sys(metadata)) # we want to drop the first result metadata._results = metadata._results[1:] with env.layers[METRICS] as metrics: metadata = metrics(metadata) assert pathlib.Path(output, "LogCatstd-output.json").exists()
def test_add_options(): mach_cmd, metadata, env = get_running_env() options = [("one", 1), ("two", 2)] add_options(env, options) extra = env.get_arg("browsertime-extra-options") assert "one=1" in extra assert "two=2" in extra
def test_attach_fails(): mach_cmd, metadata, env = get_running_env(browsertime_binary=str(DMG)) device = MacosDevice(env, mach_cmd) with pytest.raises(subprocess.CalledProcessError): try: device.run(metadata) finally: device.teardown()
def test_exception_resume(): # the last layer is called, the error is swallowed hooks = str(Path(HERE, "data", "hook_resume.py")) mach, metadata, env = get_running_env(hooks=hooks) last_layer = create_mock() env.layers = [create_mock(), Failure(env, mach), last_layer] with env: env.run(metadata) last_layer.assert_called()
def test_profile(): mach_cmd, metadata, env = get_running_env() with Profile(env, mach_cmd) as profile: profile(metadata) profile_dir = env.get_arg("profile-directory") assert os.path.exists(profile_dir) assert not os.path.exists(profile_dir)
def test_exception_raised(): # the error is raised hooks = str(Path(HERE, "data", "hook_raises.py")) mach, metadata, env = get_running_env(hooks=hooks) last_layer = create_mock() env.layers = [create_mock(), Failure(env, mach), last_layer] with env, pytest.raises(FailureException): env.run(metadata) last_layer.__call__.assert_not_called()
def test_mount_dmg(): mach_cmd, metadata, env = get_running_env(browsertime_binary=str(DMG)) device = MacosDevice(env, mach_cmd) try: device.run(metadata) finally: device.teardown() target = Path(DMG.parent, "firefox", "Contents", "MacOS", "firefox") assert env.get_arg("browsertime-binary") == str(target)
def test_replay_url(install_mozproxy): mach_cmd, metadata, env = get_running_env(proxy=True) system = env.layers[SYSTEM] env.set_arg("proxy-replay", "http://example.dump") # XXX this will run for real, we need to mock HTTP calls with system as proxy, silence(): proxy(metadata) # Give mitmproxy a bit of time to start up so we can verify that it's # actually running before we tear things down. time.sleep(5)
def test_install_url_bad(*mocked): mach, metadata, env = get_running_env(browsertime_install_url="meh") browser = env.layers[BROWSER] env.set_arg("tests", [EXAMPLE_TEST]) with pytest.raises(ValueError): try: with browser as b, silence(): b(metadata) finally: shutil.rmtree(mach._mach_context.state_dir)
def test_android(): args = { "android-install-apk": ["this"], "android": True, "android-app-name": "org.mozilla.fenix", } mach_cmd, metadata, env = get_running_env(**args) system = env.layers[SYSTEM] with system as android, silence(system): android(metadata)
def test_xpcshell_metrics_fail(*mocked): mach_cmd, metadata, env = get_running_env(flavor="xpcshell") sys = env.layers[SYSTEM] xpcshell = env.layers[TEST] env.set_arg("tests", [str(EXAMPLE_XPCSHELL_TEST)]) try: with sys as s, xpcshell as x, pytest.raises(XPCShellTestError): x(s(metadata)) finally: shutil.rmtree(mach_cmd._mach_context.state_dir)
def test_record(install_mozproxy): mach_cmd, metadata, env = get_running_env(proxy=True) system = env.layers[SYSTEM] with tempfile.TemporaryDirectory() as tmpdir: recording = os.path.join(tmpdir, "recording.dump") env.set_arg("proxy-record", recording) # XXX this will run for real, we need to mock HTTP calls with system as proxy, silence(): proxy(metadata) assert os.path.exists(recording)
def test_exception_no_user_exception(): # the last layer is called, the error is raised # because user_exception = False hooks = str(Path(HERE, "data", "hook_resume.py")) mach, metadata, env = get_running_env(hooks=hooks) last_layer = create_mock() f = Failure(env, mach) f.user_exception = False env.layers = [create_mock(), f, last_layer] with env, pytest.raises(FailureException): env.run(metadata) last_layer._call__.assert_not_called()
def test_android_failure(): # no patching so it'll try for real and fail args = { "android-install-apk": ["this"], "android": True, "android-app-name": "org.mozilla.fenix", } mach_cmd, metadata, env = get_running_env(**args) system = env.layers[SYSTEM] with system as android, silence(system), pytest.raises(DeviceError): android(metadata)
def test_browsertime_not_existing(): _, _, env = get_running_env( android=True, android_app_name="something", browsertime_geckodriver="GECKODRIVER", browsertime_iterations=1, browsertime_extra_options="one=1,two=2", tests=[EXAMPLE_TEST], ) browser = env.layers[TEST] btime_layer = browser.layers[0] assert btime_layer._should_install()
def test_visual_metrics_no_ffmpeg(device): os.environ["VISUALMETRICS_PY"] = "" mach_cmd, metadata, env = get_running_env( visualmetrics=True, perfherder=True, verbose=True ) metrics = env.layers[METRICS] env.set_arg("tests", [str(EXAMPLE_TEST)]) metadata.add_result({"results": str(BT_DATA_VIDEO.parent), "name": "browsertime"}) with pytest.raises(FileNotFoundError): with metrics as m: metadata = m(metadata)
def setup_env(options): mach_cmd, metadata, env = get_running_env(**options) runs = [] def _run_process(*args, **kw): runs.append((args, kw)) mach_cmd.run_process = _run_process metrics = env.layers[METRICS] env.set_arg("tests", [EXAMPLE_TEST]) metadata.add_result({"results": str(BT_DATA), "name": "browsertime"}) return metrics, metadata, env
def test_android_apk_alias(device): args = { "android-install-apk": ["fenix_fennec_nightly_armeabi_v7a"], "android": True, "android-app-name": "org.mozilla.fenned_aurora", } mach_cmd, metadata, env = get_running_env(**args) system = env.layers[SYSTEM] with system as android, silence(system): android(metadata) # XXX really ? assert device.mock_calls[1][1][0].endswith("target.apk")
def test_install_url(*mocked): url = "https://here/tarball/" + "".join( [random.choice(string.hexdigits[:-6]) for c in range(40)]) mach, metadata, env = get_running_env(browsertime_install_url=url) browser = env.layers[TEST] env.set_arg("tests", [EXAMPLE_TEST]) try: with temporary_env(MOZ_AUTOMATION="1"), browser as b, silence(): b(metadata) finally: shutil.rmtree(mach._mach_context.state_dir) assert mach.run_process.call_count == 1
def test_android(): args = { "flavor": "mobile-browser", "android-install-apk": ["this.apk"], "android": True, "android-timeout": 30, "android-capture-adb": "stdout", "android-app-name": "org.mozilla.fenix", } mach_cmd, metadata, env = get_running_env(**args) system = env.layers[SYSTEM] with system as android, silence(system): android(metadata)
def test_install_url_bad(*mocked): mach, metadata, env = get_running_env( browsertime_install_url="meh", tests=[EXAMPLE_TEST], ) browser = env.layers[TEST] sys = env.layers[SYSTEM] with pytest.raises(ValueError): try: with sys as s, browser as b, silence(): b(s(metadata)) finally: shutil.rmtree(mach._mach_context.state_dir)