def test_influx_service(on_try):
if ON_TRY and sys.platform == "darwin":
# macos slave in the CI are restricted
return
mocks()
with running_on_try(on_try), temp_dir() as output:
args = {
"verbose": True,
"output": output,
"perfboard-influx-password": "******",
"perfboard-grafana-key": "xxx",
"perfboard-grafana-host": "grafana",
"perfboard-influx-port": 0,
"perfboard-influx-host": "influxdb",
"tests": [EXAMPLE_TEST],
}
mach_cmd, metadata, env = get_running_env(**args)
metadata.add_result({"results": str(BT_DATA), "name": "browsertime"})
layer = Influx(env, mach_cmd)
layer.setup()
try:
metadata = layer.run(metadata)
finally:
layer.teardown()
index = on_try and 2 or 1
sent_data = responses.calls[index].request.body.split(b"\n")
fields = [line.split(b",")[0].strip() for line in sent_data]
assert b"rumspeedindex" in fields
responses.reset()
def test_xpcshell_perfherder(*mocked):
mach_cmd, metadata, env = get_running_env(flavor="xpcshell",
perfherder=True,
xpcshell_cycles=10)
sys = env.layers[SYSTEM]
xpcshell = env.layers[TEST]
env.set_arg("tests", [str(EXAMPLE_XPCSHELL_TEST)])
metrics = env.layers[METRICS]
with temp_file() as output:
env.set_arg("output", output)
try:
with sys as s, xpcshell as x, metrics as m:
m(x(s(metadata)))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "xpcshell"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 3
assert output["suites"][0]["value"] > 0
for subtest in output["suites"][0]["subtests"]:
assert subtest["name"].startswith("metrics")
def ptetls(files):
resources, dirs, output = files["resources"], files["dirs"], files[
"output"]
_, metadata, _ = get_running_env()
config = {"output": output}
file_group_list = {"group_1": list(resources.values())}
file_group_str = {"group_1": dirs["resources"].resolve().as_posix()}
yield {
"ptetl_list":
PerftestETL(
file_groups=file_group_list,
config=config,
prefix="PerftestETL",
logger=metadata,
sort_files=True,
),
"ptetl_str":
PerftestETL(
file_groups=file_group_str,
config=config,
prefix="PerftestETL",
logger=metadata,
sort_files=True,
),
}
def test_proxy(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
def test_metrics_last():
mach, metadata, env = get_running_env()
system = create_mock()
browser = create_mock()
# Check that the metrics layer is entered after
# other have finished and that the other layers
# were only called once
class M:
def __enter__(self):
system.setup.assert_called_once()
browser.setup.assert_called_once()
system.teardown.assert_called_once()
browser.teardown.assert_called_once()
return self
def __exit__(self, *args, **kw):
return
def __call__(self, metadata):
return
env.layers = [system, browser, M()]
with env:
env.run(metadata)
def test_browser_desktop(*mocked):
mach_cmd, metadata, env = get_running_env(
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
tests=[EXAMPLE_TEST],
browsertime_no_window_recorder=False,
browsertime_viewport_size="1234x567",
)
browser = env.layers[TEST]
sys = env.layers[SYSTEM]
try:
with sys as s, browser as b, silence():
# just checking that the setup_helper property gets
# correctly initialized
browsertime = browser.layers[-1]
assert browsertime.setup_helper is not None
helper = browsertime.setup_helper
assert browsertime.setup_helper is helper
b(s(metadata))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
# check that --firefox.binaryPath is set automatically
assert "--firefox.binaryPath" in cmd
def test_browser(*mocked):
mach_cmd, metadata, env = get_running_env(
android=True,
android_app_name="something",
browsertime_geckodriver="GECKODRIVER",
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
)
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
try:
with browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
# Make sure all arguments are of type str
for option in mach_cmd.run_process.call_args[0][0]:
assert isinstance(option, str)
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
assert EXAMPLE_TEST in cmd
assert "--firefox.geckodriverPath GECKODRIVER" in cmd
assert "--one 1" in cmd
assert "--two 2" in cmd
results = metadata.get_results()
assert len(results) == 1
assert set(list(results[0].keys())) - set(["name", "results"]) == set()
assert results[0]["name"] == "Example"
def test_browsertime_should_reinstall():
_, _, env = get_running_env(
android=True,
android_app_name="something",
browsertime_geckodriver="GECKODRIVER",
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
tests=[EXAMPLE_TEST],
)
with mock.patch(
"mozperftest.test.browsertime.runner.pathlib.Path.open",
build_mock_open([
{
"devDependencies": {
"browsertime":
"89771a1d6be54114db190427dbc281582cba3d47"
}
},
{
"_from":
("browsertime@https://github.com/sitespeedio/browsertime"
"/tarball/98747854be54114db190427dbc281582cba3d47")
},
]),
), mock.patch("mozperftest.test.browsertime.runner.json.load",
new=mocked_jsonload):
browser = env.layers[TEST]
btime_layer = browser.layers[0]
assert btime_layer._should_install()
def test_visual_metrics(device):
os.environ["VISUALMETRICS_PY"] = ""
mach_cmd, metadata, env = get_running_env(
visualmetrics=True,
perfherder=True,
verbose=True,
tests=[EXAMPLE_TEST],
)
metrics = env.layers[METRICS]
metadata.add_result({
"results": str(BT_DATA_VIDEO.parent),
"name": "browsertime"
})
with temp_file() as output:
env.set_arg("output", output)
with metrics as m:
metadata = m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
visual_metrics = [i["name"] for i in output["suites"][1]["subtests"]]
assert "VisualProgress96" in visual_metrics
def test_android_log(*mocked):
with temp_file() as logcat, temp_dir() as output:
args = {
"flavor": "mobile-browser",
"android-install-apk": ["this.apk"],
"android": True,
"console": True,
"android-timeout": 30,
"android-capture-adb": "stdout",
"android-capture-logcat": logcat,
"android-app-name": "org.mozilla.fenix",
"androidlog": True,
"output": output,
}
mach_cmd, metadata, env = get_running_env(**args)
env.set_arg("tests", [EXAMPLE_TEST])
with env.layers[SYSTEM] as sys, env.layers[TEST] as andro:
metadata = andro(sys(metadata))
# we want to drop the first result
metadata._results = metadata._results[1:]
with env.layers[METRICS] as metrics:
metadata = metrics(metadata)
assert pathlib.Path(output, "LogCatstd-output.json").exists()
def test_add_options():
mach_cmd, metadata, env = get_running_env()
options = [("one", 1), ("two", 2)]
add_options(env, options)
extra = env.get_arg("browsertime-extra-options")
assert "one=1" in extra
assert "two=2" in extra
def test_attach_fails():
mach_cmd, metadata, env = get_running_env(browsertime_binary=str(DMG))
device = MacosDevice(env, mach_cmd)
with pytest.raises(subprocess.CalledProcessError):
try:
device.run(metadata)
finally:
device.teardown()
def test_exception_resume():
# the last layer is called, the error is swallowed
hooks = str(Path(HERE, "data", "hook_resume.py"))
mach, metadata, env = get_running_env(hooks=hooks)
last_layer = create_mock()
env.layers = [create_mock(), Failure(env, mach), last_layer]
with env:
env.run(metadata)
last_layer.assert_called()
def test_profile():
mach_cmd, metadata, env = get_running_env()
with Profile(env, mach_cmd) as profile:
profile(metadata)
profile_dir = env.get_arg("profile-directory")
assert os.path.exists(profile_dir)
assert not os.path.exists(profile_dir)
def test_exception_raised():
# the error is raised
hooks = str(Path(HERE, "data", "hook_raises.py"))
mach, metadata, env = get_running_env(hooks=hooks)
last_layer = create_mock()
env.layers = [create_mock(), Failure(env, mach), last_layer]
with env, pytest.raises(FailureException):
env.run(metadata)
last_layer.__call__.assert_not_called()
def test_mount_dmg():
mach_cmd, metadata, env = get_running_env(browsertime_binary=str(DMG))
device = MacosDevice(env, mach_cmd)
try:
device.run(metadata)
finally:
device.teardown()
target = Path(DMG.parent, "firefox", "Contents", "MacOS", "firefox")
assert env.get_arg("browsertime-binary") == str(target)
def test_replay_url(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
env.set_arg("proxy-replay", "http://example.dump")
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
# Give mitmproxy a bit of time to start up so we can verify that it's
# actually running before we tear things down.
time.sleep(5)
def test_install_url_bad(*mocked):
mach, metadata, env = get_running_env(browsertime_install_url="meh")
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
with pytest.raises(ValueError):
try:
with browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach._mach_context.state_dir)
def test_android():
args = {
"android-install-apk": ["this"],
"android": True,
"android-app-name": "org.mozilla.fenix",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system):
android(metadata)
def test_xpcshell_metrics_fail(*mocked):
mach_cmd, metadata, env = get_running_env(flavor="xpcshell")
sys = env.layers[SYSTEM]
xpcshell = env.layers[TEST]
env.set_arg("tests", [str(EXAMPLE_XPCSHELL_TEST)])
try:
with sys as s, xpcshell as x, pytest.raises(XPCShellTestError):
x(s(metadata))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
def test_record(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
with tempfile.TemporaryDirectory() as tmpdir:
recording = os.path.join(tmpdir, "recording.dump")
env.set_arg("proxy-record", recording)
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
assert os.path.exists(recording)
def test_exception_no_user_exception():
# the last layer is called, the error is raised
# because user_exception = False
hooks = str(Path(HERE, "data", "hook_resume.py"))
mach, metadata, env = get_running_env(hooks=hooks)
last_layer = create_mock()
f = Failure(env, mach)
f.user_exception = False
env.layers = [create_mock(), f, last_layer]
with env, pytest.raises(FailureException):
env.run(metadata)
last_layer._call__.assert_not_called()
def test_android_failure():
# no patching so it'll try for real and fail
args = {
"android-install-apk": ["this"],
"android": True,
"android-app-name": "org.mozilla.fenix",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system), pytest.raises(DeviceError):
android(metadata)
def test_browsertime_not_existing():
_, _, env = get_running_env(
android=True,
android_app_name="something",
browsertime_geckodriver="GECKODRIVER",
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
tests=[EXAMPLE_TEST],
)
browser = env.layers[TEST]
btime_layer = browser.layers[0]
assert btime_layer._should_install()
def test_visual_metrics_no_ffmpeg(device):
os.environ["VISUALMETRICS_PY"] = ""
mach_cmd, metadata, env = get_running_env(
visualmetrics=True, perfherder=True, verbose=True
)
metrics = env.layers[METRICS]
env.set_arg("tests", [str(EXAMPLE_TEST)])
metadata.add_result({"results": str(BT_DATA_VIDEO.parent), "name": "browsertime"})
with pytest.raises(FileNotFoundError):
with metrics as m:
metadata = m(metadata)
def setup_env(options):
mach_cmd, metadata, env = get_running_env(**options)
runs = []
def _run_process(*args, **kw):
runs.append((args, kw))
mach_cmd.run_process = _run_process
metrics = env.layers[METRICS]
env.set_arg("tests", [EXAMPLE_TEST])
metadata.add_result({"results": str(BT_DATA), "name": "browsertime"})
return metrics, metadata, env
def test_android_apk_alias(device):
args = {
"android-install-apk": ["fenix_fennec_nightly_armeabi_v7a"],
"android": True,
"android-app-name": "org.mozilla.fenned_aurora",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system):
android(metadata)
# XXX really ?
assert device.mock_calls[1][1][0].endswith("target.apk")
def test_install_url(*mocked):
url = "https://here/tarball/" + "".join(
[random.choice(string.hexdigits[:-6]) for c in range(40)])
mach, metadata, env = get_running_env(browsertime_install_url=url)
browser = env.layers[TEST]
env.set_arg("tests", [EXAMPLE_TEST])
try:
with temporary_env(MOZ_AUTOMATION="1"), browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach._mach_context.state_dir)
assert mach.run_process.call_count == 1
def test_android():
args = {
"flavor": "mobile-browser",
"android-install-apk": ["this.apk"],
"android": True,
"android-timeout": 30,
"android-capture-adb": "stdout",
"android-app-name": "org.mozilla.fenix",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system):
android(metadata)
def test_install_url_bad(*mocked):
mach, metadata, env = get_running_env(
browsertime_install_url="meh",
tests=[EXAMPLE_TEST],
)
browser = env.layers[TEST]
sys = env.layers[SYSTEM]
with pytest.raises(ValueError):
try:
with sys as s, browser as b, silence():
b(s(metadata))
finally:
shutil.rmtree(mach._mach_context.state_dir)