def test_compare_to_success(notebook, stats):
options = {
"notebook-metrics": [metric_fields("firstPaint")],
"notebook-prefix": "",
"notebook-analysis": [],
"notebook": True,
"notebook-compare-to": [str(BT_DATA.parent)],
"notebook-stats": stats,
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
args, kwargs = notebook.call_args_list[0]
if not stats:
assert len(kwargs["data"]) == 2
assert kwargs["data"][0]["name"] == "browsertime- newest run"
assert kwargs["data"][1]["name"] == "browsertime-results"
else:
assert any("statistics" in element["subtest"]
for element in kwargs["data"])
notebook.assert_has_calls(
[mock.call().post_to_iodide(["compare"], start_local_server=True)])
def run(self, metadata):
tests = self.get_arg("tests", [])
if len(tests) != 1:
# for now we support one single test
raise NotImplementedError(str(tests))
test = Path(tests[0])
if not test.exists():
raise FileNotFoundError(str(test))
# let's grab the manifest
manifest = Path(test.parent, "xpcshell.ini")
if not manifest.exists():
raise FileNotFoundError(str(manifest))
import runxpcshelltests
xpcshell = runxpcshelltests.XPCShellTests(log=self)
kwargs = {}
kwargs["testPaths"] = test.name
kwargs["verbose"] = True
kwargs["xpcshell"] = self.mach_cmd.get_binary_path("xpcshell")
kwargs["mozInfo"] = str(Path(self.topobjdir, "mozinfo.json"))
kwargs["symbolsPath"] = str(Path(self.distdir, "crashreporter-symbols"))
kwargs["logfiles"] = True
kwargs["profileName"] = "firefox"
kwargs["pluginsPath"] = str(Path(self.distdir, "plugins"))
kwargs["testingModulesDir"] = str(Path(self.topobjdir, "_tests/modules"))
kwargs["utility_path"] = self.bindir
kwargs["manifest"] = manifest
kwargs["totalChunks"] = 1
cycles = self.get_arg("cycles", 1)
self.info("Running %d cycles" % cycles)
for cycle in range(cycles):
with temp_dir() as tmp, silence():
kwargs["tempDir"] = tmp
if not xpcshell.runTests(kwargs):
raise XPCShellTestError()
self.info("Cycle %d" % (cycle + 1))
self.info("tests done.")
results = defaultdict(list)
for m in self.metrics:
for key, val in m.items():
results[key].append(val)
metadata.add_result(
{
"name": test.name,
"framework": {"name": "xpcshell"},
"transformer": "mozperftest.test.xpcshell:XPCShellData",
"results": [
{"values": measures, "name": subtest}
for subtest, measures in results.items()
],
}
)
return metadata
def test_replay_url(killer, running_env):
mach_cmd, metadata, env = running_env
system = env.layers[SYSTEM]
env.set_arg("proxy-replay", "http://example.dump")
with system as proxy, silence():
proxy(metadata)
def test_notebookupload_with_filter(notebook, no_filter):
options = {
"notebook-metrics": [],
"notebook-prefix": "",
"notebook": True,
"notebook-analysis": ["scatterplot"],
"notebook-analyze-strings": no_filter,
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
if no_filter:
args, kwargs = notebook.call_args_list[0]
assert type(kwargs["data"][0]["data"][0]["value"]) == str
else:
for call in notebook.call_args_list:
args, kwargs = call
for a in args:
for data_dict in a:
for data in data_dict["data"]:
assert type(data["value"]) in (int, float)
notebook.assert_has_calls(
[mock.call().post_to_iodide(["scatterplot"], start_local_server=True)])
def test_run_python_script_failed(*mocked):
with _get_command(PerftestTests) as test, silence(test) as captured:
test._run_python_script("nothing")
stdout, stderr = captured
stdout.seek(0)
assert stdout.read().endswith("[FAILED]\n")
def test_browser_desktop(*mocked):
mach_cmd, metadata, env = get_running_env(
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
tests=[EXAMPLE_TEST],
browsertime_no_window_recorder=False,
browsertime_viewport_size="1234x567",
)
browser = env.layers[TEST]
sys = env.layers[SYSTEM]
try:
with sys as s, browser as b, silence():
# just checking that the setup_helper property gets
# correctly initialized
browsertime = browser.layers[-1]
assert browsertime.setup_helper is not None
helper = browsertime.setup_helper
assert browsertime.setup_helper is helper
b(s(metadata))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
# check that --firefox.binaryPath is set automatically
assert "--firefox.binaryPath" in cmd
def test_proxy(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
def test_checkout_python_script():
with silence() as captured:
assert checkout_python_script(_Venv(), "lib2to3", ["--help"])
stdout, stderr = captured
stdout.seek(0)
assert stdout.read() == "=> lib2to3 [OK]\n"
def test_perfherder_with_extra_options():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": [
metric_fields("name:firstPaint,extraOptions:['option']"),
metric_fields("name:resource,extraOptions:['second-option']"),
],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
assert len(output["suites"]) == 1
assert sorted(output["suites"][0]["extraOptions"]) == sorted(
["option", "second-option"]
)
def test_perfherder_split_by():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-app": "fenix",
"perfherder-metrics": [metric_fields("firstPaint")],
"perfherder-split-by": "browserScripts.pageinfo.url",
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Sanity check
assert len(output["suites"]) == 1
# We should have 2 subtests (1 per URL)
assert len(output["suites"][0]["subtests"]) == 2
# Check to make sure that they were properly split
names = [subtest["name"] for subtest in output["suites"][0]["subtests"]]
assert sorted(names) == [
"browserScripts.timings.firstPaint https://www.mozilla.org/en-US/",
"browserScripts.timings.firstPaint https://www.sitespeed.io/",
]
for i in range(2):
assert len(output["suites"][0]["subtests"][i]["replicates"]) == 1
def test_perfherder_with_subunits():
options = {
"perfherder":
True,
"perfherder-stats":
True,
"perfherder-prefix":
"",
"perfherder-metrics": [
metric_fields("name:firstPaint,extraOptions:['option']"),
metric_fields("name:resource,shouldAlert:True,unit:a-unit"),
],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
assert len(output["suites"]) == 1
assert all([
subtest["unit"] == "a-unit"
for subtest in output["suites"][0]["subtests"]
if "resource" in subtest["name"]
])
assert all([
subtest["unit"] == "ms" for subtest in output["suites"][0]["subtests"]
if "firstPaint" in subtest["name"]
])
def test_perfherder_metrics_filtering():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": ["I shouldn't match a metric"],
}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "nodata.json")
with nodatajson.open("w") as f:
json.dump({}, f)
metadata.add_result({
"results": str(nodatajson),
"name": "browsertime"
})
with temp_dir() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
assert not pathlib.Path(output, "perfherder-data.json").exists()
def test_perfherder():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "browsertime"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 10
assert output["suites"][0]["value"] > 0
# Check if only firstPaint metrics were obtained
for subtest in output["suites"][0]["subtests"]:
assert "firstPaint" in subtest["name"]
def test_run_python_script(*mocked):
with _get_command(PerftestTests) as test, silence(test) as captured:
test._run_python_script("lib2to3", *["--help"])
stdout, stderr = captured
stdout.seek(0)
assert stdout.read() == "=> lib2to3 [OK]\n"
def test_browser(*mocked):
mach_cmd, metadata, env = get_running_env(
android=True,
android_app_name="something",
browsertime_geckodriver="GECKODRIVER",
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
)
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
try:
with browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
# Make sure all arguments are of type str
for option in mach_cmd.run_process.call_args[0][0]:
assert isinstance(option, str)
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
assert EXAMPLE_TEST in cmd
assert "--firefox.geckodriverPath GECKODRIVER" in cmd
assert "--one 1" in cmd
assert "--two 2" in cmd
results = metadata.get_results()
assert len(results) == 1
assert set(list(results[0].keys())) - set(["name", "results"]) == set()
assert results[0]["name"] == "Example"
def test_perfherder_exlude_stats():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 1
assert output["suites"][0]["value"] > 0
# Check if only one firstPaint metric was obtained
assert ("browserScripts.timings.firstPaint" == output["suites"][0]
["subtests"][0]["name"])
def test_run_python_script_failed():
with silence() as captured:
assert not checkout_python_script(_Venv(), "nothing")
stdout, stderr = captured
stdout.seek(0)
assert stdout.read().endswith("[FAILED]\n")
def test_playback_no_file(killer):
mach_cmd, metadata, env = running_env()
system = env.layers[SYSTEM]
env.set_arg("proxy-mode", "playback")
with system as proxy, pytest.raises(ValueError) as excinfo, silence():
proxy(metadata)
assert "Proxy file not provided!!" in str(excinfo.value)
def test_hooks_state(venv, env):
kwargs = {
"tests": [EXAMPLE_TEST],
"hooks": STATE_HOOKS,
"flavor": "desktop-browser",
}
with _get_command() as test, silence(test):
test.run_perftest(**kwargs)
def test_hooks_state(venv, env):
kwargs = {
"tests": [EXAMPLE_TEST],
"hooks": STATE_HOOKS,
"flavor": "desktop-browser",
}
with _get_command() as (cmd, command_context), silence(command_context):
cmd(command_context, **kwargs)
def test_push_command(push_to_try, venv):
with _get_command() as test, silence(test):
test.run_perftest(
tests=[EXAMPLE_TEST],
flavor="desktop-browser",
push_to_try=True,
try_platform="g5",
)
push_to_try.assert_called()
def test_test_runner(*mocked):
# simulating on try to run the paths parser
old = mach_commands.ON_TRY
mach_commands.ON_TRY = True
with _get_command(PerftestTests) as test, silence(test), temporary_env(
MOZ_AUTOMATION="1"):
test.run_tests(tests=[EXAMPLE_TESTS_DIR])
mach_commands.ON_TRY = old
def test_record(killer, running_env):
mach_cmd, metadata, env = running_env
system = env.layers[SYSTEM]
with tempfile.TemporaryDirectory() as tmpdir:
recording = os.path.join(tmpdir, "recording.dump")
env.set_arg("proxy-record", recording)
with system as proxy, silence():
proxy(metadata)
def test_push_command(push_to_try, venv):
with _get_command() as (cmd, command_context), silence(command_context):
cmd(
command_context,
tests=[EXAMPLE_TEST],
flavor="desktop-browser",
push_to_try=True,
try_platform="g5",
)
push_to_try.assert_called()
def test_command_iterations(venv, env):
kwargs = {
"tests": [EXAMPLE_TEST],
"hooks": ITERATION_HOOKS,
"flavor": "desktop-browser",
}
with _get_command() as (cmd, command_context), silence(command_context):
cmd(command_context, **kwargs)
# the hook changes the iteration value to 5.
# each iteration generates 5 calls, so we want to see 25
assert len(env.mock_calls) == 25
def test_replay_url(killer):
mach_cmd, metadata, env = running_env()
system = env.layers[SYSTEM]
env.set_arg("proxy-mode", "playback")
env.set_arg("proxy-file", "http://example.dump")
with system as proxy, silence():
proxy(metadata)
browser_prefs = metadata.get_options("browser_prefs")
assert browser_prefs["network.proxy.http_port"] == 1234
def test_replay_url(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
env.set_arg("proxy-replay", "http://example.dump")
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
# Give mitmproxy a bit of time to start up so we can verify that it's
# actually running before we tear things down.
time.sleep(5)
def test_record(install_mozproxy):
mach_cmd, metadata, env = get_running_env(proxy=True)
system = env.layers[SYSTEM]
with tempfile.TemporaryDirectory() as tmpdir:
recording = os.path.join(tmpdir, "recording.dump")
env.set_arg("proxy-record", recording)
# XXX this will run for real, we need to mock HTTP calls
with system as proxy, silence():
proxy(metadata)
assert os.path.exists(recording)
def test_android():
args = {
"android-install-apk": ["this"],
"android": True,
"android-app-name": "org.mozilla.fenix",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system):
android(metadata)
def test_install_url_bad(*mocked):
mach, metadata, env = get_running_env(browsertime_install_url="meh")
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
with pytest.raises(ValueError):
try:
with browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach._mach_context.state_dir)