def test_open_file():
data = json.dumps({"1": 2})
with temp_file(name="data.json", content=data) as f:
res = open_file(f)
assert res == {"1": 2}
with temp_file(name="data.txt", content="yeah") as f:
assert open_file(f) == "yeah"
def test_xpcshell_perfherder(*mocked):
mach_cmd, metadata, env = get_running_env(flavor="xpcshell",
perfherder=True,
xpcshell_cycles=10)
sys = env.layers[SYSTEM]
xpcshell = env.layers[TEST]
env.set_arg("tests", [str(EXAMPLE_XPCSHELL_TEST)])
metrics = env.layers[METRICS]
with temp_file() as output:
env.set_arg("output", output)
try:
with sys as s, xpcshell as x, metrics as m:
m(x(s(metadata)))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "xpcshell"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 3
assert output["suites"][0]["value"] > 0
for subtest in output["suites"][0]["subtests"]:
assert subtest["name"].startswith("metrics")
def test_perfherder_exlude_stats():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 1
assert output["suites"][0]["value"] > 0
# Check if only one firstPaint metric was obtained
assert ("browserScripts.timings.firstPaint" == output["suites"][0]
["subtests"][0]["name"])
def test_perfherder():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "browsertime"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 10
assert output["suites"][0]["value"] > 0
# Check if only firstPaint metrics were obtained
for subtest in output["suites"][0]["subtests"]:
assert "firstPaint" in subtest["name"]
def test_perfherder_split_by():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-app": "fenix",
"perfherder-metrics": [metric_fields("firstPaint")],
"perfherder-split-by": "browserScripts.pageinfo.url",
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Sanity check
assert len(output["suites"]) == 1
# We should have 2 subtests (1 per URL)
assert len(output["suites"][0]["subtests"]) == 2
# Check to make sure that they were properly split
names = [subtest["name"] for subtest in output["suites"][0]["subtests"]]
assert sorted(names) == [
"browserScripts.timings.firstPaint https://www.mozilla.org/en-US/",
"browserScripts.timings.firstPaint https://www.sitespeed.io/",
]
for i in range(2):
assert len(output["suites"][0]["subtests"][i]["replicates"]) == 1
def test_notebookupload_with_filter(notebook, no_filter):
options = {
"notebook-metrics": [],
"notebook-prefix": "",
"notebook": True,
"notebook-analysis": ["scatterplot"],
"notebook-analyze-strings": no_filter,
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
if no_filter:
args, kwargs = notebook.call_args_list[0]
assert type(kwargs["data"][0]["data"][0]["value"]) == str
else:
for call in notebook.call_args_list:
args, kwargs = call
for a in args:
for data_dict in a:
for data in data_dict["data"]:
assert type(data["value"]) in (int, float)
notebook.assert_has_calls(
[mock.call().post_to_iodide(["scatterplot"], start_local_server=True)])
def test_perfherder_with_extra_options():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": [
metric_fields("name:firstPaint,extraOptions:['option']"),
metric_fields("name:resource,extraOptions:['second-option']"),
],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
assert len(output["suites"]) == 1
assert sorted(output["suites"][0]["extraOptions"]) == sorted(
["option", "second-option"]
)
def test_compare_to_success(notebook, stats):
options = {
"notebook-metrics": [metric_fields("firstPaint")],
"notebook-prefix": "",
"notebook-analysis": [],
"notebook": True,
"notebook-compare-to": [str(BT_DATA.parent)],
"notebook-stats": stats,
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
args, kwargs = notebook.call_args_list[0]
if not stats:
assert len(kwargs["data"]) == 2
assert kwargs["data"][0]["name"] == "browsertime- newest run"
assert kwargs["data"][1]["name"] == "browsertime-results"
else:
assert any("statistics" in element["subtest"]
for element in kwargs["data"])
notebook.assert_has_calls(
[mock.call().post_to_iodide(["compare"], start_local_server=True)])
def test_android_log(*mocked):
with temp_file() as logcat, temp_dir() as output:
args = {
"flavor": "mobile-browser",
"android-install-apk": ["this.apk"],
"android": True,
"console": True,
"android-timeout": 30,
"android-capture-adb": "stdout",
"android-capture-logcat": logcat,
"android-app-name": "org.mozilla.fenix",
"androidlog": True,
"output": output,
}
mach_cmd, metadata, env = get_running_env(**args)
env.set_arg("tests", [EXAMPLE_TEST])
with env.layers[SYSTEM] as sys, env.layers[TEST] as andro:
metadata = andro(sys(metadata))
# we want to drop the first result
metadata._results = metadata._results[1:]
with env.layers[METRICS] as metrics:
metadata = metrics(metadata)
assert pathlib.Path(output, "LogCatstd-output.json").exists()
def test_visual_metrics(device):
os.environ["VISUALMETRICS_PY"] = ""
mach_cmd, metadata, env = get_running_env(
visualmetrics=True,
perfherder=True,
verbose=True,
tests=[EXAMPLE_TEST],
)
metrics = env.layers[METRICS]
metadata.add_result({
"results": str(BT_DATA_VIDEO.parent),
"name": "browsertime"
})
with temp_file() as output:
env.set_arg("output", output)
with metrics as m:
metadata = m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
visual_metrics = [i["name"] for i in output["suites"][1]["subtests"]]
assert "VisualProgress96" in visual_metrics
def test_perfherder_with_subunits():
options = {
"perfherder":
True,
"perfherder-stats":
True,
"perfherder-prefix":
"",
"perfherder-metrics": [
metric_fields("name:firstPaint,extraOptions:['option']"),
metric_fields("name:resource,shouldAlert:True,unit:a-unit"),
],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
assert len(output["suites"]) == 1
assert all([
subtest["unit"] == "a-unit"
for subtest in output["suites"][0]["subtests"]
if "resource" in subtest["name"]
])
assert all([
subtest["unit"] == "ms" for subtest in output["suites"][0]["subtests"]
if "firstPaint" in subtest["name"]
])
def test_get_output_dir():
with temp_file() as temp_dir:
output_dir = get_output_dir(temp_dir)
assert output_dir.exists()
assert output_dir.is_dir()
output_dir = get_output_dir(output=temp_dir, folder="artifacts")
assert output_dir.exists()
assert output_dir.is_dir()
assert "artifacts" == output_dir.parts[-1]
def test_preview():
content = Path(EXAMPLE_TEST)
line = f"[bt][sometag] {content.name} in {content.parent}"
test_objects = [{"path": str(content)}]
cache = Path(Path.home(), ".mozbuild", ".perftestfuzzy")
with cache.open("w") as f:
f.write(json.dumps(test_objects))
with temp_file(content=str(line)) as tasklist, silence() as out:
main(args=["-t", tasklist])
stdout, __ = out
stdout.seek(0)
assert ":owner: Performance Testing Team" in stdout.read()
def test_perfherder_logcat():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": [metric_fields("TimeToDisplayed")],
}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
def processor(groups):
"""Parses the time from a displayed time string into milliseconds."""
return (float(groups[0]) * 1000) + float(groups[1])
re_w_group = r".*Displayed.*org\.mozilla\.fennec_aurora.*\+([\d]+)s([\d]+)ms.*"
metadata.add_result(
{
"results": str(HERE / "data" / "home_activity.txt"),
"transformer": "LogCatTimeTransformer",
"transformer-options": {
"first-timestamp": re_w_group,
"processor": processor,
"transform-subtest-name": "TimeToDisplayed",
},
"name": "LogCat",
}
)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m: # , silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "browsertime"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert len(output["suites"][0]["subtests"]) == 1
assert output["suites"][0]["value"] > 0
# Check if only the TimeToDisplayd metric was obtained
for subtest in output["suites"][0]["subtests"]:
assert "TimeToDisplayed" in subtest["name"]
def test_android_log_adb():
with temp_file() as log_adb:
args = {
"android-install-apk": ["gve_nightly_api16"],
"android": True,
"android-timeout": 60,
"android-app-name": "org.mozilla.geckoview_example",
"android-capture-adb": log_adb,
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
with system as android, silence(system), pytest.raises(DeviceError):
android(metadata)
with open(log_adb) as f:
assert "DEBUG ADBLoggedDevice" in f.read()
def test_perfherder_bad_app_name():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-app": "this is not an app",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
# This will raise an error because the options method
# we use in tests skips the `choices` checks.
with pytest.raises(jsonschema.ValidationError):
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_perfherder_missing_data_failure():
options = {"perfherder": True, "perfherder-prefix": ""}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "baddata.json")
with nodatajson.open("w") as f:
json.dump({"bad data": "here"}, f)
metadata.add_result({"results": str(nodatajson), "name": "browsertime"})
with pytest.raises(PerfherderValidDataError):
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_compare_to_invalid_parameter(notebook, filepath):
options = {
"notebook-metrics": [metric_fields("firstPaint")],
"notebook-prefix": "",
"notebook-analysis": [],
"notebook": True,
"notebook-compare-to": [filepath],
}
metrics, metadata, env = setup_env(options)
with pytest.raises(Exception) as einfo:
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
if filepath == "invalidPath":
assert "does not exist" in str(einfo.value)
else:
assert "not a directory" in str(einfo.value)
def test_perfherder_app_name():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-app": "fenix",
"perfherder-metrics": ["firstPaint"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Make sure that application setting is correct
assert output["application"]["name"] == "fenix"
assert "version" not in output["application"]
def test_android_log_cat(device):
with temp_file() as log_cat:
args = {
"android-install-apk": ["gve_nightly_api16"],
"android": True,
"android-timeout": 60,
"android-app-name": "org.mozilla.geckoview_example",
"android-capture-logcat": log_cat,
"android-clear-logcat": True,
"android-capture-adb": "stdout",
}
mach_cmd, metadata, env = get_running_env(**args)
system = env.layers[SYSTEM]
andro = system.layers[0]
with system as layer, silence(system):
andro.device = device
andro.device.get_logcat = mock.Mock(result_value=[])
layer(metadata)
andro.device.get_logcat.assert_called()
andro.device.clear_logcat.assert_called()
def test_perfherder_names_simplified_with_no_exclusions():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": [metric_fields("firstPaint"), metric_fields("resource")],
"perfherder-simplify-names": True,
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "browsertime"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert output["suites"][0]["value"] > 0
# In this case, some metrics will be called "median", "mean", etc.
# since those are the simplifications of the first statistics entries
# that were found.
assert not all(
[
"firstPaint" in subtest["name"]
or "duration" in subtest["name"]
or "count" in subtest["name"]
for subtest in output["suites"][0]["subtests"]
]
)
found_all = {"firstPaint": False, "count": False, "duration": False}
for subtest in output["suites"][0]["subtests"]:
if subtest["name"] in found_all:
found_all[subtest["name"]] = True
continue
for entry, value in found_all.items():
assert found_all[entry], f"Failed finding metric simplification for {entry}"
# Only a portion of the metrics should still have statistics in
# their name due to a naming conflict that only emits a warning
assert (
len(
[
subtest
for subtest in output["suites"][0]["subtests"]
if "statistics" in subtest["name"]
]
)
== 18
)
assert (
len(
[
subtest
for subtest in output["suites"][0]["subtests"]
if "statistics" not in subtest["name"]
]
)
== 12
)
def test_perfherder_simple_names():
options = {
"perfherder": True,
"perfherder-stats": True,
"perfherder-prefix": "",
"perfherder-metrics": [metric_fields("firstPaint"), metric_fields("resource")],
"perfherder-simplify-names": True,
"perfherder-simplify-exclude": ["statistics"],
}
metrics, metadata, env = setup_env(options)
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
output_file = metadata.get_output()
with open(output_file) as f:
output = json.loads(f.read())
# Check some metadata
assert output["application"]["name"] == "firefox"
assert output["framework"]["name"] == "browsertime"
# Check some numbers in our data
assert len(output["suites"]) == 1
assert output["suites"][0]["value"] > 0
# Check if only firstPaint/resource metrics were obtained and
# that simplifications occurred
assert all(
[
"firstPaint" in subtest["name"]
or "duration" in subtest["name"]
or "count" in subtest["name"]
for subtest in output["suites"][0]["subtests"]
]
)
found_all = {"firstPaint": False, "count": False, "duration": False}
for subtest in output["suites"][0]["subtests"]:
if subtest["name"] in found_all:
found_all[subtest["name"]] = True
continue
assert any([name in subtest["name"] for name in found_all.keys()])
# Statistics are not simplified so any metric that isn't
# in the list of known metrics must be a statistic
assert "statistics" in subtest["name"]
for entry, value in found_all.items():
assert found_all[entry], f"Failed finding metric simplification for {entry}"
# Statistics are not simplified by default
assert (
len(
[
subtest
for subtest in output["suites"][0]["subtests"]
if "statistics" in subtest["name"]
]
)
== 27
)
assert (
len(
[
subtest
for subtest in output["suites"][0]["subtests"]
if "statistics" not in subtest["name"]
]
)
== 3
)
def test_download_file_success():
with temp_file() as target:
download_file("http://content", Path(target), retry_sleep=0.1)
with open(target) as f:
assert f.read() == "some content"
def test_download_file_fails():
with temp_file() as target, silence(), pytest.raises(Exception):
download_file("http://I don't exist", Path(target), retry_sleep=0.1)
