def test_perfherder_metrics_filtering():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": ["I shouldn't match a metric"],
}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "nodata.json")
with nodatajson.open("w") as f:
json.dump({}, f)
metadata.add_result({
"results": str(nodatajson),
"name": "browsertime"
})
with temp_dir() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
assert not pathlib.Path(output, "perfherder-data.json").exists()
def files(data):
# Create a temporary directory.
with temp_dir() as td:
tmp_path = pathlib.Path(td)
dirs = {
"resources": tmp_path / "resources",
"output": tmp_path / "output",
}
for d in dirs.values():
d.mkdir(parents=True, exist_ok=True)
# Create temporary data files for tests.
def _create_temp_files(path, data):
path.touch(exist_ok=True)
path.write_text(data)
return path.resolve().as_posix()
resources = {}
json_1 = dirs["resources"] / "file_1.json"
resources["file_1"] = _create_temp_files(json_1,
json.dumps(data["data_1"]))
json_2 = dirs["resources"] / "file_2.json"
resources["file_2"] = _create_temp_files(json_2,
json.dumps(data["data_2"]))
txt_3 = dirs["resources"] / "file_3.txt"
resources["file_3"] = _create_temp_files(txt_3, str(data["data_3"]))
output = dirs["output"] / "output.json"
yield resources, dirs, output.resolve().as_posix()
def test_android_log(*mocked):
with temp_file() as logcat, temp_dir() as output:
args = {
"flavor": "mobile-browser",
"android-install-apk": ["this.apk"],
"android": True,
"console": True,
"android-timeout": 30,
"android-capture-adb": "stdout",
"android-capture-logcat": logcat,
"android-app-name": "org.mozilla.fenix",
"androidlog": True,
"output": output,
}
mach_cmd, metadata, env = get_running_env(**args)
env.set_arg("tests", [EXAMPLE_TEST])
with env.layers[SYSTEM] as sys, env.layers[TEST] as andro:
metadata = sys(andro(metadata))
# we want to drop the first result
metadata._results = metadata._results[1:]
with env.layers[METRICS] as metrics:
metadata = metrics(metadata)
assert pathlib.Path(output, "LogCatstd-output.json").exists()
def test_perfherder_validation_failure():
options = {"perfherder": True, "perfherder-prefix": ""}
metrics, metadata, env = setup_env(options)
# Perfherder schema has limits on min/max data values. Having
# no metrics in the options will cause a failure because of the
# timestamps that are picked up from browsertime.
with pytest.raises(jsonschema.ValidationError):
with temp_dir() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_perfherder_missing_data_failure():
options = {"perfherder": True, "perfherder-prefix": ""}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "baddata.json")
with nodatajson.open("w") as f:
json.dump({"bad data": "here"}, f)
metadata.add_result({
"results": str(nodatajson),
"name": "browsertime"
})
with pytest.raises(PerfherderValidDataError):
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_console_output(*mocked):
with temp_dir() as tempdir:
options = {
"console-prefix": "",
"console": True,
"output": tempdir,
}
mach_cmd, metadata, env = get_running_env(**options)
runs = []
def _run_process(*args, **kw):
runs.append((args, kw))
mach_cmd.run_process = _run_process
metrics = env.layers[METRICS]
env.set_arg("tests", [EXAMPLE_TEST])
res = {"name": "name", "results": [str(BT_DATA)]}
metadata.add_result(res)
with metrics as console, silence():
console(metadata)
