def test_perfherder_metrics_filtering():
options = {
"perfherder": True,
"perfherder-prefix": "",
"perfherder-metrics": [metric_fields("I shouldn't match a metric")],
}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "nodata.json")
with nodatajson.open("w") as f:
json.dump({}, f)
metadata.add_result({
"results": str(nodatajson),
"name": "browsertime"
})
with temp_dir() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
assert not pathlib.Path(output, "perfherder-data.json").exists()
def test_android_log(*mocked):
with temp_file() as logcat, temp_dir() as output:
args = {
"flavor": "mobile-browser",
"android-install-apk": ["this.apk"],
"android": True,
"console": True,
"android-timeout": 30,
"android-capture-adb": "stdout",
"android-capture-logcat": logcat,
"android-app-name": "org.mozilla.fenix",
"androidlog": True,
"output": output,
}
mach_cmd, metadata, env = get_running_env(**args)
env.set_arg("tests", [EXAMPLE_TEST])
with env.layers[SYSTEM] as sys, env.layers[TEST] as andro:
metadata = andro(sys(metadata))
# we want to drop the first result
metadata._results = metadata._results[1:]
with env.layers[METRICS] as metrics:
metadata = metrics(metadata)
assert pathlib.Path(output, "LogCatstd-output.json").exists()
def test_influx_service(on_try):
if ON_TRY and sys.platform == "darwin":
# macos slave in the CI are restricted
return
mocks()
with running_on_try(on_try), temp_dir() as output:
args = {
"verbose": True,
"output": output,
"perfboard-influx-password": "******",
"perfboard-grafana-key": "xxx",
"perfboard-grafana-host": "grafana",
"perfboard-influx-port": 0,
"perfboard-influx-host": "influxdb",
"tests": [EXAMPLE_TEST],
}
mach_cmd, metadata, env = get_running_env(**args)
metadata.add_result({"results": str(BT_DATA), "name": "browsertime"})
layer = Influx(env, mach_cmd)
layer.setup()
try:
metadata = layer.run(metadata)
finally:
layer.teardown()
index = on_try and 2 or 1
sent_data = responses.calls[index].request.body.split(b"\n")
fields = [line.split(b",")[0].strip() for line in sent_data]
assert b"rumspeedindex" in fields
responses.reset()
def run(self, metadata):
tests = self.get_arg("tests", [])
if len(tests) != 1:
# for now we support one single test
raise NotImplementedError(str(tests))
test = Path(tests[0])
if not test.exists():
raise FileNotFoundError(str(test))
# let's grab the manifest
manifest = Path(test.parent, "xpcshell.ini")
if not manifest.exists():
raise FileNotFoundError(str(manifest))
import runxpcshelltests
xpcshell = runxpcshelltests.XPCShellTests(log=self)
kwargs = {}
kwargs["testPaths"] = test.name
kwargs["verbose"] = True
kwargs["xpcshell"] = self.mach_cmd.get_binary_path("xpcshell")
kwargs["mozInfo"] = str(Path(self.topobjdir, "mozinfo.json"))
kwargs["symbolsPath"] = str(Path(self.distdir, "crashreporter-symbols"))
kwargs["logfiles"] = True
kwargs["profileName"] = "firefox"
kwargs["pluginsPath"] = str(Path(self.distdir, "plugins"))
kwargs["testingModulesDir"] = str(Path(self.topobjdir, "_tests/modules"))
kwargs["utility_path"] = self.bindir
kwargs["manifest"] = manifest
kwargs["totalChunks"] = 1
cycles = self.get_arg("cycles", 1)
self.info("Running %d cycles" % cycles)
for cycle in range(cycles):
with temp_dir() as tmp, silence():
kwargs["tempDir"] = tmp
if not xpcshell.runTests(kwargs):
raise XPCShellTestError()
self.info("Cycle %d" % (cycle + 1))
self.info("tests done.")
results = defaultdict(list)
for m in self.metrics:
for key, val in m.items():
results[key].append(val)
metadata.add_result(
{
"name": test.name,
"framework": {"name": "xpcshell"},
"transformer": "mozperftest.test.xpcshell:XPCShellData",
"results": [
{"values": measures, "name": subtest}
for subtest, measures in results.items()
],
}
)
return metadata
def test_perfherder_validation_failure():
options = {"perfherder": True, "perfherder-prefix": ""}
metrics, metadata, env = setup_env(options)
# Perfherder schema has limits on min/max data values. Having
# no metrics in the options will cause a failure because of the
# timestamps that are picked up from browsertime.
with pytest.raises(jsonschema.ValidationError):
with temp_dir() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_perfherder_missing_data_failure():
options = {"perfherder": True, "perfherder-prefix": ""}
metrics, metadata, env = setup_env(options)
metadata.clear_results()
with temp_dir() as tmpdir:
nodatajson = pathlib.Path(tmpdir, "baddata.json")
with nodatajson.open("w") as f:
json.dump({"bad data": "here"}, f)
metadata.add_result({"results": str(nodatajson), "name": "browsertime"})
with pytest.raises(PerfherderValidDataError):
with temp_file() as output:
env.set_arg("output", output)
with metrics as m, silence():
m(metadata)
def test_ping_server():
if ON_TRY and sys.platform == "darwin":
# macos slave in the CI are restricted
return
ping_data = {"some": "data"}
with temp_dir() as output:
args = {"verbose": True, "output": output}
mach_cmd, metadata, env = get_running_env(**args)
layer = PingServer(env, mach_cmd)
layer.setup()
try:
metadata = layer.run(metadata)
# simulates a ping
requests.post(
layer.endpoint + "/submit/something", data=json.dumps(ping_data)
)
finally:
layer.teardown()
with Path(output, "telemetry.json").open() as f:
assert json.loads(f.read()) == [ping_data]
def test_console_output(*mocked):
with temp_dir() as tempdir:
options = {
"console-prefix": "",
"console": True,
"output": tempdir,
}
mach_cmd, metadata, env = get_running_env(**options)
runs = []
def _run_process(*args, **kw):
runs.append((args, kw))
mach_cmd.run_process = _run_process
metrics = env.layers[METRICS]
env.set_arg("tests", [EXAMPLE_TEST])
res = {"name": "name", "results": [str(BT_DATA)]}
metadata.add_result(res)
with metrics as console, silence():
console(metadata)
def files(data):
# Create a temporary directory.
with temp_dir() as td:
tmp_path = pathlib.Path(td)
dirs = {
"resources": tmp_path / "resources",
"output": tmp_path / "output",
}
for d in dirs.values():
d.mkdir(parents=True, exist_ok=True)
# Create temporary data files for tests.
def _create_temp_files(path, data):
path.touch(exist_ok=True)
path.write_text(data)
return path.resolve().as_posix()
resources = {}
json_1 = dirs["resources"] / "file_1.json"
resources["file_1"] = _create_temp_files(json_1,
json.dumps(data["data_1"]))
json_2 = dirs["resources"] / "file_2.json"
resources["file_2"] = _create_temp_files(json_2,
json.dumps(data["data_2"]))
txt_3 = dirs["resources"] / "file_3.txt"
resources["file_3"] = _create_temp_files(txt_3, str(data["data_3"]))
output = dirs["output"] / "output.json"
yield {
"resources": resources,
"dirs": dirs,
"output": output,
}
def run(self, metadata):
tests = self.get_arg("tests", [])
if len(tests) != 1:
# for now we support one single test
raise NotImplementedError(str(tests))
test = Path(tests[0])
if not test.exists():
raise FileNotFoundError(str(test))
# let's grab the manifest
manifest = Path(test.parent, "xpcshell.ini")
if not manifest.exists():
raise FileNotFoundError(str(manifest))
nodejs = self.get_arg("nodejs")
if nodejs is not None:
os.environ["MOZ_NODE_PATH"] = nodejs
import runxpcshelltests
verbose = self.get_arg("verbose")
xpcshell = runxpcshelltests.XPCShellTests(log=self)
kwargs = {}
kwargs["testPaths"] = test.name
kwargs["verbose"] = verbose
binary = self.get_arg("binary")
if binary is None:
binary = self.mach_cmd.get_binary_path("xpcshell")
kwargs["xpcshell"] = binary
binary = Path(binary)
mozinfo = self.get_arg("mozinfo")
if mozinfo is None:
mozinfo = binary.parent / ".." / "mozinfo.json"
if not mozinfo.exists():
mozinfo = Path(self.topobjdir, "mozinfo.json")
else:
mozinfo = Path(mozinfo)
kwargs["mozInfo"] = str(mozinfo)
kwargs["symbolsPath"] = str(Path(self.distdir,
"crashreporter-symbols"))
kwargs["logfiles"] = True
kwargs["profileName"] = "firefox"
plugins = binary.parent / "plugins"
if not plugins.exists():
plugins = Path(self.distdir, "plugins")
kwargs["pluginsPath"] = str(plugins)
modules = binary.parent / "modules"
if not modules.exists():
modules = Path(self.topobjdir, "_tests", "modules")
kwargs["testingModulesDir"] = str(modules)
kwargs["utility_path"] = self.bindir
kwargs["manifest"] = str(manifest)
kwargs["totalChunks"] = 1
xre_path = self.get_arg("xre-path")
if xre_path is not None:
self.info(f"Copying {xre_path} elements to {binary.parent}")
copy_tree(xre_path, str(binary.parent), update=True)
http3server = binary.parent / "http3server"
if http3server.exists():
kwargs["http3server"] = str(http3server)
cycles = self.get_arg("cycles", 1)
self.info("Running %d cycles" % cycles)
for cycle in range(cycles):
self.info("Cycle %d" % (cycle + 1))
with temp_dir() as tmp:
kwargs["tempDir"] = tmp
if not xpcshell.runTests(kwargs):
raise XPCShellTestError()
self.info("tests done.")
results = defaultdict(list)
for m in self.metrics:
for key, val in m.items():
results[key].append(val)
metadata.add_result({
"name":
test.name,
"framework": {
"name": "mozperftest"
},
"transformer":
"mozperftest.test.xpcshell:XPCShellData",
"results": [{
"values": measures,
"name": subtest
} for subtest, measures in results.items()],
})
return metadata
