def test_perfherder_with_subunits(): options = { "perfherder": True, "perfherder-stats": True, "perfherder-prefix": "", "perfherder-metrics": [ metric_fields("name:firstPaint,extraOptions:['option']"), metric_fields("name:resource,shouldAlert:True,unit:a-unit"), ], } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) assert len(output["suites"]) == 1 assert all([ subtest["unit"] == "a-unit" for subtest in output["suites"][0]["subtests"] if "resource" in subtest["name"] ]) assert all([ subtest["unit"] == "ms" for subtest in output["suites"][0]["subtests"] if "firstPaint" in subtest["name"] ])
def test_perfherder_with_extra_options(): options = { "perfherder": True, "perfherder-stats": True, "perfherder-prefix": "", "perfherder-metrics": [ metric_fields("name:firstPaint,extraOptions:['option']"), metric_fields("name:resource,extraOptions:['second-option']"), ], } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) assert len(output["suites"]) == 1 assert sorted(output["suites"][0]["extraOptions"]) == sorted( ["option", "second-option"] )
def test_perfherder_metrics_filtering(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("I shouldn't match a metric")], } metrics, metadata, env = setup_env(options) metadata.clear_results() with temp_dir() as tmpdir: nodatajson = pathlib.Path(tmpdir, "nodata.json") with nodatajson.open("w") as f: json.dump({}, f) metadata.add_result({ "results": str(nodatajson), "name": "browsertime" }) with temp_dir() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) assert not pathlib.Path(output, "perfherder-data.json").exists()
def test_perfherder_exlude_stats(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("firstPaint")], } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some numbers in our data assert len(output["suites"]) == 1 assert len(output["suites"][0]["subtests"]) == 1 assert output["suites"][0]["value"] > 0 # Check if only firstPaint metric was obtained with 2 replicates assert len(output["suites"][0]["subtests"][0]["replicates"]) == 2 assert ( "browserScripts.timings.firstPaint" == output["suites"][0]["subtests"][0]["name"] )
def test_perfherder_split_by(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-app": "fenix", "perfherder-metrics": [metric_fields("firstPaint")], "perfherder-split-by": "browserScripts.pageinfo.url", } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Sanity check assert len(output["suites"]) == 1 # We should have 2 subtests (1 per URL) assert len(output["suites"][0]["subtests"]) == 2 # Check to make sure that they were properly split names = [subtest["name"] for subtest in output["suites"][0]["subtests"]] assert sorted(names) == [ "browserScripts.timings.firstPaint https://www.mozilla.org/en-US/", "browserScripts.timings.firstPaint https://www.sitespeed.io/", ] for i in range(2): assert len(output["suites"][0]["subtests"][i]["replicates"]) == 1
def test_compare_to_success(notebook, stats): options = { "notebook-metrics": [metric_fields("firstPaint")], "notebook-prefix": "", "notebook-analysis": [], "notebook": True, "notebook-compare-to": [str(BT_DATA.parent)], "notebook-stats": stats, } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) args, kwargs = notebook.call_args_list[0] if not stats: assert len(kwargs["data"]) == 2 assert kwargs["data"][0]["name"] == "browsertime- newest run" assert kwargs["data"][1]["name"] == "browsertime-results" else: assert any("statistics" in element["subtest"] for element in kwargs["data"]) notebook.assert_has_calls( [mock.call().post_to_iodide(["compare"], start_local_server=True)])
def test_perfherder(): options = { "perfherder": True, "perfherder-stats": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("firstPaint")], "perfherder-timestamp": 1.0, } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" assert output["framework"]["name"] == "browsertime" assert output["pushTimestamp"] == 1.0 # Check some numbers in our data assert len(output["suites"]) == 1 assert len(output["suites"][0]["subtests"]) == 10 assert output["suites"][0]["value"] > 0 # Check if only firstPaint metrics were obtained for subtest in output["suites"][0]["subtests"]: assert "firstPaint" in subtest["name"]
def test_perfherder_logcat(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("TimeToDisplayed")], } metrics, metadata, env = setup_env(options) metadata.clear_results() def processor(groups): """Parses the time from a displayed time string into milliseconds.""" return (float(groups[0]) * 1000) + float(groups[1]) re_w_group = r".*Displayed.*org\.mozilla\.fennec_aurora.*\+([\d]+)s([\d]+)ms.*" metadata.add_result( { "results": str(HERE / "data" / "home_activity.txt"), "transformer": "LogCatTimeTransformer", "transformer-options": { "first-timestamp": re_w_group, "processor": processor, "transform-subtest-name": "TimeToDisplayed", }, "name": "LogCat", } ) with temp_file() as output: env.set_arg("output", output) with metrics as m: # , silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" assert output["framework"]["name"] == "browsertime" # Check some numbers in our data assert len(output["suites"]) == 1 assert len(output["suites"][0]["subtests"]) == 1 assert output["suites"][0]["value"] > 0 # Check if only the TimeToDisplayd metric was obtained for subtest in output["suites"][0]["subtests"]: assert "TimeToDisplayed" in subtest["name"]
def test_perfherder_bad_app_name(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-app": "this is not an app", "perfherder-metrics": [metric_fields("firstPaint")], } metrics, metadata, env = setup_env(options) # This will raise an error because the options method # we use in tests skips the `choices` checks. with pytest.raises(jsonschema.ValidationError): with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata)
def test_compare_to_invalid_parameter(notebook, filepath): options = { "notebook-metrics": [metric_fields("firstPaint")], "notebook-prefix": "", "notebook-analysis": [], "notebook": True, "notebook-compare-to": [filepath], } metrics, metadata, env = setup_env(options) with pytest.raises(Exception) as einfo: with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) if filepath == "invalidPath": assert "does not exist" in str(einfo.value) else: assert "not a directory" in str(einfo.value)
def test_perfherder_app_name(): options = { "perfherder": True, "perfherder-prefix": "", "perfherder-app": "fenix", "perfherder-metrics": [metric_fields("firstPaint")], } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Make sure that application setting is correct assert output["application"]["name"] == "fenix" assert "version" not in output["application"]
def test_metric_fields_new_format(): assert metric_fields("name:foo,extraOptions:bar") == { "name": "foo", "extraOptions": "bar", }
def test_metric_fields_old_format(): assert metric_fields("firstPaint") == {"name": "firstPaint"}
def test_perfherder_simple_names(): options = { "perfherder": True, "perfherder-stats": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("firstPaint"), metric_fields("resource")], "perfherder-simplify-names": True, "perfherder-simplify-exclude": ["statistics"], } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" assert output["framework"]["name"] == "browsertime" # Check some numbers in our data assert len(output["suites"]) == 1 assert output["suites"][0]["value"] > 0 # Check if only firstPaint/resource metrics were obtained and # that simplifications occurred assert all( [ "firstPaint" in subtest["name"] or "duration" in subtest["name"] or "count" in subtest["name"] for subtest in output["suites"][0]["subtests"] ] ) found_all = {"firstPaint": False, "count": False, "duration": False} for subtest in output["suites"][0]["subtests"]: if subtest["name"] in found_all: found_all[subtest["name"]] = True continue assert any([name in subtest["name"] for name in found_all.keys()]) # Statistics are not simplified so any metric that isn't # in the list of known metrics must be a statistic assert "statistics" in subtest["name"] for entry, value in found_all.items(): assert found_all[entry], f"Failed finding metric simplification for {entry}" # Statistics are not simplified by default assert ( len( [ subtest for subtest in output["suites"][0]["subtests"] if "statistics" in subtest["name"] ] ) == 27 ) assert ( len( [ subtest for subtest in output["suites"][0]["subtests"] if "statistics" not in subtest["name"] ] ) == 3 )
def test_metric_fields_simple(metrics, expected): assert metric_fields(metrics) == expected
def test_metric_fields_complex(metrics, expected): assert metric_fields(metrics) == expected
def test_metric_fields_complex_failures(metrics): with pytest.raises(Exception): metric_fields(metrics)
def test_perfherder_names_simplified_with_no_exclusions(): options = { "perfherder": True, "perfherder-stats": True, "perfherder-prefix": "", "perfherder-metrics": [metric_fields("firstPaint"), metric_fields("resource")], "perfherder-simplify-names": True, } metrics, metadata, env = setup_env(options) with temp_file() as output: env.set_arg("output", output) with metrics as m, silence(): m(metadata) output_file = metadata.get_output() with open(output_file) as f: output = json.loads(f.read()) # Check some metadata assert output["application"]["name"] == "firefox" assert output["framework"]["name"] == "browsertime" # Check some numbers in our data assert len(output["suites"]) == 1 assert output["suites"][0]["value"] > 0 # In this case, some metrics will be called "median", "mean", etc. # since those are the simplifications of the first statistics entries # that were found. assert not all( [ "firstPaint" in subtest["name"] or "duration" in subtest["name"] or "count" in subtest["name"] for subtest in output["suites"][0]["subtests"] ] ) found_all = {"firstPaint": False, "count": False, "duration": False} for subtest in output["suites"][0]["subtests"]: if subtest["name"] in found_all: found_all[subtest["name"]] = True continue for entry, value in found_all.items(): assert found_all[entry], f"Failed finding metric simplification for {entry}" # Only a portion of the metrics should still have statistics in # their name due to a naming conflict that only emits a warning assert ( len( [ subtest for subtest in output["suites"][0]["subtests"] if "statistics" in subtest["name"] ] ) == 18 ) assert ( len( [ subtest for subtest in output["suites"][0]["subtests"] if "statistics" not in subtest["name"] ] ) == 12 )