def test_memory_distribution():
# Test that we get an error for a missing unit
contents = [{"category": {"metric": {"type": "memory_distribution"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert ("`memory_distribution` is missing required parameter `memory_unit`"
in errors[0])
# Test that memory_distribution works
contents = [{
"category": {
"metric": {
"type": "memory_distribution",
"memory_unit": "megabyte"
}
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 0
assert len(all_metrics.value) == 1
all_metrics.value["category"][
"metric"].memory_unit == metrics.MemoryUnit.megabyte
def test_all_pings_reserved():
# send_in_pings: [all_pings] is only allowed for internal metrics
contents = [{
"category": {
"metric": {
"type": "string",
"send_in_pings": ["all_pings"]
}
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert "On instance category.metric" in errors[0]
assert "Only internal metrics" in errors[0]
all_metrics = parser.parse_objects(contents, {"allow_reserved": True})
errors = list(all_metrics)
assert len(errors) == 0
# A custom ping called "all_pings" is not allowed
contents = [{"all_pings": {"include_client_id": True}}]
contents = [util.add_required_ping(x) for x in contents]
all_pings = parser.parse_objects(contents)
errors = list(all_pings)
assert len(errors) == 1
assert "is not allowed for 'all_pings'"
def test_quantity():
# Test that we get an error for a missing unit and gecko_datapoint
contents = [{"category": {"metric": {"type": "quantity"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 2
assert any("`quantity` is missing required parameter `unit`" in err
for err in errors)
assert any("is only allowed for Gecko metrics" in err for err in errors)
# Test that quantity works
contents = [{
"category": {
"metric": {
"type": "quantity",
"unit": "pixel",
"gecko_datapoint": "FOO",
}
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 0
assert len(all_metrics.value) == 1
all_metrics.value["category"]["metric"].unit == "pixel"
def test_common_prefix():
contents = [{
"telemetry": {
"network_latency": {
"type": "quantity",
"gecko_datapoint": "GC_NETWORK_LATENCY",
"unit": "ms",
},
"network_bandwidth": {
"type": "quantity",
"gecko_datapoint": "GC_NETWORK_BANDWIDTH",
"unit": "kbps",
},
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 1
assert nits[0][0] == "COMMON_PREFIX"
# Now make sure the override works
contents[0]["no_lint"] = ["COMMON_PREFIX"]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 0
def test_geckoview_only_on_valid_metrics():
for metric in [
"timing_distribution",
"custom_distributiuon",
"memory_distribution",
]:
contents = [{
"category1": {
"metric1": {
"type": metric,
"gecko_datapoint": "FOO"
}
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
contents = [{
"category1": {
"metric1": {
"type": "event",
"gecko_datapoint": "FOO"
}
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 1
assert "is only allowed for" in str(errs[0])
def test_parser_reserved():
contents = [{"glean.baseline": {"metric": {"type": "string"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert "For category 'glean.baseline'" in errors[0]
all_metrics = parser.parse_objects(contents, {"allow_reserved": True})
errors = list(all_metrics)
assert len(errors) == 0
def test_invalid_names(location, name):
contents = location(name)
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert name in errors[0]
def test_merge_metrics():
"""Merge multiple metrics.yaml files"""
contents = [
{
"category1": {
"metric1": {},
"metric2": {}
},
"category2": {
"metric3": {}
}
},
{
"category1": {
"metric4": {}
},
"category3": {
"metric5": {}
}
},
]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
list(all_metrics)
all_metrics = all_metrics.value
assert set(all_metrics["category1"].keys()) == set(
["metric1", "metric2", "metric4"])
assert set(all_metrics["category2"].keys()) == set(["metric3"])
assert set(all_metrics["category3"].keys()) == set(["metric5"])
def test_user_lifetime_expiration():
"""Test that expiring 'user' lifetime metrics generate a warning."""
contents = [{
"user_data": {
"counter": {
"type": "counter",
"lifetime": "user",
"expires": "2100-01-01",
},
"string": {
"type": "string",
"lifetime": "user",
"expires": "never"
},
"other": {
"type": "string",
"lifetime": "application"
},
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 1
assert set(["USER_LIFETIME_EXPIRATION"]) == set(v.check_name for v in nits)
def test_fake_pings():
"""Another similarly-fragile test.
It generates C++ for pings_test.yaml, comparing it byte-for-byte
with an expected output C++ file `pings_test_output_cpp`.
Expect it to be fragile.
To generate a new expected output file, edit t/c/g/metrics_index.py,
comment out all other ping yamls, and add one for
t/c/g/pytest/pings_test.yaml. Run `mach build` (it'll fail). Copy
objdir/t/c/g/GleanPings.h over pings_test_output_cpp.
(Don't forget to undo your edits to t/c/g/metrics_index.py)
"""
options = {"allow_reserved": False}
input_files = [Path(path.join(path.dirname(__file__), "pings_test.yaml"))]
all_objs = parser.parse_objects(input_files, options)
assert not util.report_validation_errors(all_objs)
assert not lint.lint_metrics(all_objs.value, options)
output_fd = io.StringIO()
cpp.output_cpp(all_objs.value, output_fd, options)
with open(path.join(path.dirname(__file__), "pings_test_output_cpp"), "r") as file:
EXPECTED_CPP = file.read()
assert output_fd.getvalue() == EXPECTED_CPP
def parse(args):
"""
Parse and lint the input files,
then return the parsed objects for further processing.
"""
# Unfortunately, GeneratedFile appends `flags` directly after `inputs`
# instead of listifying either, so we need to pull stuff from a *args.
yaml_array = args[:-1]
moz_app_version = args[-1]
input_files = [Path(x) for x in yaml_array]
# Derived heavily from glean_parser.translate.translate.
# Adapted to how mozbuild sends us a fd, and to expire on versions not dates.
options = get_parser_options(moz_app_version)
# Lint the yaml first, then lint the metrics.
if lint.lint_yaml_files(input_files, parser_config=options):
# Warnings are Errors
sys.exit(1)
all_objs = parser.parse_objects(input_files, options)
if util.report_validation_errors(all_objs):
sys.exit(1)
nits = lint.lint_metrics(all_objs.value, options)
if nits is not None and any(nit.check_name != "EXPIRED" for nit in nits):
# Treat Warnings as Errors in FOG.
# But don't fail the whole build on expired metrics (it blocks testing).
sys.exit(1)
return all_objs.value, options
def test_gifft_codegen():
"""
A regression test. Very fragile.
It generates C++ for GIFFT for metrics_test.yaml and compares it
byte-for-byte with expected output C++ files.
To generate new expected output files, edit t/c/g/metrics_index.py,
comment out the other metrics yamls, and add one for
t/c/g/pytest/metrics_test.yaml. Run `mach build` (it'll fail). Copy
objdir/t/c/g/XGIFFTMap.h to gifft_output_X.
Don't forget to undo your edits to metrics_index.py after you're done.
"""
options = {"allow_reserved": False}
here_path = Path(path.dirname(__file__))
input_files = [here_path / "metrics_test.yaml"]
all_objs = parser.parse_objects(input_files, options)
assert not util.report_validation_errors(all_objs)
assert not lint.lint_metrics(all_objs.value, options)
all_objs = all_objs.value
for probe_type in ("Event", "Histogram", "Scalar"):
output_fd = io.StringIO()
cpp_fd = io.StringIO()
run_glean_parser.output_gifft_map(output_fd, probe_type, all_objs,
cpp_fd)
with open(here_path / f"gifft_output_{probe_type}", "r") as file:
EXPECTED = file.read()
assert output_fd.getvalue() == EXPECTED
if probe_type == "Event":
with open(here_path / "gifft_output_EventExtra", "r") as file:
EXPECTED = file.read()
assert cpp_fd.getvalue() == EXPECTED
def test_translate_expires():
contents = [{
"metrics": {
"a": {
"type": "counter",
"expires": "never"
},
"b": {
"type": "counter",
"expires": "expired"
},
"c": {
"type": "counter",
"expires": "2000-01-01"
},
"d": {
"type": "counter",
"expires": "2100-01-01"
},
}
}]
contents = [util.add_required(x) for x in contents]
objs = parser.parse_objects(contents)
assert len(list(objs)) == 0
objs = objs.value
assert objs["metrics"]["a"].disabled is False
assert objs["metrics"]["b"].disabled is True
assert objs["metrics"]["c"].disabled is True
assert objs["metrics"]["d"].disabled is False
def test_combined():
contents = [{
"metrics": {
"m_network_latency_ms": {
"type": "timespan",
"time_unit": "millisecond",
},
"m_memory_usage_mb": {
"type": "memory_distribution",
"memory_unit": "megabyte",
},
"m_width_pixels": {
"type": "quantity",
"gecko_datapoint": "WIDTH_PIXELS",
"unit": "pixels",
},
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 5
assert set(["COMMON_PREFIX", "CATEGORY_GENERIC",
"UNIT_IN_NAME"]) == set(v[0] for v in nits)
def parse(self, filenames, config, repo_url=None, commit_hash=None):
config = config.copy()
config["do_not_disable_expired"] = True
paths = [Path(fname) for fname in filenames]
paths = [path for path in paths if path.is_file()]
results = parse_objects(paths, config)
errors = [err for err in results]
metrics = {
metric.identifier(): metric.serialize()
for category, probes in results.value.items()
for probe_name, metric in probes.items()
}
for v in metrics.values():
v["send_in_pings"] = [
normalize_ping_name(p) for p in v["send_in_pings"]
]
if repo_url and commit_hash:
v["source_url"] = get_source_url(v["defined_in"], repo_url,
commit_hash)
# the 'defined_in' structure is no longer needed
del v["defined_in"]
return metrics, errors
def parse(args):
"""
Parse and lint the input files,
then return the parsed objects for further processing.
"""
# Unfortunately, GeneratedFile appends `flags` directly after `inputs`
# instead of listifying either, so we need to pull stuff from a *args.
yaml_array = args[:-1]
moz_app_version = args[-1]
input_files = [Path(x) for x in yaml_array]
# Derived heavily from glean_parser.translate.translate.
# Adapted to how mozbuild sends us a fd, and to expire on versions not dates.
options = get_parser_options(moz_app_version)
all_objs = parser.parse_objects(input_files, options)
if util.report_validation_errors(all_objs):
sys.exit(1)
if lint.lint_metrics(all_objs.value, options):
# Treat Warnings as Errors in FOG
sys.exit(1)
return all_objs.value, options
def main(output_fd, metrics_index_path, which_array):
# Source the list of input files from `metrics_index.py`
sys.path.append(str(Path(metrics_index_path).parent))
from metrics_index import METRICS, PINGS
if which_array == 'METRICS':
input_files = METRICS
elif which_array == 'PINGS':
input_files = PINGS
else:
print("Build system's asking for unknown array {}".format(which_array))
sys.exit(1)
# Derived heavily from glean_parser.translate.translate.
# Adapted to how mozbuild sends us a fd.
options = {"allow_reserved": False}
input_files = [Path(x) for x in input_files]
all_objs = parser.parse_objects(input_files, options)
if util.report_validation_errors(all_objs):
sys.exit(1)
if lint.lint_metrics(all_objs.value, options):
# Treat Warnings as Errors in FOG
sys.exit(1)
rust.output_rust(all_objs.value, output_fd, options)
def test_snake_case_ping_names():
content = {"camelCasePingName": {"include_client_id": True}}
util.add_required_ping(content)
errors = list(parser.parse_objects([content]))
assert len(errors) == 1
assert "camelCasePingName" in errors[0]
def test_all_metric_types():
""" Honestly, this is a pretty bad test.
It generates Rust for a given test metrics.yaml and compares it byte-for-byte
with an expected output Rust file.
Expect it to be fragile.
To generate a new expected output file, copy the test yaml over the one in t/c/g,
run mach build, then copy the rust output from objdir/t/c/g/api/src/.
"""
options = {"allow_reserved": False}
input_files = [
Path(path.join(path.dirname(__file__), "metrics_test.yaml"))
]
all_objs = parser.parse_objects(input_files, options)
assert not util.report_validation_errors(all_objs)
assert not lint.lint_metrics(all_objs.value, options)
output_fd = io.StringIO()
rust.output_rust(all_objs.value, output_fd, options)
with open(path.join(path.dirname(__file__), "metrics_test_output"),
'r') as file:
EXPECTED_RUST = file.read()
assert output_fd.getvalue() == EXPECTED_RUST
def test_send_in_pings_restrictions():
"""Test that invalid ping names are disallowed in `send_in_pings`."""
all_metrics = parser.parse_objects(ROOT / "data" /
"invalid-ping-names.yaml")
errors = list(all_metrics)
assert len(errors) == 1
assert "'invalid_ping_name' does not match" in errors[0]
def test_misspelling_pings():
contents = [{
"user_data": {
"counter": {
"type": "counter",
"send_in_pings": ["metric"]
},
"string": {
"type": "string",
"send_in_pings": ["event"]
},
"string2": {
"type": "string",
"send_in_pings": ["metrics", "events"]
},
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 2
assert set(["MISSPELLED_PING"]) == set(v.check_name for v in nits)
def test_baseline_restriction():
contents = [{
"user_data": {
"counter": {
"type": "counter",
"send_in_pings": ["baseline"]
},
"string": {
"type": "string",
"send_in_pings": ["metrics", "baseline"]
},
"string2": {
"type": "string",
"send_in_pings": ["metrics"]
},
}
}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 2
assert set(["BASELINE_PING"]) == set(v.check_name for v in nits)
def test_multiple_errors():
"""Make sure that if there are multiple errors, we get all of them."""
contents = [{"camelCaseName": {"metric": {"type": "unknown"}}}]
contents = [util.add_required(x) for x in contents]
metrics = parser.parse_objects(contents)
errors = list(metrics)
assert len(errors) == 2
def test_timing_distribution_unit_default():
contents = [{"category1": {"metric1": {"type": "timing_distribution"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
assert (all_metrics.value["category1"]["metric1"].time_unit ==
metrics.TimeUnit.nanosecond)
def test_merge_metrics_clash():
"""Merge multiple metrics.yaml files with conflicting metric names."""
contents = [{"category1": {"metric1": {}}}, {"category1": {"metric1": {}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert "Duplicate metric name" in errors[0]
def test_required_denominator():
"""denominator is required on use_counter"""
contents = [{"category": {"metric": {"type": "use_counter"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errors = list(all_metrics)
assert len(errors) == 1
assert "denominator is required" in errors[0]
def test_reserved_metrics_category():
"""
The category "pings" can't be used by metrics -- it's reserved for pings.
"""
content = {"pings": {"metric": {"type": "string"}}}
util.add_required(content)
errors = list(parser.parse_objects(content))
assert len(errors) == 1
assert "reserved as a category name" in errors[0]
def parse(self, filenames, config):
config = config.copy()
paths = [Path(fname) for fname in filenames]
results = parse_objects(paths, config)
errors = [err for err in results]
return ({
normalize_ping_name(ping_name): ping_data.serialize()
for category, pings in results.value.items()
for ping_name, ping_data in pings.items()
}, errors)
def test_duplicate_send_in_pings():
"""Test the basics of parsing a single file."""
all_metrics = parser.parse_objects(
[ROOT / "data" / "duplicate_send_in_ping.yaml"],
config={"allow_reserved": True})
errs = list(all_metrics)
assert len(errs) == 0
metric = all_metrics.value["telemetry"]["test"]
assert metric.send_in_pings == ["core", "metrics"]
def test_category_generic():
contents = [{"metrics": {"measurement": {"type": "boolean"}}}]
contents = [util.add_required(x) for x in contents]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 1
assert nits[0][0] == "CATEGORY_GENERIC"
contents[0]["no_lint"] = ["CATEGORY_GENERIC"]
all_metrics = parser.parse_objects(contents)
errs = list(all_metrics)
assert len(errs) == 0
nits = lint.lint_metrics(all_metrics.value)
assert len(nits) == 0
