def output_gifft_map(output_fd, probe_type, all_objs, cpp_fd):
get_metric_id = generate_metric_ids(all_objs)
ids_to_probes = {}
for category_name, objs in all_objs.items():
for metric in objs.values():
if (hasattr(metric, "telemetry_mirror")
and metric.telemetry_mirror is not None):
info = (metric.telemetry_mirror,
f"{category_name}.{metric.name}")
if metric.type in GIFFT_TYPES[probe_type]:
if any(metric.telemetry_mirror == value[0]
for value in ids_to_probes.values()):
print(
f"Telemetry mirror {metric.telemetry_mirror} already registered",
file=sys.stderr,
)
sys.exit(1)
ids_to_probes[get_metric_id(metric)] = info
# If we don't support a mirror for this metric type: build error.
elif not any([
metric.type in types_for_probe
for types_for_probe in GIFFT_TYPES.values()
]):
print(
f"Glean metric {category_name}.{metric.name} is of type {metric.type}"
" which can't be mirrored (we don't know how).",
file=sys.stderr,
)
sys.exit(1)
env = jinja2.Environment(
loader=jinja2.PackageLoader("run_glean_parser", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["snake_case"] = lambda value: value.replace(".", "_").replace(
"-", "_")
env.filters["Camelize"] = util.Camelize
template = env.get_template("gifft.jinja2")
output_fd.write(
template.render(
ids_to_probes=ids_to_probes,
probe_type=probe_type,
))
output_fd.write("\n")
# Events also need to output maps from event extra enum to strings.
# Sadly we need to generate code for all possible events, not just mirrored.
# Otherwise we won't compile.
if probe_type == "Event":
template = env.get_template("gifft_events.jinja2")
cpp_fd.write(template.render(all_objs=all_objs))
cpp_fd.write("\n")
def output_cpp(objs, output_fd, options={}):
"""
Given a tree of objects, output C++ code to the file-like object `output_fd`.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_fd: Writeable file to write the output to.
:param options: options dictionary.
"""
# Monkeypatch a util.snake_case function for the templates to use
util.snake_case = lambda value: value.replace(".", "_").replace("-", "_")
# Monkeypatch util.get_jinja2_template to find templates nearby
def get_local_template(template_name, filters=()):
env = jinja2.Environment(
loader=jinja2.PackageLoader("cpp", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["camelize"] = util.camelize
env.filters["Camelize"] = util.Camelize
for filter_name, filter_func in filters:
env.filters[filter_name] = filter_func
return env.get_template(template_name)
util.get_jinja2_template = get_local_template
get_metric_id = generate_metric_ids(objs)
get_ping_id = generate_ping_ids(objs)
if len(objs) == 1 and "pings" in objs:
template_filename = "cpp_pings.jinja2"
else:
template_filename = "cpp.jinja2"
template = util.get_jinja2_template(
template_filename,
filters=(
("cpp", cpp_datatypes_filter),
("snake_case", util.snake_case),
("type_name", type_name),
("metric_id", get_metric_id),
("ping_id", get_ping_id),
("is_implemented_type", is_implemented_metric_type),
("Camelize", util.Camelize),
),
)
output_fd.write(template.render(all_objs=objs))
output_fd.write("\n")
def output_js(objs, output_fd, options={}):
"""
Given a tree of objects, output code for the JS API to the file-like object `output_fd`.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_fd: Writeable file to write the output to.
:param options: options dictionary.
"""
# Monkeypatch a util.snake_case function for the templates to use
util.snake_case = lambda value: value.replace(".", "_").replace("-", "_")
# Monkeypatch util.get_jinja2_template to find templates nearby
def get_local_template(template_name, filters=()):
env = jinja2.Environment(
loader=jinja2.PackageLoader("js", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["Camelize"] = util.Camelize
for filter_name, filter_func in filters:
env.filters[filter_name] = filter_func
return env.get_template(template_name)
util.get_jinja2_template = get_local_template
template_filename = "js.jinja2"
template = util.get_jinja2_template(
template_filename,
filters=(
("type_name", type_name),
("is_implemented_type", is_implemented_metric_type),
),
)
assert (INDEX_BITS + ID_BITS <
ENTRY_WIDTH), "INDEX_BITS or ID_BITS are larger than allowed"
get_metric_id = generate_metric_ids(objs)
# Mapping from a metric's identifier to the entry (metric ID | type id | index)
metric_id_mapping = {}
categories = []
category_string_table = StringTable()
metric_string_table = StringTable()
# Mapping from a type name to its ID
metric_type_ids = {}
for category_name, objs in objs.items():
category_name = util.snake_case(category_name)
id = category_string_table.stringIndex(category_name)
categories.append((category_name, id))
for metric in objs.values():
identifier = metric_identifier(category_name, metric.name)
if metric.type in metric_type_ids:
type_id = metric_type_ids[metric.type]
else:
type_id = len(metric_type_ids) + 1
metric_type_ids[metric.type] = type_id
idx = metric_string_table.stringIndex(identifier)
metric_id = get_metric_id(metric)
entry = create_entry(metric_id, type_id, idx)
metric_id_mapping[identifier] = entry
# Create a lookup table for the metric categories only
category_string_table = category_string_table.writeToString(
"gCategoryStringTable")
category_map = [(bytearray(category, "ascii"), id)
for (category, id) in categories]
name_phf = PerfectHash(category_map, 64)
category_by_name_lookup = name_phf.cxx_codegen(
name="CategoryByNameLookup",
entry_type="category_entry_t",
lower_entry=lambda x: str(x[1]),
key_type="const nsACString&",
key_bytes="aKey.BeginReading()",
key_length="aKey.Length()",
return_type="static Maybe<uint32_t>",
return_entry="return category_result_check(aKey, entry);",
)
# Create a lookup table for metric's identifiers.
metric_string_table = metric_string_table.writeToString(
"gMetricStringTable")
metric_map = [(bytearray(metric_name, "ascii"), metric_id)
for (metric_name, metric_id) in metric_id_mapping.items()]
metric_phf = PerfectHash(metric_map, 64)
metric_by_name_lookup = metric_phf.cxx_codegen(
name="MetricByNameLookup",
entry_type="metric_entry_t",
lower_entry=lambda x: str(x[1]),
key_type="const nsACString&",
key_bytes="aKey.BeginReading()",
key_length="aKey.Length()",
return_type="static Maybe<uint32_t>",
return_entry="return metric_result_check(aKey, entry);",
)
output_fd.write(
template.render(
categories=categories,
metric_id_mapping=metric_id_mapping,
metric_type_ids=metric_type_ids,
entry_width=ENTRY_WIDTH,
index_bits=INDEX_BITS,
id_bits=ID_BITS,
category_string_table=category_string_table,
category_by_name_lookup=category_by_name_lookup,
metric_string_table=metric_string_table,
metric_by_name_lookup=metric_by_name_lookup,
))
output_fd.write("\n")
def write_metrics(objs, output_fd, template_filename):
"""
Given a tree of objects `objs`, output metrics-only code for the JS API to the
file-like object `output_fd` using template `template_filename`
"""
template = util.get_jinja2_template(
template_filename,
filters=(
("type_name", type_name),
("is_implemented_type", is_implemented_metric_type),
),
)
assert (INDEX_BITS + ID_BITS <
ENTRY_WIDTH), "INDEX_BITS or ID_BITS are larger than allowed"
get_metric_id = generate_metric_ids(objs)
# Mapping from a metric's identifier to the entry (metric ID | type id | index)
metric_id_mapping = {}
categories = []
category_string_table = StringTable()
metric_string_table = StringTable()
# Mapping from a type name to its ID
metric_type_ids = {}
for category_name, objs in objs.items():
category_name = util.camelize(category_name)
id = category_string_table.stringIndex(category_name)
categories.append((category_name, id))
for metric in objs.values():
identifier = metric_identifier(category_name, metric.name)
if metric.type in metric_type_ids:
type_id = metric_type_ids[metric.type]
else:
type_id = len(metric_type_ids) + 1
metric_type_ids[metric.type] = type_id
idx = metric_string_table.stringIndex(identifier)
metric_id = get_metric_id(metric)
entry = create_entry(metric_id, type_id, idx)
metric_id_mapping[identifier] = entry
# Create a lookup table for the metric categories only
category_string_table = category_string_table.writeToString(
"gCategoryStringTable")
category_map = [(bytearray(category, "ascii"), id)
for (category, id) in categories]
name_phf = PerfectHash(category_map, 64)
category_by_name_lookup = name_phf.cxx_codegen(
name="CategoryByNameLookup",
entry_type="category_entry_t",
lower_entry=lambda x: str(x[1]),
key_type="const nsACString&",
key_bytes="aKey.BeginReading()",
key_length="aKey.Length()",
return_type="static Maybe<uint32_t>",
return_entry="return category_result_check(aKey, entry);",
)
# Create a lookup table for metric's identifiers.
metric_string_table = metric_string_table.writeToString(
"gMetricStringTable")
metric_map = [(bytearray(metric_name, "ascii"), metric_id)
for (metric_name, metric_id) in metric_id_mapping.items()]
metric_phf = PerfectHash(metric_map, 64)
metric_by_name_lookup = metric_phf.cxx_codegen(
name="MetricByNameLookup",
entry_type="metric_entry_t",
lower_entry=lambda x: str(x[1]),
key_type="const nsACString&",
key_bytes="aKey.BeginReading()",
key_length="aKey.Length()",
return_type="static Maybe<uint32_t>",
return_entry="return metric_result_check(aKey, entry);",
)
output_fd.write(
template.render(
categories=categories,
metric_id_mapping=metric_id_mapping,
metric_type_ids=metric_type_ids,
entry_width=ENTRY_WIDTH,
index_bits=INDEX_BITS,
id_bits=ID_BITS,
category_string_table=category_string_table,
category_by_name_lookup=category_by_name_lookup,
metric_string_table=metric_string_table,
metric_by_name_lookup=metric_by_name_lookup,
))
output_fd.write("\n")
def output_rust(objs, output_fd, options={}):
"""
Given a tree of objects, output Rust code to the file-like object `output_fd`.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_fd: Writeable file to write the output to.
:param options: options dictionary, presently unused.
"""
# Monkeypatch a util.snake_case function for the templates to use
util.snake_case = lambda value: value.replace(".", "_").replace("-", "_")
# Monkeypatch util.get_jinja2_template to find templates nearby
def get_local_template(template_name, filters=()):
env = jinja2.Environment(
loader=jinja2.PackageLoader("rust", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["camelize"] = util.camelize
env.filters["Camelize"] = util.Camelize
for filter_name, filter_func in filters:
env.filters[filter_name] = filter_func
return env.get_template(template_name)
util.get_jinja2_template = get_local_template
get_metric_id = generate_metric_ids(objs)
get_ping_id = generate_ping_ids(objs)
# Map from a tuple (const, typ) to an array of tuples (id, path)
# where:
# const: The Rust constant name to be used for the lookup map
# typ: The metric type to be stored in the lookup map
# id: The numeric metric ID
# path: The fully qualified path to the metric object in Rust
#
# This map is only filled for metrics, not for pings.
#
# Example:
#
# ("COUNTERS", "CounterMetric") -> [(1, "test_only::clicks"), ...]
objs_by_type = {}
# Map from a metric ID to the fully qualified path of the event object in Rust.
# Required for the special handling of event lookups.
#
# Example:
#
# 17 -> "test_only::an_event"
events_by_id = {}
if len(objs) == 1 and "pings" in objs:
template_filename = "rust_pings.jinja2"
else:
template_filename = "rust.jinja2"
for category_name, metrics in objs.items():
for metric in metrics.values():
# The constant is all uppercase and suffixed by `_MAP`
const_name = util.snake_case(metric.type).upper() + "_MAP"
typ = type_name(metric)
key = (const_name, typ)
metric_name = util.snake_case(metric.name)
category_name = util.snake_case(category_name)
full_path = f"{category_name}::{metric_name}"
if metric.type == "event":
events_by_id[get_metric_id(metric)] = full_path
continue
if key not in objs_by_type:
objs_by_type[key] = []
objs_by_type[key].append((get_metric_id(metric), full_path))
# Now for the modules for each category.
template = util.get_jinja2_template(
template_filename,
filters=(
("rust", rust_datatypes_filter),
("snake_case", util.snake_case),
("type_name", type_name),
("extra_type_name", extra_type_name),
("ctor", ctor),
("extra_keys", extra_keys),
("metric_id", get_metric_id),
("ping_id", get_ping_id),
),
)
output_fd.write(
template.render(
all_objs=objs,
common_metric_data_args=common_metric_data_args,
metric_by_type=objs_by_type,
extra_args=util.extra_args,
events_by_id=events_by_id,
min_submetric_id=2**27 + 1, # One more than 2**ID_BITS from js.py
))
output_fd.write("\n")
def output_rust(objs, output_fd, options={}):
"""
Given a tree of objects, output Rust code to the file-like object `output_fd`.
:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_fd: Writeable file to write the output to.
:param options: options dictionary, presently unused.
"""
# Monkeypatch a util.snake_case function for the templates to use
util.snake_case = lambda value: value.replace(".", "_").replace("-", "_")
# Monkeypatch util.get_jinja2_template to find templates nearby
def get_local_template(template_name, filters=()):
env = jinja2.Environment(
loader=jinja2.PackageLoader("rust", "templates"),
trim_blocks=True,
lstrip_blocks=True,
)
env.filters["camelize"] = util.camelize
env.filters["Camelize"] = util.Camelize
for filter_name, filter_func in filters:
env.filters[filter_name] = filter_func
return env.get_template(template_name)
util.get_jinja2_template = get_local_template
get_metric_id = generate_metric_ids(objs)
# Map from a tuple (const, typ) to an array of tuples (id, path)
# where:
# const: The Rust constant name to be used for the lookup map
# typ: The metric type to be stored in the lookup map
# id: The numeric metric ID
# path: The fully qualified path to the metric object in Rust
#
# This map is only filled for metrics, not for pings.
#
# Example:
#
# ("COUNTERS", "CounterMetric") -> [(1, "test_only::clicks"), ...]
objs_by_type = {}
if len(objs) == 1 and "pings" in objs:
template_filename = "rust_pings.jinja2"
else:
template_filename = "rust.jinja2"
for category_name, metrics in objs.items():
for metric in metrics.values():
# FIXME: Support events correctly
if metric.type == "event":
continue
# The constant is all uppercase and suffixed by `_MAP`
const_name = util.snake_case(metric.type).upper() + "_MAP"
typ = type_name(metric)
key = (const_name, typ)
if key not in objs_by_type:
objs_by_type[key] = []
metric_name = util.snake_case(metric.name)
category_name = util.snake_case(category_name)
full_path = f"{category_name}::{metric_name}"
objs_by_type[key].append((get_metric_id(metric), full_path))
# Now for the modules for each category.
template = util.get_jinja2_template(
template_filename,
filters=(
("rust", rust_datatypes_filter),
("snake_case", util.snake_case),
("type_name", type_name),
("ctor", ctor),
("extra_keys", extra_keys),
("metric_id", get_metric_id),
),
)
# The list of all args to CommonMetricData (besides category and name).
# No particular order is required, but I have these in common_metric_data.rs
# order just to be organized.
common_metric_data_args = [
"name",
"category",
"send_in_pings",
"lifetime",
"disabled",
"dynamic_label",
]
output_fd.write(
template.render(
all_objs=objs,
common_metric_data_args=common_metric_data_args,
metric_by_type=objs_by_type,
extra_args=util.extra_args,
))
output_fd.write("\n")
