def parse(args):
"""
Parse and lint the input files,
then return the parsed objects for further processing.
"""
# Unfortunately, GeneratedFile appends `flags` directly after `inputs`
# instead of listifying either, so we need to pull stuff from a *args.
yaml_array = args[:-1]
moz_app_version = args[-1]
input_files = [Path(x) for x in yaml_array]
# Derived heavily from glean_parser.translate.translate.
# Adapted to how mozbuild sends us a fd, and to expire on versions not dates.
options = get_parser_options(moz_app_version)
# Lint the yaml first, then lint the metrics.
if lint.lint_yaml_files(input_files, parser_config=options):
# Warnings are Errors
sys.exit(1)
all_objs = parser.parse_objects(input_files, options)
if util.report_validation_errors(all_objs):
sys.exit(1)
nits = lint.lint_metrics(all_objs.value, options)
if nits is not None and any(nit.check_name != "EXPIRED" for nit in nits):
# Treat Warnings as Errors in FOG.
# But don't fail the whole build on expired metrics (it blocks testing).
sys.exit(1)
return all_objs.value, options
def parse_with_options(input_files, options):
# Derived heavily from glean_parser.translate.translate.
# Adapted to how mozbuild sends us a fd, and to expire on versions not dates.
# Lint the yaml first, then lint the metrics.
if lint.lint_yaml_files(input_files, parser_config=options):
# Warnings are Errors
raise ParserError("linter found problems")
all_objs = parser.parse_objects(input_files, options)
if util.report_validation_errors(all_objs):
raise ParserError("found validation errors during parse")
nits = lint.lint_metrics(all_objs.value, options)
if nits is not None and any(nit.check_name != "EXPIRED" for nit in nits):
# Treat Warnings as Errors in FOG.
# But don't fail the whole build on expired metrics (it blocks testing).
raise ParserError("glinter nits found during parse")
objects = all_objs.value
# bug 1720494: This should be a simple call to translate.transform
counters = {}
numerators_by_denominator: Dict[str, Any] = {}
for category_val in objects.values():
for metric in category_val.values():
fqmn = metric.identifier()
if getattr(metric, "type", None) == "counter":
counters[fqmn] = metric
denominator_name = getattr(metric, "denominator_metric", None)
if denominator_name:
metric.type = "numerator"
numerators_by_denominator.setdefault(denominator_name, [])
numerators_by_denominator[denominator_name].append(metric)
for denominator_name, numerators in numerators_by_denominator.items():
if denominator_name not in counters:
print(
f"No `counter` named {denominator_name} found to be used as"
"denominator for {numerator_names}",
file=sys.stderr,
)
raise ParserError("rate couldn't find denominator")
counters[denominator_name].type = "denominator"
counters[denominator_name].numerators = numerators
return objects, options
def test_yaml_lint(capsys):
"""Tests yamllint on files with nits."""
file_paths = [
ROOT / "data" / "core.yaml", ROOT / "data" / "yaml_nits.yamlx"
]
nits = lint.lint_yaml_files(file_paths)
assert len(nits) == 3
# The second rule is empty because it's a syntax error.
assert set(["indentation", None,
"trailing-spaces"]) == set(v.rule for v in nits)
captured = capsys.readouterr()
lines = captured.out.split("\n")
for line in lines:
if line.strip() == "":
continue
assert "yaml_nits.yamlx" in line
assert "core.yaml" not in line
if metrics_files:
temp_url = (
GITHUB_RAW_URL
+ repo_url.replace("https://github.com", "")
+ "/"
+ branch
+ "/"
+ metrics_files[0]
)
response = reqs.get(temp_url)
assert response.status_code == 200
with open("temp-metrics.yaml", "w") as filehandle:
filehandle.write(response.text)
yaml_lint_errors = open("yaml-lint-errors.txt", "w")
temp_erros = lint_yaml_files(
[Path("./temp-metrics.yaml")], yaml_lint_errors, {}
)
if temp_erros:
if not repo.get("prototype", None):
validation_errors.append({"repo": repo, "errors": temp_erros})
os.remove("temp-metrics.yaml")
os.remove("yaml-lint-errors.txt")
if validation_errors:
print("\nSummary of validation errors \n====================================\n")
print(f"{len(validation_errors)} repositories had problems\n")
for error in validation_errors:
print(
f"\nErrors found in {error['repo']} \n====================================\n"
)
for line_errors in error["errors"]:
print(line_errors)