def run(self, metadata):
# Get filtered metrics
results = filtered_metrics(
metadata,
self.get_arg("output"),
self.get_arg("prefix"),
metrics=self.get_arg("metrics"),
)
if not results:
self.warning("No results left after filtering")
return metadata
analysis = self.get_arg("analysis")
data_to_post = []
for name, res in results.items():
for r in res:
val = r["data"][0]["value"]
if is_number(val):
data_to_post.append(r)
elif self.get_arg("analyze-strings"):
data_to_post.append(r)
self.ptnb = PerftestNotebook(data=data_to_post,
logger=metadata,
prefix=self.get_arg("prefix"))
self.ptnb.post_to_iodide(analysis)
return metadata
def run(self, metadata):
exclusions = None
if not self.get_arg("stats"):
exclusions = ["statistics."]
for result in metadata.get_results():
result["name"] += "- newest run"
analysis = self.get_arg("analysis")
dir_list = self.get_arg("compare-to")
if dir_list:
analysis.append("compare")
for directory in dir_list:
dirpath = pathlib.Path(directory)
if not dirpath.exists():
raise Exception(f"{dirpath} does not exist.")
if not dirpath.is_dir():
raise Exception(f"{dirpath} is not a directory")
# TODO: Handle more than just JSON data.
for jsonfile in dirpath.rglob("*.json"):
metadata.add_result({
"results": str(jsonfile.resolve()),
"name": jsonfile.parent.name,
})
results = filtered_metrics(
metadata,
self.get_arg("output"),
self.get_arg("prefix"),
metrics=self.get_arg("metrics"),
transformer=self.get_arg("transformer"),
exclude=exclusions,
split_by=self.get_arg("split-by"),
simplify_names=self.get_arg("simplify-names"),
simplify_exclude=self.get_arg("simplify-exclude"),
)
if not results:
self.warning("No results left after filtering")
return metadata
data_to_post = []
for name, res in results.items():
for r in res:
val = r["data"][0]["value"]
if is_number(val):
data_to_post.append(r)
elif self.get_arg("analyze-strings"):
data_to_post.append(r)
self.ptnb = PerftestNotebook(data=data_to_post,
logger=metadata,
prefix=self.get_arg("prefix"))
self.ptnb.post_to_iodide(
analysis, start_local_server=not self.get_arg("no-server"))
return metadata
def run(self, metadata):
"""Processes the given results into a perfherder-formatted data blob.
If the `--perfherder` flag isn't provided, then the
results won't be processed into a perfherder-data blob. If the
flavor is unknown to us, then we assume that it comes from
browsertime.
XXX If needed, make a way to do flavor-specific processing
:param results list/dict/str: Results to process.
:param perfherder bool: True if results should be processed
into a perfherder-data blob.
:param flavor str: The flavor that is being processed.
"""
prefix = self.get_arg("prefix")
output = self.get_arg("output")
# XXX Make an arugment for exclusions from metrics
# (or go directly to regex's for metrics)
exclusions = None
if not self.get_arg("stats"):
exclusions = ["statistics."]
# Get filtered metrics
metrics = self.get_arg("metrics")
results, fullsettings = filtered_metrics(
metadata,
output,
prefix,
metrics=metrics,
settings=True,
exclude=exclusions,
)
if not any([results[name] for name in results]):
self.warning("No results left after filtering")
return metadata
# XXX Add version info into this data
app_info = {"name": self.get_arg("app", default="firefox")}
# converting the metrics list into a mapping where
# keys are the metrics nane
if metrics is not None:
metrics = dict([(m["name"], m) for m in metrics])
else:
metrics = {}
all_perfherder_data = None
for name, res in results.items():
settings = dict(fullsettings[name])
# updating the settings with values provided in metrics, if any
if name in metrics:
settings.update(metrics[name])
# XXX Instead of just passing replicates here, we should build
# up a partial perfherder data blob (with options) and subtest
# overall values.
subtests = {}
for r in res:
vals = [v["value"] for v in r["data"] if is_number(v["value"])]
if vals:
subtests[r["subtest"]] = vals
perfherder_data = self._build_blob(
subtests,
name=name,
extra_options=settings.get("extraOptions"),
should_alert=strtobool(settings.get("shouldAlert", False)),
application=app_info,
alert_threshold=float(settings.get("alertThreshold", 2.0)),
lower_is_better=strtobool(settings.get("lowerIsBetter", True)),
unit=settings.get("unit", "ms"),
summary=settings.get("value"),
)
if all_perfherder_data is None:
all_perfherder_data = perfherder_data
else:
all_perfherder_data["suites"].extend(perfherder_data["suites"])
if prefix:
# If a prefix was given, store it in the perfherder data as well
all_perfherder_data["prefix"] = prefix
# Validate the final perfherder data blob
with pathlib.Path(metadata._mach_cmd.topsrcdir,
PERFHERDER_SCHEMA).open() as f:
schema = json.load(f)
jsonschema.validate(all_perfherder_data, schema)
file = "perfherder-data.json"
if prefix:
file = "{}-{}".format(prefix, file)
self.info("Writing perfherder results to {}".format(
os.path.join(output, file)))
# XXX "suites" key error occurs when using self.info so a print
# is being done for now.
print("PERFHERDER_DATA: " + json.dumps(all_perfherder_data))
metadata.set_output(write_json(all_perfherder_data, output, file))
return metadata