def run(self, metadata): # Get filtered metrics results = filtered_metrics( metadata, self.get_arg("output"), self.get_arg("prefix"), metrics=self.get_arg("metrics"), ) if not results: self.warning("No results left after filtering") return metadata analysis = self.get_arg("analysis") data_to_post = [] for name, res in results.items(): for r in res: val = r["data"][0]["value"] if is_number(val): data_to_post.append(r) elif self.get_arg("analyze-strings"): data_to_post.append(r) self.ptnb = PerftestNotebook(data=data_to_post, logger=metadata, prefix=self.get_arg("prefix")) self.ptnb.post_to_iodide(analysis) return metadata
def run(self, metadata): exclusions = None if not self.get_arg("stats"): exclusions = ["statistics."] for result in metadata.get_results(): result["name"] += "- newest run" analysis = self.get_arg("analysis") dir_list = self.get_arg("compare-to") if dir_list: analysis.append("compare") for directory in dir_list: dirpath = pathlib.Path(directory) if not dirpath.exists(): raise Exception(f"{dirpath} does not exist.") if not dirpath.is_dir(): raise Exception(f"{dirpath} is not a directory") # TODO: Handle more than just JSON data. for jsonfile in dirpath.rglob("*.json"): metadata.add_result({ "results": str(jsonfile.resolve()), "name": jsonfile.parent.name, }) results = filtered_metrics( metadata, self.get_arg("output"), self.get_arg("prefix"), metrics=self.get_arg("metrics"), transformer=self.get_arg("transformer"), exclude=exclusions, split_by=self.get_arg("split-by"), simplify_names=self.get_arg("simplify-names"), simplify_exclude=self.get_arg("simplify-exclude"), ) if not results: self.warning("No results left after filtering") return metadata data_to_post = [] for name, res in results.items(): for r in res: val = r["data"][0]["value"] if is_number(val): data_to_post.append(r) elif self.get_arg("analyze-strings"): data_to_post.append(r) self.ptnb = PerftestNotebook(data=data_to_post, logger=metadata, prefix=self.get_arg("prefix")) self.ptnb.post_to_iodide( analysis, start_local_server=not self.get_arg("no-server")) return metadata
def run(self, metadata): """Processes the given results into a perfherder-formatted data blob. If the `--perfherder` flag isn't provided, then the results won't be processed into a perfherder-data blob. If the flavor is unknown to us, then we assume that it comes from browsertime. XXX If needed, make a way to do flavor-specific processing :param results list/dict/str: Results to process. :param perfherder bool: True if results should be processed into a perfherder-data blob. :param flavor str: The flavor that is being processed. """ prefix = self.get_arg("prefix") output = self.get_arg("output") # XXX Make an arugment for exclusions from metrics # (or go directly to regex's for metrics) exclusions = None if not self.get_arg("stats"): exclusions = ["statistics."] # Get filtered metrics metrics = self.get_arg("metrics") results, fullsettings = filtered_metrics( metadata, output, prefix, metrics=metrics, settings=True, exclude=exclusions, ) if not any([results[name] for name in results]): self.warning("No results left after filtering") return metadata # XXX Add version info into this data app_info = {"name": self.get_arg("app", default="firefox")} # converting the metrics list into a mapping where # keys are the metrics nane if metrics is not None: metrics = dict([(m["name"], m) for m in metrics]) else: metrics = {} all_perfherder_data = None for name, res in results.items(): settings = dict(fullsettings[name]) # updating the settings with values provided in metrics, if any if name in metrics: settings.update(metrics[name]) # XXX Instead of just passing replicates here, we should build # up a partial perfherder data blob (with options) and subtest # overall values. subtests = {} for r in res: vals = [v["value"] for v in r["data"] if is_number(v["value"])] if vals: subtests[r["subtest"]] = vals perfherder_data = self._build_blob( subtests, name=name, extra_options=settings.get("extraOptions"), should_alert=strtobool(settings.get("shouldAlert", False)), application=app_info, alert_threshold=float(settings.get("alertThreshold", 2.0)), lower_is_better=strtobool(settings.get("lowerIsBetter", True)), unit=settings.get("unit", "ms"), summary=settings.get("value"), ) if all_perfherder_data is None: all_perfherder_data = perfherder_data else: all_perfherder_data["suites"].extend(perfherder_data["suites"]) if prefix: # If a prefix was given, store it in the perfherder data as well all_perfherder_data["prefix"] = prefix # Validate the final perfherder data blob with pathlib.Path(metadata._mach_cmd.topsrcdir, PERFHERDER_SCHEMA).open() as f: schema = json.load(f) jsonschema.validate(all_perfherder_data, schema) file = "perfherder-data.json" if prefix: file = "{}-{}".format(prefix, file) self.info("Writing perfherder results to {}".format( os.path.join(output, file))) # XXX "suites" key error occurs when using self.info so a print # is being done for now. print("PERFHERDER_DATA: " + json.dumps(all_perfherder_data)) metadata.set_output(write_json(all_perfherder_data, output, file)) return metadata