def runOneBenchmark(info, benchmark, framework, platform, backend, reporters, lock): assert "treatment" in info, "Treatment is missing in info" getLogger().info("Running {}".format(benchmark["path"])) minfo = copy.deepcopy(info["treatment"]) if "shared_libs" in info: minfo["shared_libs"] = info["shared_libs"] try: data = _runOnePass(minfo, benchmark, framework, platform) meta = None if "control" in info and benchmark["tests"][0]["metric"] == "delay": cinfo = copy.deepcopy(info["control"]) if "shared_libs" in info: cinfo["shared_libs"] = info["shared_libs"] control = _runOnePass(cinfo, benchmark, framework, platform) bname = benchmark["model"]["name"] data = _mergeDelayData(data, control, bname) if benchmark["tests"][0]["metric"] != "generic": data = _adjustData(info, data) meta = _retrieveMeta(info, benchmark, platform, framework, backend) result = {"meta": meta, "data": data} except Exception as e: # Catch all exceptions so that failure in one test does not # affect other tests getLogger().info("Exception caught when running benchmark") getLogger().info(e) data = None if data is None or len(data) == 0: name = platform.getMangledName() model_name = "" if "model" in benchmark and "name" in benchmark["model"]: model_name = benchmark["model"]["name"] commit_hash = "" if "commit" in info["treatment"]: commit_hash = info["treatment"]["commit"] getLogger().info("No data collected for ".format(model_name) + "on {}. ".format(name) + "The run may be failed for " + "{}".format(commit_hash)) return False with lock: for reporter in reporters: reporter.report(result) if "regression_commits" in info and \ info["run_type"] == "benchmark" and \ getArgs().local_reporter: from regression_detectors.regression_detectors import checkRegressions checkRegressions(info, platform, framework, benchmark, reporters, result['meta'], getArgs().local_reporter) time.sleep(5) return True
def runOneBenchmark(info, benchmark, framework, platform, backend, reporters, lock, cooldown=None, user_identifier=None, local_reporter=None): assert "treatment" in info, "Treatment is missing in info" getLogger().info("Running {}".format(benchmark["path"])) status = 0 minfo = copy.deepcopy(info["treatment"]) mbenchmark = copy.deepcopy(benchmark) if "shared_libs" in info: minfo["shared_libs"] = info["shared_libs"] try: # invalidate CPU cache [1.0 for _ in range(20 << 20)] gc.collect() data = _runOnePass(minfo, mbenchmark, framework, platform) status = status | getRunStatus() meta = None if "control" in info: cinfo = copy.deepcopy(info["control"]) if "shared_libs" in info: cinfo["shared_libs"] = info["shared_libs"] # cool down between treatment and control if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown) # invalidate CPU cache [1.0 for _ in range(20 << 20)] gc.collect() control = _runOnePass(cinfo, benchmark, framework, platform) status = status | getRunStatus() bname = benchmark["model"]["name"] data = _mergeDelayData(data, control, bname) if benchmark["tests"][0]["metric"] != "generic": data = _adjustData(info, data) meta = _retrieveMeta(info, benchmark, platform, framework, backend, user_identifier) data = _retrieveInfo(info, data) result = {"meta": meta, "data": data} except Exception as e: # Catch all exceptions so that failure in one test does not # affect other tests getLogger().info("Exception caught when running benchmark") getLogger().info(e) data = None status = 2 setRunStatus(status) getLogger().error(traceback.format_exc()) # Set result meta and data to default values to that # the reporter will not try to key into a None result = {"meta": {}, "data": []} if data is None or len(data) == 0: name = platform.getMangledName() model_name = "" if "model" in benchmark and "name" in benchmark["model"]: model_name = benchmark["model"]["name"] commit_hash = "" if "commit" in info["treatment"]: commit_hash = info["treatment"]["commit"] getLogger().info("No data collected for ".format(model_name) + "on {}. ".format(name) + "The run may be failed for " + "{}".format(commit_hash)) return status with lock: for reporter in reporters: reporter.report(result) if "regression_commits" in info and \ info["run_type"] == "benchmark" and local_reporter: from regression_detectors.regression_detectors import checkRegressions checkRegressions(info, platform, framework, benchmark, reporters, result['meta'], local_reporter) return status