def runBenchmark(self, info, platform, benchmarks): if getArgs().reboot: platform.rebootDevice() tempdir = tempfile.mkdtemp() # we need to get a different framework instance per thread # will consolidate later. For now create a new framework frameworks = getFrameworks() framework = frameworks[getArgs().framework](tempdir) reporters = getReporters() for idx in range(len(benchmarks)): benchmark = benchmarks[idx] # check the framework matches if "model" in benchmark and "framework" in benchmark["model"]: assert(benchmark["model"]["framework"] == getArgs().framework), \ "Framework specified in the json file " \ "{} ".format(benchmark["model"]["framework"]) + \ "does not match the command line argument " \ "{}".format(getArgs().framework) b = copy.deepcopy(benchmark) i = copy.deepcopy(info) success = runOneBenchmark(i, b, framework, platform, getArgs().platform, reporters, self._lock) self.success = self.success and success if idx != len(benchmarks) - 1: # cool down period between multiple benchmark runs cooldown = getArgs().cooldown if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown) shutil.rmtree(tempdir, True)
def runBenchmark(self, info, platform, benchmarks, framework): if getArgs().reboot: platform.rebootDevice() reporters = getReporters() for idx in range(len(benchmarks)): benchmark = benchmarks[idx] # check the framework matches if "model" in benchmark and "framework" in benchmark["model"]: assert(benchmark["model"]["framework"] == getArgs().framework), \ "Framework specified in the json file " \ "{} ".format(benchmark["model"]["framework"]) + \ "does not match the command line argument " \ "{}".format(getArgs().framework) b = copy.deepcopy(benchmark) i = copy.deepcopy(info) success = runOneBenchmark(i, b, framework, platform, getArgs().platform, reporters, self._lock) self.success = self.success and success if idx != len(benchmarks) - 1: # cool down period between multiple benchmark runs cooldown = getArgs().cooldown if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown)
def runBenchmark(self, info, platform, benchmarks): if self.args.reboot: platform.rebootDevice() for idx in range(len(benchmarks)): tempdir = tempfile.mkdtemp(prefix="_".join( ["aibench", str(self.args.user_identifier), ""])) # we need to get a different framework instance per thread # will consolidate later. For now create a new framework frameworks = getFrameworks() framework = frameworks[self.args.framework](tempdir, self.args) reporters = getReporters(self.args) benchmark = benchmarks[idx] # check the framework matches if "model" in benchmark and "framework" in benchmark["model"]: assert benchmark["model"][ "framework"] == self.args.framework, ( "Framework specified in the json file " "{} ".format(benchmark["model"]["framework"]) + "does not match the command line argument " "{}".format(self.args.framework)) if self.args.debug: for test in benchmark["tests"]: test["log_output"] = True if self.args.env: for test in benchmark["tests"]: cmd_env = dict(self.args.env) if "env" in test: cmd_env.update(test["env"]) test["env"] = cmd_env b = copy.deepcopy(benchmark) i = copy.deepcopy(info) status = runOneBenchmark( i, b, framework, platform, self.args.platform, reporters, self._lock, self.args.cooldown, self.args.user_identifier, self.args.local_reporter, ) self.status = self.status | status if idx != len(benchmarks) - 1: # cool down period between multiple benchmark runs cooldown = self.args.cooldown if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown) if not self.args.debug: shutil.rmtree(tempdir, True) for test in benchmark["tests"]: if "preprocess" in test and "files" in test["preprocess"]: for f in test["preprocess"]["files"].values(): shutil.rmtree(f["location"], True)
def runBenchmark(self, info, platform, benchmarks, framework): if getArgs().reboot: platform.rebootDevice() reporters = getReporters() for benchmark in benchmarks: b = copy.deepcopy(benchmark) i = copy.deepcopy(info) runOneBenchmark(i, b, framework, platform, getArgs().platform, reporters)