def runBenchmark(self, info, platform, benchmarks): if getArgs().reboot: platform.rebootDevice() tempdir = tempfile.mkdtemp() # we need to get a different framework instance per thread # will consolidate later. For now create a new framework frameworks = getFrameworks() framework = frameworks[getArgs().framework](tempdir) reporters = getReporters() for idx in range(len(benchmarks)): benchmark = benchmarks[idx] # check the framework matches if "model" in benchmark and "framework" in benchmark["model"]: assert(benchmark["model"]["framework"] == getArgs().framework), \ "Framework specified in the json file " \ "{} ".format(benchmark["model"]["framework"]) + \ "does not match the command line argument " \ "{}".format(getArgs().framework) b = copy.deepcopy(benchmark) i = copy.deepcopy(info) success = runOneBenchmark(i, b, framework, platform, getArgs().platform, reporters, self._lock) self.success = self.success and success if idx != len(benchmarks) - 1: # cool down period between multiple benchmark runs cooldown = getArgs().cooldown if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown) shutil.rmtree(tempdir, True)
def runBenchmark(self, info, platform, benchmarks): if self.args.reboot: platform.rebootDevice() for idx in range(len(benchmarks)): tempdir = tempfile.mkdtemp(prefix="_".join( ["aibench", str(self.args.user_identifier), ""])) # we need to get a different framework instance per thread # will consolidate later. For now create a new framework frameworks = getFrameworks() framework = frameworks[self.args.framework](tempdir, self.args) reporters = getReporters(self.args) benchmark = benchmarks[idx] # check the framework matches if "model" in benchmark and "framework" in benchmark["model"]: assert benchmark["model"][ "framework"] == self.args.framework, ( "Framework specified in the json file " "{} ".format(benchmark["model"]["framework"]) + "does not match the command line argument " "{}".format(self.args.framework)) if self.args.debug: for test in benchmark["tests"]: test["log_output"] = True if self.args.env: for test in benchmark["tests"]: cmd_env = dict(self.args.env) if "env" in test: cmd_env.update(test["env"]) test["env"] = cmd_env b = copy.deepcopy(benchmark) i = copy.deepcopy(info) status = runOneBenchmark( i, b, framework, platform, self.args.platform, reporters, self._lock, self.args.cooldown, self.args.user_identifier, self.args.local_reporter, ) self.status = self.status | status if idx != len(benchmarks) - 1: # cool down period between multiple benchmark runs cooldown = self.args.cooldown if "model" in benchmark and "cooldown" in benchmark["model"]: cooldown = float(benchmark["model"]["cooldown"]) time.sleep(cooldown) if not self.args.debug: shutil.rmtree(tempdir, True) for test in benchmark["tests"]: if "preprocess" in test and "files" in test["preprocess"]: for f in test["preprocess"]["files"].values(): shutil.rmtree(f["location"], True)
def run(self): tempdir = tempfile.mkdtemp(prefix="_".join( ["aibench", str(self.args.user_identifier), ""])) getLogger().info("Temp directory: {}".format(tempdir)) info = self._getInfo() frameworks = getFrameworks() assert (self.args.framework in frameworks), "Framework {} is not supported".format( self.args.framework) framework = frameworks[self.args.framework](tempdir, self.args) bcollector = BenchmarkCollector(framework, self.args.model_cache, args=self.args) benchmarks = bcollector.collectBenchmarks(info, self.args.benchmark_file, self.args.user_identifier) platforms = getPlatforms(self.args, tempdir, self.usb_controller) threads = [] for platform in platforms: t = threading.Thread(target=self.runBenchmark, args=(info, platform, benchmarks)) t.start() threads.append(t) for t in threads: t.join() if not self.args.debug: shutil.rmtree(tempdir, True) status = self.status | getRunStatus() if getRunKilled(): status_str = "killed" elif getRunTimeout(): status_str = "timeout" elif status == 0: status_str = "success" elif status == 1: status_str = "user error" elif status == 2: status_str = "harness error" else: status_str = "user and harness error" getLogger().info(" ======= {} =======".format(status_str)) if getRunKilled(): return RUN_KILLED if getRunTimeout(): return RUN_TIMEOUT return status
def run(self): tempdir = tempfile.mkdtemp() getLogger().info("Temp directory: {}".format(tempdir)) info = self._getInfo() bcollector = BenchmarkCollector(getArgs().model_cache) benchmarks = bcollector.collectBenchmarks(info, getArgs().benchmark_file) frameworks = getFrameworks() assert getArgs().framework in frameworks, \ "Framework {} is not supported".format(getArgs().framework) framework = frameworks[getArgs().framework](tempdir) platforms = getPlatforms(tempdir) threads = [] for platform in platforms: t = threading.Thread(target=self.runBenchmark, args=(info, platform, benchmarks, framework)) threads.append(t) t.start() shutil.rmtree(tempdir, True)
def run(self): tempdir = tempfile.mkdtemp() getLogger().info("Temp directory: {}".format(tempdir)) info = self._getInfo() frameworks = getFrameworks() assert self.args.framework in frameworks, \ "Framework {} is not supported".format(self.args.framework) framework = frameworks[self.args.framework](tempdir, self.args) bcollector = BenchmarkCollector(framework, self.args.model_cache, args=self.args) benchmarks = bcollector.collectBenchmarks(info, self.args.benchmark_file) platforms = getPlatforms(tempdir, self.args) threads = [] for platform in platforms: t = threading.Thread(target=self.runBenchmark, args=(info, platform, benchmarks)) t.start() threads.append(t) for t in threads: t.join() if not self.args.debug: shutil.rmtree(tempdir, True) status = self.status | getRunStatus() if status == 0: status_str = "success" elif status == 1: status_str = "user error" elif status == 2: status_str = "harness error" else: status_str = "user and harness error" getLogger().info(" ======= {} =======".format(status_str)) return status