def cugraph_update_asv(asvDir, datasetName, algoRunResults, cudaVer="", pythonVer="", osType="", machineName="", repo=""): """ algoRunResults is a list of (algoName, exeTime) tuples """ (commitHash, commitTime) = getCommitInfo() (actualRepo, branch) = getRepoInfo() repo = repo or actualRepo db = ASVDb(asvDir, repo, [branch]) uname = platform.uname() prefixDict = dict( maxGpuUtil="gpuutil", maxGpuMemUsed="gpumem", exeTime="time", ) unitsDict = dict( maxGpuUtil="percent", maxGpuMemUsed="bytes", exeTime="seconds", ) bInfo = BenchmarkInfo(machineName=machineName or uname.machine, cudaVer=cudaVer or "unknown", osType=osType or "%s %s" % (uname.system, uname.release), pythonVer=pythonVer or platform.python_version(), commitHash=commitHash, commitTime=commitTime, gpuType="unknown", cpuType=uname.processor, arch=uname.machine, ram="%d" % psutil.virtual_memory().total) validKeys = set(list(prefixDict.keys()) + list(unitsDict.keys())) for (funcName, metricsDict) in algoRunResults.items(): for (metricName, val) in metricsDict.items(): # If an invalid metricName is present (likely due to a benchmark # run error), skip if metricName in validKeys: bResult = BenchmarkResult( funcName="%s_%s" % (funcName, prefixDict[metricName]), argNameValuePairs=[("dataset", datasetName)], result=val) bResult.unit = unitsDict[metricName] db.addResult(bInfo, bResult)
def test_concurrency(): from asvdb import ASVDb, BenchmarkInfo, BenchmarkResult tmpDir = tempfile.TemporaryDirectory() asvDirName = path.join(tmpDir.name, "dir_that_does_not_exist") repo = "somerepo" branch1 = "branch1" db1 = ASVDb(asvDirName, repo, [branch1]) db2 = ASVDb(asvDirName, repo, [branch1]) db3 = ASVDb(asvDirName, repo, [branch1]) # Use the writeDelay member var to insert a delay during write to properly # test collisions by making writes slow. db1.writeDelay = 10 db2.writeDelay = 10 bInfo = BenchmarkInfo() bResult1 = BenchmarkResult(funcName="somebenchmark1", result=43) bResult2 = BenchmarkResult(funcName="somebenchmark2", result=43) bResult3 = BenchmarkResult(funcName="somebenchmark3", result=43) # db1 or db2 should be actively writing the result (because the writeDelay is long) # and db3 should be blocked. t1 = threading.Thread(target=db1.addResult, args=(bInfo, bResult1)) t2 = threading.Thread(target=db2.addResult, args=(bInfo, bResult2)) t3 = threading.Thread(target=db3.addResult, args=(bInfo, bResult3)) t1.start() t2.start() time.sleep(0.5) # ensure t3 tries to write last t3.start() # Check that db3 is blocked - if locking wasn't working, it would have # finished since it has no writeDelay. t3.join(timeout=0.5) assert t3.is_alive() is True # Cancel db1 and db2, allowing db3 to write and finish db1.cancelWrite = True db2.cancelWrite = True t3.join(timeout=11) assert t3.is_alive() is False t1.join() t2.join() t3.join() # Check that db3 wrote its result with open(path.join(asvDirName, "results", "benchmarks.json")) as fobj: jo = json.load(fobj) assert "somebenchmark3" in jo #print(jo) tmpDir.cleanup()
def addResultsForInfo(db, bInfo): from asvdb import ASVDb, BenchmarkResult for (algoName, exeTime) in algoRunResults: bResult = BenchmarkResult(funcName=algoName, argNameValuePairs=[("dataset", datasetName)], result=exeTime) db.addResult(bInfo, bResult) return db
def test_concurrency_stress(): from asvdb import ASVDb, BenchmarkInfo, BenchmarkResult tmpDir = tempfile.TemporaryDirectory() asvDirName = path.join(tmpDir.name, "dir_that_does_not_exist") repo = "somerepo" branch1 = "branch1" num = 32 dbs = [] threads = [] allFuncNames = [] bInfo = BenchmarkInfo(machineName=machineName) for i in range(num): db = ASVDb(asvDirName, repo, [branch1]) db.writeDelay = 0.5 dbs.append(db) funcName = f"somebenchmark{i}" bResult = BenchmarkResult(funcName=funcName, result=43) allFuncNames.append(funcName) t = threading.Thread(target=db.addResult, args=(bInfo, bResult)) threads.append(t) for i in range(num): threads[i].start() for i in range(num): threads[i].join() # There should be num unique results in the db after (re)reading. Pick any # of the db instances to read, they should all see the same results. results = dbs[0].getResults() assert len(results[0][1]) == num # Simply check that all unique func names were read back in. allFuncNamesCheck = [r.funcName for r in results[0][1]] assert sorted(allFuncNames) == sorted(allFuncNamesCheck) tmpDir.cleanup()
def test_addResults(): asvDir = tempfile.TemporaryDirectory() from asvdb import ASVDb, BenchmarkInfo, BenchmarkResult dbDir = asvDir.name db = ASVDb(dbDir, repo, [branch]) bInfo = BenchmarkInfo(machineName=machineName, cudaVer="9.2", osType="linux", pythonVer="3.6", commitHash=commitHash, commitTime=commitTime, branch=branch, gpuType="n/a", cpuType="x86_64", arch="my_arch", ram="123456") resultList = [] for (algoName, exeTime) in algoRunResults: bResult = BenchmarkResult(funcName=algoName, argNameValuePairs=[("dataset", datasetName)], result=exeTime) resultList.append(bResult) db.addResults(bInfo, resultList) # read back in and check dbCheck = ASVDb(dbDir, repo, [branch]) retList = dbCheck.getResults() assert len(retList) == 1 assert retList[0][0] == bInfo assert len(retList[0][1]) == len(algoRunResults) assert resultList == retList[0][1] asvDir.cleanup()
def pytest_sessionfinish(session, exitstatus): gpuBenchSess = session.config._gpubenchmarksession config = session.config asvOutputDir = config.getoption("benchmark_asv_output_dir") asvMetadata = config.getoption("benchmark_asv_metadata") gpuDeviceNums = config.getoption("benchmark_gpu_device") if asvOutputDir and gpuBenchSess.benchmarks: # FIXME: do not lookup commit metadata if already specified on the # command line. (commitHash, commitTime) = asvdbUtils.getCommitInfo() (commitRepo, commitBranch) = asvdbUtils.getRepoInfo() # FIXME: do not make pynvml calls if all the metadata provided by pynvml # was specified on the command line. smi.nvmlInit() # only supporting 1 GPU gpuDeviceHandle = smi.nvmlDeviceGetHandleByIndex(gpuDeviceNums[0]) uname = platform.uname() machineName = asvMetadata.get("machineName", uname.machine) cpuType = asvMetadata.get("cpuType", uname.processor) arch = asvMetadata.get("arch", uname.machine) pythonVer = asvMetadata.get("pythonVer", ".".join(platform.python_version_tuple()[:-1])) cudaVer = asvMetadata.get("cudaVer", _getCudaVersion() or "unknown") osType = asvMetadata.get("osType", _getOSName() or platform.linux_distribution()[0]) gpuType = asvMetadata.get("gpuType", smi.nvmlDeviceGetName(gpuDeviceHandle).decode()) ram = asvMetadata.get("ram", "%d" % psutil.virtual_memory().total) gpuRam = asvMetadata.get("gpuRam", "%d" % smi.nvmlDeviceGetMemoryInfo(gpuDeviceHandle).total) commitHash = asvMetadata.get("commitHash", commitHash) commitTime = asvMetadata.get("commitTime", commitTime) commitRepo = asvMetadata.get("commitRepo", commitRepo) commitBranch = asvMetadata.get("commitBranch", commitBranch) requirements = asvMetadata.get("requirements", "{}") suffixDict = dict(gpu_util="gpuutil", gpu_mem="gpumem", mean="time", ) unitsDict = dict(gpu_util="percent", gpu_mem="bytes", mean="seconds", ) db = ASVDb(asvOutputDir, commitRepo, [commitBranch]) bInfo = BenchmarkInfo(machineName=machineName, cudaVer=cudaVer, osType=osType, pythonVer=pythonVer, commitHash=commitHash, commitTime=commitTime, branch=commitBranch, gpuType=gpuType, cpuType=cpuType, arch=arch, ram=ram, gpuRam=gpuRam, requirements=requirements) for bench in gpuBenchSess.benchmarks: benchName = _getHierBenchNameFromFullname(bench.fullname) # build the final params dict by extracting them from the # bench.params dictionary. Not all benchmarks are parameterized params = {} bench_params = bench.params.items() if bench.params is not None else [] for (paramName, paramVal) in bench_params: # If the params are coming from a fixture, handle them # differently since they will (should be) stored in a special # variable accessible with the name of the fixture. # # NOTE: "fixture_param_names" must be manually set by the # benchmark author/user using the "request" fixture! (see below) # # @pytest.fixture(params=[1,2,3]) # def someFixture(request): # request.keywords["fixture_param_names"] = ["the_param_name"] if hasattr(bench, "fixture_param_names") and \ (bench.fixture_param_names is not None) and \ (paramName in bench.fixture_param_names): fixtureName = paramName paramNames = _ensureListLike(bench.fixture_param_names[fixtureName]) paramValues = _ensureListLike(paramVal) for (pname, pval) in zip(paramNames, paramValues): params[pname] = pval # otherwise, a benchmark/test will have params added to the # bench.params dict as a standard key:value (paramName:paramVal) else: params[paramName] = paramVal bench.stats.mean getattr(bench.stats, "gpu_mem", None) getattr(bench.stats, "gpu_util", None) resultList = [] for statType in ["mean", "gpu_mem", "gpu_util"]: bn = "%s_%s" % (benchName, suffixDict[statType]) val = getattr(bench.stats, statType, None) if val is not None: bResult = BenchmarkResult(funcName=bn, argNameValuePairs=list(params.items()), result=val) bResult.unit = unitsDict[statType] resultList.append(bResult) # If there were any custom metrics, add each of those as well as an # individual result to the same bInfo isntance. for customMetricName in bench.stats.getCustomMetricNames(): (result, unitString) = bench.stats.getCustomMetric(customMetricName) bn = "%s_%s" % (benchName, customMetricName) bResult = BenchmarkResult(funcName=bn, argNameValuePairs=list(params.items()), result=result) bResult.unit = unitString resultList.append(bResult) db.addResults(bInfo, resultList)
def genBenchmarkResults(fileList, repoName): pattern = re.compile(r"([^\/]+)") benchResults = [] for file in fileList: with open(file, 'r') as in_file: tests = json.load(in_file)["benchmarks"] # Create dictionary reference for number of parameters num_params_dict = {} for each in tests: name_and_params = pattern.findall(each["name"]) name = repoName + "." + name_and_params[0] name = name.replace("<", "[").replace(">", "]").replace("::", "_") test_params = name_and_params[1:] # Get max number of parameters for each benchmark if name not in num_params_dict: num_params_dict[name] = len(test_params) else: if len(test_params) > num_params_dict[name]: num_params_dict[name] = len(test_params) for each in tests: #Get Benchmark Name and Test Parameters name_and_params = pattern.findall(each["name"]) name = repoName + "." + name_and_params[0] name = name.replace("<", "[").replace(">", "]").replace("::", "_") test_params = name_and_params[1:] param_values = [] for idx in range(num_params_dict[name]): if idx < len(test_params): param_values.append((f"param{idx}", test_params[idx])) else: param_values.append((f"param{idx}", "None")) #Get result if "real_time" in each: bench_result = each["real_time"] elif "rms" in each: bench_result = each["rms"] if "time_unit" in each: time_unit = each["time_unit"] else: time_unit = "seconds" #Check if benchmark logged throughput if "bytes_per_second" in each: bResult = BenchmarkResult(funcName=name + "_throughput", argNameValuePairs=param_values, result=each["bytes_per_second"], unit="bps") benchResults.append(bResult) bResult = BenchmarkResult(funcName=name, argNameValuePairs=param_values, result=bench_result, unit=time_unit) benchResults.append(bResult) return benchResults
def create_bench_result(name, arg_tuple_list, result, unit): return BenchmarkResult(funcName=name, argNameValuePairs=arg_tuple_list, unit=unit, result=result)