def __runChildLoop(self, readFileNo, writeFileNo):
childReadPipe = os.fdopen(readFileNo)
childWritePipe = os.fdopen(writeFileNo, "w")
smi.nvmlInit()
# hack - get actual device ID somehow
devObj = smi.nvmlDeviceGetHandleByIndex(0)
memObj = smi.nvmlDeviceGetMemoryInfo(devObj)
utilObj = smi.nvmlDeviceGetUtilizationRates(devObj)
initialMemUsed = memObj.used
initialGpuUtil = utilObj.gpu
controlStr = self.__waitForInput(childReadPipe)
while True:
memObj = smi.nvmlDeviceGetMemoryInfo(devObj)
utilObj = smi.nvmlDeviceGetUtilizationRates(devObj)
memUsed = memObj.used - initialMemUsed
gpuUtil = utilObj.gpu - initialGpuUtil
if controlStr.strip() == "1":
self.__writeToPipe(childWritePipe, "%s %s\n"
% (memUsed, gpuUtil))
elif controlStr.strip() == "0":
break
controlStr = self.__waitForInput(childReadPipe)
smi.nvmlShutdown()
childReadPipe.close()
childWritePipe.close()
def main(args):
ns = build_argparse().parse_args(args)
testResultDir = ns.d[0]
repoName = ns.n[0]
outputDir = ns.t[0]
branchName = ns.b[0]
requirements = json.loads(ns.r[0])
gbenchFileList = os.listdir(testResultDir)
repoUrl = getCommandOutput("git remote -v").split("\n")[-1].split()[1]
db = ASVDb(outputDir, repoUrl, [branchName])
for each in gbenchFileList:
if not ".json" in each:
gbenchFileList.remove(each)
for i, val in enumerate(gbenchFileList):
gbenchFileList[i] = f"{testResultDir}/{val}"
smi.nvmlInit()
system_info = getSysInfo(requirements)
resultList = genBenchmarkResults(gbenchFileList, repoName)
db.addResults(system_info, resultList)
def pytest_sessionfinish(session, exitstatus):
gpuBenchSess = session.config._gpubenchmarksession
config = session.config
asvOutputDir = config.getoption("benchmark_asv_output_dir")
asvMetadata = config.getoption("benchmark_asv_metadata")
gpuDeviceNums = config.getoption("benchmark_gpu_device")
if asvOutputDir and gpuBenchSess.benchmarks:
# FIXME: do not lookup commit metadata if already specified on the
# command line.
(commitHash, commitTime) = asvdbUtils.getCommitInfo()
(commitRepo, commitBranch) = asvdbUtils.getRepoInfo()
# FIXME: do not make pynvml calls if all the metadata provided by pynvml
# was specified on the command line.
smi.nvmlInit()
# only supporting 1 GPU
gpuDeviceHandle = smi.nvmlDeviceGetHandleByIndex(gpuDeviceNums[0])
uname = platform.uname()
machineName = asvMetadata.get("machineName", uname.machine)
cpuType = asvMetadata.get("cpuType", uname.processor)
arch = asvMetadata.get("arch", uname.machine)
pythonVer = asvMetadata.get("pythonVer",
".".join(platform.python_version_tuple()[:-1]))
cudaVer = asvMetadata.get("cudaVer", _getCudaVersion() or "unknown")
osType = asvMetadata.get("osType",
_getOSName() or platform.linux_distribution()[0])
gpuType = asvMetadata.get("gpuType",
smi.nvmlDeviceGetName(gpuDeviceHandle).decode())
ram = asvMetadata.get("ram", "%d" % psutil.virtual_memory().total)
gpuRam = asvMetadata.get("gpuRam",
"%d" % smi.nvmlDeviceGetMemoryInfo(gpuDeviceHandle).total)
commitHash = asvMetadata.get("commitHash", commitHash)
commitTime = asvMetadata.get("commitTime", commitTime)
commitRepo = asvMetadata.get("commitRepo", commitRepo)
commitBranch = asvMetadata.get("commitBranch", commitBranch)
requirements = asvMetadata.get("requirements", "{}")
suffixDict = dict(gpu_util="gpuutil",
gpu_mem="gpumem",
mean="time",
)
unitsDict = dict(gpu_util="percent",
gpu_mem="bytes",
mean="seconds",
)
db = ASVDb(asvOutputDir, commitRepo, [commitBranch])
bInfo = BenchmarkInfo(machineName=machineName,
cudaVer=cudaVer,
osType=osType,
pythonVer=pythonVer,
commitHash=commitHash,
commitTime=commitTime,
branch=commitBranch,
gpuType=gpuType,
cpuType=cpuType,
arch=arch,
ram=ram,
gpuRam=gpuRam,
requirements=requirements)
for bench in gpuBenchSess.benchmarks:
benchName = _getHierBenchNameFromFullname(bench.fullname)
# build the final params dict by extracting them from the
# bench.params dictionary. Not all benchmarks are parameterized
params = {}
bench_params = bench.params.items() if bench.params is not None else []
for (paramName, paramVal) in bench_params:
# If the params are coming from a fixture, handle them
# differently since they will (should be) stored in a special
# variable accessible with the name of the fixture.
#
# NOTE: "fixture_param_names" must be manually set by the
# benchmark author/user using the "request" fixture! (see below)
#
# @pytest.fixture(params=[1,2,3])
# def someFixture(request):
# request.keywords["fixture_param_names"] = ["the_param_name"]
if hasattr(bench, "fixture_param_names") and \
(bench.fixture_param_names is not None) and \
(paramName in bench.fixture_param_names):
fixtureName = paramName
paramNames = _ensureListLike(bench.fixture_param_names[fixtureName])
paramValues = _ensureListLike(paramVal)
for (pname, pval) in zip(paramNames, paramValues):
params[pname] = pval
# otherwise, a benchmark/test will have params added to the
# bench.params dict as a standard key:value (paramName:paramVal)
else:
params[paramName] = paramVal
bench.stats.mean
getattr(bench.stats, "gpu_mem", None)
getattr(bench.stats, "gpu_util", None)
resultList = []
for statType in ["mean", "gpu_mem", "gpu_util"]:
bn = "%s_%s" % (benchName, suffixDict[statType])
val = getattr(bench.stats, statType, None)
if val is not None:
bResult = BenchmarkResult(funcName=bn,
argNameValuePairs=list(params.items()),
result=val)
bResult.unit = unitsDict[statType]
resultList.append(bResult)
# If there were any custom metrics, add each of those as well as an
# individual result to the same bInfo isntance.
for customMetricName in bench.stats.getCustomMetricNames():
(result, unitString) = bench.stats.getCustomMetric(customMetricName)
bn = "%s_%s" % (benchName, customMetricName)
bResult = BenchmarkResult(funcName=bn,
argNameValuePairs=list(params.items()),
result=result)
bResult.unit = unitString
resultList.append(bResult)
db.addResults(bInfo, resultList)
def stop(self):
self.__stop = True
def startGpuMetricPolling():
gpuPollObj = GPUMetricPoller()
gpuPollObj.start()
return gpuPollObj
def stopGpuMetricPolling(gpuPollObj):
gpuPollObj.stop()
gpuPollObj.join() # consider using timeout and reporting errors
"""
smi.nvmlInit()
# hack - get actual device ID somehow
devObj = smi.nvmlDeviceGetHandleByIndex(0)
memObj = smi.nvmlDeviceGetMemoryInfo(devObj)
utilObj = smi.nvmlDeviceGetUtilizationRates(devObj)
initialMemUsed = memObj.used
initialGpuUtil = utilObj.gpu
while not self.__stop:
time.sleep(0.01)
memObj = smi.nvmlDeviceGetMemoryInfo(devObj)
utilObj = smi.nvmlDeviceGetUtilizationRates(devObj)
memUsed = memObj.used - initialMemUsed
from pynvml import smi as nvidia_smi
nvidia_smi.nvmlInit()
handle = nvidia_smi.nvmlDeviceGetHandleByIndex(0)
# card id 0 hardcoded here, there is also a call to get all available card ids, so we could iterate
ans = 0
while(True):
mem_res = nvidia_smi.nvmlDeviceGetMemoryInfo(handle)
# print(mem_res.used / (1024**2)) # usage in GiB
if (mem_res.used / (1024**2) > ans):
ans = mem_res.used / (1024**2)
print(ans)
# print(f'mem: {100 * (mem_res.used / mem_res.total):.3f}%') # percentage usage