def get_available_gpus(numGPUs):
import os
from tensorflow.python.client import device_lib
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
AvailabeGPUs = []
GPUs = []
#Boolean Idicator of GPUs Availability
deviceID = GPUtil.getAvailability(GPUtil.getGPUs(), maxLoad = 0.5, maxMemory = 0.5)
print (deviceID)
# Puts each Availabe GPU ID into a list
for device in range(0, len(deviceID)):
if deviceID[device] == 1:
AvailabeGPUs.append(device)
print(AvailabeGPUs)
# Puts the apprioate amount of GPUs IDs into a list
for x in range(0, numGPUs):
GPUs.append(AvailabeGPUs[x])
print (str(GPUs))
#Format list for CUDA_VISIBLE_DEVICES
GPUstring = " ".join(str(x) for x in GPUs)
GPUstring = GPUstring.replace(" ", ",")
print (GPUstring)
os.environ["CUDA_VISIBLE_DEVICES"]= str(GPUstring)
local_device_protos = device_lib.list_local_devices()
return [x.name for x in local_device_protos if x.device_type == 'GPU']
def getComputerInfo():
# Get CPU info
# cpu = cpuinfo.get_cpu_info()
result_dict = {
# "Python version": cpu["python_version"],
"TensorFlow version": tf.version.VERSION,
# "CPU Description": cpu["brand"],
# "CPU Clock speed (advertised)": cpu["hz_advertised"],
# "CPU Clock speed (actual)": cpu["hz_actual"],
# "CPU Architecture": cpu["arch"]
}
# Get GPU info
gpus_tf = tf.config.experimental.list_physical_devices("GPU")
result_dict["Number of GPUs (tf)"] = len(gpus_tf)
gpus = GPUtil.getGPUs()
gpus_available = GPUtil.getAvailability(gpus)
for i, gpu in enumerate(gpus):
result_dict["GPU %i" % gpu.id] = gpu.name
result_dict["GPU %i (driver)" % gpu.id] = gpu.driver
result_dict["GPU %i (memory total)" % gpu.id] = gpu.memoryTotal
result_dict["GPU %i (memory free)" % gpu.id] = gpu.memoryFree
result_dict["GPU %i (available?)" % gpu.id] = gpus_available[i]
# Get RAM info
mem = virtual_memory()
result_dict["RAM (total)"] = mem.total
result_dict["RAM (available)"] = mem.available
return result_dict
def AILabUtil(pathlist=[
r'C:\tools\cuda\bin',
r'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\extras\CUPTI\libx64',
r'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\bin'
]):
import GPUtil
import os
os.environ["PATH"] += os.pathsep + os.pathsep.join(pathlist)
import tensorflow as tf
import keras
available_device = None
GPUavailability = GPUtil.getAvailability(GPUtil.getGPUs(),
maxLoad=0.4,
maxMemory=0.4,
includeNan=False,
excludeID=[],
excludeUUID=[])
for i, device in enumerate(GPUavailability):
if device == 1:
available_device = i
break
if available_device == None:
print('No available GPU devices')
return available_device
def __init__(self):
nn.Module.__init__(self)
self.sub_network1 = nn.Linear(10, 10)
self.sub_network2 = nn.Linear(10, 10)
availablity = GPUtil.getAvailability(GPUtil.getGPUs())
self.available_devices = [i for i in range(len(availablity)) if availablity[i]==1]
if len(self.available_devices) < 2:
raise ValueError(f'there are only {len(self.available_devices)} cuda devices available')
print(f'available devices: {self.available_devices}')
self.sub_network1.cuda(self.available_devices[0])
self.sub_network2.cuda(self.available_devices[1])
def checkgpu():
'''check gpu availability and utilization'''
card = gpu.getGPUs()
isavailable = gpu.getAvailability(card, maxLoad=.6)
print(time.ctime())
if isavailable == [1]:
print("can mine")
time.sleep(5)
return 'isavailable'
if isavailable == [0]:
print("gpu in use")
gpu.showUtilization()
time.sleep(5)
return 'notavailable'
def setupGPUs(RequestedGPUList=None, maxLoad=0.1, maxMemory=0.1):
if torch.cuda.is_available():
if RequestedGPUList is not None:
DeviceList = RequestedGPUList
MainGPUID = DeviceList[0]
nDevs = torch.cuda.device_count()
AvailableDevList = [i for i in range(0, nDevs)] # All GPUs
if len(DeviceList) == 1 and MainGPUID < 0:
DeviceList = AvailableDevList
if set(DeviceList).issubset(set(AvailableDevList)) == False:
raise RuntimeError(
'Unable to find requested devices {}.'.format(DeviceList))
else:
GPUs = GPUtil.getGPUs()
FreeGPUs = GPUtil.getAvailability(GPUs,
maxLoad=maxLoad,
maxMemory=maxMemory,
includeNan=False,
excludeID=[],
excludeUUID=[])
DeviceList = []
for GPU, FreeID in zip(GPUs, FreeGPUs):
if FreeID > 0:
DeviceList.append(GPU.id)
if len(DeviceList) <= 0:
raise RuntimeError(
'No GPUs with with load < {} and memory < {} found. Call with explicit GPU list to override.'
.format(maxLoad, maxMemory))
print(
'[ INFO ]: Automatically detected GPUs ({}) with load < {} and memory < {}.'
.format(DeviceList, maxLoad, maxMemory))
MainGPUID = DeviceList[0]
else:
print(
'[ WARN ]: No GPUs available. Will use device ID 0 which will default to CPU.'
)
DeviceList = [0]
MainGPUID = 0
return DeviceList, MainGPUID
import GPUtil
import argparse
import pandas as pd
import numpy as np
import os
import csv
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('-target_dir',
help='Target directory contains results of each seed')
GPUs = GPUtil.getGPUs()
GPUavailability = GPUtil.getAvailability(GPUs,
maxLoad=1,
maxMemory=1,
includeNan=False,
excludeID=[],
excludeUUID=[])
finish_exp = sum(GPUavailability) == len(GPUavailability)
def main(args):
target_dir = args.target_dir
if finish_exp:
seeds = ['2020', '2021', '2022', '2023']
acc = np.zeros(50)
invalid = np.zeros(50)
repeat = np.zeros(50)
for n, seed in enumerate(seeds):
file = target_dir + seed + '.csv'
# TODO Read this
import os
import sys
import GPUtil
limit = int(sys.argv[1]) if len(sys.argv) > 1 else 1000
requested_limit = (len(sys.argv) > 1)
# If you need one GPU, I will pick it here for you
gpu = [str(g) for g in GPUtil.getAvailable(maxMemory=0.2, limit=limit)]
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not os.environ[
'CUDA_VISIBLE_DEVICES']:
assert len(gpu) > 0, 'No available GPUs'
assert len(gpu) >= limit, 'Not enough available GPUs ({} < {})'.format(
len(gpu), limit)
print('Using GPU', ','.join(gpu))
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(gpu)
gpus = GPUtil.getGPUs()
gpu_avail = GPUtil.getAvailability(gpus, maxMemory=0.2)
free_gpus = [str(i) for i in range(len(gpu_avail)) if gpu_avail[i] > 0]
requested_gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(",")
if requested_limit:
assert len(
requested_gpus
) >= limit, "{} gpus requested from script, but only {} available from CUDA_VISIBLE_DEVICES".format(
limit, len(requested_gpus))
assert all([g in free_gpus for g in requested_gpus
]), "{} not subset of {}".format(requested_gpus, free_gpus)
