def get_available_gpus(numGPUs): import os from tensorflow.python.client import device_lib os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" AvailabeGPUs = [] GPUs = [] #Boolean Idicator of GPUs Availability deviceID = GPUtil.getAvailability(GPUtil.getGPUs(), maxLoad = 0.5, maxMemory = 0.5) print (deviceID) # Puts each Availabe GPU ID into a list for device in range(0, len(deviceID)): if deviceID[device] == 1: AvailabeGPUs.append(device) print(AvailabeGPUs) # Puts the apprioate amount of GPUs IDs into a list for x in range(0, numGPUs): GPUs.append(AvailabeGPUs[x]) print (str(GPUs)) #Format list for CUDA_VISIBLE_DEVICES GPUstring = " ".join(str(x) for x in GPUs) GPUstring = GPUstring.replace(" ", ",") print (GPUstring) os.environ["CUDA_VISIBLE_DEVICES"]= str(GPUstring) local_device_protos = device_lib.list_local_devices() return [x.name for x in local_device_protos if x.device_type == 'GPU']
def getComputerInfo(): # Get CPU info # cpu = cpuinfo.get_cpu_info() result_dict = { # "Python version": cpu["python_version"], "TensorFlow version": tf.version.VERSION, # "CPU Description": cpu["brand"], # "CPU Clock speed (advertised)": cpu["hz_advertised"], # "CPU Clock speed (actual)": cpu["hz_actual"], # "CPU Architecture": cpu["arch"] } # Get GPU info gpus_tf = tf.config.experimental.list_physical_devices("GPU") result_dict["Number of GPUs (tf)"] = len(gpus_tf) gpus = GPUtil.getGPUs() gpus_available = GPUtil.getAvailability(gpus) for i, gpu in enumerate(gpus): result_dict["GPU %i" % gpu.id] = gpu.name result_dict["GPU %i (driver)" % gpu.id] = gpu.driver result_dict["GPU %i (memory total)" % gpu.id] = gpu.memoryTotal result_dict["GPU %i (memory free)" % gpu.id] = gpu.memoryFree result_dict["GPU %i (available?)" % gpu.id] = gpus_available[i] # Get RAM info mem = virtual_memory() result_dict["RAM (total)"] = mem.total result_dict["RAM (available)"] = mem.available return result_dict
def AILabUtil(pathlist=[ r'C:\tools\cuda\bin', r'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\extras\CUPTI\libx64', r'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v9.0\bin' ]): import GPUtil import os os.environ["PATH"] += os.pathsep + os.pathsep.join(pathlist) import tensorflow as tf import keras available_device = None GPUavailability = GPUtil.getAvailability(GPUtil.getGPUs(), maxLoad=0.4, maxMemory=0.4, includeNan=False, excludeID=[], excludeUUID=[]) for i, device in enumerate(GPUavailability): if device == 1: available_device = i break if available_device == None: print('No available GPU devices') return available_device
def __init__(self): nn.Module.__init__(self) self.sub_network1 = nn.Linear(10, 10) self.sub_network2 = nn.Linear(10, 10) availablity = GPUtil.getAvailability(GPUtil.getGPUs()) self.available_devices = [i for i in range(len(availablity)) if availablity[i]==1] if len(self.available_devices) < 2: raise ValueError(f'there are only {len(self.available_devices)} cuda devices available') print(f'available devices: {self.available_devices}') self.sub_network1.cuda(self.available_devices[0]) self.sub_network2.cuda(self.available_devices[1])
def checkgpu(): '''check gpu availability and utilization''' card = gpu.getGPUs() isavailable = gpu.getAvailability(card, maxLoad=.6) print(time.ctime()) if isavailable == [1]: print("can mine") time.sleep(5) return 'isavailable' if isavailable == [0]: print("gpu in use") gpu.showUtilization() time.sleep(5) return 'notavailable'
def setupGPUs(RequestedGPUList=None, maxLoad=0.1, maxMemory=0.1): if torch.cuda.is_available(): if RequestedGPUList is not None: DeviceList = RequestedGPUList MainGPUID = DeviceList[0] nDevs = torch.cuda.device_count() AvailableDevList = [i for i in range(0, nDevs)] # All GPUs if len(DeviceList) == 1 and MainGPUID < 0: DeviceList = AvailableDevList if set(DeviceList).issubset(set(AvailableDevList)) == False: raise RuntimeError( 'Unable to find requested devices {}.'.format(DeviceList)) else: GPUs = GPUtil.getGPUs() FreeGPUs = GPUtil.getAvailability(GPUs, maxLoad=maxLoad, maxMemory=maxMemory, includeNan=False, excludeID=[], excludeUUID=[]) DeviceList = [] for GPU, FreeID in zip(GPUs, FreeGPUs): if FreeID > 0: DeviceList.append(GPU.id) if len(DeviceList) <= 0: raise RuntimeError( 'No GPUs with with load < {} and memory < {} found. Call with explicit GPU list to override.' .format(maxLoad, maxMemory)) print( '[ INFO ]: Automatically detected GPUs ({}) with load < {} and memory < {}.' .format(DeviceList, maxLoad, maxMemory)) MainGPUID = DeviceList[0] else: print( '[ WARN ]: No GPUs available. Will use device ID 0 which will default to CPU.' ) DeviceList = [0] MainGPUID = 0 return DeviceList, MainGPUID
import GPUtil import argparse import pandas as pd import numpy as np import os import csv parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('-target_dir', help='Target directory contains results of each seed') GPUs = GPUtil.getGPUs() GPUavailability = GPUtil.getAvailability(GPUs, maxLoad=1, maxMemory=1, includeNan=False, excludeID=[], excludeUUID=[]) finish_exp = sum(GPUavailability) == len(GPUavailability) def main(args): target_dir = args.target_dir if finish_exp: seeds = ['2020', '2021', '2022', '2023'] acc = np.zeros(50) invalid = np.zeros(50) repeat = np.zeros(50) for n, seed in enumerate(seeds): file = target_dir + seed + '.csv'
# TODO Read this import os import sys import GPUtil limit = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 requested_limit = (len(sys.argv) > 1) # If you need one GPU, I will pick it here for you gpu = [str(g) for g in GPUtil.getAvailable(maxMemory=0.2, limit=limit)] if 'CUDA_VISIBLE_DEVICES' not in os.environ or not os.environ[ 'CUDA_VISIBLE_DEVICES']: assert len(gpu) > 0, 'No available GPUs' assert len(gpu) >= limit, 'Not enough available GPUs ({} < {})'.format( len(gpu), limit) print('Using GPU', ','.join(gpu)) os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(gpu) gpus = GPUtil.getGPUs() gpu_avail = GPUtil.getAvailability(gpus, maxMemory=0.2) free_gpus = [str(i) for i in range(len(gpu_avail)) if gpu_avail[i] > 0] requested_gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(",") if requested_limit: assert len( requested_gpus ) >= limit, "{} gpus requested from script, but only {} available from CUDA_VISIBLE_DEVICES".format( limit, len(requested_gpus)) assert all([g in free_gpus for g in requested_gpus ]), "{} not subset of {}".format(requested_gpus, free_gpus)