def collect_system_information(trainer):
import psutil
mem = psutil.virtual_memory()
trainer.set_job_info('memory_total', mem.total)
import keras.backend as K
if K.backend() == 'theano':
# at this point, theano is already initialised through KerasLogger
from theano.sandbox import cuda
trainer.set_job_info('cuda_available', cuda.cuda_available)
if cuda.cuda_available:
trainer.on_gpu = cuda.use.device_number is not None
trainer.set_job_info('cuda_device_number', cuda.active_device_number())
trainer.set_job_info('cuda_device_name', cuda.active_device_name())
if cuda.cuda_ndarray.cuda_ndarray.mem_info:
gpu = cuda.cuda_ndarray.cuda_ndarray.mem_info()
trainer.set_job_info('cuda_device_max_memory', gpu[1])
free = gpu[0] / 1024 / 1024 / 1024
total = gpu[1] / 1024 / 1024 / 1024
used = total - free
print("%.2fGB GPU memory used of %.2fGB, %s, device id %d" % (used, total, cuda.active_device_name(), cuda.active_device_number()))
trainer.set_job_info('on_gpu', trainer.on_gpu)
import cpuinfo
cpu = cpuinfo.get_cpu_info()
trainer.set_job_info('cpu_name', cpu['brand'])
trainer.set_job_info('cpu', [cpu['hz_actual_raw'][0], cpu['count']])
def collect_system_information(trainer):
import psutil
mem = psutil.virtual_memory()
trainer.set_job_info('memory_total', mem.total)
# at this point, theano is already initialised through KerasLogger
from theano.sandbox import cuda
trainer.set_job_info('cuda_available', cuda.cuda_available)
if cuda.cuda_available:
trainer.on_gpu = cuda.use.device_number is not None
trainer.set_job_info('cuda_device_number', cuda.active_device_number())
trainer.set_job_info('cuda_device_name', cuda.active_device_name())
if cuda.cuda_ndarray.cuda_ndarray.mem_info:
gpu = cuda.cuda_ndarray.cuda_ndarray.mem_info()
trainer.set_job_info('cuda_device_max_memory', gpu[1])
free = gpu[0]/1024/1024/1024
total = gpu[1]/1024/1024/1024
used = total-free
print("%.2fGB GPU memory used of %.2fGB" %(used, total))
trainer.set_job_info('on_gpu', trainer.on_gpu)
import cpuinfo
cpu = cpuinfo.get_cpu_info()
trainer.set_job_info('cpu_name', cpu['brand'])
trainer.set_job_info('cpu', [cpu['hz_actual_raw'][0], cpu['count']])
def collect_system_information(self):
import psutil
mem = psutil.virtual_memory()
self.set_system_info('memory_total', mem.total)
on_gpu = False
import sys
if 'theano.sandbox' in sys.modules:
# at this point, theano is already initialised, so we can use it to monitor the GPU.
from theano.sandbox import cuda
self.set_system_info('cuda_available', cuda.cuda_available)
if cuda.cuda_available:
on_gpu = cuda.use.device_number is not None
self.set_system_info('cuda_device_number',
cuda.active_device_number())
self.set_system_info('cuda_device_name',
cuda.active_device_name())
if cuda.cuda_ndarray.cuda_ndarray.mem_info:
gpu = cuda.cuda_ndarray.cuda_ndarray.mem_info()
self.set_system_info('cuda_device_max_memory', gpu[1])
free = gpu[0] / 1024.0 / 1024.0 / 1024.0
total = gpu[1] / 1024.0 / 1024.0 / 1024.0
used = total - free
print(
"%.2fGB GPU memory used of %.2fGB, %s, device id %d" %
(used, total, cuda.active_device_name(),
cuda.active_device_number()))
self.set_system_info('on_gpu', on_gpu)
import cpuinfo
cpu = cpuinfo.get_cpu_info()
self.set_system_info('cpu_name', cpu['brand'])
self.set_system_info('cpu', [cpu['hz_actual_raw'][0], cpu['count']])
def configure(self, flags):
if self.configured is True:
return
self.configured = True
if 'theano' in sys.modules:
self.log.warning('Theano was already imported and cannot be reconfigured.')
return
os.environ.setdefault('THEANO_FLAGS', flags+',print_active_device=False')
cuda = logging.getLogger('theano.sandbox.cuda')
cuda.setLevel(logging.CRITICAL)
import theano
cuda.setLevel(logging.WARNING)
try:
import theano.sandbox.cuda as cd
self.log.info('Using device gpu%i: %s', cd.active_device_number(), cd.active_device_name())
except AttributeError:
self.log.info('Using device cpu0, with %r.', theano.config.floatX)