def test_maxout_conv_c01b_cifar10(self):
if cuda.cuda_available is False:
raise SkipTest('Optional package cuda disabled')
if not hasattr(cuda, 'unuse'):
raise Exception("Theano version too old to run this test!")
# Tests that we can run a small convolutional model on GPU,
assert cuda.cuda_enabled is False
# Even if there is a GPU, but the user didn't specify device=gpu
# we want to run this test.
try:
old_floatX = config.floatX
cuda.use('gpu')
config.floatX = 'float32'
try:
train = yaml_parse.load(yaml_string_maxout_conv_c01b_cifar10)
except NoDataPathError:
raise SkipTest("PYLEARN2_DATA_PATH environment variable "
"not defined")
train.main_loop()
# Check that the performance is close to the expected one:
# test_y_misclass: 0.3777000308036804
misclass_chan = train.algorithm.monitor.channels['test_y_misclass']
assert misclass_chan.val_record[-1] < 0.38, \
("misclass_chan.val_record[-1] = %g" %
misclass_chan.val_record[-1])
# test_y_nll: 1.0978516340255737
nll_chan = train.algorithm.monitor.channels['test_y_nll']
assert nll_chan.val_record[-1] < 1.1
finally:
config.floatX = old_floatX
cuda.unuse()
assert cuda.cuda_enabled is False
def test_maxout_conv_c01b_cifar10(self):
if cuda.cuda_available is False:
raise SkipTest('Optional package cuda disabled')
if not hasattr(cuda, 'unuse'):
raise Exception("Theano version too old to run this test!")
# Tests that we can run a small convolutional model on GPU,
assert cuda.cuda_enabled is False
# Even if there is a GPU, but the user didn't specify device=gpu
# we want to run this test.
try:
old_floatX = config.floatX
cuda.use('gpu')
config.floatX = 'float32'
try:
train = yaml_parse.load(yaml_string_maxout_conv_c01b_cifar10)
except NoDataPathError:
raise SkipTest("PYLEARN2_DATA_PATH environment variable "
"not defined")
train.main_loop()
# Check that the performance is close to the expected one:
# test_y_misclass: 0.3777000308036804
misclass_chan = train.algorithm.monitor.channels['test_y_misclass']
assert misclass_chan.val_record[-1] < 0.38
# test_y_nll: 1.0978516340255737
nll_chan = train.algorithm.monitor.channels['test_y_nll']
assert nll_chan.val_record[-1] < 1.1
finally:
config.floatX = old_floatX
cuda.unuse()
assert cuda.cuda_enabled is False
def init_theano():
"""Initialize Theano for Caffe
"""
global theano_initialized
if theano_initialized:
return
import caffe
from theano.sandbox.cuda import use
assert caffe.check_mode_gpu()
use('gpu%d' % caffe.get_device())
theano_initialized = True
def set_gpu_from_theano():
"""
This set the GPU used by PyCUDA to the same as the one used by Theano.
"""
#import pdb;pdb.set_trace()
if cuda.use.device_number is None:
cuda.use("gpu",
force=False,
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=True,
test_driver=True)
assert cuda.use.device_number == cuda_ndarray.active_device_number()
def test_cuda():
import theano.sandbox.cuda as theano_cuda
assert_true(theano_cuda.cuda_available, "Theano CUDA support not available. Check that nvcc is in $PATH.")
if theano_cuda.cuda_enabled: # already enabled when $THEANO_FLAGS=device=gpu
print("CUDA already enabled")
else:
print("Call theano_cuda.use")
theano_cuda.use(device="gpu", force=True)
try:
import cuda_ndarray.cuda_ndarray as cuda
except ImportError as exc:
raise Exception("Theano CUDA support seems broken: %s" % exc)
id = cuda.active_device_number(); """ :type: int """
device_name = cuda.active_device_name(); """ :type: str """
print("id: %i", id)
print("dev name: %s" % device_name)
def test_cuda():
import theano.sandbox.cuda as theano_cuda
assert_true(theano_cuda.cuda_available, "Theano CUDA support not available. Check that nvcc is in $PATH.")
if theano_cuda.cuda_enabled: # already enabled when $THEANO_FLAGS=device=gpu
print("CUDA already enabled")
else:
print("Call theano_cuda.use")
theano_cuda.use(device="gpu", force=True)
try:
import cuda_ndarray.cuda_ndarray as cuda
except ImportError as exc:
raise Exception("Theano CUDA support seems broken: %s" % exc)
id = cuda.active_device_number(); """ :type: int """
device_name = cuda.active_device_name(); """ :type: str """
print("id: %i" % id)
print("dev name: %s" % device_name)
def test_maxout_conv_c01b_basic(self):
if cuda.cuda_available is False:
raise SkipTest('Optional package cuda disabled')
if not hasattr(cuda, 'unuse'):
raise Exception("Theano version too old to run this test!")
# Tests that we can run a small convolutional model on GPU,
assert cuda.cuda_enabled is False
# Even if there is a GPU, but the user didn't specify device=gpu
# we want to run this test.
try:
old_floatX = config.floatX
cuda.use('gpu')
config.floatX = 'float32'
train = yaml_parse.load(yaml_string_maxout_conv_c01b_basic)
train.main_loop()
finally:
config.floatX = old_floatX
cuda.unuse()
assert cuda.cuda_enabled is False
def test_maxout_conv_c01b_basic(self):
if cuda.cuda_available is False:
raise SkipTest('Optional package cuda disabled')
if not hasattr(cuda, 'unuse'):
raise Exception("Theano version too old to run this test!")
# Tests that we can run a small convolutional model on GPU,
assert cuda.cuda_enabled is False
# Even if there is a GPU, but the user didn't specify device=gpu
# we want to run this test.
try:
old_floatX = config.floatX
cuda.use('gpu')
config.floatX = 'float32'
train = yaml_parse.load(yaml_string_maxout_conv_c01b_basic)
train.main_loop()
finally:
config.floatX = old_floatX
cuda.unuse()
assert cuda.cuda_enabled is False
def test_output_broadcast_cuda(self):
from theano.sandbox import cuda
if not cuda.cuda_available:
raise SkipTest("Optional package Cuda disabled")
if cuda.use.device_number is None:
# We should normally set VecAsRowAndCol as a GPUOp But we
# don't want to do this here as this will disable others
# tests in this file. So we manually init the GPU if
# needed to remove warning.
cuda.use("gpu",
force=True,
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False)
v = cuda.fvector('v')
c, r = VecAsRowAndCol()(v)
f = theano.function([v], [c, r])
v_val = cuda.CudaNdarray(self.rng.randn(5).astype('float32'))
f(v_val)
def test_output_broadcast_cuda(self):
from theano.sandbox import cuda
if not cuda.cuda_available:
raise SkipTest("Optional package Cuda disabled")
if cuda.use.device_number is None:
# We should normally set VecAsRowAndCol as a GPUOp But we
# don't want to do this here as this will disable others
# tests in this file. So we manually init the GPU if
# needed to remove warning.
cuda.use("gpu",
force=True,
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False)
v = cuda.fvector('v')
c, r = VecAsRowAndCol()(v)
f = theano.function([v], [c, r])
v_val = cuda.CudaNdarray(self.rng.randn(5).astype('float32'))
f(v_val)
def infer(datain,
model,
preptrain,
posptrain,
windowsin,
windowsout,
dataout=None,
verbose=True,
device=None):
"""
Infer on a block of data (`datain`) and write results to another block (inplace, `dataout`) with an Antipasti model
(`model`).
:type datain: numpy.ndarray
:param datain: Input block.
:type model: Antipasti.netarchs.model
:param model: Model to infer with.
:type preptrain: Antipasti.prepkit.preptrain
:param preptrain: Train of preprocessing functions to convert the input data slice to appropriate dimensions for
model to process.
:type posptrain: Antipasti.prepkit.preptrain
:param posptrain: Train of postprocessing functions to process the output of the model, i.e. convert it to the
right shape.
:type windowsin: generator or dict
:param windowsin: Sliding window generator or configuration dict for input data. See kwargs in
Antipasti.netdatautils.slidingwindowslices for more on how to populate the configuration dict.
:type windowsout: generator or dict or callable
:param windowsout: Sliding window on the output data. If callable: must accept input sliding window as an input to
produce an output sliding window. If dict: kwargs for Antipasti.netdatautils.slidingwindowslices.
If generator: Must yield the next slice.
:type dataout: numpy.ndarray
:param dataout: Output block, to be written inplace.
:return: Processed block.
"""
def smartprint(msg):
if verbose:
print(msg)
# Set up device
if device is not None:
smartprint("Process {}: Trying to initialize device {}...".format(
os.getpid(), device))
from theano.sandbox.cuda import use
use(device=device)
# Imports. All imports from now happen on 'device'.
import Antipasti.netdatautils as ndu
import Antipasti.netkit as nk
import Antipasti.netarchs as na
# Build model if required
if not isinstance(model, na.model) and callable(model):
smartprint("Process {}: Building Model...".format(os.getpid()))
model = model()
# Compile model if required
if model.classifier is None:
smartprint("Process {}: Compiling Model...".format(os.getpid()))
model.compile(what='inference')
# Build output data container if required
if dataout is None:
dataout = np.zeros_like(datain).astype('float32')
normalization = np.zeros_like(dataout).astype('float32')
# Fetch sliding window generator if windowsin is a dict
if isinstance(windowsin, dict):
windowsin = ndu.slidingwindowslices(**windowsin)
# Fetch sliding window generator if windowsout is a dict
if isinstance(windowsout, dict):
windowsout = ndu.slidingwindowslices(**windowsout)
# Loop over windowsin,
for window in windowsin:
smartprint("Process {}: Processing window {}...".format(
os.getpid(), window))
# fetch input volume,
inp = datain[window]
# apply preptrain,
inp = preptrain(inp)
# apply model,
out = model.classifier(inp)
# apply posptrain,
out = posptrain(out)
# fetch output slice from windowsout,
if callable(windowsout):
outwindow = windowsout(window)
elif hasattr(windowsout, 'next'):
outwindow = windowsout.next()
else:
raise RuntimeError(
"windowsout must be a callable or a generator or a dict.")
# add to dataout and increment normalization tensor
dataout[outwindow] += out
normalization[outwindow] += 1.
# Get rid of zeros from normalization
normalization[normalization == 0.] = 1.
# Normalize dataout
dataout = dataout / normalization
# Return dataout
return dataout
http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf
"""
import numpy
from theano.compile.sandbox import shared, pfunc
from theano import tensor
from pylearn.shared.layers import LogisticRegression, SigmoidalLayer
import theano.sandbox.softsign
import pylearn.datasets.MNIST
try:
# this tells theano to use the GPU if possible
from theano.sandbox.cuda import use
use()
except Exception, e:
print('Warning: Attempt to use GPU resulted in error "%s"' % str(e))
class LeNetConvPool(object):
"""WRITEME
Math of what the layer does, and what symbolic variables are created by the class (w, b,
output).
"""
#TODO: implement biases & scales properly. There are supposed to be more parameters.
# - one bias & scale per filter
# - one bias & scale per downsample feature location (a 2d bias)
# - more?
# coding: utf-8
# In[1]:
from theano.sandbox import cuda
cuda.use('gpu2')
# In[2]:
get_ipython().magic(u'matplotlib inline')
import utils
reload(utils)
from utils import *
from __future__ import division, print_function
# ## Setup
# In[3]:
batch_size = 64
# In[4]:
from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
# In[5]:
X_test = np.expand_dims(X_test, 1)
X_train = np.expand_dims(X_train, 1)
#needed as the gpu conv don't have a perform implementation.
if theano.config.mode == 'FAST_COMPILE':
theano_mode = theano.compile.mode.get_mode('FAST_RUN').including('gpu')
else:
theano_mode = theano.compile.mode.get_default_mode().including('gpu')
cuda_tensor4 = cuda.CudaNdarrayType([False] * 4)
device_id = theano.sandbox.cuda.use.device_number
if device_id is None:
cuda.shared_constructor(numpy.zeros(2, dtype='float32'))
device_id = theano.sandbox.cuda.use.device_number
if device_id is None:
cuda.use("gpu",
force=False,
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False,
test_driver=True)
device_id = theano.sandbox.cuda.use.device_number
cuda_ndarray = theano.sandbox.cuda.cuda_ndarray.cuda_ndarray
device_prop = cuda_ndarray.device_properties(device_id)
def py_conv_valid_numpy(img, kern):
assert img.shape[1] == kern.shape[1]
outshp = (img.shape[0], kern.shape[0],
img.shape[2] - kern.shape[2] + 1,
img.shape[3] - kern.shape[3] + 1)
out = numpy.zeros(outshp, dtype='float32')
for b in xrange(out.shape[0]):
import numpy
import theano
from theano.tests import unittest_tools as utt
from theano.sandbox.gpuarray.basic_ops import GpuReshape
import theano.sandbox.gpuarray
if theano.sandbox.gpuarray.pygpu is None:
raise SkipTest("pygpu not installed")
import theano.sandbox.cuda as cuda_ndarray
if cuda_ndarray.cuda_available and not theano.sandbox.gpuarray.pygpu_activated:
if not cuda_ndarray.use.device_number:
cuda_ndarray.use('gpu')
theano.sandbox.gpuarray.init_dev('cuda')
if not theano.sandbox.gpuarray.pygpu_activated:
raise SkipTest("pygpu disabled")
if theano.config.mode == 'FAST_COMPILE':
mode_with_gpu = theano.compile.mode.get_mode('FAST_RUN').including('gpuarray').excluding('gpu')
mode_without_gpu = theano.compile.mode.get_mode('FAST_RUN').excluding('gpuarray')
else:
mode_with_gpu = theano.compile.mode.get_default_mode().including('gpuarray').excluding('gpu')
mode_without_gpu = theano.compile.mode.get_default_mode().excluding('gpuarray')
def test_flatten():
m = theano.tensor.fmatrix()
f = theano.function([m], m.flatten(), mode=mode_with_gpu)
val = numpy.random.rand(10,11).astype("float32")
Training of Deep Networks, Advances in Neural Information Processing
Systems 19, 2007
"""
import numpy
from theano import tensor
from theano.compile.sandbox import shared, pfunc
from theano.compile.sandbox.shared_randomstreams import RandomStreams
from theano.tensor import nnet
import pylearn.datasets.MNIST
try:
#this tells theano to use the GPU if possible
from theano.sandbox.cuda import use
use()
except Exception, e:
print('Warning: Attempt to use GPU resulted in error "%s"' % str(e))
def load_mnist_batches(batch_size):
"""
We should remove the dependency on pylearn.datasets.MNIST .. and maybe
provide a pickled version of the dataset..
"""
mnist = pylearn.datasets.MNIST.train_valid_test()
train_batches = [(mnist.train.x[i:i + batch_size],
mnist.train.y[i:i + batch_size])
for i in xrange(0, len(mnist.train.x), batch_size)]
valid_batches = [(mnist.valid.x[i:i + batch_size],
mnist.valid.y[i:i + batch_size])
def run(self):
# Wait for device
self.waitforpid()
# Import theano and bind it to the GPU
if 'gpu' in self.workerconfig['device']:
self.print_("[+] Trying to initialize GPU device {}.".format(
self.workerconfig['device']))
from theano.sandbox.cuda import use
use(self.workerconfig['device'])
else:
self.print_("[-] Not using GPU. The device is set to {}.".format(
self.workerconfig['device']))
self.print_("[+] Importing theano...")
import theano as th
try:
# Build network
network = self.build()
except Exception as e:
print(
"[-] Exception raised while building network. The error message is as follows: {}"
.format(e.message))
# Send poison pill and call it a day
self.resultq.put(None)
self.resultq.close()
raise e
# Set up a poison pill
poisonpill = False
# Loop to listen for jobs
while True:
jobs = []
for _ in range(self.workerconfig['batchsize']):
# Fetch from queue
try:
jobs.append(self.jobq.get(block=False))
except q.Empty:
poisonpill = True
break
self.print_(
"[+] Fetched {} jobs from JobQ. Fetching corresponding tensor and augmenting..."
.format(len(jobs)))
try:
# Fetch tensor
inp = self.fetchtensor(jobs=jobs)
self.print_("[+] Fetch input batch of shape {}.".format(
inp.shape))
# Generate random augmentation function
self.genrandaug(batchshape=inp.shape)
# Augment inpunt
auginp = self.augfunc(inp)
self.print_(
"[+] Augmented input batch. The shape now is {}.".format(
auginp.shape))
except Exception as e:
self.print_(
"[-] Exception raised while fetching tensor and/or applying data augmentation. "
"The error message follows: {}".format(e.message))
# Send poison pill and call it a day
self.resultq.put(None)
self.resultq.close()
raise e
self.print_("[+] Inferring...")
try:
# Process
out = network.classifier(auginp)
self.print_(
"[+] Output from the network is of shape {}.".format(
out.shape))
except Exception as e:
self.print_(
"[-] Exception raised while running inference. The error message follows: {}"
.format(e.message))
# Send poison pill and call it a day
self.resultq.put(None)
self.resultq.close()
raise e
self.print_("[+] Deaugmenting...")
try:
# Deaugment output
deaugout = self.deaugfunc(out)
self.print_("[+] Deaugmented network output. "
"The shape of the deaugmented batch is {}.".format(
deaugout.shape))
except Exception as e:
self.print_(
"[-] Exception raised while deaugmenting processed data. "
"The error message follows: {}".format(e.message))
# Send poison pill and call it a day
self.resultq.put(None)
self.resultq.close()
raise e
self.print_("[+] Writing output to ResultQ.")
# Write results to the results queue
for outimg, job in zip(deaugout, jobs):
self.resultq.put({
'dataset': job['dataset'],
'slice': job['slice'],
'payload': outimg
})
# Check for death wish
if poisonpill:
self.print_("[-] Poison pill found, shutting down process.")
# Set up suicide pact
self.resultq.put(None)
self.resultq.close()
break
print("[+] Using configuration file from {}.".format(args.configset))
# Read which device to use
if args.device is None:
device = config['device']
else:
device = args.device
assert device is not None, "Please provide the device to be used as a bash argument " \
"(e.g.: python train.py /path/to/config/file.yml --device gpu0)"
print("[+] Using device {}.".format(device))
# Import shit
from theano.sandbox.cuda import use
use(device)
# Add Antipasti to path
sys.path.append(pathsy('Antipasti'))
import Antipasti.trainkit as tk
import Antipasti.netutils as nu
import Antipasti.vizkit as vz
print("[+] Importing model and datafeeders...")
# Import model
model = imp.load_source('model', pathsy(config['modelpath']))
# Import datafeeder
dpl = imp.load_source('dataplate', pathsy(config['dplpath']))
print("[+] Building network...")
# Build network
def tes_use():
tcn.use()
from numpy.testing.noseclasses import KnownFailureTest
import theano.sandbox.gpuarray
if theano.sandbox.gpuarray.pygpu is None:
raise SkipTest("pygpu not installed")
# If you are writing a new test file, don't copy this code, but rather
# import stuff from this file (like mode_with_gpu) to reuse it.
import theano.sandbox.cuda as cuda_ndarray
if cuda_ndarray.cuda_available and not theano.sandbox.gpuarray.pygpu_activated:
if not cuda_ndarray.use.device_number:
# We should not enable all the use like the flag device=gpu,
# as many tests don't work in that setup.
cuda_ndarray.use('gpu',
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False)
theano.sandbox.gpuarray.init_dev('cuda')
if not theano.sandbox.gpuarray.pygpu_activated:
raise SkipTest("pygpu disabled")
from ..type import (GpuArrayType, gpuarray_shared_constructor)
from ..basic_ops import (host_from_gpu, gpu_from_host, gpu_alloc, GpuAlloc,
gpu_from_cuda, cuda_from_gpu, HostFromGpu,
GpuContiguous, GpuFromHost, GpuReshape, gpu_join,
GpuJoin, GpuSplit, GpuEye, gpu_contiguous)
from ..subtensor import GpuSubtensor
from theano.tests import unittest_tools as utt
utt.seed_rng()
from numpy.testing.noseclasses import KnownFailureTest
import theano.sandbox.gpuarray
if theano.sandbox.gpuarray.pygpu is None:
raise SkipTest("pygpu not installed")
# If you are writing a new test file, don't copy this code, but rather
# import stuff from this file (like mode_with_gpu) to reuse it.
import theano.sandbox.cuda as cuda_ndarray
if cuda_ndarray.cuda_available and not theano.sandbox.gpuarray.pygpu_activated:
if not cuda_ndarray.use.device_number:
# We should not enable all the use like the flag device=gpu,
# as many tests don't work in that setup.
cuda_ndarray.use('gpu',
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False)
theano.sandbox.gpuarray.init_dev('cuda')
if not theano.sandbox.gpuarray.pygpu_activated:
raise SkipTest("pygpu disabled")
from ..type import (GpuArrayType,
gpuarray_shared_constructor)
from ..basic_ops import (
host_from_gpu, gpu_from_host,
gpu_alloc, GpuAlloc,
GpuAllocEmpty,
gpu_from_cuda,
cuda_from_gpu, HostFromGpu,
GpuContiguous,
# coding: utf-8
# # Enter State Farm
# In[1]:
from theano.sandbox import cuda
cuda.use('gpu1')
# In[2]:
get_ipython().magic('matplotlib inline')
from __future__ import print_function, division
#path = "data/state/"
path = "data/state/sample/"
import utils
reload(utils)
from utils import *
from IPython.display import FileLink
# In[3]:
batch_size = 64
# ## Create sample
# The following assumes you've already created your validation set - remember that the training and validation set should contain *different drivers*, as mentioned on the Kaggle competition page.
# In[ ]:
# coding: utf-8
# In[1]:
from theano.sandbox import cuda
cuda.use('gpu0')
# In[2]:
get_ipython().magic('matplotlib inline')
import utils
reload(utils)
from utils import *
from __future__ import division, print_function
# ## Setup
# In[3]:
path = "data/dogscats/"
model_path = path + 'models/'
if not os.path.exists(model_path): os.mkdir(model_path)
batch_size = 64
# In[4]:
batches = get_batches(path + 'train', shuffle=False, batch_size=batch_size)
val_batches = get_batches(path + 'valid', shuffle=False, batch_size=batch_size)
# In[6]:
from theano.sandbox.cuda.dnn import GpuDnnConv, DnnBase, dnn_conv
# needed as the gpu conv don't have a perform implementation.
if theano.config.mode == 'FAST_COMPILE':
theano_mode = theano.compile.mode.get_mode('FAST_RUN').including('gpu')
else:
theano_mode = theano.compile.mode.get_default_mode().including('gpu')
device_id = theano.sandbox.cuda.use.device_number
if device_id is None:
cuda.shared_constructor(numpy.zeros(2, dtype='float32'))
device_id = theano.sandbox.cuda.use.device_number
if device_id is None:
cuda.use("gpu",
force=False,
default_to_move_computation_to_gpu=False,
move_shared_float32_to_gpu=False,
enable_cuda=False,
test_driver=True)
device_id = theano.sandbox.cuda.use.device_number
cuda_ndarray = theano.sandbox.cuda.cuda_ndarray.cuda_ndarray
device_prop = cuda_ndarray.device_properties(device_id)
def py_conv_valid_numpy(img, kern):
assert img.shape[1] == kern.shape[1]
outshp = (img.shape[0], kern.shape[0], img.shape[2] - kern.shape[2] + 1,
img.shape[3] - kern.shape[3] + 1)
out = numpy.zeros(outshp, dtype='float32')
for b in xrange(out.shape[0]):
for k in xrange(out.shape[1]):
def tes_use():
tcn.use()
