def __init__(self,
layerID,
inputSize,
kernelSize,
downsampleFactor,
learningRate=0.001,
momentumRate=0.9,
dropout=None,
initialWeights=None,
initialThresholds=None,
activation=tanh,
randomNumGen=None):
Layer.__init__(self, layerID, learningRate, momentumRate, dropout,
activation)
# TODO: this check is likely unnecessary
if inputSize[2] == kernelSize[2] or inputSize[3] == kernelSize[3]:
raise ValueError('ConvolutionalLayer Error: ' +
'inputSize cannot equal kernelSize')
if inputSize[1] != kernelSize[1]:
raise ValueError('ConvolutionalLayer Error: ' +
'Number of Channels must match in ' +
'inputSize and kernelSize')
self._inputSize = inputSize
self._kernelSize = kernelSize
self._downsampleFactor = downsampleFactor
# create weights based on the optimal distribution for the activation
if initialWeights is None or initialThresholds is None:
self._initializeWeights(size=self._kernelSize,
fanIn=np.prod(self._inputSize[1:]),
fanOut=self._kernelSize[0],
randomNumGen=randomNumGen)
def __init__(self, n_in, n_hidden, n_out, scale=0.5):
Layer.__init__(self)
self.n_in = n_in
self.n_hidden = n_hidden
self.n_out = n_out
self.scale = sqrt(1.0 / n_in) or scale
self.scale_out = sqrt(1.0 / n_hidden) or scale
self.W_hi = shared_normal((n_hidden, n_in), scale=self.scale)
self.W_ci = shared_normal((n_hidden, n_hidden), scale=self.scale)
self.b_i = shared_zeros((n_hidden, ))
self.W_hf = shared_normal((n_hidden, n_in), scale=self.scale)
self.W_cf = shared_normal((n_hidden, n_hidden), scale=self.scale)
self.b_f = shared_zeros((n_hidden, ))
self.W_hc = shared_normal((n_hidden, n_in), scale=self.scale)
self.b_c = shared_zeros((n_hidden, ))
self.W_ho = shared_normal((n_hidden, n_in), scale=self.scale)
self.W_co = shared_normal((n_hidden, n_hidden), scale=self.scale)
self.b_o = shared_zeros((n_hidden, ))
self.W_od = shared_normal((n_out, n_hidden),
scale=self.scale_out) #output decoder
self.b_od = shared_zeros((n_out, n_hidden))
self.params = [
self.W_hi, self.W_ci, self.b_i, self.W_hf, self.W_cf, self.b_f,
self.W_hc, self.b_c, self.W_ho, self.W_co, self.b_o, self.W_od,
self.b_od
]
def __init__ (self, layerID, inputSize, kernelSize,
downsampleFactor, learningRate=0.001, momentumRate=0.9,
dropout=None, initialWeights=None, initialThresholds=None,
activation=tanh, randomNumGen=None) :
Layer.__init__(self, layerID, learningRate, momentumRate, dropout,
activation)
if inputSize[1] != kernelSize[1] :
raise ValueError('ConvolutionalLayer Error: ' +
'Number of Channels must match in ' +
'inputSize and kernelSize')
# NOTE: use None instead of the batch size to allow variable batch
# sizes during deployment.
self._inputSize = tuple([None] + list(inputSize[1:]))
self._kernelSize = tuple(kernelSize)
self._downsampleFactor = tuple(downsampleFactor)
# create weights based on the optimal distribution for the activation
if initialWeights is None or initialThresholds is None :
self._initializeWeights(
size=self._kernelSize,
fanIn=np.prod(self._inputSize[1:]),
fanOut=self._kernelSize[0],
randomNumGen=randomNumGen)
def __init__(self, n_in, n_out):
Layer.__init__(self)
self.n_in = n_in
self.n_out = n_out
scale = sqrt(1.0 / n_in)
self.W = shared_normal((self.n_in, self.n_out), scale=scale)
self.b = shared_zeros((self.n_out, ))
self.params = [self.W, self.b]
def __init__ (self, layerID, inputSize, numNeurons,
learningRate=0.001, momentumRate=0.9, dropout=None,
initialWeights=None, initialThresholds=None, activation=tanh,
randomNumGen=None) :
Layer.__init__(self, layerID, learningRate, momentumRate,
dropout, activation)
self._inputSize = (None, inputSize[1])
self._numNeurons = numNeurons
# create weights based on the optimal distribution for the activation
if initialWeights is None or initialThresholds is None :
self._initializeWeights(
size=(self._inputSize[1], self._numNeurons),
fanIn=self._inputSize[1],
fanOut=self._numNeurons,
randomNumGen=randomNumGen)
def __init__ (self, layerID, inputSize, numNeurons,
learningRate=0.001, momentumRate=0.9, dropout=None,
initialWeights=None, initialThresholds=None, activation=tanh,
randomNumGen=None) :
Layer.__init__(self, layerID, learningRate, momentumRate,
dropout, activation)
self._inputSize = inputSize
if isinstance(self._inputSize, six.integer_types) or \
len(self._inputSize) is not 2 :
self._inputSize = (1, inputSize)
self._numNeurons = numNeurons
# create weights based on the optimal distribution for the activation
if initialWeights is None or initialThresholds is None :
self._initializeWeights(
size=(self._inputSize[1], self._numNeurons),
fanIn=self._inputSize[1],
fanOut=self._numNeurons,
randomNumGen=randomNumGen)
def __init__(self, f):
Layer.__init__(self)
self.f = f
def __init__ (self, layerID, input, inputSize, numNeurons,
learningRate=0.001, momentumRate=0.9, dropout=None,
initialWeights=None, initialThresholds=None, activation=tanh,
randomNumGen=None) :
Layer.__init__(self, layerID, learningRate, momentumRate, dropout)
# adjust the input for the correct number of dimensions
if isinstance(input, tuple) :
if input[1].ndim > 2 : input = input[0].flatten(2), \
input[1].flatten(2)
else :
if input.ndim > 2 : input = input.flatten(2)
# store the input buffer -- this can either be a tuple or scalar
# The input layer will only have a scalar so its duplicated here
self.input = input if isinstance(input, tuple) else (input, input)
self._inputSize = inputSize
if isinstance(self._inputSize, six.integer_types) or \
len(self._inputSize) is not 2 :
self._inputSize = (1, inputSize)
self._numNeurons = numNeurons
# setup initial values for the weights
if initialWeights is None :
# create a rng if its needed
if randomNumGen is None :
from numpy.random import RandomState
from time import time
randomNumGen = RandomState(int(time()))
initialWeights = np.asarray(randomNumGen.uniform(
low=-np.sqrt(6. / (self._inputSize[1] + self._numNeurons)),
high=np.sqrt(6. / (self._inputSize[1] + self._numNeurons)),
size=(self._inputSize[1], self._numNeurons)),
dtype=config.floatX)
if activation == sigmoid :
initialWeights *= 4.
self._weights = shared(value=initialWeights, borrow=True)
# setup initial values for the thresholds
if initialThresholds is None :
initialThresholds = np.zeros((self._numNeurons,),
dtype=config.floatX)
self._thresholds = shared(value=initialThresholds, borrow=True)
# create the logits
def findLogit(input, weights, thresholds) :
return dot(input, weights) + thresholds
outClass = findLogit(self.input[0], self._weights, self._thresholds)
outTrain = findLogit(self.input[1], self._weights, self._thresholds)
# determine dropout if requested
if self._dropout is not None :
# here there are two possible paths --
# outClass : path of execution intended for classification. Here
# all neurons are present and weights must be scaled by
# the dropout factor. This ensures resultant
# probabilities fall within intended bounds when all
# neurons are present.
# outTrain : path of execution for training with dropout. Here each
# neuron's output goes through a Bernoulli Trial. This
# retains a neuron with the probability specified by the
# dropout factor.
outClass = outClass / self._dropout
outTrain = switch(self._randStream.binomial(
size=(self._numNeurons,), p=self._dropout), outTrain, 0)
# activate the layer --
# output is a tuple to represent two possible paths through the
# computation graph.
self.output = (outClass, outTrain) if activation is None else \
(activation(outClass), activation(outTrain))
# create a convenience function
self.activate = function([self.input[0]], self.output[0])
class Identity(Layer):
__init__ = lambda self: Layer.__init__(self)
forward = lambda self, input: input
def __init__(self, p=0.5):
Layer.__init__(self)
self.p = p
self.training = True
self.rng = RandomStreams()
def __init__(self, pattern):
Layer.__init__(self)
self.pattern = pattern
def __init__(self, shape):
Layer.__init__(self)
self.shape = shape
def __init__ (self, layerID, input, inputSize, kernelSize,
downsampleFactor, learningRate=0.001, momentumRate=0.9,
dropout=None, initialWeights=None, initialThresholds=None,
activation=tanh, randomNumGen=None) :
Layer.__init__(self, layerID, learningRate, momentumRate, dropout)
# TODO: this check is likely unnecessary
if inputSize[2] == kernelSize[2] or inputSize[3] == kernelSize[3] :
raise ValueError('ConvolutionalLayer Error: ' +
'inputSize cannot equal kernelSize')
if inputSize[1] != kernelSize[1] :
raise ValueError('ConvolutionalLayer Error: ' +
'Number of Channels must match in ' +
'inputSize and kernelSize')
from theano.tensor.nnet.conv import conv2d
from theano.tensor.signal.downsample import max_pool_2d
# theano variables don't actually preserve buffer sizing
self.input = input if isinstance(input, tuple) else (input, input)
self._inputSize = inputSize
self._kernelSize = kernelSize
self._downsampleFactor = downsampleFactor
# setup initial values for the weights -- if necessary
if initialWeights is None :
# create a rng if its needed
if randomNumGen is None :
from numpy.random import RandomState
from time import time
randomNumGen = RandomState(int(time()))
# this creates optimal initial weights by randomizing them
# to an appropriate range around zero, which leads to better
# convergence.
downRate = np.prod(self._downsampleFactor)
fanIn = np.prod(self._kernelSize[1:])
fanOut = self._kernelSize[0] * \
np.prod(self._kernelSize[2:]) / downRate
scaleFactor = np.sqrt(6. / (fanIn + fanOut))
initialWeights = np.asarray(randomNumGen.uniform(
low=-scaleFactor, high=scaleFactor, size=self._kernelSize),
dtype=config.floatX)
self._weights = shared(value=initialWeights, borrow=True)
# setup initial values for the thresholds -- if necessary
if initialThresholds is None :
initialThresholds = np.zeros((self._kernelSize[0],),
dtype=config.floatX)
self._thresholds = shared(value=initialThresholds, borrow=True)
def findLogits(input, weights,
inputSize, kernelSize, downsampleFactor, thresholds) :
# create a function to perform the convolution
convolve = conv2d(input, weights, inputSize, kernelSize)
# create a function to perform the max pooling
pooling = max_pool_2d(convolve, downsampleFactor, True)
# the output buffer is now connected to a sequence of operations
return pooling + thresholds.dimshuffle('x', 0, 'x', 'x')
outClass = findLogits(self.input[0], self._weights,
self._inputSize, self._kernelSize,
self._downsampleFactor, self._thresholds)
outTrain = findLogits(self.input[1], self._weights,
self._inputSize, self._kernelSize,
self._downsampleFactor, self._thresholds)
# determine dropout if requested
if self._dropout is not None :
# here there are two possible paths --
# outClass : path of execution intended for classification. Here
# all neurons are present and weights must be scaled by
# the dropout factor. This ensures resultant
# probabilities fall within intended bounds when all
# neurons are present.
# outTrain : path of execution for training with dropout. Here each
# neuron's output goes through a Bernoulli Trial. This
# retains a neuron with the probability specified by the
# dropout factor.
outClass = outClass / self._dropout
outTrain = switch(self._randStream.binomial(
size=self.getOutputSize()[1:], p=self._dropout), outTrain, 0)
# activate the layer --
# output is a tuple to represent two possible paths through the
# computation graph.
self.output = (outClass, outTrain) if activation is None else \
(activation(outClass), activation(outTrain))
# we can call this method to activate the layer
self.activate = function([self.input[0]], self.output[0])