def randomizeWeights(self, scale='glorot', mode='uni'):
n_out = self.filter_shape[0]
if self.activation_func == 'relu':
norm = self.filter_shape[1] * self.filter_shape[
3] * self.filter_shape[4]
b_values = np.asarray(initWeights((n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif self.activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: #self.activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
W_values = np.asarray(initWeights(self.filter_shape,
scale,
mode,
pool=self.pool),
dtype='float32')
self.W.set_value(W_values)
self.b.set_value(b_values)
def randomizeWeights(self, scale='glorot', mode='uni'):
n_out = self.filter_shape[0]
if self.activation_func == 'relu':
norm = self.filter_shape[1] * self.filter_shape[3] * self.filter_shape[4]
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif self.activation_func == 'sigmoid':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: #self.activation_func=='tanh':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
W_values = np.asarray(
initWeights(self.filter_shape,
scale,
mode,
pool=self.pool),
dtype='float32')
self.W.set_value(W_values)
self.b.set_value(b_values)
def __init__(self, input, n_in, n_hid, batch_size, activation_func='tanh'):
assert input.ndim == 3
input = input.dimshuffle(1, 0, 2) # exchange batch and time
self.n_in = n_in
self.n_hid = n_hid
self.activation_func = activation_func
self.output_shape = (batch_size, n_hid)
self.hid_lin = None
self.output = None
print "RecurrentLayer( #Inputs =", n_in, "#Hidden = ", n_hid, ")"
W_in_values = np.asarray(initWeights((n_in, n_hid), scale='glorot', mode='uni'), dtype='float32')
self.W_in = theano.shared(W_in_values, name='W_in', borrow=True)
W_hid_values = np.asarray(initWeights((n_hid, n_hid), mode='rnn'), dtype='float32')
self.W_hid = theano.shared(W_hid_values, name='W_hid', borrow=True)
b_hid_values = np.asarray(initWeights((n_hid, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b_hid = theano.shared(b_hid_values, name='b_hid', borrow=True)
hid_0_values = np.zeros(n_hid, dtype='float32')
self.hid_0 = theano.shared(hid_0_values, name='hid_0', borrow=True)
W_in, W_hid, b_hid, hid_0 = self.W_in, self.W_hid, self.b_hid, self.hid_0
self.params = [W_in, W_hid, b_hid, hid_0]
# Select non-linearities
if activation_func == 'tanh': # range = [-1,1]
act = T.tanh # shape: (batch_size, num_outputs)
elif activation_func == 'relu': # rectified linear unit ,range = [0,inf]
act = lambda x: x * (x > 0) #T.maximum(lin_output,T.zeros_like(lin_output))
elif activation_func == 'abs': # abs unit ,range = [0,inf]
act = T.abs_
elif activation_func == 'sigmoid': # range = [0,1]
print "WARNING: sig() used!"
#print "WARNING: consider using tanh(.) or relu(.) instead! Sigmoid is really BAD! (relu > tanh >> sigmoid)"
act = T.nnet.sigmoid #1/(1 + T.exp(-lin_output))
elif activation_func == 'linear':
print "Warning: linear activation in recurrent layer with fanout-%i! Is this the output layer?" % n_hid
act = lambda x: x
else:
raise NotImplementedError("options are: activation_func=('relu'|'sigmoid'|'tanh'|'abs')")
def recurrence(x_t, hid_prev):
hid_lin_t = T.dot(x_t, W_in) + T.dot(hid_prev, W_hid) + b_hid
hid_t = act(hid_lin_t)
return [hid_t, hid_lin_t]
outputs_info = [dict(initial=T.alloc(hid_0, input.shape[1], n_hid), taps=[-1]), dict()]
# shapes are [time, batch, feat]
([hid, hid_lin], updates) = theano.scan(fn=recurrence,
sequences=input,
outputs_info=outputs_info,
name='Recurrence')
hid_lin = hid_lin.dimshuffle(1, 0, 2) # exchange batch and time again --> [batch, time, hid/feat]
hid = act(hid_lin) # I think this is needed for structural damping (calculating grad wrt hid_lin)
self.output = hid[:, -1] # [batch, hid/feat]
self.hid = hid
self.hid_lin = hid_lin
def __init__(self, input, n_in, n_hid, batch_size, activation_func='tanh'):
assert input.ndim == 3
input = input.dimshuffle(1, 0, 2) # exchange batch and time
self.n_in = n_in
self.n_hid = n_hid
self.activation_func = activation_func
self.output_shape = (batch_size, n_hid)
self.hid_lin = None
self.output = None
print "RecurrentLayer( #Inputs =", n_in, "#Hidden = ", n_hid, ")"
W_in_values = np.asarray(initWeights((n_in, n_hid), scale='glorot', mode='uni'), dtype='float32')
self.W_in = theano.shared(W_in_values, name='W_in', borrow=True)
W_hid_values = np.asarray(initWeights((n_hid, n_hid), mode='rnn'), dtype='float32')
self.W_hid = theano.shared(W_hid_values, name='W_hid', borrow=True)
b_hid_values = np.asarray(initWeights((n_hid, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b_hid = theano.shared(b_hid_values, name='b_hid', borrow=True)
hid_0_values = np.zeros(n_hid, dtype='float32')
self.hid_0 = theano.shared(hid_0_values, name='hid_0', borrow=True)
W_in, W_hid, b_hid, hid_0 = self.W_in, self.W_hid, self.b_hid, self.hid_0
self.params = [W_in, W_hid, b_hid, hid_0]
# Select non-linearities
if activation_func == 'tanh': # range = [-1,1]
act = T.tanh # shape: (batch_size, num_outputs)
elif activation_func == 'relu': # rectified linear unit ,range = [0,inf]
act = lambda x: x * (x > 0) #T.maximum(lin_output,T.zeros_like(lin_output))
elif activation_func == 'abs': # abs unit ,range = [0,inf]
act = T.abs_
elif activation_func == 'sigmoid': # range = [0,1]
print "WARNING: sig() used!"
#print "WARNING: consider using tanh(.) or relu(.) instead! Sigmoid is really BAD! (relu > tanh >> sigmoid)"
act = T.nnet.sigmoid #1/(1 + T.exp(-lin_output))
elif activation_func == 'linear':
print "Warning: linear activation in recurrent layer with fanout-%i! Is this the output layer?" % n_hid
act = lambda x: x
else:
raise NotImplementedError("options are: activation_func=('relu'|'sigmoid'|'tanh'|'abs')")
def recurrence(x_t, hid_prev):
hid_lin_t = T.dot(x_t, W_in) + T.dot(hid_prev, W_hid) + b_hid
hid_t = act(hid_lin_t)
return [hid_t, hid_lin_t]
outputs_info = [dict(initial=T.alloc(hid_0, input.shape[1], n_hid), taps=[-1]), dict()]
# shapes are [time, batch, feat]
([hid, hid_lin], updates) = theano.scan(fn=recurrence,
sequences=input,
outputs_info=outputs_info,
name='Recurrence')
hid_lin = hid_lin.dimshuffle(1, 0, 2) # exchange batch and time again --> [batch, time, hid/feat]
hid = act(hid_lin) # I think this is needed for structural damping (calculating grad wrt hid_lin)
self.output = hid[:, -1] # [batch, hid/feat]
self.hid = hid
self.hid_lin = hid_lin
def randomizeWeights(self, scale_w=1.0):
n_in, n_hid = self.n_in, self.n_hid
W_in_values = np.asarray(initWeights((n_in, n_hid), scale='glorot', mode='uni'), dtype='float32')
self.W_in.set_value(W_in_values)
W_hid_values = np.asarray(initWeights((n_in, n_hid), mode='rnn'), dtype='float32')
self.W_hid.set_value(W_hid_values)
b_hid_values = np.asarray(initWeights((n_hid, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b_hid.set_value(b_hid_values)
hid_0_values = np.zeros(n_hid, dtype='float32')
self.hid_0.set_value(hid_0_values)
def randomizeWeights(self, scale_w=1.0):
n_in, n_hid = self.n_in, self.n_hid
W_in_values = np.asarray(initWeights((n_in, n_hid), scale='glorot', mode='uni'), dtype='float32')
self.W_in.set_value(W_in_values)
W_hid_values = np.asarray(initWeights((n_in, n_hid), mode='rnn'), dtype='float32')
self.W_hid.set_value(W_hid_values)
b_hid_values = np.asarray(initWeights((n_hid, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b_hid.set_value(b_hid_values)
hid_0_values = np.zeros(n_hid, dtype='float32')
self.hid_0.set_value(hid_0_values)
def randomizeWeights(self, scale='glorot', mode='uni'):
n_in = self.n_in
n_out = self.output_shape[1]
if self.activation_func == 'relu':
b_values = np.asarray(initWeights((n_out, ), scale=1.0, mode='const'), dtype='float32')
elif self.activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ), scale=0.5, mode='const'), dtype='float32')
else: #self.activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ), scale=1e-6, mode='fix-uni'), dtype='float32')
W_values = np.asarray(initWeights((n_in, n_out), scale, mode), dtype='float32')
self.W.set_value(W_values)
self.b.set_value(b_values)
def randomizeWeights(self, scale='glorot', mode='uni'):
n_in = self.n_in
n_out = self.output_shape[1]
if self.activation_func == 'relu':
b_values = np.asarray(initWeights((n_out, ), scale=1.0, mode='const'), dtype='float32')
elif self.activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ), scale=0.5, mode='const'), dtype='float32')
else: #self.activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ), scale=1e-6, mode='fix-uni'), dtype='float32')
W_values = np.asarray(initWeights((n_in, n_out), scale, mode), dtype='float32')
self.W.set_value(W_values)
self.b.set_value(b_values)
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max'):
W_values1 = np.asarray(
initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values2 = np.asarray(
initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values3 = np.asarray(
initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values = np.concatenate([W_values1, W_values2, W_values3], axis=0)
self.W = theano.shared(W_values, name='W_conv', borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
if activation_func in ['ReLU', 'relu']:
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.ones((filter_shape[0], ), dtype='float32') / norm
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
b_values = np.concatenate([b_values, b_values, b_values], axis=0)
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
N = filter_shape[0]
l1 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[0:N],
b=self.b[0:N])
l2 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[N:2 * N],
b=self.b[N:2 * N])
l3 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[2 * N:3 * N],
b=self.b[2 * N:3 * N])
self.params = [self.W, self.b]
self.mfp_strides = l1.mfp_strides
self.mfp_offsets = l1.mfp_offsets
self.pool = l1.pool
self.output_shape = list(l1.output_shape)
self.output_shape[2] = self.output_shape[2] * 3
self.prob_shape = list(l1.prob_shape)
self.prob_shape[1] = self.output_shape[1] * 3
self.activation_func = l1.activation_func
self.class_probabilities = T.concatenate(
[l1.class_probabilities, l2.class_probabilities, l3.class_probabilities], axis=1)
self.class_prediction = T.concatenate(
[l1.class_prediction, l2.class_prediction, l3.class_prediction], axis=1)
self.l1 = l1
self.l2 = l2
self.l3 = l3
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max',
affinity=False):
assert len(filter_shape) == 5
assert input_shape[2] == filter_shape[2]
self.input = input
self.pool = pool
self.number_of_filters = filter_shape[0]
self.filter_shape = filter_shape
self.activation_func = activation_func
self.input_shape = input_shape
self.input_layer = input_layer
self.mfp_strides = mfp_strides
self.mfp_offsets = mfp_offsets
self.reshape_output = reshape_output
print "3DConv: input=", input_shape, "\tfilter=", filter_shape #,"@std=",W_bound
if W is None:
W_values = np.asarray(
initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
self.W = theano.shared(W_values, name='W_conv', borrow=True)
else:
if isinstance(W, np.ndarray):
self.W = theano.shared(
W.astype(np.float32),
name='W_conv',
borrow=True)
else:
assert isinstance(
W,
T.TensorVariable), "W must be either np.ndarray or theano var"
self.W = W
# the bias is a 1D tensor -- one bias per output feature map
if activation_func in ['ReLU', 'relu']:
norm = filter_shape[1] * filter_shape[3] * filter_shape[4] #
b_values = np.ones(
(filter_shape[0], ),
dtype='float32') / norm
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
else:
if isinstance(b, np.ndarray):
self.b = theano.shared(
b.astype(np.float32),
name='b_conv',
borrow=True)
else:
assert isinstance(
b,
T.TensorVariable), "b must be either np.ndarray or theano var"
self.b = b
# store parameters of this layer
self.params = [self.W, self.b]
# convolve input feature maps with filters
self.mode = theano.compile.get_default_mode()
self.conv_out = conv3d(
signals=input,
filters=self.W,
border_mode='valid',
filters_shape=filter_shape
) # signals_shape=input_shape if input_shape[0] is not None else None)
# down-sample each feature map individually, using maxpooling
if np.any(pool != 1):
pool_func = lambda x: pooling.pooling3d(x, pool_shape=pool, mode=pooling_mode)
if use_fragment_pooling:
pooled_out, self.mfp_offsets, self.mfp_strides = self.fragmentpool(
self.conv_out, pool, mfp_offsets, mfp_strides, pool_func)
else:
pooled_out = pool_func(self.conv_out)
else:
pooled_out = self.conv_out
if enable_dropout:
print "Dropout: ACTIVE"
self.activation_noise = theano.shared(
np.float32(0.5),
name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial(
(pooled_out.shape[1], pooled_out.shape[3],
pooled_out.shape[4]),
1,
p,
dtype='float32')
pooled_out = pooled_out * self.dropout_gate.dimshuffle(('x', 0, 'x', 1, 2))
lin_output = pooled_out + self.b.dimshuffle('x', 'x', 0, 'x', 'x')
self.lin_output = lin_output
r = 1
if activation_func == 'tanh':
self.activation_func = 'tanh'
self.output = T.tanh(lin_output) # shape: (batch_size, num_outputs)
elif activation_func in ['ReLU', 'relu']: #rectified linear unit
self.activation_func = 'relu'
self.output = lin_output * (lin_output > 0) # shape: (batch_size, num_outputs)
elif activation_func in ['linear', 'none', 'None', None]:
self.activation_func = 'linear'
self.output = lin_output
elif activation_func in ['abs']:
self.activation_func = 'abs'
self.output = T.abs_(lin_output)
elif activation_func in ['sigmoid']:
self.activation_func = 'sigmoid'
self.output = T.nnet.sigmoid(lin_output)
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
self.output = pooling.maxout(lin_output, factor=r, axis=2)
else:
raise NotImplementedError()
output_shape = getOutputShape(
(1 if input_shape[0] is None else input_shape[0], ) +
input_shape[1:], filter_shape, pool, use_fragment_pooling, r)
print "Output=", output_shape, "Dropout", (
"ON," if enable_dropout else
"OFF,"), "Act:", activation_func, "pool:", pooling_mode
self.output_shape = output_shape # e.g. (None, 16, 100, 100)
if affinity:
raise RuntimeError("Dont use this code")
# self.class_probabilities = T.nnet.sigmoid(lin_output) # (bs, z, 3, x, y)
# self.class_probabilities = self.class_probabilities.dimshuffle((0,2,1,3,4))
# sh = lin_output.shape
# if use_fragment_pooling:
# self.fragmentstodense(sh) # works on
#
# self.prob_shape = getProbShape(output_shape, self.mfp_strides)
# self.class_prediction = T.gt(self.class_probabilities, 0.5)
# # self.class_probabilities = (bs,3,z,x,y)
else:
sh = lin_output.shape #(bs,x,ch,y,z) # use this shape to reshape the output to image-shape after softmax
sh = (sh[2], sh[0], sh[1], sh[3], sh[4]) #(ch, x, y, bs)
# put spatial, at back --> (ch,bs,x,y,z), flatten this --> (ch, bs*x*y*z), swap labels --> (bs*x*y*z, ch)
self.class_probabilities = T.nnet.softmax(
lin_output.dimshuffle((2, 0, 1, 3, 4)).flatten(2).dimshuffle((1, 0)))
if reshape_output:
self.reshapeoutput(sh)
if use_fragment_pooling:
self.fragmentstodense(sh)
self.prob_shape = getProbShape(output_shape, self.mfp_strides)
print "Class Prob Output =", self.prob_shape
# compute prediction as class whose "probability" is maximal in symbolic form
self.class_prediction = T.argmax(self.class_probabilities, axis=1)
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max'):
W_values1 = np.asarray(initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values2 = np.asarray(initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values3 = np.asarray(initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
W_values = np.concatenate([W_values1, W_values2, W_values3], axis=0)
self.W = theano.shared(W_values, name='W_conv', borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
if activation_func in ['ReLU', 'relu']:
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.ones((filter_shape[0], ), dtype='float32') / norm
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.asarray(initWeights((n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
b_values = np.concatenate([b_values, b_values, b_values], axis=0)
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
N = filter_shape[0]
l1 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[0:N],
b=self.b[0:N])
l2 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[N:2 * N],
b=self.b[N:2 * N])
l3 = ConvLayer3d(input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
affinity=False,
W=self.W[2 * N:3 * N],
b=self.b[2 * N:3 * N])
self.params = [self.W, self.b]
self.mfp_strides = l1.mfp_strides
self.mfp_offsets = l1.mfp_offsets
self.pool = l1.pool
self.output_shape = list(l1.output_shape)
self.output_shape[2] = self.output_shape[2] * 3
self.prob_shape = list(l1.prob_shape)
self.prob_shape[1] = self.output_shape[1] * 3
self.activation_func = l1.activation_func
self.class_probabilities = T.concatenate([
l1.class_probabilities, l2.class_probabilities,
l3.class_probabilities
],
axis=1)
self.class_prediction = T.concatenate(
[l1.class_prediction, l2.class_prediction, l3.class_prediction],
axis=1)
self.l1 = l1
self.l2 = l2
self.l3 = l3
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max',
affinity=False):
assert len(filter_shape) == 5
assert input_shape[2] == filter_shape[2]
self.input = input
self.pool = pool
self.number_of_filters = filter_shape[0]
self.filter_shape = filter_shape
self.activation_func = activation_func
self.input_shape = input_shape
self.input_layer = input_layer
self.mfp_strides = mfp_strides
self.mfp_offsets = mfp_offsets
self.reshape_output = reshape_output
print "3DConv: input=", input_shape, "\tfilter=", filter_shape #,"@std=",W_bound
if W is None:
W_values = np.asarray(initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
self.W = theano.shared(W_values, name='W_conv', borrow=True)
else:
if isinstance(W, np.ndarray):
self.W = theano.shared(W.astype(np.float32),
name='W_conv',
borrow=True)
else:
assert isinstance(
W, T.TensorVariable
), "W must be either np.ndarray or theano var"
self.W = W
# the bias is a 1D tensor -- one bias per output feature map
if activation_func in ['ReLU', 'relu']:
norm = filter_shape[1] * filter_shape[3] * filter_shape[4] #
b_values = np.ones((filter_shape[0], ), dtype='float32') / norm
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[1] * filter_shape[3] * filter_shape[4]
b_values = np.asarray(initWeights((n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
else:
if isinstance(b, np.ndarray):
self.b = theano.shared(b.astype(np.float32),
name='b_conv',
borrow=True)
else:
assert isinstance(
b, T.TensorVariable
), "b must be either np.ndarray or theano var"
self.b = b
# store parameters of this layer
self.params = [self.W, self.b]
# convolve input feature maps with filters
self.mode = theano.compile.get_default_mode()
self.conv_out = conv3d(
signals=input,
filters=self.W,
border_mode='valid',
filters_shape=filter_shape
) # signals_shape=input_shape if input_shape[0] is not None else None)
# down-sample each feature map individually, using maxpooling
if np.any(pool != 1):
pool_func = lambda x: pooling.pooling3d(
x, pool_shape=pool, mode=pooling_mode)
if use_fragment_pooling:
pooled_out, self.mfp_offsets, self.mfp_strides = self.fragmentpool(
self.conv_out, pool, mfp_offsets, mfp_strides, pool_func)
else:
pooled_out = pool_func(self.conv_out)
else:
pooled_out = self.conv_out
if enable_dropout:
print "Dropout: ACTIVE"
self.activation_noise = theano.shared(np.float32(0.5),
name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial(
(pooled_out.shape[1], pooled_out.shape[3],
pooled_out.shape[4]),
1,
p,
dtype='float32')
pooled_out = pooled_out * self.dropout_gate.dimshuffle(
('x', 0, 'x', 1, 2))
lin_output = pooled_out + self.b.dimshuffle('x', 'x', 0, 'x', 'x')
self.lin_output = lin_output
r = 1
if activation_func == 'tanh':
self.activation_func = 'tanh'
self.output = T.tanh(
lin_output) # shape: (batch_size, num_outputs)
elif activation_func in ['ReLU', 'relu']: #rectified linear unit
self.activation_func = 'relu'
self.output = lin_output * (lin_output > 0
) # shape: (batch_size, num_outputs)
elif activation_func in ['linear', 'none', 'None', None]:
self.activation_func = 'linear'
self.output = lin_output
elif activation_func in ['abs']:
self.activation_func = 'abs'
self.output = T.abs_(lin_output)
elif activation_func in ['sigmoid']:
self.activation_func = 'sigmoid'
self.output = T.nnet.sigmoid(lin_output)
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
self.output = pooling.maxout(lin_output, factor=r, axis=2)
else:
raise NotImplementedError()
output_shape = getOutputShape(
(1 if input_shape[0] is None else input_shape[0], ) +
input_shape[1:], filter_shape, pool, use_fragment_pooling, r)
print "Output=", output_shape, "Dropout", (
"ON," if enable_dropout else
"OFF,"), "Act:", activation_func, "pool:", pooling_mode
self.output_shape = output_shape # e.g. (None, 16, 100, 100)
if affinity:
raise RuntimeError("Dont use this code")
# self.class_probabilities = T.nnet.sigmoid(lin_output) # (bs, z, 3, x, y)
# self.class_probabilities = self.class_probabilities.dimshuffle((0,2,1,3,4))
# sh = lin_output.shape
# if use_fragment_pooling:
# self.fragmentstodense(sh) # works on
#
# self.prob_shape = getProbShape(output_shape, self.mfp_strides)
# self.class_prediction = T.gt(self.class_probabilities, 0.5)
# # self.class_probabilities = (bs,3,z,x,y)
else:
sh = lin_output.shape #(bs,x,ch,y,z) # use this shape to reshape the output to image-shape after softmax
sh = (sh[2], sh[0], sh[1], sh[3], sh[4]) #(ch, x, y, bs)
# put spatial, at back --> (ch,bs,x,y,z), flatten this --> (ch, bs*x*y*z), swap labels --> (bs*x*y*z, ch)
self.class_probabilities = T.nnet.softmax(
lin_output.dimshuffle((2, 0, 1, 3, 4)).flatten(2).dimshuffle(
(1, 0)))
if reshape_output:
self.reshapeoutput(sh)
if use_fragment_pooling:
self.fragmentstodense(sh)
self.prob_shape = getProbShape(output_shape, self.mfp_strides)
print "Class Prob Output =", self.prob_shape
# compute prediction as class whose "probability" is maximal in symbolic form
self.class_prediction = T.argmax(self.class_probabilities, axis=1)
def __init__(self,
input,
n_in,
n_out,
batch_size,
enable_dropout,
activation_func='tanh',
input_noise=None,
input_layer=None,
W=None,
b=None):
self.input_layer = input_layer # only for autoencoder
self.activation_func = activation_func
self.output_shape = (batch_size, n_out)
self.n_in = n_in
self.lin_output = None
self.output = None
self.last_grads = [] # only for autoencoder
print "PerceptronLayer( #Inputs =", n_in, "#Outputs =", n_out, ")"
if input_noise is not None:
self.input_noise = theano.shared(np.float32(input_noise), name='Input Noise')
print "Input_noise active, p=" + str(self.input_noise.get_value())
rng = np.random.RandomState(int(time.time()))
theano_rng = RandomStreams(rng.randint(2**30))
# apply multiplicative noise to input
#self.input = theano_rng.binomial(size=input.shape, n=1, p=1-self.input_noise,
# dtype='float32') * input
# apply additive noise to input
self.input = input + theano_rng.normal(size=input.shape,
avg=0,
std=input_noise,
dtype='float32')
else: # no input noise
self.input = input
if W is None:
W_values = np.asarray(initWeights((n_in, n_out), scale='glorot', mode='uni'), dtype='float32')
self.W = theano.shared(value=W_values, name='W_perceptron' + str(n_in) + '.' + str(n_out), borrow=True)
else:
print "Directly using fixed/shared W (", W, "), no Training on it in this layer!"
if isinstance(W, np.ndarray):
self.W = theano.shared(value=W.astype(np.float32),
name='W_perceptron' + str(n_in) + '.' + str(n_out),
borrow=True)
else:
assert isinstance(W, T.TensorVariable), "W must be either np.ndarray or theano var"
self.W = W
if b is None:
#b_values = np.asarray(np.random.uniform(-1e-8,1e-8,(n_out,)), dtype='float32')
if activation_func == 'relu' or activation_func == 'ReLU':
b_values = np.asarray(initWeights((n_out, ), scale=1.0, mode='const'), dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ), scale=0.5, mode='const'), dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b = theano.shared(value=b_values, name='b_perceptron' + str(n_in) + '.' + str(n_out), borrow=True)
else:
print "Directly using fixed given b (", b, "), no Training on it in this layer!"
if isinstance(b, np.ndarray):
self.b = theano.shared(value=b.astype(np.float32),
name='b_perceptron' + str(n_in) + '.' + str(n_out),
borrow=True)
else:
assert isinstance(b, T.TensorVariable), "b must be either np.ndarray or theano var"
self.b = b
lin_output = T.dot(self.input, self.W)
if enable_dropout:
print "Dropout ON"
self.activation_noise = theano.shared(np.float32(0.5), name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial((n_out, ), 1, p, dtype='float32')
lin_output = lin_output * self.dropout_gate.dimshuffle(('x', 0))
lin_output = lin_output + self.b
# Apply non-linearities and ggf. change bias-initialisations
if activation_func == 'tanh': # range = [-1,1]
self.output = T.tanh(lin_output) # shape: (batch_size, num_outputs)
elif activation_func == 'relu' or activation_func == 'ReLU': # rectified linear unit ,range = [0,inf]
self.activation_func = 'relu'
self.output = lin_output * (lin_output > 0) #T.maximum(lin_output,T.zeros_like(lin_output))
elif activation_func == 'abs': # abs unit ,range = [0,inf]
self.output = T.abs_(lin_output)
elif activation_func == 'sigmoid': # range = [0,1]
#print "WARNING: consider using tanh(.) or relu(.) instead! Sigmoid is BAD! (relu > tanh >> sigmoid)"
lin_output = T.dot(self.input, self.W) + self.b
self.output = T.nnet.sigmoid(lin_output) #1/(1 + T.exp(-lin_output))
elif activation_func == 'linear':
self.output = (lin_output)
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
n_out = n_out / r
self.output = pooling.maxout(lin_output, factor=r)
else:
raise NotImplementedError("Options are: activation_func=('relu'|'sigmoid'|'tanh'|'abs')")
self.lin_output = lin_output
self.params = [self.b if b is None else []] + ([self.W]
if W is None else [])
self.class_probabilities = T.nnet.softmax(lin_output) # shape: (batch_size, num_outputs)
#self.class_probabilities = T.exp(lin_output) / T.sum(T.exp(lin_output), axis=1, keepdims=True) # For Hessian
self.class_prediction = T.argmax(self.class_probabilities, axis=1) # shape: (batch_size,)
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max'):
assert input_shape[1] == filter_shape[1]
self.input = input
self.pool = pool
self.number_of_filters = filter_shape[0]
self.filter_shape = filter_shape
self.activation_func = activation_func
self.input_shape = input_shape
self.input_layer = input_layer
self.mfp_strides = mfp_strides
self.mfp_offsets = mfp_offsets
self.reshape_output = reshape_output
print "2DConv: input=", input_shape, "\tfilter=", filter_shape #,"@std=",W_bound
if W is None:
W_values = np.asarray(
initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype=theano.config.floatX)
self.W = theano.shared(W_values, name='W_conv', borrow=True)
else:
if isinstance(W, np.ndarray):
self.W = theano.shared(
W.astype(np.float32),
name='W_conv',
borrow=True)
else:
assert isinstance(W, T.TensorVariable), "W must be either np.ndarray or theano var"
self.W = W
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[2] * filter_shape[3]
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1.0 / norm,
mode='const'),
dtype=theano.config.floatX)
elif activation_func == 'sigmoid':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=0.5,
mode='const'),
dtype=theano.config.floatX)
else: # activation_func=='tanh':
b_values = np.asarray(
initWeights(
(n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
else:
if isinstance(b, np.ndarray):
self.b = theano.shared(
b.astype(np.float32),
name='b_conv',
borrow=True)
else:
assert isinstance(b, T.TensorVariable), "b must be either np.ndarray or theano var"
self.b = b
# store parameters of this layer
self.params = [self.W, self.b]
# convolve input feature maps with filters
# shape of pooled_out , e.g.: (1,2,27,27) for 2 class-output
self.conv_out = conv.conv2d(
input=input,
filters=self.W,
border_mode='valid',
filter_shape=filter_shape
) # image_shape = input_shape if input_shape[0] is not None else None)
# down-sample each feature map individually, using maxpooling
if np.any(pool != 1):
pool_func = lambda x: pooling.pooling2d(x, pool_shape=pool, mode=pooling_mode)
if use_fragment_pooling:
pooled_out, self.mfp_offsets, self.mfp_strides = self.fragmentpool(
self.conv_out, pool, mfp_offsets, mfp_strides, pool_func)
else:
#pooled_out = downsample.max_pool_2d(input=self.conv_out, ds=pool, ignore_border=True)
pooled_out = pool_func(self.conv_out)
else:
pooled_out = self.conv_out
if enable_dropout:
#print "Dropout: ACTIVE"
self.activation_noise = theano.shared(np.float32(0.5), name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial(
(pooled_out.shape[2], pooled_out.shape[3]),
1,
p,
dtype=theano.config.floatX)
pooled_out = pooled_out * self.dropout_gate.dimshuffle(('x', 'x', 0, 1))
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature maps
# width & height
r = 1 # maxout factor
lin_output = pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')
if activation_func == 'tanh':
self.output = T.tanh(lin_output) # shape: (batch_size, num_outputs)
elif activation_func in ['ReLU', 'relu']: #rectified linear unit
self.output = lin_output * (lin_output > 0) # shape: (batch_size, num_outputs)
elif activation_func in ['sigmoid']:
self.output = T.nnet.sigmoid(lin_output)
elif activation_func in ['abs']:
self.output = T.abs_(lin_output)
elif activation_func in ['linear']:
self.output = lin_output
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
self.output = pooling.maxout(lin_output, factor=r)
else:
raise NotImplementedError()
output_shape = getOutputShape(input_shape, filter_shape, pool, use_fragment_pooling, r)
print "Output =", output_shape, "Dropout", ("ON," if enable_dropout else "OFF,"), "Act:",\
activation_func, "pool:", pooling_mode
self.output_shape = output_shape # e.g. (None, 16, 100, 100)
sh = lin_output.shape # (bs,ch,x,y) # use this shape to reshape the output to image-shape after softmax
sh = (sh[1], sh[2], sh[3], sh[0]) #(ch, x, y, bs)
# put batchsize, at back --> (ch,x,y,bs), flatten this --> (ch, x*y*bs), swap labels --> (x*y*bs, ch)
self.class_probabilities = T.nnet.softmax(lin_output.dimshuffle((1, 2, 3, 0)).flatten(2).dimshuffle((1, 0)))
if reshape_output:
self.reshapeoutput(sh)
if use_fragment_pooling:
self.fragmentstodense(sh)
self.prob_shape = getProbShape(output_shape, self.mfp_strides)
print "Class Prob Output =", self.prob_shape
# compute prediction as class whose "probability" is maximal in symbolic form
self.class_prediction = T.argmax(self.class_probabilities, axis=1)
def __init__(self,
input,
n_in,
n_out,
batch_size,
enable_dropout,
activation_func='tanh',
input_noise=None,
input_layer=None,
W=None,
b=None):
self.input_layer = input_layer # only for autoencoder
self.activation_func = activation_func
self.output_shape = (batch_size, n_out)
self.n_in = n_in
self.lin_output = None
self.output = None
self.last_grads = [] # only for autoencoder
print "PerceptronLayer( #Inputs =", n_in, "#Outputs =", n_out, ")"
if input_noise is not None:
self.input_noise = theano.shared(np.float32(input_noise), name='Input Noise')
print "Input_noise active, p=" + str(self.input_noise.get_value())
rng = np.random.RandomState(int(time.time()))
theano_rng = RandomStreams(rng.randint(2**30))
# apply multiplicative noise to input
#self.input = theano_rng.binomial(size=input.shape, n=1, p=1-self.input_noise,
# dtype='float32') * input
# apply additive noise to input
self.input = input + theano_rng.normal(size=input.shape,
avg=0,
std=input_noise,
dtype='float32')
else: # no input noise
self.input = input
if W is None:
W_values = np.asarray(initWeights((n_in, n_out), scale='glorot', mode='uni'), dtype='float32')
self.W = theano.shared(value=W_values, name='W_perceptron' + str(n_in) + '.' + str(n_out), borrow=True)
else:
print "Directly using fixed/shared W (", W, "), no Training on it in this layer!"
if isinstance(W, np.ndarray):
self.W = theano.shared(value=W.astype(np.float32),
name='W_perceptron' + str(n_in) + '.' + str(n_out),
borrow=True)
else:
assert isinstance(W, T.TensorVariable), "W must be either np.ndarray or theano var"
self.W = W
if b is None:
#b_values = np.asarray(np.random.uniform(-1e-8,1e-8,(n_out,)), dtype='float32')
if activation_func == 'relu' or activation_func == 'ReLU':
b_values = np.asarray(initWeights((n_out, ), scale=1.0, mode='const'), dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ), scale=0.5, mode='const'), dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ), scale=1e-6, mode='fix-uni'), dtype='float32')
self.b = theano.shared(value=b_values, name='b_perceptron' + str(n_in) + '.' + str(n_out), borrow=True)
else:
print "Directly using fixed given b (", b, "), no Training on it in this layer!"
if isinstance(b, np.ndarray):
self.b = theano.shared(value=b.astype(np.float32),
name='b_perceptron' + str(n_in) + '.' + str(n_out),
borrow=True)
else:
assert isinstance(b, T.TensorVariable), "b must be either np.ndarray or theano var"
self.b = b
lin_output = T.dot(self.input, self.W)
if enable_dropout:
print "Dropout ON"
self.activation_noise = theano.shared(np.float32(0.5), name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial((n_out, ), 1, p, dtype='float32')
lin_output = lin_output * self.dropout_gate.dimshuffle(('x', 0))
lin_output = lin_output + self.b
# Apply non-linearities and ggf. change bias-initialisations
if activation_func == 'tanh': # range = [-1,1]
self.output = T.tanh(lin_output) # shape: (batch_size, num_outputs)
elif activation_func == 'relu' or activation_func == 'ReLU': # rectified linear unit ,range = [0,inf]
self.activation_func = 'relu'
self.output = lin_output * (lin_output > 0) #T.maximum(lin_output,T.zeros_like(lin_output))
elif activation_func == 'abs': # abs unit ,range = [0,inf]
self.output = T.abs_(lin_output)
elif activation_func == 'sigmoid': # range = [0,1]
#print "WARNING: consider using tanh(.) or relu(.) instead! Sigmoid is BAD! (relu > tanh >> sigmoid)"
lin_output = T.dot(self.input, self.W) + self.b
self.output = T.nnet.sigmoid(lin_output) #1/(1 + T.exp(-lin_output))
elif activation_func == 'linear':
self.output = (lin_output)
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
n_out = n_out / r
self.output = pooling.maxout(lin_output, factor=r)
else:
raise NotImplementedError("Options are: activation_func=('relu'|'sigmoid'|'tanh'|'abs')")
self.lin_output = lin_output
self.params = [self.b if b is None else []] + ([self.W]
if W is None else [])
self.class_probabilities = T.nnet.softmax(lin_output) # shape: (batch_size, num_outputs)
#self.class_probabilities = T.exp(lin_output) / T.sum(T.exp(lin_output), axis=1, keepdims=True) # For Hessian
self.class_prediction = T.argmax(self.class_probabilities, axis=1) # shape: (batch_size,)
def __init__(self,
input,
input_shape,
filter_shape,
pool,
activation_func,
enable_dropout,
use_fragment_pooling,
reshape_output,
mfp_offsets,
mfp_strides,
input_layer=None,
W=None,
b=None,
pooling_mode='max'):
assert input_shape[1] == filter_shape[1]
self.input = input
self.pool = pool
self.number_of_filters = filter_shape[0]
self.filter_shape = filter_shape
self.activation_func = activation_func
self.input_shape = input_shape
self.input_layer = input_layer
self.mfp_strides = mfp_strides
self.mfp_offsets = mfp_offsets
self.reshape_output = reshape_output
print "2DConv: input=", input_shape, "\tfilter=", filter_shape #,"@std=",W_bound
if W is None:
W_values = np.asarray(initWeights(filter_shape,
scale='glorot',
mode='normal',
pool=pool),
dtype='float32')
self.W = theano.shared(W_values, name='W_conv', borrow=True)
else:
if isinstance(W, np.ndarray):
self.W = theano.shared(W.astype(np.float32),
name='W_conv',
borrow=True)
else:
assert isinstance(
W, T.TensorVariable
), "W must be either np.ndarray or theano var"
self.W = W
if b is None:
n_out = filter_shape[0]
if activation_func == 'relu' or activation_func == 'ReLU':
norm = filter_shape[2] * filter_shape[3]
b_values = np.asarray(initWeights((n_out, ),
scale=1.0 / norm,
mode='const'),
dtype='float32')
elif activation_func == 'sigmoid':
b_values = np.asarray(initWeights((n_out, ),
scale=0.5,
mode='const'),
dtype='float32')
else: # activation_func=='tanh':
b_values = np.asarray(initWeights((n_out, ),
scale=1e-6,
mode='fix-uni'),
dtype='float32')
self.b = theano.shared(value=b_values, borrow=True, name='b_conv')
else:
if isinstance(b, np.ndarray):
self.b = theano.shared(b.astype(np.float32),
name='b_conv',
borrow=True)
else:
assert isinstance(
b, T.TensorVariable
), "b must be either np.ndarray or theano var"
self.b = b
# store parameters of this layer
self.params = [self.W, self.b]
# convolve input feature maps with filters
# shape of pooled_out , e.g.: (1,2,27,27) for 2 class-output
self.conv_out = conv.conv2d(
input=input,
filters=self.W,
border_mode='valid',
filter_shape=filter_shape
) # image_shape = input_shape if input_shape[0] is not None else None)
# down-sample each feature map individually, using maxpooling
if np.any(pool != 1):
pool_func = lambda x: pooling.pooling2d(
x, pool_shape=pool, mode=pooling_mode)
if use_fragment_pooling:
pooled_out, self.mfp_offsets, self.mfp_strides = self.fragmentpool(
self.conv_out, pool, mfp_offsets, mfp_strides, pool_func)
else:
#pooled_out = downsample.max_pool_2d(input=self.conv_out, ds=pool, ignore_border=True)
pooled_out = pool_func(self.conv_out)
else:
pooled_out = self.conv_out
if enable_dropout:
#print "Dropout: ACTIVE"
self.activation_noise = theano.shared(np.float32(0.5),
name='Dropout Rate')
rng = T.shared_randomstreams.RandomStreams(int(time.time()))
p = 1 - self.activation_noise
self.dropout_gate = 1.0 / p * rng.binomial(
(pooled_out.shape[2], pooled_out.shape[3]),
1,
p,
dtype='float32')
pooled_out = pooled_out * self.dropout_gate.dimshuffle(
('x', 'x', 0, 1))
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature maps
# width & height
r = 1 # maxout factor
lin_output = pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')
if activation_func == 'tanh':
self.output = T.tanh(
lin_output) # shape: (batch_size, num_outputs)
elif activation_func in ['ReLU', 'relu']: #rectified linear unit
self.output = lin_output * (lin_output > 0
) # shape: (batch_size, num_outputs)
elif activation_func in ['sigmoid']:
self.output = T.nnet.sigmoid(lin_output)
elif activation_func in ['abs']:
self.output = T.abs_(lin_output)
elif activation_func in ['linear']:
self.output = lin_output
elif activation_func.startswith("maxout"):
r = int(activation_func.split(" ")[1])
assert r >= 2
self.output = pooling.maxout(lin_output, factor=r)
else:
raise NotImplementedError()
output_shape = getOutputShape(input_shape, filter_shape, pool,
use_fragment_pooling, r)
print "Output =", output_shape, "Dropout", ("ON," if enable_dropout else "OFF,"), "Act:",\
activation_func, "pool:", pooling_mode
self.output_shape = output_shape # e.g. (None, 16, 100, 100)
sh = lin_output.shape # (bs,ch,x,y) # use this shape to reshape the output to image-shape after softmax
sh = (sh[1], sh[2], sh[3], sh[0]) #(ch, x, y, bs)
# put batchsize, at back --> (ch,x,y,bs), flatten this --> (ch, x*y*bs), swap labels --> (x*y*bs, ch)
self.class_probabilities = T.nnet.softmax(
lin_output.dimshuffle((1, 2, 3, 0)).flatten(2).dimshuffle((1, 0)))
if reshape_output:
self.reshapeoutput(sh)
if use_fragment_pooling:
self.fragmentstodense(sh)
self.prob_shape = getProbShape(output_shape, self.mfp_strides)
print "Class Prob Output =", self.prob_shape
# compute prediction as class whose "probability" is maximal in symbolic form
self.class_prediction = T.argmax(self.class_probabilities, axis=1)
