def __init__(
self,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None,
draw=None):
self.cfg = cfg
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = 1
print self.max_col_norm
self.layers = []
self.lstm_layers = []
self.fc_layers = []
# 1. lstm
self.lstm_layers_sizes = cfg.lstm_layers_sizes
self.lstm_layers_number = len(self.lstm_layers_sizes)
# 2. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.ivector('y')
#######################
# build lstm layers #
#######################
print '1. start to build AttendLSTMLayer : ' + str(
self.lstm_layers_number) + ', n_attendout: ' + str(cfg.batch_size)
for i in xrange(1):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.layers[-1].output
lstm_layer = AttendLSTMLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.lstm_layers_sizes[i],
steps=cfg.batch_size,
draw=draw)
print '\tbuild AttendLSTMLayer: ' + str(input_size) + ' x ' + str(
lstm_layer.n_out)
self.layers.append(lstm_layer)
self.lstm_layers.append(lstm_layer)
self.params.extend(lstm_layer.params)
self.delta_params.extend(lstm_layer.delta_params)
print '1. finish AttendLSTMLayer: ' + str(self.layers[-1].n_out)
print '2. start to build LSTMLayer : ' + str(self.lstm_layers_number)
for i in xrange(1, self.lstm_layers_number, 1):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.layers[-1].output
lstm_layer = LSTMLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.lstm_layers_sizes[i])
print '\tbuild LSTMLayer: ' + str(input_size) + ' x ' + str(
lstm_layer.n_out)
self.layers.append(lstm_layer)
self.lstm_layers.append(lstm_layer)
self.params.extend(lstm_layer.params)
self.delta_params.extend(lstm_layer.delta_params)
print '2. finish LSTMLayer: ' + str(self.layers[-1].n_out)
#######################
# build dnnv layers #
#######################
#print '2. start to build dnnv layer: '+ str(self.hidden_layers_number)
#for i in xrange(self.hidden_layers_number):
# if i == 0:
# input_size = self.layers[-1].n_out
# else:
# input_size = self.hidden_layers_sizes[i - 1]
# input = self.layers[-1].output
# fc_layer = HiddenLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.hidden_layers_sizes[i], activation=self.activation)
# print '\tbuild dnnv layer: ' + str(input_size) +' x '+ str(fc_layer.n_out)
# self.layers.append(fc_layer)
# self.fc_layers.append(fc_layer)
# self.params.extend(fc_layer.params)
# self.delta_params.extend(fc_layer.delta_params)
#print '2. finish dnnv layer: '+ str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '3. start to build log layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
logLayer = LogisticRegression(input=input,
n_in=input_size,
n_out=self.n_outs)
print '\tbuild final layer: ' + str(input_size) + ' x ' + str(
self.n_outs)
self.layers.append(logLayer)
self.params.extend(logLayer.params)
self.delta_params.extend(logLayer.delta_params)
print '3. finish log layer: ' + str(self.layers[-1].n_out)
print 'Total layers: ' + str(len(self.layers))
sys.stdout.flush()
self.finetune_cost = logLayer.negative_log_likelihood(self.y)
self.errors = logLayer.errors(self.y)
def __init__(self, numpy_rng, theano_rng=None,
cfg = None, # the network configuration
dnn_shared = None, shared_layers=[], input = None):
self.cfg = cfg
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins; self.n_outs = cfg.n_outs
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.do_maxout = cfg.do_maxout; self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
print self.max_col_norm
self.layers = []
self.bilayers = []
self.lstm_layers = []
self.fc_layers = []
# 1. lstm
self.lstm_layers_sizes = cfg.lstm_layers_sizes
self.lstm_layers_number = len(self.lstm_layers_sizes)
# 2. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.ivector('y')
#######################
# build lstm layers #
#######################
print '1. start to build lstm layer: '+ str(self.lstm_layers_number)
for i in xrange(self.lstm_layers_number):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.bilayers[-1].output
# Forward
f_lstm_layer = LSTMLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.lstm_layers_sizes[i])
print '\tbuild f_lstm layer: ' + str(input_size) +' x '+ str(f_lstm_layer.n_out)
self.layers.append(f_lstm_layer)
self.lstm_layers.append(f_lstm_layer)
self.params.extend(f_lstm_layer.params)
self.delta_params.extend(f_lstm_layer.delta_params)
# Backward
b_lstm_layer = LSTMLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.lstm_layers_sizes[i], backwards=True)
print '\tbuild b_lstm layer: ' + str(input_size) +' x '+ str(b_lstm_layer.n_out)
self.layers.append(b_lstm_layer)
self.lstm_layers.append(b_lstm_layer)
self.params.extend(b_lstm_layer.params)
self.delta_params.extend(b_lstm_layer.delta_params)
# Sum forward + backward
bi_layer = SUMLayer(finput=f_lstm_layer.output,binput=b_lstm_layer.output[::-1], n_out=self.lstm_layers_sizes[i - 1])
self.bilayers.append(bi_layer)
print '\tbuild sum layer: ' + str(input_size) +' x '+ str(bi_layer.n_out)
print '1. finish lstm layer: '+ str(self.bilayers[-1].n_out)
#######################
# build log layers #
#######################
print '3. start to build log layer: 1'
input_size = self.bilayers[-1].n_out
input = self.bilayers[-1].output
logLayer = LogisticRegression(input=input, n_in=input_size, n_out=self.n_outs)
print '\tbuild final layer: ' + str(input_size) +' x '+ str(self.n_outs)
self.layers.append(logLayer)
self.params.extend(logLayer.params)
self.delta_params.extend(logLayer.delta_params)
print '3. finish log layer: '+ str(self.bilayers[-1].n_out)
print 'Total layers: '+ str(len(self.layers))
sys.stdout.flush()
self.finetune_cost = logLayer.negative_log_likelihood(self.y)
self.errors = logLayer.errors(self.y)
def __init__(self,
numpy_rng,
theano_rng=None,
batch_size=256,
n_outs=500,
conv_layer_configs=[],
hidden_layers_sizes=[500, 500],
ivec_layers_sizes=[500, 500],
conv_activation=T.nnet.sigmoid,
full_activation=T.nnet.sigmoid,
use_fast=False,
update_part=[0, 1],
ivec_dim=100):
self.conv_layers = []
self.full_layers = []
self.ivec_layers = []
self.params = []
self.delta_params = []
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
input_shape = conv_layer_configs[0]['input_shape']
n_ins = input_shape[-1] * input_shape[-2] * input_shape[-3]
self.iv = self.x[:, n_ins:n_ins + ivec_dim]
self.raw = self.x[:, 0:n_ins]
self.conv_layer_num = len(conv_layer_configs)
self.full_layer_num = len(hidden_layers_sizes)
self.ivec_layer_num = len(ivec_layers_sizes)
# construct the adaptation NN
for i in xrange(self.ivec_layer_num):
if i == 0:
input_size = ivec_dim
layer_input = self.iv
else:
input_size = ivec_layers_sizes[i - 1]
layer_input = self.ivec_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=ivec_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.ivec_layers.append(sigmoid_layer)
if 0 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
linear_func = lambda x: x
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=self.ivec_layers[-1].output,
n_in=ivec_layers_sizes[-1],
n_out=n_ins,
activation=linear_func)
self.ivec_layers.append(sigmoid_layer)
if 0 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.raw + self.ivec_layers[-1].output
else:
input = self.conv_layers[-1].output
config = conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng,
input=input,
input_shape=config['input_shape'],
filter_shape=config['filter_shape'],
poolsize=config['poolsize'],
activation=conv_activation,
flatten=config['flatten'],
use_fast=use_fast)
self.conv_layers.append(conv_layer)
if 1 in update_part:
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config[
'output_shape'][2] * config['output_shape'][3]
for i in xrange(self.full_layer_num):
# construct the sigmoidal layer
if i == 0:
input_size = self.conv_output_dim
layer_input = self.conv_layers[-1].output
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = self.full_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=full_activation)
# add the layer to our list of layers
self.full_layers.append(sigmoid_layer)
if 1 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.full_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.full_layers.append(self.logLayer)
if 1 in update_part:
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def __init__(
self,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None):
self.layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.non_updated_layers = cfg.non_updated_layers
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.ivector('y')
for i in xrange(self.hidden_layers_number):
# construct the hidden layer
if i == 0:
input_size = self.n_ins
layer_input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
W = None
b = None
if (i in shared_layers):
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
if self.do_maxout == True:
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] *
self.pool_size,
W=W,
b=b,
activation=(lambda x: 1.0 * x),
do_maxout=True,
pool_size=self.pool_size)
else:
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
# add the layer to our list of layers
self.layers.append(hidden_layer)
# if the layer index is included in self.non_updated_layers, parameters of this layer will not be updated
if (i not in self.non_updated_layers):
self.params.extend(hidden_layer.params)
self.delta_params.extend(hidden_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
if self.n_outs > 0:
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
if self.l1_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l1_reg * (abs(W).sum())
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng,
theano_rng=None,
upper_hidden_layers_sizes=[500, 500],
n_outs=10,
tower1_hidden_layers_sizes=[500, 500],
tower1_n_ins=100,
tower2_hidden_layers_sizes=[500, 500],
tower2_n_ins=100,
activation=T.nnet.sigmoid,
do_maxout=False,
pool_size=1,
do_pnorm=False,
pnorm_order=1,
max_col_norm=None,
l1_reg=None,
l2_reg=None):
self.tower1_layers = []
self.tower2_layers = []
self.upper_layers = []
self.params = []
self.delta_params = []
self.max_col_norm = max_col_norm
self.l1_reg = l1_reg
self.l2_reg = l2_reg
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
self.tower1_input = self.x[:, 0:tower1_n_ins]
self.tower2_input = self.x[:,
tower1_n_ins:(tower1_n_ins + tower2_n_ins)]
# build tower1
for i in xrange(len(tower1_hidden_layers_sizes)):
if i == 0:
input_size = tower1_n_ins
layer_input = self.tower1_input
else:
input_size = tower1_hidden_layers_sizes[i - 1]
layer_input = self.tower1_layers[-1].output
layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=tower1_hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.tower1_layers.append(layer)
self.params.extend(layer.params)
self.delta_params.extend(layer.delta_params)
# build tower2
for i in xrange(len(tower2_hidden_layers_sizes)):
if i == 0:
input_size = tower2_n_ins
layer_input = self.tower2_input
else:
input_size = tower2_hidden_layers_sizes[i - 1]
layer_input = self.tower2_layers[-1].output
layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=tower2_hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.tower2_layers.append(layer)
self.params.extend(layer.params)
self.delta_params.extend(layer.delta_params)
for i in xrange(len(upper_hidden_layers_sizes)):
# construct the sigmoidal layer
if i == 0:
input_size = tower1_hidden_layers_sizes[
-1] + tower2_hidden_layers_sizes[-1]
layer_input = T.concatenate([
self.tower1_layers[-1].output,
self.tower2_layers[-1].output
],
axis=1)
else:
input_size = upper_hidden_layers_sizes[i - 1]
layer_input = self.upper_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=upper_hidden_layers_sizes[i],
activation=activation)
# add the layer to our list of layers
self.upper_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.upper_layers[-1].output,
n_in=upper_hidden_layers_sizes[-1],
n_out=n_outs)
self.upper_layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def __init__(
self,
task_id,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None):
self.layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.non_updated_layers = cfg.non_updated_layers
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
if input == None:
self.x = T.matrix('x')
else:
self.x = input
if task_id == 0:
self.y = T.ivector('y')
else:
self.y = T.matrix('y')
#######################
# build dnnv layers #
#######################
print "=============="
print "Task ID: %d" % (task_id)
print "=============="
print '1. start to build dnn layer: ' + str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
input = self.layers[-1].output
W = None
b = None
if (i in shared_layers):
print "shared layer = %d" % (i)
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
hidden_layer = HiddenLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
print '\tbuild lstm layer: ' + str(input_size) + ' x ' + str(
hidden_layer.n_out)
self.layers.append(hidden_layer)
self.params.extend(hidden_layer.params)
self.delta_params.extend(hidden_layer.delta_params)
print '1. finish dnnv layer: ' + str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '2. start to build final layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
if task_id == 0:
self.logLayer = LogisticRegression(
input=self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
print '\tbuild final layer (classification): ' + str(
input_size) + ' x ' + str(self.logLayer.n_out)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
else:
self.logLayer = OutputLayer(input=input,
n_in=input_size,
n_out=self.n_outs)
print '\tbuild final layer (regression): ' + str(
input_size) + ' x ' + str(self.logLayer.n_out)
self.finetune_cost = self.logLayer.l2(self.y)
self.errors = self.logLayer.errors(self.y)
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
print '2. finish log layer: ' + str(self.layers[-1].n_out)
print 'Total layers: ' + str(len(self.layers))
sys.stdout.flush()
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10,
corruption_levels=[0.1, 0.1],
pool_size=3,
sparsity=None,
sparsity_weight=None,
first_reconstruct_activation=T.tanh):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i] *
pool_size,
activation=(lambda x: 1.0 * x),
do_maxout=True,
pool_size=pool_size)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
# Construct a denoising autoencoder that shared weights with this layer
if i == 0:
reconstruct_activation = first_reconstruct_activation
else:
reconstruct_activation = (lambda x: 1.0 * x)
# reconstruct_activation = first_reconstruct_activation
dA_layer = dA_maxout(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i] * pool_size,
W=sigmoid_layer.W,
bhid=sigmoid_layer.b,
sparsity=sparsity,
sparsity_weight=sparsity_weight,
pool_size=pool_size,
reconstruct_activation=reconstruct_activation)
self.dA_layers.append(dA_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.sigmoid_layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
def __init__(self,
numpy_rng,
theano_rng=None,
cfg=None,
dnn_shared=None,
shared_layers=[]):
self.layers = []
self.dropout_layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.input_dropout_factor = cfg.input_dropout_factor
self.dropout_factor = cfg.dropout_factor
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.hidden_layers_number):
# construct the hidden layer
if i == 0:
input_size = self.n_ins
layer_input = self.x
if self.input_dropout_factor > 0.0:
dropout_layer_input = _dropout_from_layer(
theano_rng, self.x, self.input_dropout_factor)
else:
dropout_layer_input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
layer_input = (
1 - self.dropout_factor[i - 1]) * self.layers[-1].output
dropout_layer_input = self.dropout_layers[-1].dropout_output
W = None
b = None
if (i in shared_layers):
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
if self.do_maxout == False:
dropout_layer = DropoutHiddenLayer(
rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation,
dropout_factor=self.dropout_factor[i])
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
activation=self.activation,
W=dropout_layer.W,
b=dropout_layer.b)
else:
dropout_layer = DropoutHiddenLayer(
rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] * self.pool_size,
W=W,
b=b,
activation=(lambda x: 1.0 * x),
dropout_factor=self.dropout_factor[i],
do_maxout=True,
pool_size=self.pool_size)
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] *
self.pool_size,
activation=(lambda x: 1.0 * x),
W=dropout_layer.W,
b=dropout_layer.b,
do_maxout=True,
pool_size=self.pool_size)
# add the layer to our list of layers
self.layers.append(hidden_layer)
self.dropout_layers.append(dropout_layer)
self.params.extend(dropout_layer.params)
self.delta_params.extend(dropout_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.dropout_logLayer = LogisticRegression(
input=self.dropout_layers[-1].dropout_output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
self.logLayer = LogisticRegression(
input=(1 - self.dropout_factor[-1]) * self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs,
W=self.dropout_logLayer.W,
b=self.dropout_logLayer.b)
self.dropout_layers.append(self.dropout_logLayer)
self.layers.append(self.logLayer)
self.params.extend(self.dropout_logLayer.params)
self.delta_params.extend(self.dropout_logLayer.delta_params)
# compute the cost
self.finetune_cost = self.dropout_logLayer.negative_log_likelihood(
self.y)
self.errors = self.logLayer.errors(self.y)
if self.l1_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l1_reg * (abs(W).sum())
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self, numpy_rng, theano_rng=None,
batch_size = 256, n_outs=500,
sparsity = None, sparsity_weight = None, sparse_layer = 3,
conv_layer_configs = [],
hidden_layers_sizes=[500, 500],
conv_activation = T.nnet.sigmoid,
full_activation = T.nnet.sigmoid,
use_fast = False):
self.layers = []
self.params = []
self.delta_params = []
self.sparsity = sparsity
self.sparsity_weight = sparsity_weight
self.sparse_layer = sparse_layer
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
self.conv_layer_num = len(conv_layer_configs)
self.full_layer_num = len(hidden_layers_sizes)
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.x
is_input_layer = True
else:
input = self.layers[-1].output
is_input_layer = False
config = conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng, input=input, is_input_layer = is_input_layer,
input_shape = config['input_shape'], filter_shape = config['filter_shape'], poolsize = config['poolsize'],
activation = conv_activation, flatten = config['flatten'])
self.layers.append(conv_layer)
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config['output_shape'][2] * config['output_shape'][3]
for i in xrange(self.full_layer_num):
# construct the sigmoidal layer
if i == 0:
input_size = self.conv_output_dim
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=full_activation)
# add the layer to our list of layers
self.layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
if self.sparsity_weight is not None:
sparsity_level = T.extra_ops.repeat(self.sparsity, 630)
avg_act = self.sigmoid_layers[sparse_layer].output.mean(axis=0)
kl_div = self.kl_divergence(sparsity_level, avg_act)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) + self.sparsity_weight * kl_div.sum()
else:
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
