def __init__(self,
numpy_rng,
theano_rng=None,
cfg_si=None,
cfg_adapt=None):
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
# we assume that i-vectors are appended to speech features in a frame-wise manner
self.feat_dim = cfg_si.n_ins
self.ivec_dim = cfg_adapt.n_ins
self.iv = self.x[:, self.feat_dim:self.feat_dim + self.ivec_dim]
self.feat = self.x[:, 0:self.feat_dim]
# the parameters
self.params = [] # the params to be updated in the current training
self.delta_params = []
# the i-vector network
dnn_adapt = DNN(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg=cfg_adapt,
input=self.iv)
self.dnn_adapt = dnn_adapt
# the final output layer which has the same dimension as the input features
linear_func = lambda x: x
adapt_output_layer = HiddenLayer(
rng=numpy_rng,
input=dnn_adapt.layers[-1].output,
n_in=cfg_adapt.hidden_layers_sizes[-1],
n_out=self.feat_dim,
activation=linear_func)
dnn_adapt.layers.append(adapt_output_layer)
dnn_adapt.params.extend(adapt_output_layer.params)
dnn_adapt.delta_params.extend(adapt_output_layer.delta_params)
dnn_si = DNN(numpy_rng=numpy_rng,
theano_rng=theano_rng,
cfg=cfg_si,
input=self.feat + dnn_adapt.layers[-1].output)
self.dnn_si = dnn_si
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = dnn_si.logLayer.negative_log_likelihood(self.y)
self.errors = dnn_si.logLayer.errors(self.y)
def __init__(
self,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None):
self.layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.non_updated_layers = cfg.non_updated_layers
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.ivector('y')
for i in xrange(self.hidden_layers_number):
# construct the hidden layer
if i == 0:
input_size = self.n_ins
layer_input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
W = None
b = None
if (i in shared_layers):
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
if self.do_maxout == True:
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] *
self.pool_size,
W=W,
b=b,
activation=(lambda x: 1.0 * x),
do_maxout=True,
pool_size=self.pool_size)
else:
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
# add the layer to our list of layers
self.layers.append(hidden_layer)
# if the layer index is included in self.non_updated_layers, parameters of this layer will not be updated
if (i not in self.non_updated_layers):
self.params.extend(hidden_layer.params)
self.delta_params.extend(hidden_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
if self.n_outs > 0:
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
if self.l1_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l1_reg * (abs(W).sum())
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng,
theano_rng=None,
batch_size=256,
n_outs=500,
conv_layer_configs=[],
hidden_layers_sizes=[500, 500],
ivec_layers_sizes=[500, 500],
conv_activation=T.nnet.sigmoid,
full_activation=T.nnet.sigmoid,
use_fast=False,
update_part=[0, 1],
ivec_dim=100):
self.conv_layers = []
self.full_layers = []
self.ivec_layers = []
self.params = []
self.delta_params = []
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
input_shape = conv_layer_configs[0]['input_shape']
n_ins = input_shape[-1] * input_shape[-2] * input_shape[-3]
self.iv = self.x[:, n_ins:n_ins + ivec_dim]
self.raw = self.x[:, 0:n_ins]
self.conv_layer_num = len(conv_layer_configs)
self.full_layer_num = len(hidden_layers_sizes)
self.ivec_layer_num = len(ivec_layers_sizes)
# construct the adaptation NN
for i in xrange(self.ivec_layer_num):
if i == 0:
input_size = ivec_dim
layer_input = self.iv
else:
input_size = ivec_layers_sizes[i - 1]
layer_input = self.ivec_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=ivec_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.ivec_layers.append(sigmoid_layer)
if 0 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
linear_func = lambda x: x
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=self.ivec_layers[-1].output,
n_in=ivec_layers_sizes[-1],
n_out=n_ins,
activation=linear_func)
self.ivec_layers.append(sigmoid_layer)
if 0 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.raw + self.ivec_layers[-1].output
else:
input = self.conv_layers[-1].output
config = conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng,
input=input,
input_shape=config['input_shape'],
filter_shape=config['filter_shape'],
poolsize=config['poolsize'],
activation=conv_activation,
flatten=config['flatten'],
use_fast=use_fast)
self.conv_layers.append(conv_layer)
if 1 in update_part:
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config[
'output_shape'][2] * config['output_shape'][3]
for i in xrange(self.full_layer_num):
# construct the sigmoidal layer
if i == 0:
input_size = self.conv_output_dim
layer_input = self.conv_layers[-1].output
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = self.full_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=full_activation)
# add the layer to our list of layers
self.full_layers.append(sigmoid_layer)
if 1 in update_part:
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.full_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.full_layers.append(self.logLayer)
if 1 in update_part:
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def __init__(
self,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None):
self.cfg = cfg
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.layers = []
self.lstm_layers = []
self.fc_layers = []
# 1. lstm
self.lstm_layers_sizes = cfg.lstm_layers_sizes
self.lstm_layers_number = len(self.lstm_layers_sizes)
# 2. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.matrix('y')
#######################
# build lstm layers #
#######################
print '1. start to build attend-lstm layer: ' + str(
self.lstm_layers_number) + ', n_attendout: ' + str(cfg.batch_size)
for i in xrange(self.lstm_layers_number):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.layers[-1].output
lstm_layer = AttendRnnLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.lstm_layers_sizes[i],
n_attendout=cfg.batch_size)
print '\tbuild attend-lstm layer: ' + str(
input_size) + ' x ' + str(lstm_layer.n_out)
self.layers.append(lstm_layer)
self.lstm_layers.append(lstm_layer)
self.params.extend(lstm_layer.params)
self.delta_params.extend(lstm_layer.delta_params)
print '1. finish attend-lstm layer: ' + str(self.layers[-1].n_out)
#######################
# build dnnv layers #
#######################
print '2. start to build dnnv layer: ' + str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
if i == 0:
input_size = self.layers[-1].n_out
else:
input_size = self.hidden_layers_sizes[i - 1]
input = self.layers[-1].output
fc_layer = HiddenLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i])
print '\tbuild dnnv layer: ' + str(input_size) + ' x ' + str(
fc_layer.n_out)
self.layers.append(fc_layer)
self.fc_layers.append(fc_layer)
self.params.extend(fc_layer.params)
self.delta_params.extend(fc_layer.delta_params)
print '2. finish dnnv layer: ' + str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '3. start to build log layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
logLayer = OutputLayer(input=input, n_in=input_size, n_out=self.n_outs)
print '\tbuild final layer: ' + str(input_size) + ' x ' + str(
fc_layer.n_out)
self.layers.append(logLayer)
self.params.extend(logLayer.params)
self.delta_params.extend(logLayer.delta_params)
print '3. finish log layer: ' + str(self.layers[-1].n_out)
print 'Total layers: ' + str(len(self.layers))
sys.stdout.flush()
self.finetune_cost = self.layers[-1].l2(self.y)
self.errors = self.layers[-1].errors(self.y)
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self, numpy_rng, theano_rng=None,
cfg = None, # the network configuration
dnn_shared = None, shared_layers=[], input = None, extra_input = None):
self.cfg = cfg
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins; self.n_outs = cfg.n_outs
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.do_maxout = cfg.do_maxout; self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
print self.max_col_norm
self.layers = []
self.extra_layers = []
self.lstm_layers = []
self.fc_layers = []
# 1. lstm
self.lstm_layers_sizes = cfg.lstm_layers_sizes
self.lstm_layers_number = len(self.lstm_layers_sizes)
# 1.5 attention
self.extra_dim = cfg.extra_dim
print 'Extra dim: '+str(cfg.extra_dim)
self.extra_layers_sizes = cfg.extra_layers_sizes
# 2. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
if input == None:
self.x = T.matrix('x')
self.extra_x = T.matrix('extra_x')
else:
self.x = input
self.extra_x = extra_input
self.y = T.matrix('y')
#######################################
# build phase-based attention layer #
#######################################
# 0. phase-based attention
#self.extra_layers_sizes.extend([self.conv_output_dim])
#print '0. start to build attend layer: '+ str(self.extra_layers_sizes)
#for i in xrange(len(self.extra_layers_sizes)):
# if i == 0:
# input_size = 6400*5
# input_size = cfg.extra_dim
# layer_input = self.extra_x
# else:
# input_size = self.extra_layers_sizes[i - 1]
# layer_input = self.extra_layers[-1].output
#
# W = None; b = None
# attend_layer = HiddenLayer(rng=numpy_rng,
# input=layer_input,
# n_in=input_size,
# n_out=self.extra_layers_sizes[i],
# W = W, b = b,
# activation=self.activation)
# print '\tbuild attend layer: ' + str(input_size) +' x '+ str(attend_layer.n_out)
# self.extra_layers.append(attend_layer)
# self.params.extend(attend_layer.params)
# self.delta_params.extend(attend_layer.delta_params)
# self.extra_layers[-1].att_e_tl = self.extra_layers[-1].output
# self.extra_layers[-1].att_a_tl = T.nnet.softmax(self.extra_layers[-1].att_e_tl)
# #self.extra_layers[-1].att_a_tl = T.exp(self.extra_layers[-1].att_e_tl)/(T.exp(self.extra_layers[-1].att_e_tl)).sum(0,keepdims=True)
# print '0. finish attend layer: '+ str(self.extra_layers[-1].n_out)
#######################
# build lstm layers #
#######################
#print '1. start to build PhaseAttendLSTMLayer : '+ str(self.lstm_layers_number) + ', n_attendout: '+ str(cfg.batch_size)
print '1. start to build PhaseAttendLSTMLayer : '+ str(self.lstm_layers_number) + ', n_attendout: '+ str(self.n_ins)
for i in xrange(self.lstm_layers_number):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.layers[-1].output
lstm_layer = PhaseAttendLSTMLayer(rng=numpy_rng, input=input, n_in=input_size,
extra_input = extra_input,
n_out=self.lstm_layers_sizes[i])
print '\tbuild PhaseAttendLSTMLayer: ' + str(input_size) +' x '+ str(lstm_layer.n_out)
self.layers.append(lstm_layer)
self.lstm_layers.append(lstm_layer)
self.params.extend(lstm_layer.params)
self.delta_params.extend(lstm_layer.delta_params)
print '1. finish PhaseAttendLSTMLayer: '+ str(self.layers[-1].n_out)
#######################
# build dnnv layers #
#######################
print '2. start to build dnnv layer: '+ str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
if i == 0:
input_size = self.layers[-1].n_out
else:
input_size = self.hidden_layers_sizes[i - 1]
input = self.layers[-1].output
fc_layer = HiddenLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.hidden_layers_sizes[i], activation=self.activation)
print '\tbuild dnnv layer: ' + str(input_size) +' x '+ str(fc_layer.n_out)
self.layers.append(fc_layer)
self.fc_layers.append(fc_layer)
self.params.extend(fc_layer.params)
self.delta_params.extend(fc_layer.delta_params)
print '2. finish dnnv layer: '+ str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '3. start to build log layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
logLayer = OutputLayer(input=input, n_in=input_size, n_out=self.n_outs)
print '\tbuild final layer: ' + str(input_size) +' x '+ str(fc_layer.n_out)
self.layers.append(logLayer)
self.params.extend(logLayer.params)
self.delta_params.extend(logLayer.delta_params)
print '3. finish log layer: '+ str(self.layers[-1].n_out)
print 'Total layers: '+ str(len(self.layers))
sys.stdout.flush()
self.finetune_cost = self.layers[-1].l2(self.y)
self.errors = self.layers[-1].errors(self.y)
if self.l2_reg is not None:
#for i in xrange(self.lstm_layers_number):
# W = self.lstm_layers[i].W_xi
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_hi
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_xf
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_hf
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_xc
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_hc
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_xo
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
# W = self.lstm_layers[i].W_ho
# self.finetune_cost += self.l2_reg * T.sqr(W).sum()
for i in xrange(self.hidden_layers_number):
W = self.fc_layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng=None,
theano_rng=None,
cfg=[],
non_maximum_erasing=False,
use_fast=False):
self.conv_layers = []
self.n_outs = cfg.n_outs
self.layers = []
self.extra_layers = []
self.conv_layer_num = len(cfg.conv_layer_configs)
self.dnn_layer_num = len(cfg.hidden_layers_sizes)
self.extra_layers_sizes = cfg.extra_layers_sizes
self.x = T.tensor4('x')
self.extra_x = T.matrix('extra_x')
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.x
else:
input = self.conv_layers[-1].output
config = cfg.conv_layer_configs[i]
print config['filter_shape']
conv_layer = ConvLayerForward(numpy_rng=numpy_rng,
input=input,
filter_shape=config['filter_shape'],
poolsize=config['poolsize'],
activation=config['activation'],
flatten=config['flatten'],
use_fast=use_fast)
self.layers.append(conv_layer)
self.conv_layers.append(conv_layer)
self.conv_output_dim = config['output_shape'][1] * config[
'output_shape'][2] * config['output_shape'][3]
cfg.n_ins = config['output_shape'][1] * config['output_shape'][
2] * config['output_shape'][3]
print self.conv_output_dim
print cfg.n_ins
print 'Extra input dimension: ' + str(cfg.extra_dim)
for i in xrange(len(self.extra_layers_sizes)):
if i == 0:
input_size = cfg.extra_dim
layer_input = self.extra_x
else:
input_size = self.extra_layers_sizes[i - 1]
layer_input = self.extra_layers[-1].output
W = None
b = None
attend_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.extra_layers_sizes[i],
W=W,
b=b)
self.extra_layers.append(attend_layer)
self.extra_output = self.extra_layers[-1].output
self.extra_output = T.nnet.softmax(self.extra_layers[-1].output)
print 'layer num: ' + str(len(self.layers) - 1)
for i in xrange(self.dnn_layer_num):
if i == 0:
# 1. Join two features (magnitude + phase)
input_size = self.conv_output_dim + self.extra_layers_sizes[-1]
layer_input = T.join(1, self.layers[-1].output,
self.extra_output)
# 2. Weighted Sum (magnitude * phase)
#input_size = self.conv_output_dim
#layer_input = self.layers[-1].output * self.extra_output
else:
input_size = cfg.hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
W = None
b = None
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=cfg.hidden_layers_sizes[i],
W=W,
b=b)
self.layers.append(hidden_layer)
print 'layer num: ' + str(len(self.layers) - 1)
logLayer = OutputLayer(input=self.layers[-1].output,
n_in=cfg.hidden_layers_sizes[-1],
n_out=self.n_outs)
self.layers.append(logLayer)
print 'layer num: ' + str(len(self.layers) - 1)
def __init__(self,
numpy_rng=None,
theano_rng=None,
cfg=None,
non_maximum_erasing=False,
use_fast=False):
self.n_outs = cfg.n_outs
self.layers = []
self.extra_layers = []
self.conv_layer_num = len(cfg.conv_layer_configs)
self.dnn_layer_num = len(cfg.hidden_layers_sizes)
self.extra_layers_sizes = cfg.extra_layers_sizes
self.x = T.tensor4('x')
self.extra_x = T.matrix('extra_x')
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.x
else:
input = self.layers[-1].output
config = cfg.conv_layer_configs[i]
conv_layer = ConvLayerForward(numpy_rng=numpy_rng,
input=input,
filter_shape=config['filter_shape'],
poolsize=config['poolsize'],
activation=config['activation'],
flatten=config['flatten'],
use_fast=use_fast)
self.layers.append(conv_layer)
self.conv_output_dim = config['output_shape'][1] * config[
'output_shape'][2] * config['output_shape'][3]
cfg.n_ins = config['output_shape'][1] * config['output_shape'][
2] * config['output_shape'][3]
for i in xrange(len(self.extra_layers_sizes)):
if i == 0:
input_size = 6400 * 5
input_size = cfg.extra_dim
layer_input = self.extra_x
else:
input_size = self.extra_layers_sizes[i - 1]
layer_input = self.extra_layers[-1].output
W = None
b = None
attend_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.extra_layers_sizes[i],
W=W,
b=b)
self.extra_layers.append(attend_layer)
self.extra_layers[-1].att_e_tl = self.extra_layers[-1].output
self.extra_layers[-1].att_a_tl = T.nnet.softmax(
self.extra_layers[-1].att_e_tl)
#self.extra_layers[-1].att_a_tl = T.exp(self.extra_layers[-1].att_e_tl)/(T.exp(self.extra_layers[-1].att_e_tl)).sum(0,keepdims=True)
for i in xrange(self.dnn_layer_num):
if i == 0:
#input_size = self.conv_output_dim
#layer_input = (self.extra_layers[-1].att_a_tl*self.layers[-1].output)
input_size = self.conv_output_dim + self.extra_layers_sizes[-1]
layer_input = T.join(1, self.extra_layers[-1].att_a_tl,
self.layers[-1].output)
else:
input_size = cfg.hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
W = None
b = None
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=cfg.hidden_layers_sizes[i],
W=W,
b=b)
self.layers.append(hidden_layer)
logLayer = OutputLayer(input=self.layers[-1].output,
n_in=cfg.hidden_layers_sizes[-1],
n_out=self.n_outs)
self.layers.append(logLayer)
def __init__(self,
numpy_rng,
theano_rng=None,
cfg=None,
testing=False,
input=None):
self.layers = []
self.extra_layers = []
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.conv_layers = []
self.cfg = cfg
self.conv_layer_configs = cfg.conv_layer_configs
self.conv_activation = cfg.conv_activation
self.use_fast = cfg.use_fast
self.extra_x = T.matrix('extra_x')
# 1.5 attention
self.extra_dim = cfg.extra_dim
print 'Extra input dimension: ' + str(cfg.extra_dim)
self.extra_layers_sizes = cfg.extra_layers_sizes
# 2. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.matrix('y')
#######################
# build cnn layers #
#######################
print '1. start to build cnn mag layer: ' + str(
self.conv_layer_configs)
self.conv_layer_num = len(self.conv_layer_configs)
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.x
else:
input = self.layers[-1].output
config = self.conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng,
input=input,
input_shape=config['input_shape'],
filter_shape=config['filter_shape'],
poolsize=config['poolsize'],
activation=self.conv_activation,
flatten=config['flatten'],
use_fast=self.use_fast,
testing=testing)
self.layers.append(conv_layer)
self.conv_layers.append(conv_layer)
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config[
'output_shape'][2] * config['output_shape'][3]
cfg.n_ins = config['output_shape'][1] * config['output_shape'][
2] * config['output_shape'][3]
#######################################
# build phase-based attention layer #
#######################################
# 0. phase-based attention
print '2. start to build attend layer: ' + str(self.extra_layers_sizes)
for i in xrange(len(self.extra_layers_sizes)):
if i == 0:
input_size = cfg.extra_dim
layer_input = self.extra_x
else:
input_size = self.extra_layers_sizes[i - 1]
layer_input = self.extra_layers[-1].output
W = None
b = None
attend_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.extra_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
print '\tbuild attend layer: ' + str(input_size) + ' x ' + str(
attend_layer.n_out)
self.extra_layers.append(attend_layer)
self.params.extend(attend_layer.params)
self.delta_params.extend(attend_layer.delta_params)
self.extra_output = self.extra_layers[-1].output
self.extra_output = T.nnet.softmax(self.extra_layers[-1].output)
#self.extra_output_rand = numpy.asarray(numpy_rng.uniform(
# low=-0.1,
# high=1.0,
# size=(32,20)), dtype=theano.config.floatX)
#self.extra_output = theano.shared(value=self.extra_output_rand, name='rand', borrow=True)
print '2. finish attend layer softmax(0): ' + str(
self.extra_layers[-1].n_out)
#######################################
# build dnnv #
#######################################
print '3. start to build dnnv layer: ' + str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
# construct the hidden layer
if i == 0:
# 1. Join two features (magnitude + phase)
input_size = self.conv_output_dim + self.extra_layers_sizes[-1]
layer_input = T.join(1, self.layers[-1].output,
self.extra_output)
# 2. Weighted Sum (magnitude * phase)
#input_size = self.conv_output_dim
#layer_input = self.layers[-1].output * self.extra_output
else:
input_size = self.hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
W = None
b = None
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
print '\tbuild dnnv layer: ' + str(input_size) + ' x ' + str(
hidden_layer.n_out)
# add the layer to our list of layers
self.layers.append(hidden_layer)
self.params.extend(hidden_layer.params)
self.delta_params.extend(hidden_layer.delta_params)
print '3. finish dnnv layer: ' + str(self.layers[-1].n_out)
#######################################
# build logistic regression layer #
#######################################
print '4. start to build log layer: 1'
# We now need to add a logistic layer on top of the MLP
self.logLayer = OutputLayer(input=self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
print '\tbuild final layer: ' + str(
self.layers[-1].n_out) + ' x ' + str(self.n_outs)
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
print '4. finish log layer: ' + str(self.layers[-1].n_out)
print 'Total layers: ' + str(len(self.layers))
self.finetune_cost = self.logLayer.l2(self.y)
self.errors = self.logLayer.errors(self.y)
sys.stdout.flush()
def __init__(self, numpy_rng, theano_rng=None, cfg = None, testing = False, input = None):
self.cfg = cfg
self.params = []
self.delta_params = []
self.n_ins = cfg.n_ins; self.n_outs = cfg.n_outs
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.do_maxout = cfg.do_maxout; self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.layers = []
self.conv_layers = []
self.lstm_layers = []
self.fc_layers = []
# 1. conv
self.conv_layer_configs = cfg.conv_layer_configs
self.conv_activation = cfg.conv_activation
self.conv_layers_number = len(self.conv_layer_configs)
self.use_fast = cfg.use_fast
# 2. lstm
self.lstm_layers_sizes = cfg.lstm_layers_sizes
self.lstm_layers_number = len(self.lstm_layers_sizes)
# 3. dnn
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
if input == None:
self.x = T.matrix('x')
else:
self.x = input
self.y = T.matrix('y')
#######################
# build conv layers #
#######################
print '1. start to build conv layer: '+ str(self.conv_layers_number)
for i in xrange(self.conv_layers_number):
if i == 0:
input = self.x
else:
input = self.conv_layers[-1].output
config = self.conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng, input=input,
input_shape = config['input_shape'],
filter_shape = config['filter_shape'],
poolsize = config['poolsize'],
activation = self.conv_activation,
flatten = config['flatten'],
use_fast = self.use_fast, testing = testing)
print '\tbuild conv layer: ' +str(config['input_shape'])
self.layers.append(conv_layer)
self.conv_layers.append(conv_layer)
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config['output_shape'][2] * config['output_shape'][3]
print '\t cnn out: '+ str(self.conv_output_dim)
cfg.n_ins = config['output_shape'][1] * config['output_shape'][2] * config['output_shape'][3]
print '1. finish conv layer: '+ str(self.layers[-1].n_out)
#######################
# build lstm layers #
#######################
print '2. start to build lstm layer: '+ str(self.lstm_layers_number)
for i in xrange(self.lstm_layers_number):
if i == 0:
input_size = self.conv_output_dim
input = self.layers[-1].output
else:
input_size = self.lstm_layers_sizes[i - 1]
input = self.layers[-1].output
print 'build lstm layer: ' + str(input_size)
lstm_layer = LSTMLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.lstm_layers_sizes[i])
print '\tbuild lstm layer: ' + str(input_size) +' x '+ str(lstm_layer.n_out)
self.layers.append(lstm_layer)
self.lstm_layers.append(lstm_layer)
self.params.extend(lstm_layer.params)
self.delta_params.extend(lstm_layer.delta_params)
print '2. finish lstm layer: '+ str(self.layers[-1].n_out)
#######################
# build dnnv layers #
#######################
print '3. start to build dnnv layer: '+ str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
if i == 0:
input_size = self.layers[-1].n_out
else:
input_size = self.hidden_layers_sizes[i - 1]
input = self.layers[-1].output
fc_layer = HiddenLayer(rng=numpy_rng, input=input, n_in=input_size, n_out=self.hidden_layers_sizes[i])
print '\tbuild dnnv layer: ' + str(input_size) +' x '+ str(fc_layer.n_out)
self.layers.append(fc_layer)
self.fc_layers.append(fc_layer)
self.params.extend(fc_layer.params)
self.delta_params.extend(fc_layer.delta_params)
print '3. finish dnnv layer: '+ str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '4. start to build log layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
logLayer = OutputLayer(input=input, n_in=input_size, n_out=self.n_outs)
print '\tbuild final layer: ' + str(input_size) +' x '+ str(fc_layer.n_out)
self.layers.append(logLayer)
self.params.extend(logLayer.params)
self.delta_params.extend(logLayer.delta_params)
print '4. finish log layer: '+ str(self.layers[-1].n_out)
print 'Total layers: '+ str(len(self.layers))
sys.stdout.flush()
self.finetune_cost = self.layers[-1].l2(self.y)
self.errors = self.layers[-1].errors(self.y)
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng,
theano_rng=None,
upper_hidden_layers_sizes=[500, 500],
n_outs=10,
tower1_hidden_layers_sizes=[500, 500],
tower1_n_ins=100,
tower2_hidden_layers_sizes=[500, 500],
tower2_n_ins=100,
activation=T.nnet.sigmoid,
do_maxout=False,
pool_size=1,
do_pnorm=False,
pnorm_order=1,
max_col_norm=None,
l1_reg=None,
l2_reg=None):
self.tower1_layers = []
self.tower2_layers = []
self.upper_layers = []
self.params = []
self.delta_params = []
self.max_col_norm = max_col_norm
self.l1_reg = l1_reg
self.l2_reg = l2_reg
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
self.tower1_input = self.x[:, 0:tower1_n_ins]
self.tower2_input = self.x[:,
tower1_n_ins:(tower1_n_ins + tower2_n_ins)]
# build tower1
for i in xrange(len(tower1_hidden_layers_sizes)):
if i == 0:
input_size = tower1_n_ins
layer_input = self.tower1_input
else:
input_size = tower1_hidden_layers_sizes[i - 1]
layer_input = self.tower1_layers[-1].output
layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=tower1_hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.tower1_layers.append(layer)
self.params.extend(layer.params)
self.delta_params.extend(layer.delta_params)
# build tower2
for i in xrange(len(tower2_hidden_layers_sizes)):
if i == 0:
input_size = tower2_n_ins
layer_input = self.tower2_input
else:
input_size = tower2_hidden_layers_sizes[i - 1]
layer_input = self.tower2_layers[-1].output
layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=tower2_hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.tower2_layers.append(layer)
self.params.extend(layer.params)
self.delta_params.extend(layer.delta_params)
for i in xrange(len(upper_hidden_layers_sizes)):
# construct the sigmoidal layer
if i == 0:
input_size = tower1_hidden_layers_sizes[
-1] + tower2_hidden_layers_sizes[-1]
layer_input = T.concatenate([
self.tower1_layers[-1].output,
self.tower2_layers[-1].output
],
axis=1)
else:
input_size = upper_hidden_layers_sizes[i - 1]
layer_input = self.upper_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=upper_hidden_layers_sizes[i],
activation=activation)
# add the layer to our list of layers
self.upper_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(input=self.upper_layers[-1].output,
n_in=upper_hidden_layers_sizes[-1],
n_out=n_outs)
self.upper_layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
def __init__(
self,
task_id,
numpy_rng,
theano_rng=None,
cfg=None, # the network configuration
dnn_shared=None,
shared_layers=[],
input=None):
self.layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
self.non_updated_layers = cfg.non_updated_layers
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
if input == None:
self.x = T.matrix('x')
else:
self.x = input
if task_id == 0:
self.y = T.ivector('y')
else:
self.y = T.matrix('y')
#######################
# build dnnv layers #
#######################
print "=============="
print "Task ID: %d" % (task_id)
print "=============="
print '1. start to build dnn layer: ' + str(self.hidden_layers_number)
for i in xrange(self.hidden_layers_number):
if i == 0:
input_size = self.n_ins
input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
input = self.layers[-1].output
W = None
b = None
if (i in shared_layers):
print "shared layer = %d" % (i)
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
hidden_layer = HiddenLayer(rng=numpy_rng,
input=input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation)
print '\tbuild lstm layer: ' + str(input_size) + ' x ' + str(
hidden_layer.n_out)
self.layers.append(hidden_layer)
self.params.extend(hidden_layer.params)
self.delta_params.extend(hidden_layer.delta_params)
print '1. finish dnnv layer: ' + str(self.layers[-1].n_out)
#######################
# build log layers #
#######################
print '2. start to build final layer: 1'
input_size = self.layers[-1].n_out
input = self.layers[-1].output
if task_id == 0:
self.logLayer = LogisticRegression(
input=self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
print '\tbuild final layer (classification): ' + str(
input_size) + ' x ' + str(self.logLayer.n_out)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
else:
self.logLayer = OutputLayer(input=input,
n_in=input_size,
n_out=self.n_outs)
print '\tbuild final layer (regression): ' + str(
input_size) + ' x ' + str(self.logLayer.n_out)
self.finetune_cost = self.logLayer.l2(self.y)
self.errors = self.logLayer.errors(self.y)
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
print '2. finish log layer: ' + str(self.layers[-1].n_out)
print 'Total layers: ' + str(len(self.layers))
sys.stdout.flush()
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10,
corruption_levels=[0.1, 0.1],
pool_size=3,
sparsity=None,
sparsity_weight=None,
first_reconstruct_activation=T.tanh):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i] *
pool_size,
activation=(lambda x: 1.0 * x),
do_maxout=True,
pool_size=pool_size)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
# Construct a denoising autoencoder that shared weights with this layer
if i == 0:
reconstruct_activation = first_reconstruct_activation
else:
reconstruct_activation = (lambda x: 1.0 * x)
# reconstruct_activation = first_reconstruct_activation
dA_layer = dA_maxout(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i] * pool_size,
W=sigmoid_layer.W,
bhid=sigmoid_layer.b,
sparsity=sparsity,
sparsity_weight=sparsity_weight,
pool_size=pool_size,
reconstruct_activation=reconstruct_activation)
self.dA_layers.append(dA_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.sigmoid_layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
def __init__(self,
numpy_rng,
theano_rng=None,
cfg=None,
dnn_shared=None,
shared_layers=[]):
self.layers = []
self.dropout_layers = []
self.params = []
self.delta_params = []
self.cfg = cfg
self.n_ins = cfg.n_ins
self.n_outs = cfg.n_outs
self.hidden_layers_sizes = cfg.hidden_layers_sizes
self.hidden_layers_number = len(self.hidden_layers_sizes)
self.activation = cfg.activation
self.do_maxout = cfg.do_maxout
self.pool_size = cfg.pool_size
self.input_dropout_factor = cfg.input_dropout_factor
self.dropout_factor = cfg.dropout_factor
self.max_col_norm = cfg.max_col_norm
self.l1_reg = cfg.l1_reg
self.l2_reg = cfg.l2_reg
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.hidden_layers_number):
# construct the hidden layer
if i == 0:
input_size = self.n_ins
layer_input = self.x
if self.input_dropout_factor > 0.0:
dropout_layer_input = _dropout_from_layer(
theano_rng, self.x, self.input_dropout_factor)
else:
dropout_layer_input = self.x
else:
input_size = self.hidden_layers_sizes[i - 1]
layer_input = (
1 - self.dropout_factor[i - 1]) * self.layers[-1].output
dropout_layer_input = self.dropout_layers[-1].dropout_output
W = None
b = None
if (i in shared_layers):
W = dnn_shared.layers[i].W
b = dnn_shared.layers[i].b
if self.do_maxout == False:
dropout_layer = DropoutHiddenLayer(
rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
W=W,
b=b,
activation=self.activation,
dropout_factor=self.dropout_factor[i])
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
activation=self.activation,
W=dropout_layer.W,
b=dropout_layer.b)
else:
dropout_layer = DropoutHiddenLayer(
rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] * self.pool_size,
W=W,
b=b,
activation=(lambda x: 1.0 * x),
dropout_factor=self.dropout_factor[i],
do_maxout=True,
pool_size=self.pool_size)
hidden_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i] *
self.pool_size,
activation=(lambda x: 1.0 * x),
W=dropout_layer.W,
b=dropout_layer.b,
do_maxout=True,
pool_size=self.pool_size)
# add the layer to our list of layers
self.layers.append(hidden_layer)
self.dropout_layers.append(dropout_layer)
self.params.extend(dropout_layer.params)
self.delta_params.extend(dropout_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.dropout_logLayer = LogisticRegression(
input=self.dropout_layers[-1].dropout_output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs)
self.logLayer = LogisticRegression(
input=(1 - self.dropout_factor[-1]) * self.layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_outs,
W=self.dropout_logLayer.W,
b=self.dropout_logLayer.b)
self.dropout_layers.append(self.dropout_logLayer)
self.layers.append(self.logLayer)
self.params.extend(self.dropout_logLayer.params)
self.delta_params.extend(self.dropout_logLayer.delta_params)
# compute the cost
self.finetune_cost = self.dropout_logLayer.negative_log_likelihood(
self.y)
self.errors = self.logLayer.errors(self.y)
if self.l1_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l1_reg * (abs(W).sum())
if self.l2_reg is not None:
for i in xrange(self.hidden_layers_number):
W = self.layers[i].W
self.finetune_cost += self.l2_reg * T.sqr(W).sum()
def __init__(self, numpy_rng, theano_rng=None,
batch_size = 256, n_outs=500,
sparsity = None, sparsity_weight = None, sparse_layer = 3,
conv_layer_configs = [],
hidden_layers_sizes=[500, 500],
conv_activation = T.nnet.sigmoid,
full_activation = T.nnet.sigmoid,
use_fast = False):
self.layers = []
self.params = []
self.delta_params = []
self.sparsity = sparsity
self.sparsity_weight = sparsity_weight
self.sparse_layer = sparse_layer
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
self.conv_layer_num = len(conv_layer_configs)
self.full_layer_num = len(hidden_layers_sizes)
for i in xrange(self.conv_layer_num):
if i == 0:
input = self.x
is_input_layer = True
else:
input = self.layers[-1].output
is_input_layer = False
config = conv_layer_configs[i]
conv_layer = ConvLayer(numpy_rng=numpy_rng, input=input, is_input_layer = is_input_layer,
input_shape = config['input_shape'], filter_shape = config['filter_shape'], poolsize = config['poolsize'],
activation = conv_activation, flatten = config['flatten'])
self.layers.append(conv_layer)
self.params.extend(conv_layer.params)
self.delta_params.extend(conv_layer.delta_params)
self.conv_output_dim = config['output_shape'][1] * config['output_shape'][2] * config['output_shape'][3]
for i in xrange(self.full_layer_num):
# construct the sigmoidal layer
if i == 0:
input_size = self.conv_output_dim
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = self.layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=full_activation)
# add the layer to our list of layers
self.layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
self.delta_params.extend(sigmoid_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.layers.append(self.logLayer)
self.params.extend(self.logLayer.params)
self.delta_params.extend(self.logLayer.delta_params)
if self.sparsity_weight is not None:
sparsity_level = T.extra_ops.repeat(self.sparsity, 630)
avg_act = self.sigmoid_layers[sparse_layer].output.mean(axis=0)
kl_div = self.kl_divergence(sparsity_level, avg_act)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) + self.sparsity_weight * kl_div.sum()
else:
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)