def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10,
activation = T.nnet.sigmoid, input_dropout_factor = 0,
dropout_factor = [0.2,0.2,0.2,0.2,0.2,0.2,0.2],
adv_activation = None, max_col_norm = None,
l1_reg = None, l2_reg = None):
super(DNN_Dropout, self).__init__()
self.layers = []
self.dropout_layers = []
self.n_layers = len(hidden_layers_sizes)
self.max_col_norm = max_col_norm
self.l1_reg = l1_reg
self.l2_reg = l2_reg
self.input_dropout_factor = input_dropout_factor
self.dropout_factor = dropout_factor
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.n_layers):
# construct the sigmoidal layer
if i == 0:
input_size = n_ins
layer_input = self.x
if input_dropout_factor > 0.0:
dropout_layer_input = _dropout_from_layer(theano_rng, self.x, input_dropout_factor)
else:
dropout_layer_input = self.x
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = (1 - self.dropout_factor[i - 1]) * self.layers[-1].output
dropout_layer_input = self.dropout_layers[-1].dropout_output
if not adv_activation is None:
dropout_layer = DropoutHiddenLayer(rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i] * adv_activation['pool_size'],
activation= activation,
adv_activation_method = adv_activation['method'],
pool_size = adv_activation['pool_size'],
pnorm_order = adv_activation['pnorm_order'],
dropout_factor=self.dropout_factor[i])
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i] * adv_activation['pool_size'],
activation=activation,
adv_activation_method = adv_activation['method'],
pool_size = adv_activation['pool_size'],
pnorm_order = adv_activation['pnorm_order'],
W=dropout_layer.W, b=dropout_layer.b)
else:
dropout_layer = DropoutHiddenLayer(rng=numpy_rng,
input=dropout_layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation= activation,
dropout_factor=self.dropout_factor[i])
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i] ,
activation= activation,
W=dropout_layer.W, b=dropout_layer.b)
# add the layer to our list of layers
self.layers.append(sigmoid_layer)
self.dropout_layers.append(dropout_layer)
self.params.extend(dropout_layer.params)
self.delta_params.extend(dropout_layer.delta_params)
# We now need to add a logistic layer on top of the MLP
self.dropout_logLayer = LogisticRegression(
input=self.dropout_layers[-1].dropout_output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.logLayer = LogisticRegression(
input=(1 - self.dropout_factor[-1]) * self.layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs,
W=self.dropout_logLayer.W, b=self.dropout_logLayer.b)
self.dropout_layers.append(self.dropout_logLayer)
self.layers.append(self.logLayer)
self.params.extend(self.dropout_logLayer.params)
self.delta_params.extend(self.dropout_logLayer.delta_params)
# compute the cost
self.finetune_cost = self.dropout_logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
self.output = self.logLayer.prediction();
self.features = self.layers[-2].output;
self.features_dim = self.layers[-2].n_out
if self.l1_reg is not None:
self.__l1Regularization__();
if self.l2_reg is not None:
self.__l2Regularization__();
def __init__(self, numpy_rng, theano_rng, batch_size, n_outs,conv_layer_configs, hidden_layer_configs,
use_fast=False,hidden_activation = T.nnet.sigmoid,l1_reg=None,l2_reg=None,
max_col_norm=None,input_dropout_factor=0.0):
super(DropoutCNN, self).__init__(conv_layer_configs,hidden_layer_configs,l1_reg,l2_reg,max_col_norm)
self.input_dropout_factor = input_dropout_factor;
self.dropout_layers = []
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
for i in xrange(self.conv_layer_num): # construct the convolution layer
if i == 0: #is_input layer
conv_input = self.x
if self.input_dropout_factor > 0.0:
dropout_conv_input = _dropout_from_layer(theano_rng, self.x,self.input_dropout_factor)
else:
dropout_conv_input = self.x;
else:
conv_input = (1-conv_layer_configs[i-1]['dropout_factor'])*self.layers[-1].output #output of previous layer
dropout_conv_input = self.dropout_layers[-1].dropout_output;
config = conv_layer_configs[i]
conv_activation= parse_activation(config['activation'])
dropout_conv_layer = DropoutConvLayer(numpy_rng=numpy_rng, input=dropout_conv_input,
input_shape=config['input_shape'],filter_shape=config['filter_shape'],poolsize=config['poolsize'],
activation = conv_activation, use_fast = use_fast,dropout_factor=conv_layer_configs[i]['dropout_factor'])
conv_layer = ConvLayer(numpy_rng=numpy_rng, input=conv_input,input_shape=config['input_shape'],
filter_shape=config['filter_shape'],poolsize=config['poolsize'],activation = conv_activation,
use_fast = use_fast, W = dropout_conv_layer.W, b = dropout_conv_layer.b)
self.dropout_layers.append(dropout_conv_layer);
self.layers.append(conv_layer)
self.conv_layers.append(conv_layer)
if config['update']==True: # only few layers of convolution layer are considered for updation
self.params.extend(dropout_conv_layer.params)
self.delta_params.extend(dropout_conv_layer.delta_params)
hidden_layers = hidden_layer_configs['hidden_layers'];
self.conv_output_dim = config['output_shape'][1] * config['output_shape'][2] * config['output_shape'][3]
adv_activation_configs = hidden_layer_configs['adv_activation']
#flattening the last convolution output layer
self.dropout_features = self.dropout_layers[-1].dropout_output.flatten(2);
self.features = self.conv_layers[-1].output.flatten(2);
self.features_dim = self.conv_output_dim;
self.dropout_layers = [];
self.dropout_factor = hidden_layer_configs['dropout_factor'];
for i in xrange(self.hidden_layer_num): # construct the hidden layer
if i == 0: # is first sigmoidal layer
input_size = self.conv_output_dim
dropout_layer_input = self.dropout_features
layer_input = self.features
else:
input_size = hidden_layers[i - 1] # number of hidden neurons in previous layers
dropout_layer_input = self.dropout_layers[-1].dropout_output
layer_input = (1 - self.dropout_factor[i-1]) * self.layers[-1].output
if adv_activation_configs is None:
dropout_sigmoid_layer = DropoutHiddenLayer(rng=numpy_rng, input=layer_input,n_in=input_size,
n_out = hidden_layers[i], activation=hidden_activation,
dropout_factor = self.dropout_factor[i]);
sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input,n_in=input_size,
n_out = hidden_layers[i], activation=hidden_activation,
W=dropout_sigmoid_layer.W, b=dropout_sigmoid_layer.b);
else:
dropout_sigmoid_layer = DropoutHiddenLayer(rng=numpy_rng, input=layer_input,n_in=input_size,
n_out = hidden_layers[i]*adv_activation_configs['pool_size'], activation=hidden_activation,
adv_activation_method = adv_activation_configs['method'],
pool_size = adv_activation_configs['pool_size'],
pnorm_order = adv_activation_configs['pnorm_order'],
dropout_factor = self.dropout_factor[i]);
sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input,n_in=input_size,
n_out = hidden_layers[i]*adv_activation_configs['pool_size'], activation=hidden_activation,
adv_activation_method = adv_activation_configs['method'],
pool_size = adv_activation_configs['pool_size'],
pnorm_order = adv_activation_configs['pnorm_order'],
W=dropout_sigmoid_layer.W, b=dropout_sigmoid_layer.b);
self.layers.append(sigmoid_layer)
self.dropout_layers.append(dropout_sigmoid_layer)
self.mlp_layers.append(sigmoid_layer)
if config['update']==True: # only few layers of hidden layer are considered for updation
self.params.extend(dropout_sigmoid_layer.params)
self.delta_params.extend(dropout_sigmoid_layer.delta_params)
self.dropout_logLayer = LogisticRegression(input=self.dropout_layers[-1].dropout_output,n_in=hidden_layers[-1],n_out=n_outs)
self.logLayer = LogisticRegression(
input=(1 - self.dropout_factor[-1]) * self.layers[-1].output,
n_in=hidden_layers[-1],n_out=n_outs,
W=self.dropout_logLayer.W, b=self.dropout_logLayer.b)
self.dropout_layers.append(self.dropout_logLayer)
self.layers.append(self.logLayer)
self.params.extend(self.dropout_logLayer.params)
self.delta_params.extend(self.dropout_logLayer.delta_params)
self.finetune_cost = self.dropout_logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)
self.output = self.logLayer.prediction()
#regularization
if self.l1_reg is not None:
self.__l1Regularization__(self.hidden_layer_num*2);
if self.l2_reg is not None:
self.__l2Regularization__(self.hidden_layer_num*2);
