def _layer_stats(self, state_below, layer_output):
max_act = theano.max(self.activity)
min_act = theano.min(self.activity)
mean_act = theano.mean(self.activity)
max_thresh = theano.max(self.threshold)
min_thresh = theano.min(self.threshold)
mean_thresh = theano.mean(self.threshold)
ls1 = super(Noisy_RELU, self)._layer_stats(state_below, layer_output)
ls2 = [('max_activity', max_act),
('min_activity', min_act),
('mean_activity', mean_act),
('max_threshold', max_thresh),
('min_threshold', min_thresh),
('mean_threshold', mean_thresh)]
return ls1 + ls2
def _train_fprop(self, state_below):
if self.batch_count > self.num_batch:
return state_below * (state_below > self.threshold)
else:
self.batch_count += 1
state_below = state_below + theano_rand.normal(size=state_below.shape, std=self.std, dtype=floatX)
state_below = state_below * (state_below > self.threshold)
activity = theano.mean(state_below > 0, axis=0)
self.activity = self.alpha * activity + (1-self.alpha) * self.activity
self.threshold += self.threshold_lr * (self.activity - self.sparsity_factor)
return state_below * (state_below > self.threshold)
def _train_fprop(self, state_below):
if self.batch_count > self.num_batch:
return state_below * (state_below > self.threshold)
else:
self.batch_count += 1
state_below = state_below + theano_rand.normal(size=state_below.shape, std=self.std, dtype=floatX)
state_below = state_below * (state_below > self.threshold)
activity = theano.mean(state_below > 0, axis=0)
self.activity = self.alpha * activity + (1-self.alpha) * self.activity
self.threshold += self.threshold_lr * (self.activity - self.sparsity_factor)
return state_below * (state_below > self.threshold)
def _train_fprop(self, state_below):
output = super(Noisy_RELU, self)._train_fprop(state_below)
if self.batch_count > self.num_batch:
return output * (output > self.threshold)
else:
self.batch_count += 1
output = output + theano_rand.normal(size=output.shape, std=self.std, dtype=floatX)
output = output * (output > self.threshold)
activity = theano.mean(output > 0, axis=0)
self.activity = self.alpha * activity + (1-self.alpha) * self.activity
self.threshold += self.threshold_lr * (self.activity - self.sparsity_factor)
return output * (output > self.threshold)
def errors(self, y):
"""Return a float representing the number of errors in the minibatch over the total number of examples of the minibatch ; zero one
loss over the size of the minibatch
:type y: theano.tensor.TensorType
:param y: corresponds to a vector that gives for each example the
correct label
"""
# check if y has same dimension of y_pred
if y.ndim != self.y_pred.ndim: raise TypeError(
'y should have the same shape as self.y_pred',('y', y.type, 'y_pred', self.y_pred.type)
)
# check if y is of the correct datatype
if y.dtype.startswith('int'):
# the T.neq operator returns a vector of 0s and 1s, where 1 # represents a mistake in prediction
return theano.mean(theano.neq(self.y_pred, y))
else:
raise NotImplementedError()
