def test_shuffle_row_elements(self):
"""Test that RandomStreams.shuffle_row_elements generates the right results"""
# Check over two calls to see if the random state is correctly updated.
# On matrices, for each row, the elements of that row should be shuffled.
# Note that this differs from numpy.random.shuffle, where all the elements
# of the matrix are shuffled.
random = RandomStreams(utt.fetch_seed())
m_input = tensor.dmatrix()
f = function([m_input],
random.shuffle_row_elements(m_input),
updates=random.updates())
# Generate the elements to be shuffled
val_rng = numpy.random.RandomState(utt.fetch_seed() + 42)
in_mval = val_rng.uniform(-2, 2, size=(20, 5))
fn_mval0 = f(in_mval)
fn_mval1 = f(in_mval)
print(in_mval[0])
print(fn_mval0[0])
print(fn_mval1[0])
assert not numpy.all(in_mval == fn_mval0)
assert not numpy.all(in_mval == fn_mval1)
assert not numpy.all(fn_mval0 == fn_mval1)
rng_seed = numpy.random.RandomState(utt.fetch_seed()).randint(2**30)
rng = numpy.random.RandomState(int(rng_seed))
numpy_mval0 = in_mval.copy()
numpy_mval1 = in_mval.copy()
for row in numpy_mval0:
rng.shuffle(row)
for row in numpy_mval1:
rng.shuffle(row)
assert numpy.all(numpy_mval0 == fn_mval0)
assert numpy.all(numpy_mval1 == fn_mval1)
# On vectors, the behaviour is the same as numpy.random.shuffle,
# except that it does not work in place, but returns a shuffled vector.
random1 = RandomStreams(utt.fetch_seed())
v_input = tensor.dvector()
f1 = function([v_input], random1.shuffle_row_elements(v_input))
in_vval = val_rng.uniform(-3, 3, size=(12, ))
fn_vval = f1(in_vval)
numpy_vval = in_vval.copy()
vrng = numpy.random.RandomState(int(rng_seed))
vrng.shuffle(numpy_vval)
print(in_vval)
print(fn_vval)
print(numpy_vval)
assert numpy.all(numpy_vval == fn_vval)
# Trying to shuffle a vector with function that should shuffle
# matrices, or vice versa, raises a TypeError
self.assertRaises(TypeError, f1, in_mval)
self.assertRaises(TypeError, f, in_vval)
def test_shuffle_row_elements(self):
"""Test that RandomStreams.shuffle_row_elements generates the right results"""
# Check over two calls to see if the random state is correctly updated.
# On matrices, for each row, the elements of that row should be shuffled.
# Note that this differs from numpy.random.shuffle, where all the elements
# of the matrix are shuffled.
random = RandomStreams(utt.fetch_seed())
m_input = tensor.dmatrix()
f = function([m_input], random.shuffle_row_elements(m_input), updates=random.updates())
# Generate the elements to be shuffled
val_rng = numpy.random.RandomState(utt.fetch_seed()+42)
in_mval = val_rng.uniform(-2, 2, size=(20, 5))
fn_mval0 = f(in_mval)
fn_mval1 = f(in_mval)
print(in_mval[0])
print(fn_mval0[0])
print(fn_mval1[0])
assert not numpy.all(in_mval == fn_mval0)
assert not numpy.all(in_mval == fn_mval1)
assert not numpy.all(fn_mval0 == fn_mval1)
rng_seed = numpy.random.RandomState(utt.fetch_seed()).randint(2**30)
rng = numpy.random.RandomState(int(rng_seed))
numpy_mval0 = in_mval.copy()
numpy_mval1 = in_mval.copy()
for row in numpy_mval0:
rng.shuffle(row)
for row in numpy_mval1:
rng.shuffle(row)
assert numpy.all(numpy_mval0 == fn_mval0)
assert numpy.all(numpy_mval1 == fn_mval1)
# On vectors, the behaviour is the same as numpy.random.shuffle,
# except that it does not work in place, but returns a shuffled vector.
random1 = RandomStreams(utt.fetch_seed())
v_input = tensor.dvector()
f1 = function([v_input], random1.shuffle_row_elements(v_input))
in_vval = val_rng.uniform(-3, 3, size=(12,))
fn_vval = f1(in_vval)
numpy_vval = in_vval.copy()
vrng = numpy.random.RandomState(int(rng_seed))
vrng.shuffle(numpy_vval)
print(in_vval)
print(fn_vval)
print(numpy_vval)
assert numpy.all(numpy_vval == fn_vval)
# Trying to shuffle a vector with function that should shuffle
# matrices, or vice versa, raises a TypeError
self.assertRaises(TypeError, f1, in_mval)
self.assertRaises(TypeError, f, in_vval)
def call(self, x, **kwargs):
from theano import tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
if K.backend() == "theano":
import theano
mask_rng = RandomStreams(self.seed)
ints = mask_rng.random_integers(size=K.expand_dims(x.shape[0], 0),
high=x.shape[1] - 1)
def set_value_at_position(i, ns_x):
zeros = T.zeros_like(ns_x[0, :])
return T.set_subtensor(zeros[:i], 1)
result, updates = theano.scan(fn=set_value_at_position,
outputs_info=None,
sequences=ints,
non_sequences=x)
mask = mask_rng.shuffle_row_elements(result)
elif K.backend() == "tensorflow":
import tensorflow as tf
tf.set_random_seed(self.seed)
ints = tf.random_uniform(shape=K.expand_dims(tf.shape(x)[0], 0),
maxval=x.shape[1],
dtype=tf.int32)
result = tf.sequence_mask(ints, maxlen=x.shape[1])
parallel_iterations = self._deterministic and 1 or 10
mask = tf.cast(
tf.map_fn(tf.random_shuffle,
result,
parallel_iterations=parallel_iterations), K.floatx())
else:
raise NotImplementedError()
return K.concatenate([x * mask, mask])
class MaskGenerator(object):
def __init__(self, input_size, hidden_sizes, l, random_seed=1234):
self._random_seed = random_seed
self._mrng = MRG_RandomStreams(seed=random_seed)
self._rng = RandomStreams(seed=random_seed)
self._hidden_sizes = hidden_sizes
self._input_size = input_size
self._l = l
self.ordering = theano.shared(value=np.arange(
input_size, dtype=theano.config.floatX),
name='ordering',
borrow=False)
# Initial layer connectivity
self.layers_connectivity = [
theano.shared(value=(self.ordering + 1).eval(),
name='layer_connectivity_input',
borrow=False)
]
for i in range(len(self._hidden_sizes)):
self.layers_connectivity += [
theano.shared(value=np.zeros((self._hidden_sizes[i]),
dtype=theano.config.floatX),
name='layer_connectivity_hidden{0}'.format(i),
borrow=False)
]
self.layers_connectivity += [self.ordering]
## Theano functions
new_ordering = self._rng.shuffle_row_elements(self.ordering)
self.shuffle_ordering = theano.function(
name='shuffle_ordering',
inputs=[],
updates=[(self.ordering, new_ordering),
(self.layers_connectivity[0], new_ordering + 1)])
self.layers_connectivity_updates = []
for i in range(len(self._hidden_sizes)):
self.layers_connectivity_updates += [
self._get_hidden_layer_connectivity(i)
]
# self.layers_connectivity_updates = [self._get_hidden_layer_connectivity(i) for i in range(len(self._hidden_sizes))] # WTF THIS DO NOT WORK
self.sample_connectivity = theano.function(
name='sample_connectivity',
inputs=[],
updates=[(self.layers_connectivity[i + 1],
self.layers_connectivity_updates[i])
for i in range(len(self._hidden_sizes))])
# Save random initial state
self._initial_mrng_rstate = copy.deepcopy(self._mrng.rstate)
self._initial_mrng_state_updates = [
state_update[0].get_value()
for state_update in self._mrng.state_updates
]
# Ensuring valid initial connectivity
self.sample_connectivity()
def reset(self):
# Set Original ordering
self.ordering.set_value(
np.arange(self._input_size, dtype=theano.config.floatX))
# Reset RandomStreams
self._rng.seed(self._random_seed)
# Initial layer connectivity
self.layers_connectivity[0].set_value((self.ordering + 1).eval())
for i in range(1, len(self.layers_connectivity) - 1):
self.layers_connectivity[i].set_value(
np.zeros((self._hidden_sizes[i - 1]),
dtype=theano.config.floatX))
self.layers_connectivity[-1].set_value(self.ordering.get_value())
# Reset MRG_RandomStreams (GPU)
self._mrng.rstate = self._initial_mrng_rstate
for state, value in zip(self._mrng.state_updates,
self._initial_mrng_state_updates):
state[0].set_value(value)
self.sample_connectivity()
def _get_p(self, start_choice):
start_choice_idx = (start_choice - 1).astype('int32')
p_vals = T.concatenate([
T.zeros((start_choice_idx, )),
T.nnet.nnet.softmax(self._l * T.arange(
start_choice, self._input_size, dtype=theano.config.floatX))[0]
])
p_vals = T.inc_subtensor(
p_vals[start_choice_idx], 1.
) # Stupid hack because de multinomial does not contain a safety for numerical imprecision.
return p_vals
def _get_hidden_layer_connectivity(self, layerIdx):
layer_size = self._hidden_sizes[layerIdx]
if layerIdx == 0:
p_vals = self._get_p(T.min(self.layers_connectivity[layerIdx]))
else:
p_vals = self._get_p(
T.min(self.layers_connectivity_updates[layerIdx - 1]))
# #Implementations of np.choose in theano GPU
# return T.nonzero(self._mrng.multinomial(pvals=[self._p_vals] * layer_size, dtype=theano.config.floatX))[1].astype(dtype=theano.config.floatX)
# return T.argmax(self._mrng.multinomial(pvals=[self._p_vals] * layer_size, dtype=theano.config.floatX), axis=1)
return T.sum(T.cumsum(self._mrng.multinomial(
pvals=T.tile(p_vals[::-1][None, :], (layer_size, 1)),
dtype=theano.config.floatX),
axis=1),
axis=1)
def _get_mask(self, layerIdxIn, layerIdxOut):
return (self.layers_connectivity[layerIdxIn][:, None] <=
self.layers_connectivity[layerIdxOut][None, :]).astype(
theano.config.floatX)
def get_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, layerIdx + 1)
def get_direct_input_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(0, layerIdx)
def get_direct_output_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, -1)
def run(self):
worker = Worker(self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.stride,
self.img_size,
self.classifier,
self.n_train_files,
self.n_test_files,
self.samples_per_image,
self.on_ratio,
self.directory_input,
self.directory_labels,
self.membrane_edges,
self.layers_3D,
self.pre_processed_folder,
self.batch_size,
self.num_kernels,
self.kernel_sizes,
self.maxoutsize,
self.params,
self.eval_window_size,
config_file,
self.n_validation_samples)
if self.pre_process == True:
print "Generating Train/Test Set..."
worker.generate_train_data()
data,n_train_samples = worker.get_train_data()
# Load weight layers
self.load_layers(self.load_n_layers)
print 'Loading data ...'
if self.predict_only == False:
train_set_x,train_set_y = data[0],data[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
valid_set_x,valid_set_y = data[1],data[3]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
print 'Initializing neural network ...'
# print error if batch size is to large
if valid_set_y.eval().size<self.batch_size:
print 'Error: Batch size is larger than size of validation set.'
print 'Batch size: ',self.batch_size
n_train_batches /= self.batch_size
n_valid_batches /= self.batch_size
# symbolic variables
x = T.matrix('x') # input image data
y = T.matrix('y') # input label data
# Initialize networks
if self.net == 'ConvNet':
model = ConvNet(rng,
self.batch_size,
self.layers_3D,
self.num_kernels,
self.kernel_sizes,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
maxoutsize = self.maxoutsize,
params = self.params,
dropout = self.dropout)
model_val = model.TestVersion(rng,
self.batch_size,
self.layers_3D,
self.num_kernels,
self.kernel_sizes,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
maxoutsize = self.maxoutsize,
params = self.params,
network = model,
dropout = [0.,0.,0.,0.0])
elif self.net == "FullyCon":
model = FullyCon(rng,
self.batch_size,
self.layers_3D,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
params = self.params)
model_val= model.TestVersion(rng,
self.batch_size,
self.layers_3D,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
params = self.params,
network = model)
elif self.net == "FullyConCompressed":
model = FullyConCompressed(rng,
self.batch_size,
self.layers_3D,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
params = self.params)
model_val= model.TestVersion(rng,
self.batch_size,
self.layers_3D,
x,
y,
self.in_window_shape,
self.out_window_shape,
self.pred_window_size,
self.classifier,
params = self.params,
network = model)
else:
raise RuntimeError('Unable to load network: ' + str(self.net))
# Initialize parameters and functions
cost = model.layer4.negative_log_likelihood(self.penalty_factor) # Cost function
self.params = model.params # List of parameters
grads = T.grad(cost, self.params) # Gradient
index = T.lscalar() # Index
# Intialize optimizera
optimizer = Optimizer()
updates = optimizer.init_optimizer(self.optimizer, cost, self.params, self.optimizerData)
srng = RandomStreams(seed=234)
perm = theano.shared(np.arange(train_set_x.eval().shape[0]))
# Train functions
if self.predict_only == False:
train_model = theano.function(
[index],
cost,
updates = updates,
givens = {
x: train_set_x[perm[index * self.batch_size: (index + 1) * self.batch_size]],
y: train_set_y[perm[index * self.batch_size: (index + 1) * self.batch_size]]
}
)
# Initialize result arrays
cost_results = []
val_results_pixel = []
time_results = []
predict_val = f.init_predict(valid_set_x, model_val,self.batch_size,x,index)
# Solver
try:
print '... Solving'
start_time = time.time()
for epoch in range(self.epochs):
t1 = time.time()
perm = srng.shuffle_row_elements(perm)
train_set_x,train_set_y = f.flip_rotate(train_set_x,
train_set_y,
self.in_window_shape,
self.out_window_shape,
perm,
index,
cost,
updates,
self.batch_size,
x,
y,
self.classifier,
self.layers_3D)
costs = [train_model(i) for i in xrange(n_train_batches)]
epoch_cost = np.mean(costs)
output_val = f.predict_set(predict_val,n_valid_batches,self.classifier, self.pred_window_size)
error_pixel,error_window = f.evaluate(output_val,valid_set_y.get_value(borrow=True),self.eval_window_size,self.classifier)
#error_pixel = 0.
#error_window = 0.
t2 = time.time()
epoch_time = (t2-t1)/60.
cost_results.append(epoch_cost)
val_results_pixel.append(error_pixel)
time_results.append(epoch_time)
# store parameters
self.save_params(self.get_params(), self.path)
if self.classifier in ["membrane","synapse"]:
print "Epoch {} Training Cost: {:.5} Validation Error (pixel/window): {:.5}/{:.5} Time (epoch/total): {:.2} mins".format(epoch + 1, epoch_cost, error_pixel,error_window, epoch_time)
else:
print "Epoch {} Training Cost: {:.5} Validation Error, Membrane (pixel/window): {:.5}/{:.5} Validation Error, Synapse (pixel/window): {:.5}/{:.5} Time (epoch/total): {:.2} mins".format(epoch + 1, epoch_cost, error_pixel[0],error_window[0],error_pixel[1],error_window[1], epoch_time)
except KeyboardInterrupt:
print 'Exiting solver ...'
print ''
# End timer
end_time = time.time()
end_epochs = epoch+1
try:
results = np.zeros((4, len(cost_results)))
results[0] = np.array(cost_results)
results[1] = np.array(val_results_pixel)
results[2] = np.array(time_results)
np.save(self.results_folder + 'results.npy', results)
except:
pass
(output,
y,
error_pixel_before,
error_window_before,
error_pixel_after,
error_window_after) = worker.generate_test_data(model,x,y,index,self.net)
print 'Error before averaging (pixel/window): ' + str(error_pixel_before) + "/" + str(error_window_before)
print 'Error after averaging (pixel/window): ' + str(error_pixel_after) + "/" + str(error_window_after)
# Save and write
self.write_results(error_pixel_before,error_window_before,error_pixel_after,error_window_after)
self.write_parameters(end_epochs,n_train_samples)
np.save(self.results_folder + 'output.npy', output)
np.save(self.results_folder + 'y.npy', y)
self.write_last_run(self.ID_folder)
make_set = True
if make_set == True:
from PIL import Image
set_folder = self.results_folder+"/set/"
if os.path.isdir(set_folder) != True:
os.makedirs(set_folder)
for n in xrange(output.shape[0]):
im = Image.fromarray(np.uint8(output[n]*255))
im.save(set_folder + "set_" + str(n) + ".tif")
class MaskGenerator(object):
def __init__(self, input_size, hidden_sizes, l, random_seed=1234):
self._random_seed = random_seed
self._mrng = MRG_RandomStreams(seed=random_seed)
self._rng = RandomStreams(seed=random_seed)
self._hidden_sizes = hidden_sizes
self._input_size = input_size
self._l = l
self.ordering = theano.shared(value=np.arange(input_size, dtype=theano.config.floatX), name='ordering', borrow=False)
# Initial layer connectivity
self.layers_connectivity = [theano.shared(value=(self.ordering + 1).eval(), name='layer_connectivity_input', borrow=False)]
for i in range(len(self._hidden_sizes)):
self.layers_connectivity += [theano.shared(value=np.zeros((self._hidden_sizes[i]), dtype=theano.config.floatX), name='layer_connectivity_hidden{0}'.format(i), borrow=False)]
self.layers_connectivity += [self.ordering]
## Theano functions
new_ordering = self._rng.shuffle_row_elements(self.ordering)
self.shuffle_ordering = theano.function(name='shuffle_ordering',
inputs=[],
updates=[(self.ordering, new_ordering), (self.layers_connectivity[0], new_ordering + 1)])
self.layers_connectivity_updates = []
for i in range(len(self._hidden_sizes)):
self.layers_connectivity_updates += [self._get_hidden_layer_connectivity(i)]
# self.layers_connectivity_updates = [self._get_hidden_layer_connectivity(i) for i in range(len(self._hidden_sizes))] # WTF THIS DO NOT WORK
self.sample_connectivity = theano.function(name='sample_connectivity',
inputs=[],
updates=[(self.layers_connectivity[i+1], self.layers_connectivity_updates[i]) for i in range(len(self._hidden_sizes))])
# Save random initial state
self._initial_mrng_rstate = copy.deepcopy(self._mrng.rstate)
self._initial_mrng_state_updates = [state_update[0].get_value() for state_update in self._mrng.state_updates]
# Ensuring valid initial connectivity
self.sample_connectivity()
def reset(self):
# Set Original ordering
self.ordering.set_value(np.arange(self._input_size, dtype=theano.config.floatX))
# Reset RandomStreams
self._rng.seed(self._random_seed)
# Initial layer connectivity
self.layers_connectivity[0].set_value((self.ordering + 1).eval())
for i in range(1, len(self.layers_connectivity)-1):
self.layers_connectivity[i].set_value(np.zeros((self._hidden_sizes[i-1]), dtype=theano.config.floatX))
self.layers_connectivity[-1].set_value(self.ordering.get_value())
# Reset MRG_RandomStreams (GPU)
self._mrng.rstate = self._initial_mrng_rstate
for state, value in zip(self._mrng.state_updates, self._initial_mrng_state_updates):
state[0].set_value(value)
self.sample_connectivity()
def _get_p(self, start_choice):
start_choice_idx = (start_choice-1).astype('int32')
p_vals = T.concatenate([T.zeros((start_choice_idx,)), T.nnet.nnet.softmax(self._l * T.arange(start_choice, self._input_size, dtype=theano.config.floatX))[0]])
p_vals = T.inc_subtensor(p_vals[start_choice_idx], 1.) # Stupid hack because de multinomial does not contain a safety for numerical imprecision.
return p_vals
def _get_hidden_layer_connectivity(self, layerIdx):
layer_size = self._hidden_sizes[layerIdx]
if layerIdx == 0:
p_vals = self._get_p(T.min(self.layers_connectivity[layerIdx]))
else:
p_vals = self._get_p(T.min(self.layers_connectivity_updates[layerIdx-1]))
# #Implementations of np.choose in theano GPU
# return T.nonzero(self._mrng.multinomial(pvals=[self._p_vals] * layer_size, dtype=theano.config.floatX))[1].astype(dtype=theano.config.floatX)
# return T.argmax(self._mrng.multinomial(pvals=[self._p_vals] * layer_size, dtype=theano.config.floatX), axis=1)
return T.sum(T.cumsum(self._mrng.multinomial(pvals=T.tile(p_vals[::-1][None, :], (layer_size, 1)), dtype=theano.config.floatX), axis=1), axis=1)
def _get_mask(self, layerIdxIn, layerIdxOut):
return (self.layers_connectivity[layerIdxIn][:, None] <= self.layers_connectivity[layerIdxOut][None, :]).astype(theano.config.floatX)
def get_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, layerIdx + 1)
def get_direct_input_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(0, layerIdx)
def get_direct_output_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, -1)
class MaskGenerator(object):
def __init__(self, input_size, hidden_sizes, l, random_seed=1234):
self._random_seed = random_seed
self._mrng = MRG_RandomStreams(seed=random_seed)
self._rng = RandomStreams(seed=random_seed)
self._hidden_sizes = hidden_sizes
self._input_size = input_size
self._l = l
self.ordering = theano.shared(np.arange(input_size,
dtype=theano.config.floatX),
'ordering',
borrow=False)
# Initial layer connectivity
self.layers_connectivity = [theano.shared((self.ordering + 1).eval(),
'layer_connectivity_input',
borrow=False)]
for i in range(len(self._hidden_sizes)):
lc = theano.shared(np.zeros((self._hidden_sizes[i]),dtype=floatX),
'layer_connectivity_hidden{0}'.format(i),
borrow=False)
self.layers_connectivity += [lc]
self.layers_connectivity += [self.ordering]
## Theano functions
new_ordering = self._rng.shuffle_row_elements(self.ordering)
updates = [(self.ordering, new_ordering),
(self.layers_connectivity[0], new_ordering + 1)]
self.shuffle_ordering = theano.function(name='shuffle_ordering',
inputs=[],
updates=updates)
self.layers_connectivity_updates = []
for i in range(len(self._hidden_sizes)):
lcu = self._get_hidden_layer_connectivity(i)
self.layers_connectivity_updates += [lcu]
hsizes = range(len(self._hidden_sizes))
updates = [(self.layers_connectivity[i+1],
self.layers_connectivity_updates[i]) for i in hsizes]
self.sample_connectivity = theano.function(name='sample_connectivity',
inputs=[],
updates=updates)
# Save random initial state
self._initial_mrng_rstate = copy.deepcopy(self._mrng.rstate)
self._initial_mrng_state_updates = [sup[0].get_value() for sup in
self._mrng.state_updates]
# Ensuring valid initial connectivity
self.sample_connectivity()
def reset(self):
# Set Original ordering
self.ordering.set_value(np.arange(self._input_size,
dtype=theano.config.floatX))
# Reset RandomStreams
self._rng.seed(self._random_seed)
# Initial layer connectivity
self.layers_connectivity[0].set_value((self.ordering + 1).eval())
for i in range(1, len(self.layers_connectivity)-1):
value = np.zeros((self._hidden_sizes[i-1]),
dtype=theano.config.floatX)
self.layers_connectivity[i].set_value(value)
self.layers_connectivity[-1].set_value(self.ordering.get_value())
# Reset MRG_RandomStreams (GPU)
self._mrng.rstate = self._initial_mrng_rstate
states_values = zip(self._mrng.state_updates,
self._initial_mrng_state_updates)
for state, value in states_values:
state[0].set_value(value)
self.sample_connectivity()
def _get_p(self, start_choice):
start_choice_idx = (start_choice-1).astype('int32')
prob = T.nnet.nnet.softmax(self._l * T.arange(start_choice,
self._input_size,
dtype=floatX))[0]
p_vals = T.concatenate([T.zeros((start_choice_idx,)),prob])
p_vals = T.inc_subtensor(p_vals[start_choice_idx], 1.)
return p_vals
def _get_hidden_layer_connectivity(self, layerIdx):
layer_size = self._hidden_sizes[layerIdx]
if layerIdx == 0:
lc = self.layers_connectivity[layerIdx]
p_vals = self._get_p(T.min(lc))
else:
lc = self.layers_connectivity_updates[layerIdx-1]
p_vals = self._get_p(T.min(lc))
return T.sum(
T.cumsum(self._mrng.multinomial(
pvals=T.tile(p_vals[::-1][None, :],(layer_size, 1)),
dtype=floatX), axis=1), axis=1
)
def _get_mask(self, layerIdxIn, layerIdxOut):
return (self.layers_connectivity[layerIdxIn][:, None] <=
self.layers_connectivity[layerIdxOut][None, :]).astype(floatX)
def get_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, layerIdx + 1)
def get_direct_input_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(0, layerIdx)
def get_direct_output_mask_layer_UPDATE(self, layerIdx):
return self._get_mask(layerIdx, -1)
