def test_sum(self):
vl = A([[[-3, 2],
[5, 6]],
[[1, -1],
[9, 8]]])
n = 10
vu = A([[[n, n],
[n, n]],
[[n, n],
[n, n]]])
tvl, tvu = T.tensor3s('tvl', 'tvu')
d = {tvl: vl, tvu: vu}
itv = TheanoInterval(tvl, tvu)
res1 = itv.sum(axis=0, keepdims=False)
res2 = itv.sum(axis=1, keepdims=False)
res3 = itv.sum(axis=2, keepdims=False)
res4 = itv.sum(axis=0, keepdims=True)
res5 = itv.sum(axis=1, keepdims=True)
res6 = itv.sum(axis=2, keepdims=True)
l1, _ = res1.eval(d)
l2, _ = res2.eval(d)
l3, _ = res3.eval(d)
l4, _ = res4.eval(d)
l5, _ = res5.eval(d)
l6, _ = res6.eval(d)
array_almost_equal(l1, A([[-2, 1], [14, 14]]))
array_almost_equal(l2, A([[2, 8], [10, 7]]))
array_almost_equal(l3, A([[-1, 11], [0, 17]]))
array_almost_equal(l4, A([[[-2, 1], [14, 14]]]))
array_almost_equal(l5, A([[[2, 8]], [[10, 7]]]))
array_almost_equal(l6, A([[[-1], [11]], [[0], [17]]]))
def test_op_relu(self):
inpl = A([[[-3, -1, 1]]])
inpu = A([[[-2, 3, 2]]])
tinpl, tinpu = T.tensor3s('tinpl', 'tinpu')
iinp = TheanoInterval(tinpl, tinpu)
res = iinp.op_relu()
d = {tinpl: inpl, tinpu: inpu}
rl, ru = res.eval(d)
array_almost_equal(rl, A([[[0, 0, 1]]]))
array_almost_equal(ru, A([[[0, 3, 2]]]))
def _get_theano_interval_result(self, lower, upper, function, *args,
**kwargs):
dim = len(lower.shape)
if dim == 1:
t_lower, t_upper = T.dvectors('inpl', 'inpu')
elif dim == 3:
t_lower, t_upper = T.tensor3s('inpl', 'inpu')
else:
raise NotImplementedError
iinp = TheanoInterval(t_lower, t_upper)
res = function(iinp, *args, **kwargs)
d = {t_lower: lower, t_upper: upper}
return res.eval(d)
def test_convolutional_shift():
weights_var, shift_var = T.tensor3s('weights', 'shift')
num_shifts = 3
weights_reshaped = weights_var.reshape((16 * 4, 128))
weights_reshaped = weights_reshaped.dimshuffle(0, 'x', 'x', 1)
shift_reshaped = shift_var.reshape((16 * 4, num_shifts))
shift_reshaped = shift_reshaped.dimshuffle(0, 'x', 'x', 1)
pad = (num_shifts // 2, (num_shifts - 1) // 2)
weights_padded = padding.pad(weights_reshaped, [pad], batch_ndim=3)
convolution = T.nnet.conv2d(weights_padded,
shift_reshaped,
input_shape=(16 * 4, 1, 1,
128 + pad[0] + pad[1]),
filter_shape=(16 * 4, 1, 1, num_shifts),
subsample=(1, 1),
border_mode='valid')
w_tilde = convolution[T.arange(16 * 4), T.arange(16 * 4), 0, :]
w_tilde = w_tilde.reshape((16, 4, 128))
convolutional_shift_fn = theano.function([weights_var, shift_var], w_tilde)
weights = np.random.rand(16, 4, 128)
shift = np.random.rand(16, 4, 3)
weight_tilde = convolutional_shift_fn(weights, shift)
weight_tilde_manual = np.zeros_like(weight_tilde)
for i in range(16):
for j in range(4):
for k in range(128):
# Filters in T.nnet.conv2d are reversed
if (k - 1) >= 0:
weight_tilde_manual[i, j,
k] += shift[i, j, 2] * weights[i, j,
k - 1]
weight_tilde_manual[i, j,
k] += shift[i, j, 1] * weights[i, j, k]
if (k + 1) < 128:
weight_tilde_manual[i, j,
k] += shift[i, j, 0] * weights[i, j,
k + 1]
assert weight_tilde.shape == (16, 4, 128)
assert np.allclose(weight_tilde, weight_tilde_manual)
def test_abs(self):
vl = A([[[-3, 2],
[5, 6]],
[[1, -1],
[9, 8]]])
vu = A([[[-2, 3],
[5, 7]],
[[1, 1],
[9, 9]]])
tvl, tvu = T.tensor3s('tvl', 'tvu')
d = {tvl: vl, tvu: vu}
itv = TheanoInterval(tvl, tvu)
res = itv.abs()
l, u = res.eval(d)
array_almost_equal(l, A([[[2, 2], [5, 6]],
[[1, 0], [9, 8]]]))
array_almost_equal(u, A([[[3, 3], [5, 7]],
[[1, 1], [9, 9]]]))
def test_dot(self):
inpl = A([[[0, 1]]])
inpu = A([[[2, 3]]])
w = A([[4, -5, 6], [7, 8, 9]])
b = A([1, 3, 5])
crl = A([0 * 4 + 1 * 7 + 1,
2 * (-5) + 1 * 8 + 3,
0 * 6 + 1 * 9 + 5])
cru = A([2 * 4 + 3 * 7 + 1,
0 * (-5) + 3 * 8 + 3,
2 * 6 + 3 * 9 + 5])
tinpl, tinpu = T.tensor3s('inpl', 'inpu')
iinp = TheanoInterval(tinpl, tinpu)
res = iinp.flatten().dot(w)
res += b
d = {tinpl: inpl, tinpu: inpu}
rl, ru = res.eval(d)
array_almost_equal(rl, crl)
array_almost_equal(ru, cru)
def test_batch_size():
input_var_1, input_var_2 = T.tensor3s('input1', 'input2')
target_var_1, target_var_2 = T.imatrices('target1', 'target2')
# First model with `batch_size=16`
output_var_1, _, params1 = memory_augmented_neural_network(
input_var_1,
target_var_1,
batch_size=16,
nb_class=5,
memory_shape=(128, 40),
controller_size=200,
input_size=20 * 20,
nb_reads=4)
# Second model with `batch_size=1`
output_var_2, _, params2 = memory_augmented_neural_network(
input_var_2,
target_var_2,
batch_size=1,
nb_class=5,
memory_shape=(128, 40),
controller_size=200,
input_size=20 * 20,
nb_reads=4)
for (param1, param2) in zip(params1, params2):
param2.set_value(param1.get_value())
posterior_fn1 = theano.function([input_var_1, target_var_1], output_var_1)
posterior_fn2 = theano.function([input_var_2, target_var_2], output_var_2)
# Input has shape (batch_size, timesteps, vocabulary_size + actions_vocabulary_size + 3)
test_input = np.random.rand(16, 50, 20 * 20)
test_target = np.random.randint(5, size=(16, 50)).astype('int32')
test_output1 = posterior_fn1(test_input, test_target)
test_output2 = np.zeros_like(test_output1)
for i in range(16):
test_output2[i] = posterior_fn2(test_input[i][np.newaxis, :, :],
test_target[i][np.newaxis, :])
assert np.allclose(test_output1, test_output2)
def test_batch_size():
input_var01, input_var16 = T.tensor3s('input01', 'input16')
l_output01 = model(input_var01, batch_size=1)
l_output16 = model(input_var16, batch_size=16)
# Share the parameters for both models
params01 = get_all_param_values(l_output01)
set_all_param_values(l_output16, params01)
posterior_fn01 = theano.function([input_var01], get_output(l_output01))
posterior_fn16 = theano.function([input_var16], get_output(l_output16))
example_input = np.random.rand(16, 30, 8)
example_output16 = posterior_fn16(example_input)
example_output01 = np.zeros_like(example_output16)
for i in range(16):
example_output01[i] = posterior_fn01(example_input[i][np.newaxis, :, :])
assert example_output16.shape == (16, 30, 8)
assert np.allclose(example_output16, example_output01, atol=1e-3)
def test_cosine_similarity():
from similarities import cosine_similarity
key_var, memory_var = T.tensor3s('key', 'memory')
cosine_similarity_fn = theano.function([key_var, memory_var], \
cosine_similarity(key_var, memory_var, eps=1e-6))
test_key = np.random.rand(16, 4, 20)
test_memory = np.random.rand(16, 128, 20)
test_output = cosine_similarity_fn(test_key, test_memory)
test_output_manual = np.zeros_like(test_output)
for i in range(16):
for j in range(4):
for k in range(128):
test_output_manual[i, j, k] = np.dot(test_key[i, j], test_memory[i, k]) / \
np.sqrt(np.sum(test_key[i, j] * test_key[i, j]) * np.sum(test_memory[i, k] * \
test_memory[i, k]) + 1e-6)
assert np.allclose(test_output, test_output_manual)
def test_batch_size():
input_var_1, input_var_2 = T.tensor3s('input1', 'input2')
target_var_1, target_var_2 = T.imatrices('target1', 'target2')
# First model with `batch_size=16`
output_var_1, _, params1 = memory_augmented_neural_network(
input_var_1, target_var_1,
batch_size=16,
nb_class=5,
memory_shape=(128, 40),
controller_size=200,
input_size=20 * 20,
nb_reads=4)
# Second model with `batch_size=1`
output_var_2, _, params2 = memory_augmented_neural_network(
input_var_2, target_var_2,
batch_size=1,
nb_class=5,
memory_shape=(128, 40),
controller_size=200,
input_size=20 * 20,
nb_reads=4)
for (param1, param2) in zip(params1, params2):
param2.set_value(param1.get_value())
posterior_fn1 = theano.function([input_var_1, target_var_1], output_var_1)
posterior_fn2 = theano.function([input_var_2, target_var_2], output_var_2)
# Input has shape (batch_size, timesteps, vocabulary_size + actions_vocabulary_size + 3)
test_input = np.random.rand(16, 50, 20 * 20)
test_target = np.random.randint(5, size=(16, 50)).astype('int32')
test_output1 = posterior_fn1(test_input, test_target)
test_output2 = np.zeros_like(test_output1)
for i in range(16):
test_output2[i] = posterior_fn2(test_input[i][np.newaxis, :, :], test_target[i][np.newaxis, :])
assert np.allclose(test_output1, test_output2)
def test_convolutional_shift():
weights_var, shift_var = T.tensor3s('weights', 'shift')
num_shifts = 3
weights_reshaped = weights_var.reshape((16 * 4, 128))
weights_reshaped = weights_reshaped.dimshuffle(0, 'x', 'x', 1)
shift_reshaped = shift_var.reshape((16 * 4, num_shifts))
shift_reshaped = shift_reshaped.dimshuffle(0, 'x', 'x', 1)
pad = (num_shifts // 2, (num_shifts - 1) // 2)
weights_padded = padding.pad(weights_reshaped, [pad], batch_ndim=3)
convolution = T.nnet.conv2d(weights_padded, shift_reshaped,
input_shape=(16 * 4, 1, 1, 128 + pad[0] + pad[1]),
filter_shape=(16 * 4, 1, 1, num_shifts),
subsample=(1, 1),
border_mode='valid')
w_tilde = convolution[T.arange(16 * 4), T.arange(16 * 4), 0, :]
w_tilde = w_tilde.reshape((16, 4, 128))
convolutional_shift_fn = theano.function([weights_var, shift_var], w_tilde)
weights = np.random.rand(16, 4, 128)
shift = np.random.rand(16, 4, 3)
weight_tilde = convolutional_shift_fn(weights, shift)
weight_tilde_manual = np.zeros_like(weight_tilde)
for i in range(16):
for j in range(4):
for k in range(128):
# Filters in T.nnet.conv2d are reversed
if (k - 1) >= 0:
weight_tilde_manual[i, j, k] += shift[i, j, 2] * weights[i, j, k - 1]
weight_tilde_manual[i, j, k] += shift[i, j, 1] * weights[i, j, k]
if (k + 1) < 128:
weight_tilde_manual[i, j, k] += shift[i, j, 0] * weights[i, j, k + 1]
assert weight_tilde.shape == (16, 4, 128)
assert np.allclose(weight_tilde, weight_tilde_manual)
def test_sharpening():
weight_var, gamma_var = T.tensor3s('weight', 'gamma')
gamma_var = T.addbroadcast(gamma_var, 2)
w = T.pow(weight_var + 1e-6, gamma_var)
w /= T.sum(w, axis=2).dimshuffle(0, 1, 'x')
sharpening_fn = theano.function([weight_var, gamma_var], w)
weights = np.random.rand(16, 4, 128)
gamma = np.random.rand(16, 4, 1)
weight_t = sharpening_fn(weights, gamma)
weight_t_manual = np.zeros_like(weight_t)
for i in range(16):
for j in range(4):
for k in range(128):
weight_t_manual[i, j, k] = np.power(weights[i, j, k] + 1e-6, gamma[i, j])
weight_t_manual[i, j] /= np.sum(weight_t_manual[i, j])
assert weight_t.shape == (16, 4, 128)
assert np.allclose(weight_t, weight_t_manual)
def test_bitonicity_and_extremas_interval(self):
shp = (5, 1, 1)
tinpl, tinpu = T.tensor3s('tinpl', 'tinpu')
iinp = TheanoInterval(tinpl, tinpu)
out = a_norm(iinp, shp)
b = 200.0
a = (2.0 * (50000.0 + b * b))**0.5
inp = A([[[b]], [[0.0]], [[a]], [[0.0]], [[0.0]]])
d = {tinpl: inp, tinpu: inp}
l1, _ = out.eval(d)
inp[2, 0, 0] = a - 20.0
l2, _ = out.eval(d)
inp[2, 0, 0] = a + 20.0
l3, _ = out.eval(d)
inp[2, 0, 0] = a - 19.0
l4, _ = out.eval(d)
inp[2, 0, 0] = a - 18.0
l5, _ = out.eval(d)
inpl = inp.copy()
inpu = inp.copy()
d2 = {tinpl: inpl, tinpu: inpu}
inpl[2, 0, 0] = a - 18.0
inpu[2, 0, 0] = a + 20.0
l6, _ = out.eval(d2)
inpl[2, 0, 0] = a - 19.0
inpu[2, 0, 0] = a + 20.0
l7, _ = out.eval(d2)
inpl[2, 0, 0] = a - 20.0
inpu[2, 0, 0] = a + 20.0
l8, _ = out.eval(d2)
inpl[2, 0, 0] = a - 19.0
inpu[2, 0, 0] = a + 20.0
assert_greater(l1[2, 0, 0], l2[2, 0, 0])
assert_greater(l1[2, 0, 0], l3[2, 0, 0])
assert_greater(l5[2, 0, 0], l3[2, 0, 0])
assert_almost_equal(l3[2, 0, 0], l6[2, 0, 0], places=2)
assert_almost_equal(l3[2, 0, 0], l7[2, 0, 0], places=2)
def test_content_addressing():
from ntm.similarities import cosine_similarity
beta_var, key_var, memory_var = T.tensor3s('beta', 'key', 'memory')
beta_var = T.addbroadcast(beta_var, 2)
betaK = beta_var * cosine_similarity(key_var, memory_var)
w_c = lasagne.nonlinearities.softmax(betaK.reshape((16 * 4, 128)))
w_c = w_c.reshape(betaK.shape)
content_addressing_fn = theano.function([beta_var, key_var, memory_var],
w_c)
beta = np.random.rand(16, 4, 1)
key = np.random.rand(16, 4, 20)
memory = np.random.rand(16, 128, 20)
weights = content_addressing_fn(beta, key, memory)
weights_manual = np.zeros_like(weights)
def softmax(x):
y = np.exp(x.T - np.max(x, axis=1))
z = y / np.sum(y, axis=0)
return z.T
betaK_manual = np.zeros((16, 4, 128))
for i in range(16):
for j in range(4):
for k in range(128):
betaK_manual[i, j, k] = beta[i, j, 0] * np.dot(key[i, j], \
memory[i, k]) / np.sqrt(np.sum(key[i, j] * key[i, j]) * \
np.sum(memory[i, k] * memory[i, k]) + 1e-6)
for i in range(16):
weights_manual[i] = softmax(betaK_manual[i])
assert weights.shape == (16, 4, 128)
assert np.allclose(np.sum(weights, axis=2), np.ones((16, 4)))
assert np.allclose(weights, weights_manual)
def test_content_addressing():
from ntm.similarities import cosine_similarity
beta_var, key_var, memory_var = T.tensor3s('beta', 'key', 'memory')
beta_var = T.addbroadcast(beta_var, 2)
betaK = beta_var * cosine_similarity(key_var, memory_var)
w_c = lasagne.nonlinearities.softmax(betaK.reshape((16 * 4, 128)))
w_c = w_c.reshape(betaK.shape)
content_addressing_fn = theano.function([beta_var, key_var, memory_var], w_c)
beta = np.random.rand(16, 4, 1)
key = np.random.rand(16, 4, 20)
memory = np.random.rand(16, 128, 20)
weights = content_addressing_fn(beta, key, memory)
weights_manual = np.zeros_like(weights)
def softmax(x):
y = np.exp(x.T - np.max(x, axis=1))
z = y / np.sum(y, axis=0)
return z.T
betaK_manual = np.zeros((16, 4, 128))
for i in range(16):
for j in range(4):
for k in range(128):
betaK_manual[i, j, k] = beta[i, j, 0] * np.dot(key[i, j], \
memory[i, k]) / np.sqrt(np.sum(key[i, j] * key[i, j]) * \
np.sum(memory[i, k] * memory[i, k]) + 1e-6)
for i in range(16):
weights_manual[i] = softmax(betaK_manual[i])
assert weights.shape == (16, 4, 128)
assert np.allclose(np.sum(weights, axis=2), np.ones((16, 4)))
assert np.allclose(weights, weights_manual)
def test_sharpening():
weight_var, gamma_var = T.tensor3s('weight', 'gamma')
gamma_var = T.addbroadcast(gamma_var, 2)
w = T.pow(weight_var + 1e-6, gamma_var)
w /= T.sum(w, axis=2).dimshuffle(0, 1, 'x')
sharpening_fn = theano.function([weight_var, gamma_var], w)
weights = np.random.rand(16, 4, 128)
gamma = np.random.rand(16, 4, 1)
weight_t = sharpening_fn(weights, gamma)
weight_t_manual = np.zeros_like(weight_t)
for i in range(16):
for j in range(4):
for k in range(128):
weight_t_manual[i, j, k] = np.power(weights[i, j, k] + 1e-6,
gamma[i, j])
weight_t_manual[i, j] /= np.sum(weight_t_manual[i, j])
assert weight_t.shape == (16, 4, 128)
assert np.allclose(weight_t, weight_t_manual)
"n_in": 50,
"batch_size": 50,
"stim_dur": 25,
"delay_dur": 100,
"resp_dur": 25,
"kappa": 2.0,
"spon_rate": 0.1,
"tr_max_iter": 25001,
"test_max_iter": 2501
}
# Build task generators
generator, test_generator = build_generators(ExptDict)
# Define the input and expected output variable
input_var, target_var = T.tensor3s('input', 'target')
# Build the model
l_out, l_rec = build_model(input_var, ExptDict)
# The generated output variable and the loss function
if ExptDict["task"]["task_id"] in ['DE1', 'DE2', 'GDE2', 'VDE1', 'SINE']:
pred_var = lasagne.layers.get_output(l_out)
elif ExptDict["task"]["task_id"] in [
'CD1', 'CD2', 'Harvey2012', 'Harvey2012Dynamic', 'Harvey2016',
'COMP'
]:
pred_var = T.clip(lasagne.layers.get_output(l_out), 1e-6, 1.0 - 1e-6)
# Build loss
rec_act = lasagne.layers.get_output(l_rec)
_, seqlen, _ = l_in.input_var.shape
# Recurrent EI Net
l_in_hid = DenseLayer(lasagne.layers.InputLayer((None, n_in)), n_hid, b=None, nonlinearity=lasagne.nonlinearities.linear)
l_hid_hid = DenseLayer(lasagne.layers.InputLayer((None, n_hid)), n_hid, nonlinearity=lasagne.nonlinearities.linear)
l_rec = lasagne.layers.CustomRecurrentLayer(l_in, l_in_hid, l_hid_hid, nonlinearity=lasagne.nonlinearities.rectify)
# Output Layer
l_shp = ReshapeLayer(l_rec, (-1, n_hid))
l_dense = DenseLayer(l_shp, num_units=n_out, nonlinearity=lasagne.nonlinearities.linear)
# To reshape back to our original shape, we can use the symbolic shape variables we retrieved above.
l_out = ReshapeLayer(l_dense, (batch_size, seqlen, n_out))
return l_out, l_rec
if __name__ == '__main__':
# Define the input and expected output variable
input_var, target_var = T.tensor3s('input', 'target')
# The generator to sample examples from
tr_cond = 'all_gains'
test_cond = 'all_gains'
generator = KalmanFilteringTaskFFWD(max_iter=50001, batch_size=10, n_in=n_in, n_out=1, stim_dur=25, sigtc_sq=4.0, signu_sq=1.0, gamma=0.1, tr_cond=tr_cond)
# The model
l_out, l_rec = model(input_var, batch_size=generator.batch_size, n_in=generator.n_in, n_out=generator.n_out, n_hid=n_hid)
# The generated output variable and the loss function
# all_layers = lasagne.layers.get_all_layers(l_out)
# l2_penalty = lasagne.regularization.regularize_layer_params(all_layers, lasagne.regularization.l2) * 1e-6
pred_var = lasagne.layers.get_output(l_out)
loss = T.mean(lasagne.objectives.squared_error(pred_var[:,:,-1], target_var[:,:,-1])) # + l2_penalty
