def test_sum(self): vl = A([[[-3, 2], [5, 6]], [[1, -1], [9, 8]]]) n = 10 vu = A([[[n, n], [n, n]], [[n, n], [n, n]]]) tvl, tvu = T.tensor3s('tvl', 'tvu') d = {tvl: vl, tvu: vu} itv = TheanoInterval(tvl, tvu) res1 = itv.sum(axis=0, keepdims=False) res2 = itv.sum(axis=1, keepdims=False) res3 = itv.sum(axis=2, keepdims=False) res4 = itv.sum(axis=0, keepdims=True) res5 = itv.sum(axis=1, keepdims=True) res6 = itv.sum(axis=2, keepdims=True) l1, _ = res1.eval(d) l2, _ = res2.eval(d) l3, _ = res3.eval(d) l4, _ = res4.eval(d) l5, _ = res5.eval(d) l6, _ = res6.eval(d) array_almost_equal(l1, A([[-2, 1], [14, 14]])) array_almost_equal(l2, A([[2, 8], [10, 7]])) array_almost_equal(l3, A([[-1, 11], [0, 17]])) array_almost_equal(l4, A([[[-2, 1], [14, 14]]])) array_almost_equal(l5, A([[[2, 8]], [[10, 7]]])) array_almost_equal(l6, A([[[-1], [11]], [[0], [17]]]))
def test_op_relu(self): inpl = A([[[-3, -1, 1]]]) inpu = A([[[-2, 3, 2]]]) tinpl, tinpu = T.tensor3s('tinpl', 'tinpu') iinp = TheanoInterval(tinpl, tinpu) res = iinp.op_relu() d = {tinpl: inpl, tinpu: inpu} rl, ru = res.eval(d) array_almost_equal(rl, A([[[0, 0, 1]]])) array_almost_equal(ru, A([[[0, 3, 2]]]))
def _get_theano_interval_result(self, lower, upper, function, *args, **kwargs): dim = len(lower.shape) if dim == 1: t_lower, t_upper = T.dvectors('inpl', 'inpu') elif dim == 3: t_lower, t_upper = T.tensor3s('inpl', 'inpu') else: raise NotImplementedError iinp = TheanoInterval(t_lower, t_upper) res = function(iinp, *args, **kwargs) d = {t_lower: lower, t_upper: upper} return res.eval(d)
def test_convolutional_shift(): weights_var, shift_var = T.tensor3s('weights', 'shift') num_shifts = 3 weights_reshaped = weights_var.reshape((16 * 4, 128)) weights_reshaped = weights_reshaped.dimshuffle(0, 'x', 'x', 1) shift_reshaped = shift_var.reshape((16 * 4, num_shifts)) shift_reshaped = shift_reshaped.dimshuffle(0, 'x', 'x', 1) pad = (num_shifts // 2, (num_shifts - 1) // 2) weights_padded = padding.pad(weights_reshaped, [pad], batch_ndim=3) convolution = T.nnet.conv2d(weights_padded, shift_reshaped, input_shape=(16 * 4, 1, 1, 128 + pad[0] + pad[1]), filter_shape=(16 * 4, 1, 1, num_shifts), subsample=(1, 1), border_mode='valid') w_tilde = convolution[T.arange(16 * 4), T.arange(16 * 4), 0, :] w_tilde = w_tilde.reshape((16, 4, 128)) convolutional_shift_fn = theano.function([weights_var, shift_var], w_tilde) weights = np.random.rand(16, 4, 128) shift = np.random.rand(16, 4, 3) weight_tilde = convolutional_shift_fn(weights, shift) weight_tilde_manual = np.zeros_like(weight_tilde) for i in range(16): for j in range(4): for k in range(128): # Filters in T.nnet.conv2d are reversed if (k - 1) >= 0: weight_tilde_manual[i, j, k] += shift[i, j, 2] * weights[i, j, k - 1] weight_tilde_manual[i, j, k] += shift[i, j, 1] * weights[i, j, k] if (k + 1) < 128: weight_tilde_manual[i, j, k] += shift[i, j, 0] * weights[i, j, k + 1] assert weight_tilde.shape == (16, 4, 128) assert np.allclose(weight_tilde, weight_tilde_manual)
def test_abs(self): vl = A([[[-3, 2], [5, 6]], [[1, -1], [9, 8]]]) vu = A([[[-2, 3], [5, 7]], [[1, 1], [9, 9]]]) tvl, tvu = T.tensor3s('tvl', 'tvu') d = {tvl: vl, tvu: vu} itv = TheanoInterval(tvl, tvu) res = itv.abs() l, u = res.eval(d) array_almost_equal(l, A([[[2, 2], [5, 6]], [[1, 0], [9, 8]]])) array_almost_equal(u, A([[[3, 3], [5, 7]], [[1, 1], [9, 9]]]))
def test_dot(self): inpl = A([[[0, 1]]]) inpu = A([[[2, 3]]]) w = A([[4, -5, 6], [7, 8, 9]]) b = A([1, 3, 5]) crl = A([0 * 4 + 1 * 7 + 1, 2 * (-5) + 1 * 8 + 3, 0 * 6 + 1 * 9 + 5]) cru = A([2 * 4 + 3 * 7 + 1, 0 * (-5) + 3 * 8 + 3, 2 * 6 + 3 * 9 + 5]) tinpl, tinpu = T.tensor3s('inpl', 'inpu') iinp = TheanoInterval(tinpl, tinpu) res = iinp.flatten().dot(w) res += b d = {tinpl: inpl, tinpu: inpu} rl, ru = res.eval(d) array_almost_equal(rl, crl) array_almost_equal(ru, cru)
def test_batch_size(): input_var_1, input_var_2 = T.tensor3s('input1', 'input2') target_var_1, target_var_2 = T.imatrices('target1', 'target2') # First model with `batch_size=16` output_var_1, _, params1 = memory_augmented_neural_network( input_var_1, target_var_1, batch_size=16, nb_class=5, memory_shape=(128, 40), controller_size=200, input_size=20 * 20, nb_reads=4) # Second model with `batch_size=1` output_var_2, _, params2 = memory_augmented_neural_network( input_var_2, target_var_2, batch_size=1, nb_class=5, memory_shape=(128, 40), controller_size=200, input_size=20 * 20, nb_reads=4) for (param1, param2) in zip(params1, params2): param2.set_value(param1.get_value()) posterior_fn1 = theano.function([input_var_1, target_var_1], output_var_1) posterior_fn2 = theano.function([input_var_2, target_var_2], output_var_2) # Input has shape (batch_size, timesteps, vocabulary_size + actions_vocabulary_size + 3) test_input = np.random.rand(16, 50, 20 * 20) test_target = np.random.randint(5, size=(16, 50)).astype('int32') test_output1 = posterior_fn1(test_input, test_target) test_output2 = np.zeros_like(test_output1) for i in range(16): test_output2[i] = posterior_fn2(test_input[i][np.newaxis, :, :], test_target[i][np.newaxis, :]) assert np.allclose(test_output1, test_output2)
def test_batch_size(): input_var01, input_var16 = T.tensor3s('input01', 'input16') l_output01 = model(input_var01, batch_size=1) l_output16 = model(input_var16, batch_size=16) # Share the parameters for both models params01 = get_all_param_values(l_output01) set_all_param_values(l_output16, params01) posterior_fn01 = theano.function([input_var01], get_output(l_output01)) posterior_fn16 = theano.function([input_var16], get_output(l_output16)) example_input = np.random.rand(16, 30, 8) example_output16 = posterior_fn16(example_input) example_output01 = np.zeros_like(example_output16) for i in range(16): example_output01[i] = posterior_fn01(example_input[i][np.newaxis, :, :]) assert example_output16.shape == (16, 30, 8) assert np.allclose(example_output16, example_output01, atol=1e-3)
def test_cosine_similarity(): from similarities import cosine_similarity key_var, memory_var = T.tensor3s('key', 'memory') cosine_similarity_fn = theano.function([key_var, memory_var], \ cosine_similarity(key_var, memory_var, eps=1e-6)) test_key = np.random.rand(16, 4, 20) test_memory = np.random.rand(16, 128, 20) test_output = cosine_similarity_fn(test_key, test_memory) test_output_manual = np.zeros_like(test_output) for i in range(16): for j in range(4): for k in range(128): test_output_manual[i, j, k] = np.dot(test_key[i, j], test_memory[i, k]) / \ np.sqrt(np.sum(test_key[i, j] * test_key[i, j]) * np.sum(test_memory[i, k] * \ test_memory[i, k]) + 1e-6) assert np.allclose(test_output, test_output_manual)
def test_sharpening(): weight_var, gamma_var = T.tensor3s('weight', 'gamma') gamma_var = T.addbroadcast(gamma_var, 2) w = T.pow(weight_var + 1e-6, gamma_var) w /= T.sum(w, axis=2).dimshuffle(0, 1, 'x') sharpening_fn = theano.function([weight_var, gamma_var], w) weights = np.random.rand(16, 4, 128) gamma = np.random.rand(16, 4, 1) weight_t = sharpening_fn(weights, gamma) weight_t_manual = np.zeros_like(weight_t) for i in range(16): for j in range(4): for k in range(128): weight_t_manual[i, j, k] = np.power(weights[i, j, k] + 1e-6, gamma[i, j]) weight_t_manual[i, j] /= np.sum(weight_t_manual[i, j]) assert weight_t.shape == (16, 4, 128) assert np.allclose(weight_t, weight_t_manual)
def test_bitonicity_and_extremas_interval(self): shp = (5, 1, 1) tinpl, tinpu = T.tensor3s('tinpl', 'tinpu') iinp = TheanoInterval(tinpl, tinpu) out = a_norm(iinp, shp) b = 200.0 a = (2.0 * (50000.0 + b * b))**0.5 inp = A([[[b]], [[0.0]], [[a]], [[0.0]], [[0.0]]]) d = {tinpl: inp, tinpu: inp} l1, _ = out.eval(d) inp[2, 0, 0] = a - 20.0 l2, _ = out.eval(d) inp[2, 0, 0] = a + 20.0 l3, _ = out.eval(d) inp[2, 0, 0] = a - 19.0 l4, _ = out.eval(d) inp[2, 0, 0] = a - 18.0 l5, _ = out.eval(d) inpl = inp.copy() inpu = inp.copy() d2 = {tinpl: inpl, tinpu: inpu} inpl[2, 0, 0] = a - 18.0 inpu[2, 0, 0] = a + 20.0 l6, _ = out.eval(d2) inpl[2, 0, 0] = a - 19.0 inpu[2, 0, 0] = a + 20.0 l7, _ = out.eval(d2) inpl[2, 0, 0] = a - 20.0 inpu[2, 0, 0] = a + 20.0 l8, _ = out.eval(d2) inpl[2, 0, 0] = a - 19.0 inpu[2, 0, 0] = a + 20.0 assert_greater(l1[2, 0, 0], l2[2, 0, 0]) assert_greater(l1[2, 0, 0], l3[2, 0, 0]) assert_greater(l5[2, 0, 0], l3[2, 0, 0]) assert_almost_equal(l3[2, 0, 0], l6[2, 0, 0], places=2) assert_almost_equal(l3[2, 0, 0], l7[2, 0, 0], places=2)
def test_content_addressing(): from ntm.similarities import cosine_similarity beta_var, key_var, memory_var = T.tensor3s('beta', 'key', 'memory') beta_var = T.addbroadcast(beta_var, 2) betaK = beta_var * cosine_similarity(key_var, memory_var) w_c = lasagne.nonlinearities.softmax(betaK.reshape((16 * 4, 128))) w_c = w_c.reshape(betaK.shape) content_addressing_fn = theano.function([beta_var, key_var, memory_var], w_c) beta = np.random.rand(16, 4, 1) key = np.random.rand(16, 4, 20) memory = np.random.rand(16, 128, 20) weights = content_addressing_fn(beta, key, memory) weights_manual = np.zeros_like(weights) def softmax(x): y = np.exp(x.T - np.max(x, axis=1)) z = y / np.sum(y, axis=0) return z.T betaK_manual = np.zeros((16, 4, 128)) for i in range(16): for j in range(4): for k in range(128): betaK_manual[i, j, k] = beta[i, j, 0] * np.dot(key[i, j], \ memory[i, k]) / np.sqrt(np.sum(key[i, j] * key[i, j]) * \ np.sum(memory[i, k] * memory[i, k]) + 1e-6) for i in range(16): weights_manual[i] = softmax(betaK_manual[i]) assert weights.shape == (16, 4, 128) assert np.allclose(np.sum(weights, axis=2), np.ones((16, 4))) assert np.allclose(weights, weights_manual)
"n_in": 50, "batch_size": 50, "stim_dur": 25, "delay_dur": 100, "resp_dur": 25, "kappa": 2.0, "spon_rate": 0.1, "tr_max_iter": 25001, "test_max_iter": 2501 } # Build task generators generator, test_generator = build_generators(ExptDict) # Define the input and expected output variable input_var, target_var = T.tensor3s('input', 'target') # Build the model l_out, l_rec = build_model(input_var, ExptDict) # The generated output variable and the loss function if ExptDict["task"]["task_id"] in ['DE1', 'DE2', 'GDE2', 'VDE1', 'SINE']: pred_var = lasagne.layers.get_output(l_out) elif ExptDict["task"]["task_id"] in [ 'CD1', 'CD2', 'Harvey2012', 'Harvey2012Dynamic', 'Harvey2016', 'COMP' ]: pred_var = T.clip(lasagne.layers.get_output(l_out), 1e-6, 1.0 - 1e-6) # Build loss rec_act = lasagne.layers.get_output(l_rec)
_, seqlen, _ = l_in.input_var.shape # Recurrent EI Net l_in_hid = DenseLayer(lasagne.layers.InputLayer((None, n_in)), n_hid, b=None, nonlinearity=lasagne.nonlinearities.linear) l_hid_hid = DenseLayer(lasagne.layers.InputLayer((None, n_hid)), n_hid, nonlinearity=lasagne.nonlinearities.linear) l_rec = lasagne.layers.CustomRecurrentLayer(l_in, l_in_hid, l_hid_hid, nonlinearity=lasagne.nonlinearities.rectify) # Output Layer l_shp = ReshapeLayer(l_rec, (-1, n_hid)) l_dense = DenseLayer(l_shp, num_units=n_out, nonlinearity=lasagne.nonlinearities.linear) # To reshape back to our original shape, we can use the symbolic shape variables we retrieved above. l_out = ReshapeLayer(l_dense, (batch_size, seqlen, n_out)) return l_out, l_rec if __name__ == '__main__': # Define the input and expected output variable input_var, target_var = T.tensor3s('input', 'target') # The generator to sample examples from tr_cond = 'all_gains' test_cond = 'all_gains' generator = KalmanFilteringTaskFFWD(max_iter=50001, batch_size=10, n_in=n_in, n_out=1, stim_dur=25, sigtc_sq=4.0, signu_sq=1.0, gamma=0.1, tr_cond=tr_cond) # The model l_out, l_rec = model(input_var, batch_size=generator.batch_size, n_in=generator.n_in, n_out=generator.n_out, n_hid=n_hid) # The generated output variable and the loss function # all_layers = lasagne.layers.get_all_layers(l_out) # l2_penalty = lasagne.regularization.regularize_layer_params(all_layers, lasagne.regularization.l2) * 1e-6 pred_var = lasagne.layers.get_output(l_out) loss = T.mean(lasagne.objectives.squared_error(pred_var[:,:,-1], target_var[:,:,-1])) # + l2_penalty