def step(in_h1_t, ingate_h1_t, h1_tm1, h2_tm1, h3_tm1):
h1_t = GRU(in_h1_t, ingate_h1_t, h1_tm1, n_hid, n_hid, random_state)
h1_h2_t, h1gate_h2_t = GRUFork([h1_t], [n_hid], n_hid, random_state)
h2_t = GRU(h1_h2_t, h1gate_h2_t, h2_tm1, n_hid, n_hid, random_state)
h2_h3_t, h2gate_h3_t = GRUFork([h2_t], [n_hid], n_hid, random_state)
h3_t = GRU(h2_h3_t, h2gate_h3_t, h3_tm1, n_hid, n_hid, random_state)
return h1_t, h2_t, h3_t
init_w = tensor.matrix("init_w")
init_w.tag.test_value = np_zeros((minibatch_size, n_chars))
params = []
biases = []
cell1 = GRU(input_dim, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
cell3 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
params += cell3.get_params()
inp_to_h1 = GRUFork(input_dim, n_hid, random_state)
inp_to_h2 = GRUFork(input_dim, n_hid, random_state)
inp_to_h3 = GRUFork(input_dim, n_hid, random_state)
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
att_to_h3 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
h1_to_h3 = GRUFork(n_hid, n_hid, random_state)
h2_to_h3 = GRUFork(n_hid, n_hid, random_state)
params += inp_to_h1.get_params()
params += inp_to_h2.get_params()
params += inp_to_h3.get_params()
params += att_to_h1.get_params()
params += att_to_h2.get_params()
params += att_to_h3.get_params()
init_kappa.tag.test_value = np_zeros((minibatch_size, att_size))
init_w = tensor.matrix("init_w")
init_w.tag.test_value = np_zeros((minibatch_size, n_chars))
params = []
w_conv1, = make_conv_weights(1, (n_kernels,), (conv_size1, input_dim),
random_state)
b_conv1, = make_biases((n_kernels,))
w_conv2, = make_conv_weights(n_kernels, (n_kernels,),
(conv_size2, 1), random_state)
b_conv2, = make_biases((n_kernels,))
params += [w_conv1, b_conv1, w_conv2, b_conv2]
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
conv_to_h1 = GRUFork(n_kernels, n_hid, random_state)
conv_to_h2 = GRUFork(n_kernels, n_hid, random_state)
params += conv_to_h1.get_params()
params += conv_to_h2.get_params()
cell1 = GRU(n_kernels, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
params += att_to_h1.get_params()
init_w = tensor.matrix("init_w")
init_w.tag.test_value = np_zeros((minibatch_size, n_chars))
params = []
biases = []
cell1 = GRU(input_dim, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
cell3 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
params += cell3.get_params()
inp_to_h1 = GRUFork(input_dim, n_hid, random_state)
inp_to_h2 = GRUFork(input_dim, n_hid, random_state)
inp_to_h3 = GRUFork(input_dim, n_hid, random_state)
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
att_to_h3 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
h1_to_h3 = GRUFork(n_hid, n_hid, random_state)
h2_to_h3 = GRUFork(n_hid, n_hid, random_state)
params += inp_to_h1.get_params()
params += inp_to_h2.get_params()
params += inp_to_h3.get_params()
params += att_to_h1.get_params()
params += att_to_h2.get_params()
params += att_to_h3.get_params()
init_w.tag.test_value = np_zeros((minibatch_size, n_chars))
params = []
cell1 = GRU(input_dim, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
cell3 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
params += cell3.get_params()
v_cell1 = GRU(1, n_v_hid, random_state)
params += v_cell1.get_params()
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
inp_to_h1 = GRUFork(input_dim, n_hid, random_state)
inp_to_h2 = GRUFork(input_dim, n_hid, random_state)
inp_to_h3 = GRUFork(input_dim, n_hid, random_state)
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
att_to_h3 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
h1_to_h3 = GRUFork(n_hid, n_hid, random_state)
h2_to_h3 = GRUFork(n_hid, n_hid, random_state)
params += inp_to_h1.get_params()
params += inp_to_h2.get_params()
params += inp_to_h3.get_params()
params += att_to_h1.get_params()
params += att_to_h2.get_params()
params += att_to_h3.get_params()
init_kappa.tag.test_value = np_zeros((minibatch_size, att_size))
init_w = tensor.matrix("init_w")
init_w.tag.test_value = np_zeros((minibatch_size, n_chars))
params = []
w_conv1, = make_conv_weights(1, (n_kernels, ), (conv_size1, input_dim),
random_state)
b_conv1, = make_biases((n_kernels, ))
w_conv2, = make_conv_weights(n_kernels, (n_kernels, ), (conv_size2, 1),
random_state)
b_conv2, = make_biases((n_kernels, ))
params += [w_conv1, b_conv1, w_conv2, b_conv2]
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
conv_to_h1 = GRUFork(n_kernels, n_hid, random_state)
conv_to_h2 = GRUFork(n_kernels, n_hid, random_state)
params += conv_to_h1.get_params()
params += conv_to_h2.get_params()
cell1 = GRU(n_kernels, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
params += att_to_h1.get_params()
cell1 = GRU(input_dim, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
cell3 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
params += cell3.get_params()
# Use GRU classes only to fork 1 inp to 2 inp:gate pairs
inp_proj, = make_weights(input_dim, [n_hid], random_state)
inp_b, = make_biases([n_hid])
params += [inp_proj, inp_b]
biases += [inp_b]
inp_to_h1 = GRUFork(n_hid, n_hid, random_state)
inp_to_h2 = GRUFork(n_hid, n_hid, random_state)
inp_to_h3 = GRUFork(n_hid, n_hid, random_state)
att_to_h1 = GRUFork(n_chars, n_hid, random_state)
att_to_h2 = GRUFork(n_chars, n_hid, random_state)
att_to_h3 = GRUFork(n_chars, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
h1_to_h3 = GRUFork(n_hid, n_hid, random_state)
h2_to_h3 = GRUFork(n_hid, n_hid, random_state)
params += inp_to_h1.get_params()
params += inp_to_h2.get_params()
params += inp_to_h3.get_params()
params += att_to_h1.get_params()
params += att_to_h2.get_params()
params += att_to_h3.get_params()
init_h3 = tensor.matrix("init_h3")
init_h3.tag.test_value = np_zeros((minibatch_size, n_hid))
params = []
biases = []
cell1 = GRU(input_dim, n_hid, random_state)
cell2 = GRU(n_hid, n_hid, random_state)
cell3 = GRU(n_hid, n_hid, random_state)
params += cell1.get_params()
params += cell2.get_params()
params += cell3.get_params()
inp_to_h1 = GRUFork(input_dim, n_hid, random_state)
inp_to_h2 = GRUFork(input_dim, n_hid, random_state)
inp_to_h3 = GRUFork(input_dim, n_hid, random_state)
h1_to_h2 = GRUFork(n_hid, n_hid, random_state)
h1_to_h3 = GRUFork(n_hid, n_hid, random_state)
h2_to_h3 = GRUFork(n_hid, n_hid, random_state)
params += inp_to_h1.get_params()
params += inp_to_h2.get_params()
params += inp_to_h3.get_params()
params += h1_to_h2.get_params()
params += h1_to_h3.get_params()
params += h2_to_h3.get_params()
biases += inp_to_h1.get_biases()
biases += inp_to_h2.get_biases()
init_h2_i = tensor.matrix("init_h2")
init_h2_i.tag.test_value = np_zeros((minibatch_size, n_hid))
init_h3_i = tensor.matrix("init_h3")
init_h3_i.tag.test_value = np_zeros((minibatch_size, n_hid))
init_h1, init_h2, init_h3 = LearnedInitHidden(
[init_h1_i, init_h2_i, init_h3_i], 3 * [(minibatch_size, n_hid)])
inpt = X_sym[:-1]
target = X_sym[1:]
mask = X_mask_sym[:-1]
embed_dim = 256
embed1 = Embedding(inpt, 256, embed_dim, random_state)
in_h1, ingate_h1 = GRUFork([embed1], [embed_dim], n_hid, random_state)
def step(in_h1_t, ingate_h1_t, h1_tm1, h2_tm1, h3_tm1):
h1_t = GRU(in_h1_t, ingate_h1_t, h1_tm1, n_hid, n_hid, random_state)
h1_h2_t, h1gate_h2_t = GRUFork([h1_t], [n_hid], n_hid, random_state)
h2_t = GRU(h1_h2_t, h1gate_h2_t, h2_tm1, n_hid, n_hid, random_state)
h2_h3_t, h2gate_h3_t = GRUFork([h2_t], [n_hid], n_hid, random_state)
h3_t = GRU(h2_h3_t, h2gate_h3_t, h3_tm1, n_hid, n_hid, random_state)
return h1_t, h2_t, h3_t
(h1, h2,
h3), updates = theano.scan(fn=step,
sequences=[in_h1, ingate_h1],
