def RNN(x, weights, biases):
with WeightsInitializer(initializer=init_ops.Constant(0.1)) as vs:
cell1 = LSTMCell(n_hidden,debug=True)
cell2 = LSTMCell(n_hidden,debug=True)
cell= MultiRNNCell([cell1, cell2])
result, state = dynamic_rnn(cell, symbols_in_keys)
"Dense in this case should be out of WeightsInitializer scope because we are passing constants"
out_l = Dense(10,kernel_initializer=init_ops.Constant(out_weights),bias_initializer=init_ops.Constant(out_biases))
return out_l(state[-1].h)
def encoding_layer(rnn_cell_size, sequence_len, n_layers, rnn_inputs, dropout_prob):
if(encoder_type=="bi" and n_layers%2 == 0):
n_bi_layer=int(n_layers/2)
encoding_output, encoding_state=bidirectional_dynamic_rnn(get_rnn_cell(rnn_cell_size, dr_prob,n_bi_layer,debug),get_rnn_cell(rnn_cell_size, dr_prob,n_bi_layer,debug), rnn_inputs)
print("encoding_state:",encoding_state)
if(n_bi_layer > 1):
#layers/2
"""
Forward-First(0)
((LSTMStateTuple({'c': array([[0.30450274, 0.30450274, 0.30450274, 0.30450274, 0.30450274]]),
'h': array([[0.16661529, 0.16661529, 0.16661529, 0.16661529, 0.16661529]])}),
Forward-Second(1)
LSTMStateTuple({'c': array([[0.27710986, 0.07844026, 0.18714019, 0.28426586, 0.28426586]]),
'h': array([[0.15019765, 0.04329417, 0.10251247, 0.1539225 , 0.1539225 ]])})),
Backward-First(0)
(LSTMStateTuple({'c': array([[0.30499766, 0.30499766, 0.30499766, 0.30499766, 0.30499766]]),
'h': array([[0.16688152, 0.16688152, 0.16688152, 0.16688152, 0.16688152]])}),
Backward-Second(1)
LSTMStateTuple({'c': array([[0.25328871, 0.17537864, 0.21700339, 0.25627687, 0.25627687]]),
'h': array([[0.13779658, 0.09631104, 0.11861721, 0.1393639 , 0.1393639 ]])})))
"""
encoder_state = []
for layer_id in range(n_bi_layer):
encoder_state.append(encoding_state[0][layer_id]) # forward
encoder_state.append(encoding_state[1][layer_id]) # backward
encoding_state = tuple(encoder_state)
"""
First(0)
((LSTMStateTuple({'c': array([[0.30450274, 0.30450274, 0.30450274, 0.30450274, 0.30450274]]),
'h': array([[0.16661529, 0.16661529, 0.16661529, 0.16661529, 0.16661529]])}),
Second(1)
LSTMStateTuple({'c': array([[0.30499766, 0.30499766, 0.30499766, 0.30499766, 0.30499766]]),
'h': array([[0.16688152, 0.16688152, 0.16688152, 0.16688152, 0.16688152]])})),
Third(2)
(LSTMStateTuple({'c': array([[0.27710986, 0.07844026, 0.18714019, 0.28426586, 0.28426586]]),
'h': array([[0.15019765, 0.04329417, 0.10251247, 0.1539225 , 0.1539225 ]])}),
Fourth(3)
LSTMStateTuple({'c': array([[0.25328871, 0.17537864, 0.21700339, 0.25627687, 0.25627687]]),
'h': array([[0.13779658, 0.09631104, 0.11861721, 0.1393639 , 0.1393639 ]])})))
"""
else:
encoding_output, encoding_state=dynamic_rnn(get_rnn_cell(rnn_cell_size, dr_prob,n_layers,debug), rnn_inputs)
return encoding_output, encoding_state
kernel_initializer=init_ops.Constant(out_weights),
bias_initializer=init_ops.Constant(out_biases))
while step < training_iters:
if offset > (len(train_data) - end_offset):
offset = rnd.randint(0, n_input + 1)
print("offset:", offset)
symbols_in_keys = [
input_one_hot(dictionary[str(train_data[i])], vocab_size)
for i in range(offset, offset + n_input)
]
symbols_in_keys = np.reshape(np.array(symbols_in_keys),
[-1, n_input, vocab_size])
target = dictionary[str(train_data[offset + n_input])]
result, state = dynamic_rnn(cell, symbols_in_keys, initstate)
(c, h) = state.c, state.h
print("final:", result, state, h.shape)
#last layer of Feed Forward to compare to transform result to the shape of target
pred = out_l(h)
print("pred:", pred)
#cross_entropy_loss internally calculates the same softmax as and then the loss as above but for a batch and sequence
#pred- batch,seq,input_size
#labels-batch,seq(has to be transformed before comparision with preds(line-43).)
yhat, cel = cross_entropy_loss(pred.reshape([1, 1, vocab_size]),
np.array([[target]]))
print("yhat:", yhat)
print("CEL:", cel)