def simple_rnn(rnn_input,
init_hidden,
hidden_size,
kernel_param_attr=None,
recurrent_param_attr=None,
bias_attr=None,
act='relu',
sequence_length=None,
name='simple_rnn'):
# Transpose (sequence x batch x hidden)
rnn_input = layers.transpose(rnn_input, [1, 0, 2])
# Generate Mask
mask = None
if sequence_length:
max_seq_len = layers.shape(rnn_input)[0]
mask = layers.sequence_mask(sequence_length,
maxlen=max_seq_len,
dtype='float32')
mask = layers.transpose(mask, [1, 0])
# Init
simple_rnn = SimpleRNN_unit(rnn_input, hidden_size, kernel_param_attr,
recurrent_param_attr, bias_attr, act)
rnn = PaddingRNN()
with rnn.step():
step_in = rnn.step_input(rnn_input)
if mask:
step_mask = rnn.step_input(mask)
if init_hidden:
pre_hidden = rnn.memory(init=init_hidden)
else:
pre_hidden = rnn.memory(batch_ref=rnn_input,
shape=[-1, hidden_size])
last_hidden = simple_rnn(step_in, pre_hidden)
rnn.update_memory(pre_hidden, last_hidden)
rnn.step_output(last_hidden)
step_input = last_hidden
rnn_out = rnn()
last_hidden = rnn_out[-1]
last_hidden = layers.reshape(last_hidden, shape=[1, -1, hidden_size])
rnn_output = layers.transpose(rnn_out, [1, 0, 2])
last_hidden = layers.transpose(last_hidden, [1, 0, 2])
return rnn_out, last_hidden
def rnn_decoder(gru_unit,
cue_gru_unit,
input,
input_size,
hidden_size,
num_layers,
memory,
memory_mask,
knowledge,
output_size,
init_hidden=None,
mask=None,
dropout=0.0,
batch_first=True,
name="decoder"):
""" rnn decoder """
input_emb = get_embedding(input, input_size, output_size)
if batch_first:
input_emb = layers.transpose(input_emb, perm=[1, 0, 2])
if mask:
trans_mask = layers.transpose(mask, perm=[1, 0])
rnn = PaddingRNN()
with rnn.step():
step_in = rnn.step_input(input_emb)
step_mask = None
if mask:
step_mask = rnn.step_input(trans_mask)
# split pre_hidden
pre_hidden_list = []
pre_hidden = rnn.memory(init=init_hidden)
real_out, last_hidden = \
decoder_step(gru_unit, cue_gru_unit, step_in, pre_hidden, input_size,
hidden_size, memory, memory_mask, knowledge, mask=step_mask)
rnn.update_memory(pre_hidden, last_hidden)
step_in = layers.squeeze(real_out, axes=[1])
rnn.step_output(step_in)
rnnout = rnn()
rnnout = layers.transpose(rnnout, perm=[1, 0, 2])
rnnout = layers.elementwise_mul(rnnout, mask, axis=0)
output_in_size = hidden_size + hidden_size
rnnout = layers.dropout(rnnout, dropout_prob=dropout)
rnnout = fc(rnnout, output_in_size, hidden_size, name='dec_out_fc1')
rnnout = fc(rnnout, hidden_size, output_size, name='dec_out_fc2')
softmax_out = layers.softmax(rnnout)
return softmax_out
def gru_rnn(input,
input_size,
hidden_size,
init_hidden=None,
batch_first=False,
mask=None,
num_layers=1,
dropout=0.0,
name="gru"):
""" gru rnn """
gru_unit = GRU_unit(input_size,
hidden_size,
num_layers=num_layers,
dropout=dropout,
name=name + "_gru_unit")
if batch_first:
input = layers.transpose(x=input, perm=[1, 0, 2])
if mask:
mask = layers.transpose(mask, perm=[1, 0])
rnn = PaddingRNN()
with rnn.step():
step_in = rnn.step_input(input)
step_mask = None
if mask:
step_mask = rnn.step_input(mask)
pre_hidden = rnn.memory(init=init_hidden)
new_hidden, last_hidden = gru_unit(step_in, pre_hidden, step_mask)
rnn.update_memory(pre_hidden, last_hidden)
step_in = new_hidden
rnn.step_output(step_in)
rnn.step_output(last_hidden)
rnn_res = rnn()
rnn_out = rnn_res[0]
last_hidden = layers.slice(rnn_res[1],
axes=[0],
starts=[-1],
ends=[1000000000])
last_hidden = layers.reshape(last_hidden,
shape=[num_layers, -1, hidden_size])
if batch_first:
rnnout = layers.transpose(x=rnn_out, perm=[1, 0, 2])
return rnnout, last_hidden
def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None):
weight_1_arr = []
weight_2_arr = []
bias_arr = []
hidden_array = []
cell_array = []
mask_array = []
for i in range(num_layers):
weight_1 = layers.create_parameter(
[hidden_size * 2, hidden_size * 4],
dtype="float32",
name="fc_weight1_" + str(i),
default_initializer=fluid.initializer.UniformInitializer(
low=-init_scale, high=init_scale))
weight_1_arr.append(weight_1)
bias_1 = layers.create_parameter(
[hidden_size * 4],
dtype="float32",
name="fc_bias1_" + str(i),
default_initializer=fluid.initializer.Constant(0.0))
bias_arr.append(bias_1)
pre_hidden = layers.slice(
init_hidden, axes=[0], starts=[i], ends=[i + 1])
pre_cell = layers.slice(
init_cell, axes=[0], starts=[i], ends=[i + 1])
pre_hidden = layers.reshape(pre_hidden, shape=[-1, hidden_size])
pre_cell = layers.reshape(pre_cell, shape=[-1, hidden_size])
hidden_array.append(pre_hidden)
cell_array.append(pre_cell)
input_embedding = layers.transpose(input_embedding, perm=[1, 0, 2])
rnn = PaddingRNN()
with rnn.step():
input = rnn.step_input(input_embedding)
for k in range(num_layers):
pre_hidden = rnn.memory(init=hidden_array[k])
pre_cell = rnn.memory(init=cell_array[k])
weight_1 = weight_1_arr[k]
bias = bias_arr[k]
nn = layers.concat([input, pre_hidden], 1)
gate_input = layers.matmul(x=nn, y=weight_1)
gate_input = layers.elementwise_add(gate_input, bias)
i = layers.slice(
gate_input, axes=[1], starts=[0], ends=[hidden_size])
j = layers.slice(
gate_input,
axes=[1],
starts=[hidden_size],
ends=[hidden_size * 2])
f = layers.slice(
gate_input,
axes=[1],
starts=[hidden_size * 2],
ends=[hidden_size * 3])
o = layers.slice(
gate_input,
axes=[1],
starts=[hidden_size * 3],
ends=[hidden_size * 4])
c = pre_cell * layers.sigmoid(f) + layers.sigmoid(
i) * layers.tanh(j)
m = layers.tanh(c) * layers.sigmoid(o)
rnn.update_memory(pre_hidden, m)
rnn.update_memory(pre_cell, c)
rnn.step_output(m)
rnn.step_output(c)
input = m
if dropout != None and dropout > 0.0:
input = layers.dropout(
input,
dropout_prob=dropout,
dropout_implementation='upscale_in_train')
rnn.step_output(input)
rnnout = rnn()
last_hidden_array = []
last_cell_array = []
real_res = rnnout[-1]
for i in range(num_layers):
m = rnnout[i * 2]
c = rnnout[i * 2 + 1]
m.stop_gradient = True
c.stop_gradient = True
last_h = layers.slice(
m, axes=[0], starts=[num_steps - 1], ends=[num_steps])
last_hidden_array.append(last_h)
last_c = layers.slice(
c, axes=[0], starts=[num_steps - 1], ends=[num_steps])
last_cell_array.append(last_c)
real_res = layers.transpose(x=real_res, perm=[1, 0, 2])
last_hidden = layers.concat(last_hidden_array, 0)
last_cell = layers.concat(last_cell_array, 0)
return real_res, last_hidden, last_cell
def convlstm2d_rnn(rnn_input,
init_hidden,
init_cell,
padding,
hidden_h,
hidden_w,
filters,
filter_size,
drop_out=None,
sequence_length=None,
name='conv_lstm_2d'):
# transpose : (sequence x batch x C x H x W)
rnn_input = layers.transpose(rnn_input, [1, 0, 4, 2, 3])
# generate mask
mask = None
if sequence_length:
max_seq_len = layers.shape(rnn_input)[0]
mask = layers.sequence_mask(sequence_length,
maxlen=max_seq_len,
dtype='float32')
mask = layers.transpose(mask, [1, 0])
# init
conv_lstm_2d = ConvLSTM2D_unit(filters, filter_size, padding)
rnn = PaddingRNN()
with rnn.step():
step_in = rnn.step_input(rnn_input)
if mask:
step_mask = rnn.step_input(mask)
if init_hidden and init_cell:
pre_hidden = rnn.memory(init=init_hidden)
pre_cell = rnn.memory(init=init_cell)
else:
pre_hidden = rnn.memory(batch_ref=rnn_input,
shape=[-1, filters, hidden_h, hidden_w])
pre_cell = rnn.memory(batch_ref=rnn_input,
shape=[-1, filters, hidden_h, hidden_w])
real_out, last_hidden, last_cell = conv_lstm_2d(
step_in, pre_hidden, pre_cell)
if mask:
last_hidden = dot(last_hidden, step_mask, axis=0) - dot(
pre_hidden, (step_mask - 1), axis=0)
last_cell = dot(last_cell, step_mask, axis=0) - dot(
pre_cell, (step_mask - 1), axis=0)
rnn.update_memory(pre_hidden, last_hidden)
rnn.update_memory(pre_cell, last_cell)
rnn.step_output(last_hidden)
rnn.step_output(last_cell)
step_input = last_hidden
if drop_out != None and drop_out > 0.0:
step_input = layers.dropout(
step_input,
dropout_prob=drop_out,
dropout_implementation='upscale_in_train')
rnn_res = rnn()
rnn_out = rnn_res[0]
last_hidden = layers.slice(rnn_res[1],
axes=[0],
starts=[-1],
ends=[1000000000])
rnn_out = layers.transpose(rnn_out, [1, 0, 3, 4, 2])
last_hidden = layers.transpose(last_hidden, [1, 0, 3, 4, 2])
# print('rnn_out ', rnn_out.shape)
# print('last_hidden ', last_hidden.shape)
return rnn_out, last_hidden