def bi_lstm_encoder(input_seq, gate_size, para_name, args): # A bi-directional lstm encoder implementation. # Linear transformation part for input gate, output gate, forget gate # and cell activation vectors need be done outside of dynamic_lstm. # So the output size is 4 times of gate_size. input_forward_proj = layers.fc(input=input_seq, param_attr=fluid.ParamAttr(name=para_name + '_fw_gate_w'), size=gate_size * 4, act=None, bias_attr=False) input_reversed_proj = layers.fc(input=input_seq, param_attr=fluid.ParamAttr(name=para_name + '_bw_gate_w'), size=gate_size * 4, act=None, bias_attr=False) forward, _ = layers.dynamic_lstm( input=input_forward_proj, size=gate_size * 4, use_peepholes=False, param_attr=fluid.ParamAttr(name=para_name + '_fw_lstm_w'), bias_attr=fluid.ParamAttr(name=para_name + '_fw_lstm_b')) reversed, _ = layers.dynamic_lstm( input=input_reversed_proj, param_attr=fluid.ParamAttr(name=para_name + '_bw_lstm_w'), bias_attr=fluid.ParamAttr(name=para_name + '_bw_lstm_b'), size=gate_size * 4, is_reverse=True, use_peepholes=False) encoder_out = layers.concat(input=[forward, reversed], axis=1) return encoder_out
def __call__(self, inputs): lstm_list = [] name_prefix = "lstm" rnn_hidden_size = self.rnn_hidden_size for no in range(1, 3): if no == 1: is_reverse = False else: is_reverse = True name = "%s_st1_fc%d" % (name_prefix, no) fc = layers.fc(input=inputs, size=rnn_hidden_size * 4, param_attr=fluid.ParamAttr(name=name + "_w"), bias_attr=fluid.ParamAttr(name=name + "_b"), name=name) name = "%s_st1_out%d" % (name_prefix, no) lstm, _ = layers.dynamic_lstm( input=fc, size=rnn_hidden_size * 4, is_reverse=is_reverse, param_attr=fluid.ParamAttr(name=name + "_w"), bias_attr=fluid.ParamAttr(name=name + "_b"), use_peepholes=False) name = "%s_st2_fc%d" % (name_prefix, no) fc = layers.fc(input=lstm, size=rnn_hidden_size * 4, param_attr=fluid.ParamAttr(name=name + "_w"), bias_attr=fluid.ParamAttr(name=name + "_b"), name=name) name = "%s_st2_out%d" % (name_prefix, no) lstm, _ = layers.dynamic_lstm( input=fc, size=rnn_hidden_size * 4, is_reverse=is_reverse, param_attr=fluid.ParamAttr(name=name + "_w"), bias_attr=fluid.ParamAttr(name=name + "_b"), use_peepholes=False) lstm_list.append(lstm) return lstm_list
def bi_lstm_encoder(input_seq, gate_size): # A bi-directional lstm encoder implementation. # Linear transformation part for input gate, output gate, forget gate # and cell activation vectors need be done outside of dynamic_lstm. # So the output size is 4 times of gate_size. input_forward_proj = layers.fc(input=input_seq, size=gate_size * 4, act='tanh', bias_attr=False) forward, _ = layers.dynamic_lstm( input=input_forward_proj, size=gate_size * 4, use_peepholes=False) input_reversed_proj = layers.fc(input=input_seq, size=gate_size * 4, act='tanh', bias_attr=False) reversed, _ = layers.dynamic_lstm( input=input_reversed_proj, size=gate_size * 4, is_reverse=True, use_peepholes=False) encoder_out = layers.concat(input=[forward, reversed], axis = 1) return encoder_out
def encoder(): # encoder src_word = layers.data( name="src_word", shape=[1], dtype='int64', lod_level=1) src_embedding = layers.embedding( input=src_word, size=[dict_size, word_dim], dtype='float32', is_sparse=IS_SPARSE) fc1 = layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) encoder_out = layers.sequence_last_step(input=lstm_hidden0) return encoder_out
def __call__(self, input, h_0=None, c_0=None): return layers.dynamic_lstm( input=input, h_0=h_0, c_0=c_0, size=size, param_attr=self.attr_holder.param_attr, bias_attr=self.attr_holder.bias_attr, use_peepholes=use_peepholes, is_reverse=is_reverse, gate_activation=gate_activation, cell_activation=cell_activation, candidate_activation=candidate_activation, dtype=dtype)
def encoder(): src_word_id = pd.data( name="src_word_id", shape=[1], dtype='int64', lod_level=1) src_embedding = pd.embedding( input=src_word_id, size=[dict_size, word_dim], dtype='float32', is_sparse=is_sparse, param_attr=fluid.ParamAttr(name='vemb')) fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4) encoder_out = pd.sequence_last_step(input=lstm_hidden0) return encoder_out
def encoder(is_sparse): # encoder src_word_id = pd.data( name="src_word_id", shape=[1], dtype='int64', lod_level=1) src_embedding = pd.embedding( input=src_word_id, size=[dict_size, word_dim], dtype='float32', is_sparse=is_sparse, param_attr=fluid.ParamAttr(name='vemb')) fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4) encoder_out = pd.sequence_last_step(input=lstm_hidden0) return encoder_out
def encoder(): # Encoder implementation of RNN translation src_word = layers.data( name="src_word", shape=[1], dtype='int64', lod_level=1) src_embedding = layers.embedding( input=src_word, size=[source_dict_dim, embedding_dim], dtype='float32', is_sparse=True) fc1 = layers.fc(input=src_embedding, size=encoder_size * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm( input=fc1, size=encoder_size * 4) encoder_out = layers.sequence_last_step(input=lstm_hidden0) return encoder_out
def encoder_decoder(): # encoder src_word_id = layers.data(name="src_word_id", shape=[1], dtype='int64', lod_level=1) src_embedding = layers.embedding(input=src_word_id, size=[dict_size, word_dim], dtype='float32', is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr(name='vemb')) fc1 = fluid.layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) encoder_out = layers.sequence_last_step(input=lstm_hidden0) # decoder trg_language_word = layers.data(name="target_language_word", shape=[1], dtype='int64', lod_level=1) trg_embedding = layers.embedding(input=trg_language_word, size=[dict_size, word_dim], dtype='float32', is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr(name='vemb')) rnn = fluid.layers.DynamicRNN() with rnn.block(): current_word = rnn.step_input(trg_embedding) mem = rnn.memory(init=encoder_out) fc1 = fluid.layers.fc(input=[current_word, mem], size=decoder_size, act='tanh') out = fluid.layers.fc(input=fc1, size=target_dict_dim, act='softmax') rnn.update_memory(mem, fc1) rnn.output(out) return rnn()
def encoder_decoder(): # encoder src_word_id = layers.data( name="src_word_id", shape=[1], dtype='int64', lod_level=1) src_embedding = layers.embedding( input=src_word_id, size=[dict_size, word_dim], dtype='float32', is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr(name='vemb')) fc1 = fluid.layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) encoder_out = layers.sequence_last_step(input=lstm_hidden0) # decoder trg_language_word = layers.data( name="target_language_word", shape=[1], dtype='int64', lod_level=1) trg_embedding = layers.embedding( input=trg_language_word, size=[dict_size, word_dim], dtype='float32', is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr(name='vemb')) rnn = fluid.layers.DynamicRNN() with rnn.block(): current_word = rnn.step_input(trg_embedding) mem = rnn.memory(init=encoder_out) fc1 = fluid.layers.fc(input=[current_word, mem], size=decoder_size, act='tanh') out = fluid.layers.fc(input=fc1, size=target_dict_dim, act='softmax') rnn.update_memory(mem, fc1) rnn.output(out) return rnn()
def lstm_recv_inside(feat): forward, _ = L.dynamic_lstm( input=feat, size=hidden_dim * 4, use_peepholes=False) output = L.sequence_last_step(forward) return output