def encoder_init(input,
postcshape,
hidden_dim,
depth,
dropout=0,
seq2seq=True,
bidirectional=True,
unroll=False,
stateful=False,
Encoder=None,
global_name="",
return_model=False):
if Encoder == None:
Encoder = [hidden_dim] * depth[0]
else:
if len(Encoder) < depth[0]:
Encoder = Encoder + [hidden_dim] * (depth[0] - len(Encoder))
encoder = RecurrentSequential(
unroll=unroll,
stateful=stateful,
# return_states=True, return_all_states=True, AllStateTransfer needs modification in the tensorflow backend
return_sequences=True,
name=global_name + 'encoder')
encoder.add(LSTMCell(Encoder[0], batch_input_shape=postcshape[1:]))
for k in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(Encoder[k]))
if bidirectional:
encoder = Bidirectional(encoder,
merge_mode='sum',
name=global_name + 'encoder')
encoder.forward_layer.build(postcshape)
encoder.backward_layer.build(postcshape)
# patch
encoder.layer = encoder.forward_layer
if return_model:
enc_input = Input(shape=postcshape[1:], name='encoder_input')
encoded_out = encoder(enc_input)
encoder_model = Model(inputs=[enc_input], outputs=[encoded_out])
return encoder_model(input)
return encoder(input)
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
# encoded = encoder(_input)
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# inputs = [_input]
# decoded = decoder(encoded)
# model = Model(inputs, decoded)
return encoder, decoder
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
is_embedding=True,
embedding_dim=None,
n_tokens=1000,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=False,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
elif input_length:
if is_embedding == False and n_tokens > 0:
pass
else:
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
if is_embedding:
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
else:
i = Input(shape=(input_length, ), name='sentence_input', dtype='int32')
i._keras_history[0].supports_masking = True
if embedding_dim is None:
embedding_dim = hidden_dim
_input = Embedding(input_dim=n_tokens,
output_dim=embedding_dim,
input_length=input_length)(i)
shape = (batch_size, ) + (input_length, ) + (embedding_dim, )
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
x = encoder(_input)
decoder_outputs = decoder(x)
output = TimeDistributed(Dense(n_tokens,
activation='softmax'))(decoder_outputs)
if is_embedding:
return Model(_input, output)
else:
return Model(i, output)
def paired_trimodal_model(output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0):
"""
One modal translates into two other modalities, no cycle involved
The model has 1 encoder and 2 decoders
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# encoder phase
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
# encoder phase
encoder_2 = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder_2.add(
LSTMCell(hidden_dim, batch_input_shape=(shape[0], output_dim)))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
encoder_2.add(Dropout(dropout))
encoder_2.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoder_2 = Bidirectional(encoder_2, merge_mode='sum')
encoder_2.forward_layer.build(shape)
encoder_2.backward_layer.build(shape)
# patch
encoder_2.layer = encoder_2.forward_layer
encoded_one = encoder(_input)
# decoder phase
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
decoder_2 = RecurrentSequential(decode=True,
output_length=input_length,
unroll=unroll,
stateful=stateful)
decoder_2.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
if depth[1] == 1:
decoder_2.add(
AttentionDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
else:
decoder_2.add(
AttentionDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder_2.add(Dropout(dropout))
decoder_2.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder_2.add(Dropout(dropout))
decoder_2.add(
LSTMDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
inputs = [_input]
decoded_one = decoder(encoded_one)
encoded_two = encoder_2(decoded_one)
decoded_two = decoder_2(encoded_two)
return inputs, encoded_one, encoded_two, decoded_one, decoded_two
def mctn_model(output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0,
is_cycled=True):
"""
MCTN Model (by default with Cycle Consistency Loss)
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# encoder phase
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
# encoder.add(Dropout(dropout))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(_input)
# decoder phase
decoder = RecurrentSequential(
decode=True,
output_length=1, #output_length
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
inputs = [_input]
decoded_0 = decoder(encoded)
decoded = Reshape((output_dim, ))(decoded_0)
# cycle phase
cycled_decoded = None
if is_cycled:
cycled_encoded = encoder(decoded_0)
cycled_decoded = decoder(cycled_encoded)
return inputs, encoded, decoded, cycled_decoded
def mctn_level2_model(input,
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0):
"""
Level 2 MCTN used for translation between the joint embedded of
2 modalities to the third one. Due to the lack of ground truth, no
cycle phase happens
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise
if hidden_dim is None:
hidden_dim = output_dim
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(input)
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
inputs = [input]
decoded = decoder(encoded)
return inputs, encoded, decoded
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
[1] Sequence to Sequence Learning with Neural Networks
[2] Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation
[3] Neural Machine Translation by Jointly Learning to Align and Translate
[4] A Neural Conversational Model
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
# shape:[batch, max_encoder_length, input_dim]
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# 1.定义encoder
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(
hidden_dim,
batch_input_shape=(shape[0],
shape[2]))) # shape[0]:batch, shape[2]:input_dim
for _ in range(1, depth[0]): # 所谓的depth,就是lstm堆叠的层数
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(
shape) # [batch, max_encoder_length, input_dim]
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
# 2.encode
# _input:[batch, max_encoder_length, input_dim]
# encoded: [batch, max_encoder_length, hidden]
encoded = encoder(_input)
# 3.定义decoder
decoder = RecurrentSequential(
decode=True,
output_length=output_length,
unroll=unroll, # False
stateful=stateful) # False
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
# attention
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
if depth[1] != 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# 4. decode
decoded = decoder(encoded)
inputs = [_input]
model = Model(inputs=inputs, outputs=decoded)
return model
def Pointer(output_dim, output_length, batch_input_shape=None,
batch_size=None, input_shape=None, input_length=None,
input_dim=None, hidden_dim=None, depth=1,
bidirectional=True, unroll=False, stateful=False, dropout=0.0,):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward networ k.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size,) + input_shape
elif input_dim:
if input_length:
shape = (batch_size,) + (input_length,) + (input_dim,)
else:
shape = (batch_size,) + (None,) + (input_dim,)
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
# print shape
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoder = RecurrentSequential(unroll=unroll, stateful=False,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(_input)
decoder = RecurrentSequential(decode=True, output_length=output_length,
unroll=unroll, stateful=stateful,return_sequences=True)
# decoder.add(Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
# if depth[1] == 1:
# decoder.add(PointerDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
decoder.add(PointerDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim,batch_input_shape=(shape[0], shape[1], hidden_dim)))
# decoder.add(TimeDistributed(Activation('softmax')))
# decoder.add(TimeDistributed(Activation('softmax')))
# output = TimeDistributed(Dense(output_dim, activation='softmax'))
# output = TimeDistributed(Activation='softmax')
# output = TimeDistributed(Dense(output_dim, activation='softmax'))
# Dense(class_count, activation='softmax')(x)
# decoder.add(Dense(output_dim, activation='softmax')(x))
# else:
# decoder.add(PointerDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# for _ in range(depth[1] - 2):
# decoder.add(Dropout(dropout))
# decoder.add(LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
# decoder.add(Dropout(dropout))
# decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# Softmax is outside
# inputs = [_input]
# decoded = decoder(encoded)
# outputs = output(decoded)
# model = Model(inputs, outputs)
# return model
# Softmax is inside cell
inputs = [_input]
decoded = decoder(encoded)
model = Model(inputs, decoded)
return model
def AttentionSeqtoSeq(output_dim, output_length, batch_input_shape=None,
batch_size=None, input_shape=None, input_length=None,
input_dim=None, hidden_dim=None, depth=1,
bidirectional=True, unroll=False, stateful=False, dropout=0.0,
):
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size,) + input_shape
elif input_dim:
if input_length:
shape = (batch_size,) + (input_length,) + (input_dim,)
else:
shape = (batch_size,) + (None,) + (input_dim,)
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoder = RecurrentSequential(unroll=unroll, stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
#encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(_input)
#decoder_input = Input(batch_shape = encoded.shape)#新加的
decoder = RecurrentSequential(decode=True, output_length=output_length,
unroll=unroll, stateful=stateful)
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
#decoder.add(Dropout(dropout))
decoder.add(LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
#decoder.add(Dropout(dropout))
decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
decoder.add(Dense(output_dim*2))
decoder.add(Dense(output_dim,activation = "softmax"))
inputs = [_input]
decoded = decoder(encoded)
model = Model(inputs, decoded)
return model