def __build_readout_decoder__(self):
self.decoder.add(
RepeatVector(
self.sequence_len,
input_shape=(self.enc_layer_output[-1],
))) # Repeat the final vector for answer input
# Using recurrentshop's container with readout
container = RecurrentContainer(readout=True,
return_sequences=True,
output_length=self.sequence_len)
if len(self.dec_layer_output) > 1:
container.add(
LSTMCell(output_dim=self.dec_layer_output[0],
input_dim=self.enc_layer_output[-1]))
for dl in self.dec_layer_output[1:-1]:
container.add(LSTMCell(output_dim=dl))
container.add(LSTMCell(output_dim=self.enc_layer_output[-1]))
else:
container.add(
LSTMCell(input_dim=self.enc_layer_output[-1],
output_dim=self.enc_layer_output[-1]))
if self.enc_layer_output[-1] != self.dec_layer_output[-1]:
print(
'WARNING: Overriding final decoder output to %s for readout compatibility'
% self.enc_layer_output[-1])
self.decoder.add(container)
def __build_seq2seq_decoder__(self):
# Using recurrentshop's decoder container
container = RecurrentContainer(
return_sequences=True,
readout='add',
output_length=self.sequence_len,
input_shape=(self.enc_layer_output[-1], ),
decode=True)
if len(self.dec_layer_output) > 1:
container.add(
LSTMCell(output_dim=self.dec_layer_output[0],
input_dim=self.enc_layer_output[-1]))
for dl in self.dec_layer_output[1:-1]:
container.add(LSTMCell(output_dim=dl))
container.add(LSTMCell(output_dim=self.enc_layer_output[-1]))
else:
container.add(
LSTMCell(input_dim=self.enc_layer_output[-1],
output_dim=self.enc_layer_output[-1]))
if self.enc_layer_output[-1] != self.dec_layer_output[-1]:
print(
'WARNING: Overriding final decoder output to %s for readout compatibility'
% self.enc_layer_output[-1])
self.decoder.add(container)
def encoder_init(input,
postcshape,
hidden_dim,
depth,
dropout=0,
seq2seq=True,
bidirectional=True,
unroll=False,
stateful=False,
Encoder=None,
global_name="",
return_model=False):
if Encoder == None:
Encoder = [hidden_dim] * depth[0]
else:
if len(Encoder) < depth[0]:
Encoder = Encoder + [hidden_dim] * (depth[0] - len(Encoder))
encoder = RecurrentSequential(
unroll=unroll,
stateful=stateful,
# return_states=True, return_all_states=True, AllStateTransfer needs modification in the tensorflow backend
return_sequences=True,
name=global_name + 'encoder')
encoder.add(LSTMCell(Encoder[0], batch_input_shape=postcshape[1:]))
for k in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(Encoder[k]))
if bidirectional:
encoder = Bidirectional(encoder,
merge_mode='sum',
name=global_name + 'encoder')
encoder.forward_layer.build(postcshape)
encoder.backward_layer.build(postcshape)
# patch
encoder.layer = encoder.forward_layer
if return_model:
enc_input = Input(shape=postcshape[1:], name='encoder_input')
encoded_out = encoder(enc_input)
encoder_model = Model(inputs=[enc_input], outputs=[encoded_out])
return encoder_model(input)
return encoder(input)
def Seq2Seq(output_dim,
output_length,
batch_input_shape=None,
input_shape=None,
batch_size=None,
input_dim=None,
input_length=None,
hidden_dim=None,
depth=1,
broadcast_state=True,
unroll=False,
stateful=False,
inner_broadcast_state=True,
teacher_force=False,
peek=False,
dropout=0.):
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
encoder = RecurrentSequential(readout=True,
state_sync=inner_broadcast_state,
unroll=unroll,
stateful=stateful,
return_states=broadcast_state)
for _ in range(depth[0]):
encoder.add(
LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
encoder.add(Dropout(dropout))
dense1 = TimeDistributed(Dense(hidden_dim))
dense1.supports_masking = True
dense2 = Dense(output_dim)
decoder = RecurrentSequential(readout='add' if peek else 'readout_only',
state_sync=inner_broadcast_state,
decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful,
teacher_force=teacher_force)
for _ in range(depth[1]):
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
decoder.add(
LSTMDecoderCell(output_dim=output_dim,
hidden_dim=hidden_dim,
batch_input_shape=(shape[0], output_dim)))
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoded_seq = dense1(_input)
encoded_seq = encoder(encoded_seq)
if broadcast_state:
assert type(encoded_seq) is list
states = encoded_seq[-2:]
encoded_seq = encoded_seq[0]
else:
states = None
encoded_seq = dense2(encoded_seq)
inputs = [_input]
if teacher_force:
truth_tensor = Input(batch_shape=(shape[0], output_length, output_dim))
truth_tensor._keras_history[0].supports_masking = True
inputs += [truth_tensor]
decoded_seq = decoder(encoded_seq,
ground_truth=inputs[1] if teacher_force else None,
initial_readout=encoded_seq,
initial_state=states)
model = Model(inputs, decoded_seq)
model.encoder = encoder
model.decoder = decoder
return model
def SimpleSeq2Seq(output_dim,
output_length,
hidden_dim=None,
input_shape=None,
batch_size=None,
batch_input_shape=None,
input_dim=None,
input_length=None,
depth=1,
dropout=0.0,
unroll=False,
stateful=False):
'''
Simple model for sequence to sequence learning.
The encoder encodes the input sequence to vector (called context vector)
The decoder decodes the context vector in to a sequence of vectors.
There is no one on one relation between the input and output sequence
elements. The input sequence and output sequence may differ in length.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
dropout : Dropout probability in between layers.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
encoder = RecurrentSequential(unroll=unroll, stateful=stateful)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
decoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
decode=True,
output_length=output_length)
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim)))
if depth[1] == 1:
decoder.add(LSTMCell(output_dim))
else:
decoder.add(LSTMCell(hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(output_dim))
return encoder, decoder
def Seq2Seq(output_dim,
output_length,
batch_input_shape=None,
input_shape=None,
batch_size=None,
input_dim=None,
input_length=None,
hidden_dim=None,
depth=1,
broadcast_state=True,
unroll=False,
stateful=False,
inner_broadcast_state=True,
teacher_force=False,
peek=False,
dropout=0.):
'''
Seq2seq model based on [1] and [2].
This model has the ability to transfer the encoder hidden state to the decoder's
hidden state(specified by the broadcast_state argument). Also, in deep models
(depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
the inner_broadcast_state argument. You can switch between [1] based model and [2]
based model using the peek argument.(peek = True for [2], peek = False for [1]).
When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
[1] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
y(0) = LSTM(s0, C); C is the context vector from the encoder.
[2] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1), C)
y(0) = LSTM(s0, C, C)
Where s is the hidden state of the LSTM (h and c), and C is the context vector
from the encoder.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
broadcast_state : Specifies whether the hidden state from encoder should be
transfered to the deocder.
inner_broadcast_state : Specifies whether hidden states should be propogated
throughout the LSTM stack in deep models.
peek : Specifies if the decoder should be able to peek at the context vector
at every timestep.
dropout : Dropout probability in between layers.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
encoder = RecurrentSequential(readout=True,
state_sync=inner_broadcast_state,
unroll=unroll,
stateful=stateful,
return_states=broadcast_state)
for _ in range(depth[0]):
encoder.add(
LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
encoder.add(Dropout(dropout))
dense1 = TimeDistributed(Dense(hidden_dim))
dense1.supports_masking = True
dense2 = Dense(output_dim)
decoder = RecurrentSequential(readout='add' if peek else 'readout_only',
state_sync=inner_broadcast_state,
decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful,
teacher_force=teacher_force)
for _ in range(depth[1]):
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
decoder.add(
LSTMDecoderCell(output_dim=output_dim,
hidden_dim=hidden_dim,
batch_input_shape=(shape[0], output_dim)))
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoded_seq = dense1(_input)
encoded_seq = encoder(encoded_seq)
if broadcast_state:
assert type(encoded_seq) is list
states = encoded_seq[-2:]
encoded_seq = encoded_seq[0]
else:
states = None
encoded_seq = dense2(encoded_seq)
inputs = [_input]
if teacher_force:
truth_tensor = Input(batch_shape=(shape[0], output_length, output_dim))
truth_tensor._keras_history[0].supports_masking = True
inputs += [truth_tensor]
decoded_seq = decoder(encoded_seq,
ground_truth=inputs[1] if teacher_force else None,
initial_readout=encoded_seq,
initial_state=states)
model = Model(inputs, decoded_seq)
model.encoder = encoder
model.decoder = decoder
return model
def Seq2Seq(output_dim, output_length, hidden_dim=None, depth=1, broadcast_state=True, inner_broadcast_state=True, peek=False, dropout=0., **kwargs):
'''
Seq2seq model based on [1] and [2].
This model has the ability to transfer the encoder hidden state to the decoder's
hidden state(specified by the broadcast_state argument). Also, in deep models
(depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
the inner_broadcast_state argument. You can switch between [1] based model and [2]
based model using the peek argument.(peek = True for [2], peek = False for [1]).
When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
[1] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
y(0) = LSTM(s0, C); C is the context vector from the encoder.
[2] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1), C)
y(0) = LSTM(s0, C, C)
Where s is the hidden state of the LSTM (h and c), and C is the context vector
from the encoder.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
broadcast_state : Specifies whether the hidden state from encoder should be
transfered to the deocder.
inner_broadcast_state : Specifies whether hidden states should be propogated
throughout the LSTM stack in deep models.
peek : Specifies if the decoder should be able to peek at the context vector
at every timestep.
dropout : Dropout probability in between layers.
'''
if type(depth) == int:
depth = [depth, depth]
if 'batch_input_shape' in kwargs:
shape = kwargs['batch_input_shape']
del kwargs['batch_input_shape']
elif 'input_shape' in kwargs:
shape = (None,) + tuple(kwargs['input_shape'])
del kwargs['input_shape']
elif 'input_dim' in kwargs:
if 'input_length' in kwargs:
shape = (None, kwargs['input_length'], kwargs['input_dim'])
del kwargs['input_length']
else:
shape = (None, None, kwargs['input_dim'])
del kwargs['input_dim']
if 'unroll' in kwargs:
unroll = kwargs['unroll']
del kwargs['unroll']
else:
unroll = False
if 'stateful' in kwargs:
stateful = kwargs['stateful']
del kwargs['stateful']
else:
stateful = False
if not hidden_dim:
hidden_dim = output_dim
encoder = RecurrentContainer(readout=True, state_sync=inner_broadcast_state, input_length=shape[1], unroll=unroll, stateful=stateful)
for i in range(depth[0]):
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim), **kwargs))
encoder.add(Dropout(dropout))
dense1 = TimeDistributed(Dense(hidden_dim))
dense2 = Dense(output_dim)
decoder = RecurrentContainer(readout='add' if peek else 'readout_only', state_sync=inner_broadcast_state, output_length=output_length, unroll=unroll, stateful=stateful, decode=True)
for i in range(depth[1]):
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], output_dim), **kwargs))
input = Input(batch_shape=shape)
encoded_seq = dense1(input)
encoded_seq = encoder(encoded_seq)
if broadcast_state:
decoder.model.layers[1].states[:2] = encoder.state_outputs[-3:-1]
encoded_seq = dense2(encoded_seq)
decoder.initial_readout = encoded_seq
decoded_seq = decoder(encoded_seq)
model = Model(input, decoded_seq)
model.encoder = encoder
model.decoder = decoder
return model
def SimpleSeq2Seq(output_dim, output_length, hidden_dim=None, depth=1, dropout=0., **kwargs): ''' Simple model for sequence to sequence learning. The encoder encodes the input sequence to vector (called context vector) The decoder decodes the context vector in to a sequence of vectors. There is no one on one relation between the input and output sequence elements. The input sequence and output sequence may differ in length. Arguments: output_dim : Required output dimension. hidden_dim : The dimension of the internal representations of the model. output_length : Length of the required output sequence. depth : Used to create a deep Seq2seq model. For example, if depth = 3, there will be 3 LSTMs on the enoding side and 3 LSTMs on the decoding side. You can also specify depth as a tuple. For example, if depth = (4, 5), 4 LSTMs will be added to the encoding side and 5 LSTMs will be added to the decoding side. dropout : Dropout probability in between layers. ''' if type(depth) == int: depth = [depth, depth] if 'batch_input_shape' in kwargs: shape = kwargs['batch_input_shape'] del kwargs['batch_input_shape'] elif 'input_shape' in kwargs: shape = (None,) + tuple(kwargs['input_shape']) del kwargs['input_shape'] elif 'input_dim' in kwargs: if 'input_length' in kwargs: shape = (None, kwargs['input_length'], kwargs['input_dim']) del kwargs['input_length'] else: shape = (None, None, kwargs['input_dim']) del kwargs['input_dim'] if 'unroll' in kwargs: unroll = kwargs['unroll'] del kwargs['unroll'] else: unroll = False if 'stateful' in kwargs: stateful = kwargs['stateful'] del kwargs['stateful'] else: stateful = False if not hidden_dim: hidden_dim = output_dim encoder = RecurrentContainer(unroll=unroll, stateful=stateful, input_length=shape[1]) encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2]), **kwargs)) for _ in range(1, depth[0]): encoder.add(Dropout(dropout)) encoder.add(LSTMCell(hidden_dim, **kwargs)) decoder = RecurrentContainer(unroll=unroll, stateful=stateful, decode=True, output_length=output_length) decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim))) decoder.add(LSTMCell(hidden_dim, **kwargs)) for _ in range(1, depth[1]): decoder.add(Dropout(dropout)) decoder.add(LSTMCell(hidden_dim, **kwargs)) model = Sequential() model.add(encoder) model.add(decoder) return model
def SimpleSeq2Seq(output_dim,
output_length,
hidden_dim=None,
input_shape=None,
batch_size=None,
batch_input_shape=None,
input_dim=None,
input_length=None,
is_embedding=True,
embedding_dim=None,
n_tokens=None,
depth=1,
dropout=0.0,
unroll=False,
stateful=False):
'''
Simple model for sequence to sequence learning.
The encoder encodes the input sequence to vector (called context vector)
The decoder decodes the context vector in to a sequence of vectors.
There is no one on one relation between the input and output sequence
elements. The input sequence and output sequence may differ in length.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
dropout : Dropout probability in between layers.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
elif input_length:
if is_embedding == False and n_tokens > 0:
pass
else:
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
if is_embedding:
_input = Input(batch_shape=shape)
else:
i = Input(shape=(input_length, ), name='sentence_input', dtype='int32')
if embedding_dim is None:
embedding_dim = hidden_dim
_input = Embedding(input_dim=n_tokens,
output_dim=embedding_dim,
mask_zero=True,
input_length=input_length)(i)
shape = (batch_size, ) + (input_length, ) + (embedding_dim, )
encoder = RecurrentSequential(unroll=unroll, stateful=stateful)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
decoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
decode=True,
output_length=output_length)
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim)))
if depth[1] == 1:
decoder.add(LSTMCell(output_dim))
else:
decoder.add(LSTMCell(hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(output_dim))
x = encoder(_input)
decoder_outputs = decoder(x)
output = TimeDistributed(Dense(n_tokens,
activation='softmax'))(decoder_outputs)
if is_embedding:
return Model(_input, output)
else:
return Model(i, output)
def __init__(self, vocab, config, hps):
super(SummarizationModel, self).__init__(config)
self.__name = 'pointer_generator_summarizer'
self.config = config
self.hps = hps
self.mode = config['mode']
self.use_coverage = config['use_coverage']
self.pointer_gen = config['pointer_gen']
self.embed_trainable = config['train_embed']
self.embedding_size = config['embed_size']
self.vsize = config['vocab_size']
self.rand_unif_init_mag = config['rand_unif_init_mag']
self.trunc_norm_init_std = config['trunc_norm_init_std']
self.hidden_units = self.config['hidden_units']
self.cov_loss_wt = self.config['cov_loss_wt']
# Initializers:
self.rand_unif_init = RandomUniform(minval=-self.rand_unif_init_mag,
maxval=self.rand_unif_init_mag,
seed=123)
self.trunc_norm_init = TruncatedNormal(stddev=self.trunc_norm_init_std)
# Optimizers:
self.adg = optimizers.TFOptimizer(
K.tf.train.AdagradOptimizer(
self.hps.lr,
initial_accumulator_value=self.hps.adagrad_init_acc))
# Layers
self.Emb = Embedding(self.vsize,
self.embedding_size,
weights=config['embed'],
trainable=self.embed_trainable)
# different dictionary for source and target
# Bi-directional lstm encoder, return (output, states)
# Dimension: 2*hidden_units
# concatenated forward and backward vectors
self.Encoder = Bidirectional(
CuDNNLSTM(self.hidden_units,
return_state=True,
return_sequences=True,
kernel_initializer=self.rand_unif_init))
# Decoder is not bi-directional, perform linear reduction...
# Dense_layer_dimension=encoder_hidden_units
# Encoder states and output tensors are separated...
# to initialize decoder
# Decoder cell input: [input, state_h, state_c]
self.DecoderCell = LSTMCell(self.hidden_units,
kernel_initializer=self.rand_unif_init,
bias_initializer="zeros",
recurrent_initializer=self.rand_unif_init)
# Decoder output projector
# to probabilities[word_index]
self.DecoderOutputProjector = Dense(
self.vsize,
kernel_initializer=self.trunc_norm_init,
bias_initializer=self.trunc_norm_init,
activation=None)
self.ConcatenateAxis1 = Concatenate(axis=1)
self.ConcatenateLastDim = Concatenate(axis=-1)
self.StackSecondDim = Lambda(lambda x: K.tf.stack(x, axis=1))
self.SoftmaxforScore = Softmax(axis=-1)
self._batch_size = None
self._enc_batch = None
self._enc_lens = None
self._enc_padding_mask = None
self._enc_batch_extend_vocab = None
self._max_art_oovs = None
self._max_art_oovs_inp = None
self._dec_batch = None
self._target_batch = None
self._dec_padding_mask = None
self._dec_in_state = None
self._enc_states = None
self._dec_out_state = None
self.p_gens = None
self.prev_coverage = None
self.coverage = None
self._coverage_loss = None
self.check_list = []
if not self.check():
pass
pass
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
[1] Sequence to Sequence Learning with Neural Networks
[2] Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation
[3] Neural Machine Translation by Jointly Learning to Align and Translate
[4] A Neural Conversational Model
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
# shape:[batch, max_encoder_length, input_dim]
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# 1.定义encoder
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(
hidden_dim,
batch_input_shape=(shape[0],
shape[2]))) # shape[0]:batch, shape[2]:input_dim
for _ in range(1, depth[0]): # 所谓的depth,就是lstm堆叠的层数
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(
shape) # [batch, max_encoder_length, input_dim]
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
# 2.encode
# _input:[batch, max_encoder_length, input_dim]
# encoded: [batch, max_encoder_length, hidden]
encoded = encoder(_input)
# 3.定义decoder
decoder = RecurrentSequential(
decode=True,
output_length=output_length,
unroll=unroll, # False
stateful=stateful) # False
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
# attention
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
if depth[1] != 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# 4. decode
decoded = decoder(encoded)
inputs = [_input]
model = Model(inputs=inputs, outputs=decoded)
return model
def SimpleSeq2Seq(output_dim, output_length, hidden_dim=None, input_shape=None,
batch_size=None, batch_input_shape=None, input_dim=None,
input_length=None, depth=1, dropout=0.0, unroll=False,
stateful=False):
'''
Simple model for sequence to sequence learning.
The encoder encodes the input sequence to vector (called context vector)
The decoder decodes the context vector in to a sequence of vectors.
There is no one on one relation between the input and output sequence
elements. The input sequence and output sequence may differ in length.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
dropout : Dropout probability in between layers.
'''
'''
Below block is trying to construct the shape tuple which is (batch_size,input_dim,input_length)
batch_shape(batch_size,input_dim,input_length)
# TODO: Add more information about the statefulness of the LSTM and the input_shape expected by the same.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size,) + input_shape
elif input_dim:
if input_length:
shape = (batch_size,) + (input_length,) + (input_dim,)
else:
shape = (batch_size,) + (None,) + (input_dim,)
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
'''
batch_input_shape=(batch_size, timesteps, data_dim)
for lstm arguments :- https://keras.io/layers/recurrent/#lstm
>Specifically hidden_dim = units and it is a Positive integer representing dimensionality of the output space.
'''
encoder = RecurrentSequential(unroll=unroll, stateful=stateful)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
decoder = RecurrentSequential(unroll=unroll, stateful=stateful,
decode=True, output_length=output_length)
'''
hidden_dim is here equal to the output dim of the encoder thing. Hence should not have shape error.
Also the second arguments is the expected dimension of the input. See below comment
'''
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim)))
if depth[1] == 1:
decoder.add(LSTMCell(output_dim))
else:
decoder.add(LSTMCell(hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(output_dim))
'''
This returns a tensor
>batch_shape(batch_size,input_dim,input_length)
https://keras.io/layers/recurrent/ :- Note on using statefulness in RNNs
'''
_input = Input(batch_shape=shape)
x = encoder(_input)
output = decoder(x)
'''
This Model thing; takes _input as the input and applies everything to reach output.
This model will include all layers required in the computation of output given _input. So one can
think of them as Theano function.
'''
return Model(_input, output)
def Seq2Seq(output_dim, output_length, batch_input_shape=None,
input_shape=None, batch_size=None, input_dim=None, input_length=None,
hidden_dim=None, depth=1, broadcast_state=True, unroll=False,
stateful=False, inner_broadcast_state=True, teacher_force=False,
peek=False, dropout=0.):
'''
Seq2seq model based on [1] and [2].
This model has the ability to transfer the encoder hidden state to the decoder's
hidden state(specified by the broadcast_state argument). Also, in deep models
(depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
the inner_broadcast_state argument. You can switch between [1] based model and [2]
based model using the peek argument.(peek = True for [2], peek = False for [1]).
When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
[1] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
y(0) = LSTM(s0, C); C is the context vector from the encoder.
[2] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1), C)
y(0) = LSTM(s0, C, C)
Where s is the hidden state of the LSTM (h and c), and C is the context vector
from the encoder.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
broadcast_state : Specifies whether the hidden state from encoder should be
transfered to the deocder.
inner_broadcast_state : Specifies whether hidden states should be propogated
throughout the LSTM stack in deep models.
peek : Specifies if the decoder should be able to peek at the context vector
at every timestep.
dropout : Dropout probability in between layers.
'''
if isinstance(depth, int):
depth = (depth, depth)
# depth是整数时，相当于编码器和解码器都有相同的层数
if batch_input_shape:
shape = batch_input_shape
# 批输入的shape作为模型输入的shape
elif input_shape:
shape = (batch_size,) + input_shape
# 不指定batch input shape，则用批大小拼接input shape，如batch size为32，input为768，拼接后就是（32，768）
# input shape 必须是一个元组
elif input_dim:
if input_length:
shape = (batch_size,) + (input_length,) + (input_dim,)
# 一般情况下通用的shape（批大小，输入序列长度，输入维度）
else:
shape = (batch_size,) + (None,) + (input_dim,)
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
# 隐藏层的维度如果也是None？那代表什么呢
encoder = RecurrentSequential(readout=True, state_sync=inner_broadcast_state,
unroll=unroll, stateful=stateful,
return_states=broadcast_state)
'''
参数：
readout：是否额外将输出进行处理 选项有add（True），multiply，average，maximum等
state_sync：状态是否在内部传播，源码中对initial_states的处理不一样，对每个cell的state都进行传播到下一个batch
stateful：keras特性，在不同的batch之间传递cells的状态，而不是仅仅在cell之间传递状态，即stateful
在stateful = True 时，我们要在fit中手动使得shuffle = False。随后，在X[i]（表示输入矩阵中第
i个sample）这个小序列训练完之后，Keras会将将训练完的记忆参数传递给X[i+bs]（表示第i+bs个sample）,
作为其初始的记忆参数。
unroll：keras特性，将LSTM网络展开，也就是原本的时序序列直接展开成多个cell拼接，可以加快速度，但是占用更多内存
'''
for _ in range(depth[0]):
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
encoder.add(Dropout(dropout))
# 根据depth[0]指定编码器深度
dense1 = TimeDistributed(Dense(hidden_dim))
'''
# dence1：
# 使用TimeDistributed层对1个batch中样本（input_length，input_dim）每个向量都进行Dense操作，在整个length长度下，这个样本
# 都共享TimeDistributed层的权重，即输出后变成（batch_size，input_length，hidden_dim）
'''
dense1.supports_masking = True
dense2 = Dense(output_dim)
'''
dence2:
处理从encoder之后的编码，整型为output_dim，再送给decoder
'''
decoder = RecurrentSequential(readout='add' if peek else 'readout_only',
state_sync=inner_broadcast_state, decode=True,
output_length=output_length, unroll=unroll,
stateful=stateful, teacher_force=teacher_force)
'''
参数：
teaching force ：它每次不使用上一个state的输出作为下一个state的输入，而是直接
使用训练数据的标准答案(ground truth)的对应上一项作为下一个state的输入。
结合beam search和计划抽样，使用一个概率p来决定使用teaching还是free training，随着训练epoch增加，
概率p也会减少，相当于逐步的减小teaching的采样频率，确保模型既能快速学习，又有泛化能力
'''
for _ in range(depth[1]):
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim,
batch_input_shape=(shape[0], output_dim)))
# 根据depth[1]指定解码器的深度
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoded_seq = dense1(_input)
# 对输入数据先通过TimeDistributed层，处理成hidden_dim的向量维度
encoded_seq = encoder(encoded_seq)
# 再通过encoder编码
# 以下是一些选项的处理，是否广播状态，是否teaching模式等
if broadcast_state:
assert type(encoded_seq) is list
states = encoded_seq[-2:]
encoded_seq = encoded_seq[0]
else:
states = None
encoded_seq = dense2(encoded_seq)
inputs = [_input]
if teacher_force:
truth_tensor = Input(batch_shape=(shape[0], output_length, output_dim))
truth_tensor._keras_history[0].supports_masking = True
inputs += [truth_tensor]
# 编码之后的后续处理
# 解码，initial_state是否接受从编码器传递过来的状态，
decoded_seq = decoder(encoded_seq,
ground_truth=inputs[1] if teacher_force else None,
initial_readout=encoded_seq, initial_state=states)
seq2seq_model = Model(inputs, decoded_seq)
# 整个模型就是从输入到解码seq，可以将编码器单独拿出来，使用其中的编码
# 另外，模型处理的实时新闻序列到股价波动序列，如果要将休盘期内新闻信息也纳入训练，
# 则需要共享编码器和解码权重，并增加新的Flatten和Dence层，将解码器输出序列视为波动编码，再进入Dence输出标量
# 涉及到，在RecurrentSequential后增加Sequencial序列
seq2seq_model.encoder = encoder
seq2seq_model.decoder = decoder
decoded_vec = Flatten()(decoded_seq)
decoded_vec = Dense(1, activation='tanh')(decoded_vec)
seq2vec_model = Model(inputs, [decoded_seq, decoded_vec])
# 最终模型有1个输入，2个输出
return seq2vec_model
def AttentionSeqtoSeq(output_dim, output_length, batch_input_shape=None,
batch_size=None, input_shape=None, input_length=None,
input_dim=None, hidden_dim=None, depth=1,
bidirectional=True, unroll=False, stateful=False, dropout=0.0,
):
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size,) + input_shape
elif input_dim:
if input_length:
shape = (batch_size,) + (input_length,) + (input_dim,)
else:
shape = (batch_size,) + (None,) + (input_dim,)
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoder = RecurrentSequential(unroll=unroll, stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
#encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(_input)
#decoder_input = Input(batch_shape = encoded.shape)#新加的
decoder = RecurrentSequential(decode=True, output_length=output_length,
unroll=unroll, stateful=stateful)
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
#decoder.add(Dropout(dropout))
decoder.add(LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
#decoder.add(Dropout(dropout))
decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
decoder.add(Dense(output_dim*2))
decoder.add(Dense(output_dim,activation = "softmax"))
inputs = [_input]
decoded = decoder(encoded)
model = Model(inputs, decoded)
return model
from recurrentshop import LSTMCell, RecurrentModel
""" Example: Create a simple LSTM that supports readout with an initial readout
input. """
# Main cell input.
cell_input = layers.Input(shape=(10, ))
# Readout input.
readout_input = layers.Input(shape=(100, ))
# Internal inputs for the LSTM cell.
last_state = layers.Input(shape=(100, ))
last_output = layers.Input(shape=(100, ))
# Create the LSTM layer.
fused_inputs = layers.concatenate([cell_input, readout_input])
lstm1_o, lstm1_h, lstm1_c = LSTMCell(100)(
[cell_input, last_state, last_output])
# Build the RNN.
rnn = RecurrentModel(input=cell_input,
output=lstm1_o,
initial_states=[last_state, last_output],
final_states=[lstm1_h, lstm1_c],
readout_input=readout_input)
# Main sequence input.
sequence_input = layers.Input(shape=(50, 10))
# Initial readout input.
initial_readout = layers.Input(shape=(100, ))
rnn_output = rnn(sequence_input, initial_readout=initial_readout)
def mctn_model(output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0,
is_cycled=True):
"""
MCTN Model (by default with Cycle Consistency Loss)
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# encoder phase
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
# encoder.add(Dropout(dropout))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(_input)
# decoder phase
decoder = RecurrentSequential(
decode=True,
output_length=1, #output_length
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
inputs = [_input]
decoded_0 = decoder(encoded)
decoded = Reshape((output_dim, ))(decoded_0)
# cycle phase
cycled_decoded = None
if is_cycled:
cycled_encoded = encoder(decoded_0)
cycled_decoded = decoder(cycled_encoded)
return inputs, encoded, decoded, cycled_decoded
def paired_trimodal_model(output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0):
"""
One modal translates into two other modalities, no cycle involved
The model has 1 encoder and 2 decoders
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
# encoder phase
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
# encoder phase
encoder_2 = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder_2.add(
LSTMCell(hidden_dim, batch_input_shape=(shape[0], output_dim)))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
encoder_2.add(Dropout(dropout))
encoder_2.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoder_2 = Bidirectional(encoder_2, merge_mode='sum')
encoder_2.forward_layer.build(shape)
encoder_2.backward_layer.build(shape)
# patch
encoder_2.layer = encoder_2.forward_layer
encoded_one = encoder(_input)
# decoder phase
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
decoder_2 = RecurrentSequential(decode=True,
output_length=input_length,
unroll=unroll,
stateful=stateful)
decoder_2.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
if depth[1] == 1:
decoder_2.add(
AttentionDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
else:
decoder_2.add(
AttentionDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder_2.add(Dropout(dropout))
decoder_2.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder_2.add(Dropout(dropout))
decoder_2.add(
LSTMDecoderCell(output_dim=input_dim, hidden_dim=hidden_dim))
inputs = [_input]
decoded_one = decoder(encoded_one)
encoded_two = encoder_2(decoded_one)
decoded_two = decoder_2(encoded_two)
return inputs, encoded_one, encoded_two, decoded_one, decoded_two
def Seq2Seq(output_dim, output_length, lookup_matrix, hidden_dim=None, depth=1, broadcast_state=True, inner_broadcast_state=True, teacher_force=False, peek=False, dropout=0., **kwargs):
'''
Seq2seq model based on [1] and [2].
This model has the ability to transfer the encoder hidden state to the decoder's
hidden state(specified by the broadcast_state argument). Also, in deep models
(depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
the inner_broadcast_state argument. You can switch between [1] based model and [2]
based model using the peek argument.(peek = True for [2], peek = False for [1]).
When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
[1] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
y(0) = LSTM(s0, C); C is the context vector from the encoder.
[2] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1), C)
y(0) = LSTM(s0, C, C)
Where s is the hidden state of the LSTM (h and c), and C is the context vector
from the encoder.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
broadcast_state : Specifies whether the hidden state from encoder should be
transfered to the deocder.
inner_broadcast_state : Specifies whether hidden states should be propogated
throughout the LSTM stack in deep models.
peek : Specifies if the decoder should be able to peek at the context vector
at every timestep.
dropout : Dropout probability in between layers.
'''
if type(depth) == int:
depth = [depth, depth]
if 'batch_input_shape' in kwargs:
shape = kwargs['batch_input_shape']
del kwargs['batch_input_shape']
elif 'input_shape' in kwargs:
shape = (None,) + tuple(kwargs['input_shape'])
del kwargs['input_shape']
elif 'input_dim' in kwargs:
if 'input_length' in kwargs:
shape = (None, kwargs['input_length'], kwargs['input_dim'])
del kwargs['input_length']
else:
shape = (None, None, kwargs['input_dim'])
del kwargs['input_dim']
if 'unroll' in kwargs:
unroll = kwargs['unroll']
del kwargs['unroll']
else:
unroll = False
if 'stateful' in kwargs:
stateful = kwargs['stateful']
del kwargs['stateful']
else:
stateful = False
if not hidden_dim:
hidden_dim = output_dim
encoder = RecurrentContainer(readout=True, state_sync=inner_broadcast_state, input_length=shape[1], unroll=unroll, stateful=stateful, return_states=broadcast_state)
for i in range(depth[0]):
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], hidden_dim), **kwargs))
encoder.add(Dropout(dropout))
dense1 = TimeDistributed(Dense(hidden_dim))
dense1.supports_masking = True
dense2 = Dense(output_dim)
decoder = RecurrentContainer(readout='add' if peek else 'readout_only', state_sync=inner_broadcast_state, output_length=output_length, unroll=unroll, stateful=stateful, decode=True, input_length=shape[1])
for i in range(depth[1]):
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], output_dim)))
decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], output_dim), **kwargs))
input = Input(batch_shape=(shape[0],shape[1]))
print input.shape
embedded_input = Embedding(input_dim=lookup_matrix.shape[0], output_dim=lookup_matrix.shape[1], weights=[lookup_matrix])(input)
print embedded_input.shape
input._keras_history[0].supports_masking = True
encoded_seq = dense1(embedded_input)
# print encoded_seq.shape
encoded_seq = encoder(encoded_seq)
print encoded_seq
if broadcast_state:
states = encoded_seq[-2:]
encoded_seq = encoded_seq[0]
else:
states = [None] * 2
encoded_seq = dense2(encoded_seq)
inputs = [input]
if teacher_force:
truth_tensor = Input(batch_shape=(shape[0], output_length, output_dim))
truth_tensor._keras_history[0].supports_masking = True
inputs += [truth_tensor]
decoded_seq = decoder({'input': encoded_seq, 'ground_truth': inputs[1] if teacher_force else None, 'initial_readout': encoded_seq, 'states': states})
model = Model(inputs, decoded_seq)
model.encoder = encoder
model.decoder = decoder
print "==========Input========="
print model.input
print "==========Input========="
print model.output
return model
def Seq2Seq(output_dim,
output_length,
hidden_dim=None,
batch_input_shape=None,
input_shape=None,
batch_size=None,
input_dim=None,
input_length=None,
is_embedding=True,
embedding_dim=None,
n_tokens=None,
depth=1,
broadcast_state=True,
unroll=False,
stateful=False,
inner_broadcast_state=False,
teacher_force=False,
peek=False,
dropout=0.):
'''
Seq2seq model based on [1] and [2].
This model has the ability to transfer the encoder hidden state to the decoder's
hidden state(specified by the broadcast_state argument). Also, in deep models
(depth > 1), the hidden state is propogated throughout the LSTM stack(specified by
the inner_broadcast_state argument. You can switch between [1] based model and [2]
based model using the peek argument.(peek = True for [2], peek = False for [1]).
When peek = True, the decoder gets a 'peek' at the context vector at every timestep.
[1] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1)); Where s is the hidden state of the LSTM (h and c)
y(0) = LSTM(s0, C); C is the context vector from the encoder.
[2] based model:
Encoder:
X = Input sequence
C = LSTM(X); The context vector
Decoder:
y(t) = LSTM(s(t-1), y(t-1), C)
y(0) = LSTM(s0, C, C)
Where s is the hidden state of the LSTM (h and c), and C is the context vector
from the encoder.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
broadcast_state : Specifies whether the hidden state from encoder should be
transfered to the deocder.
inner_broadcast_state : Specifies whether hidden states should be propogated
throughout the LSTM stack in deep models.
peek : Specifies if the decoder should be able to peek at the context vector
at every timestep.
dropout : Dropout probability in between layers.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
elif input_length:
if is_embedding == False and n_tokens > 0:
pass
else:
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
if is_embedding:
_input = Input(batch_shape=shape)
else:
i = Input(shape=(input_length, ), name='sentence_input', dtype='int32')
if embedding_dim is None:
embedding_dim = hidden_dim
_input = Embedding(input_dim=n_tokens,
output_dim=embedding_dim,
mask_zero=True,
input_length=input_length)(i)
shape = (batch_size, ) + (input_length, ) + (embedding_dim, )
encoder = RecurrentSequential(readout=True,
unroll=unroll,
stateful=stateful,
return_states=broadcast_state)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
# dense2 = Dense(output_dim)
decoder = RecurrentSequential(readout='add' if peek else 'readout_only',
decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim)))
if depth[1] == 1:
#decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
decoder.add(LSTMCell(output_dim))
else:
#decoder.add(LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
decoder.add(LSTMCell(hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
#decoder.add(LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
decoder.add(LSTMCell(hidden_dim))
decoder.add(Dropout(dropout))
#decoder.add(LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], hidden_dim)))
decoder.add(LSTMCell(output_dim))
x = encoder(_input)
if broadcast_state:
assert type(x) is list
states = x[-2:]
x = x[0]
else:
states = None
decoder_outputs = decoder(x, initial_state=states, initial_readout=x)
output = TimeDistributed(Dense(n_tokens,
activation='softmax'))(decoder_outputs)
if is_embedding:
return Model(_input, output)
else:
return Model(i, output)
def SimpleSeq2Seq(output_dim, output_length, latent_dim, batch_size, epsilon_std,
lookup_matrix=None, hidden_dim=None, depth=1, dropout=0., **kwargs):
'''
Simple model for sequence to sequence learning.
The encoder encodes the input sequence to vector (called context vector)
The decoder decodes the context vector in to a sequence of vectors.
There is no one on one relation between the input and output sequence elements.
The input sequence and output sequence may differ in length.
Arguments:
output_dim : Required output dimension.
hidden_dim : The dimension of the internal representations of the model.
output_length : Length of the required output sequence.
depth : Used to create a deep Seq2seq model. For example, if depth = 3,
there will be 3 LSTMs on the enoding side and 3 LSTMs on the
decoding side. You can also specify depth as a tuple. For example,
if depth = (4, 5), 4 LSTMs will be added to the encoding side and
5 LSTMs will be added to the decoding side.
dropout : Dropout probability in between layers.
'''
if type(depth) == int:
depth = [depth, depth]
if 'batch_input_shape' in kwargs:
shape = kwargs['batch_input_shape']
del kwargs['batch_input_shape']
elif 'input_shape' in kwargs:
shape = (None,) + tuple(kwargs['input_shape'])
del kwargs['input_shape']
elif 'input_dim' in kwargs:
if 'input_length' in kwargs:
shape = (None, kwargs['input_length'], kwargs['input_dim'])
del kwargs['input_length']
else:
shape = (None, None, kwargs['input_dim'])
del kwargs['input_dim']
if 'unroll' in kwargs:
unroll = kwargs['unroll']
del kwargs['unroll']
else:
unroll = False
if 'stateful' in kwargs:
stateful = kwargs['stateful']
del kwargs['stateful']
else:
stateful = False
if not hidden_dim:
hidden_dim = output_dim
embedder = Embedding(input_dim=lookup_matrix.shape[0], output_dim=lookup_matrix.shape[1], \
input_length=output_length, weights=[lookup_matrix])
encoder = RecurrentContainer(unroll=unroll, stateful=stateful, input_length=shape[1])
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2]), **kwargs))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim, **kwargs))
decoder = RecurrentContainer(unroll=unroll, stateful=stateful, decode=True, output_length=output_length, input_length=shape[1])
decoder.add(Dropout(dropout, batch_input_shape=(shape[0], hidden_dim)))
if depth[1] == 1:
decoder.add(LSTMCell(output_dim, **kwargs))
else:
decoder.add(LSTMCell(hidden_dim, **kwargs))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(hidden_dim, **kwargs))
decoder.add(Dropout(dropout))
decoder.add(LSTMCell(output_dim, **kwargs))
x = Input(batch_shape=(None,output_length))
embedded_x = embedder(x)
h_encoded = encoder(embedded_x)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0., std=epsilon_std)
return z_mean + K.exp(z_log_var / 2) * epsilon
z_mean = Dense(latent_dim)(h_encoded)
z_log_var = Dense(latent_dim)(h_encoded)
z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
print z
h_0 = Dense(hidden_dim, activation='relu')(z)
print h_0
model_out = decoder(h_0)
print model_out
# model_out = decoder(h_encoded)
y = Input(batch_shape=(None,output_length))
embedded_y = embedder(y)
print embedded_y
loss = tf.reduce_sum(mean_squared_error(model_out, embedded_y))
adam = tf.train.AdamOptimizer()
optimizer = adam.minimize(loss)
return optimizer, loss, x, y
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
is_embedding=True,
embedding_dim=None,
n_tokens=1000,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=False,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
elif input_length:
if is_embedding == False and n_tokens > 0:
pass
else:
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
if is_embedding:
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
else:
i = Input(shape=(input_length, ), name='sentence_input', dtype='int32')
i._keras_history[0].supports_masking = True
if embedding_dim is None:
embedding_dim = hidden_dim
_input = Embedding(input_dim=n_tokens,
output_dim=embedding_dim,
input_length=input_length)(i)
shape = (batch_size, ) + (input_length, ) + (embedding_dim, )
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[-1])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
x = encoder(_input)
decoder_outputs = decoder(x)
output = TimeDistributed(Dense(n_tokens,
activation='softmax'))(decoder_outputs)
if is_embedding:
return Model(_input, output)
else:
return Model(i, output)
Created : 22nd Oct, 2018 """ import keras from keras.layers import Input, Multiply from keras.layers import LSTM, ConvLSTM2D # from keras.layers import LSTMCell from CustomNets import make_from_mobilenet import code import numpy as np import cv2 image_nrows = 240 image_ncols = 320 image_nchnl = 3 input_img = Input(shape=(50, ), name='feature_cube') att = Input(shape=(50, ), name='attention') u = Multiply()([input_img, att]) from recurrentshop import LSTMCell lstm_output, state1_t, state2_t = LSTMCell(10)(u, state1_tm1, state_tm1) model = keras.models.Model(inputs=[input_img, att], outputs=u) model.summary() keras.utils.plot_model(model, to_file='./model_att.png', show_shapes=True)
def AttentionSeq2Seq(output_dim, output_length, hidden_dim=None, depth=1, bidirectional=True, dropout=0., **kwargs):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alhpa = softmax(energy)
Where a is a feed forward network.
'''
if type(depth) == int:
depth = [depth, depth]
if 'batch_input_shape' in kwargs:
shape = kwargs['batch_input_shape']
del kwargs['batch_input_shape']
elif 'input_shape' in kwargs:
shape = (None,) + tuple(kwargs['input_shape'])
del kwargs['input_shape']
elif 'input_dim' in kwargs:
if 'input_length' in kwargs:
shape = (None, kwargs['input_length'], kwargs['input_dim'])
del kwargs['input_length']
else:
shape = (None, None, kwargs['input_dim'])
del kwargs['input_dim']
if 'unroll' in kwargs:
unroll = kwargs['unroll']
del kwargs['unroll']
else:
unroll = False
if 'stateful' in kwargs:
stateful = kwargs['stateful']
del kwargs['stateful']
else:
stateful = False
if not hidden_dim:
hidden_dim = output_dim
encoder = RecurrentContainer(unroll=unroll, stateful=stateful, return_sequences=True, input_length=shape[1])
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2]), **kwargs))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim, **kwargs))
input = Input(batch_shape=shape)
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoded = encoder(input)
decoded = encoded
for _ in range(1, depth[1]):
decoder = AttentionDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], shape[1], hidden_dim)).get_layer(decode=True, output_length=output_length, unroll=unroll, stateful=stateful)
decoded = Dropout(dropout)(decoded)
decoded = decoder(decoded)
decoder = AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim, batch_input_shape=(shape[0], output_length if depth[1] > 1 else shape[1], hidden_dim)).get_layer(decode=True, output_length=output_length, unroll=unroll, stateful=stateful)
decoded = Dropout(dropout)(decoded)
decoded = decoder(decoded)
model = Model(input, decoded)
return model
print(out.shape) #->(1,10)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.fit(x=Dx, y=Dy, batch_size=4, epochs=20)
quit()
if False:
from keras.layers import Input, add, Activation
from recurrentshop import LSTMCell, GRUCell, RecurrentModel
# RNNCell - Using RNNCells in Functional API
input = Input(shape=(5, ))
state1_tm1 = Input((10, ))
state2_tm1 = Input((10, ))
state3_tm1 = Input((10, ))
lstm_output, state1_t, state2_t = LSTMCell(10)(
[input, state1_tm1, state2_tm1])
gru_output, state3_t = GRUCell(10)([input, state3_tm1])
output = add([lstm_output, gru_output])
output = Activation('tanh')(output)
rnn = RecurrentModel(input=input,
initial_states=[state1_tm1, state2_tm1, state3_tm1],
output=output,
final_states=[state1_t, state2_t, state3_t])
def __init__(self, config):
self.model = None
self.check_list = {
'text_maxlen', 'sentence_maxnum', 'sentence_maxlen', 'hidden_size',
'delimiter', 'pad_word', 'unk_word', 'start_sent', 'end_sent',
'vocab_size', 'embed_size', 'learning_rate'
}
self.config = config
assert self.check(), 'parametre check failed'
self.size = self.config['hidden_size']
self.Emb = Embedding(self.config['vocab_size'],
self.config['embed_size'],
trainable=True)
self.Splitlayer_keephead = SplitLayer(
delimiter=self.config['delimiter'],
output_sentence_len=self.config['sentence_maxlen'],
output_sentence_num=self.config['sentence_maxnum'],
pad_word=self.config['pad_word'],
cut_head=False,
name='Split_Layer_keep_head')
self.Splitlayer_cuthead = SplitLayer(
delimiter=self.config['delimiter'],
output_sentence_len=self.config['sentence_maxlen'],
output_sentence_num=self.config['sentence_maxnum'],
pad_word=self.config['pad_word'],
cut_head=True,
name='Split_Layer_cut_head')
self.Sentence_reshape1D = Reshape((self.config['sentence_maxnum'] *
self.config['sentence_maxlen'], ),
name='Sentence_reshape1D')
self.Sentence_reshape2D = Reshape((
self.config['sentence_maxnum'],
self.config['sentence_maxlen'],
self.config['embed_size'],
),
name='Sentence_reshape2D')
self.Encoder_word = CuDNNLSTM(units=self.size,
name='Encoder_word',
return_state=True)
self.Decoder_word_cell = LSTMCell(units=self.size,
name='Decoder_word_cell')
self.AttentionMapper = Linear(output_size=self.size,
bias=True,
bias_start=0.0,
activation='tanh')
self.Join = Dense(units=1, use_bias=False,
name='Join') # shape : [attention_vec_size]
self.Exp = Lambda(lambda x: K.exp(x), name='Exp')
self.Calcprob = Dense(units=self.config['vocab_size'],
activation='softmax',
name='Calcprob')
self.ArgMax = Lambda(lambda x: K.argmax(x, axis=-1), dtype='int32')
self.Printer = Lambda(lambda x: K.tf.Print(x, [x]))
self.Identical = Lambda(lambda x: x, name='Identical')
self.EncoderModel = None
self.DecoderModel_onestep = None
self._mask = None
self._targets = None
self.optim = optimizers.SGD(config['learning_rate'])
return
def Seq2SeqtableQA(row_maxlen,
question_maxlen,
answer_maxlen,
len_dic,
hidden_dim,
batch_size,
depth=(1, 1),
dropout=0.0,
unroll=False,
stateful=False):
# def Seq2SeqtableQA(output_dim, output_length, hidden_dim=None, input_shape=None,
# batch_size=None, batch_input_shape=None, input_dim=None,
# input_length=None, depth=1, dropout=0.0, unroll=False,
# stateful=False):
'''
Based on SimpleSeq2Seq
from https://github.com/farizrahman4u/seq2seq/blob/master/seq2seq/models.py
'''
# input placeholders
table = Input((row_maxlen, len_dic))
question = Input((question_maxlen, len_dic))
# table encoder
table_encoder = RecurrentSequential(unroll=unroll, stateful=stateful)
# table_encoder.add(Embedding(input_dim=len_dic,
# output_dim=hidden_dim,
# input_length=row_maxlen,
# # weights = [embedding_matrix],
# mask_zero=True,
# trainable=False))
table_encoder.add(
LSTMCell(hidden_dim, batch_input_shape=(row_maxlen, len_dic)))
for _ in range(1, depth[0]):
table_encoder.add(Dropout(dropout))
table_encoder.add(LSTMCell(hidden_dim))
table_encoded = table_encoder(table)
# question encoder
question_encoder = RecurrentSequential(unroll=unroll, stateful=stateful)
question_encoder.add(
LSTMCell(hidden_dim, batch_input_shape=(question_maxlen, len_dic)))
for _ in range(1, depth[0]):
question_encoder.add(Dropout(dropout))
question_encoder.add(LSTMCell(hidden_dim))
question_encoded = question_encoder(question)
# match table and question
match = dot([table_encoded, question_encoded], axes=(1, 1))
# match = Activation('softmax')(match)
# answer decoder
answer_decoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
decode=True,
output_length=answer_maxlen)
answer_decoder.add(Dropout(dropout, input_shape=(batch_size, hidden_dim)))
if depth[1] == 1:
answer_decoder.add(LSTMCell(len_dic))
else:
answer_decoder.add(LSTMCell(hidden_dim))
for _ in range(depth[1] - 2):
answer_decoder.add(Dropout(dropout))
answer_decoder.add(LSTMCell(hidden_dim))
answer_decoder.add(Dropout(dropout))
answer_decoder.add(LSTMCell(len_dic))
answer_decoded = answer_decoder(match)
return Model(inputs=[table, question], outputs=answer_decoded)
def AttentionSeq2Seq(
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0,
):
'''
This is an attention Seq2seq model based on [3].
Here, there is a soft allignment between the input and output sequence elements.
A bidirection encoder is used by default. There is no hidden state transfer in this
model.
The math:
Encoder:
X = Input Sequence of length m.
H = Bidirection_LSTM(X); Note that here the LSTM has return_sequences = True,
so H is a sequence of vectors of length m.
Decoder:
y(i) = LSTM(s(i-1), y(i-1), v(i)); Where s is the hidden state of the LSTM (h and c)
and v (called the context vector) is a weighted sum over H:
v(i) = sigma(j = 0 to m-1) alpha(i, j) * H(j)
The weight alpha[i, j] for each hj is computed as follows:
energy = a(s(i-1), H(j))
alpha = softmax(energy)
Where a is a feed forward network.
'''
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise TypeError
if hidden_dim is None:
hidden_dim = output_dim
_input = Input(batch_shape=shape)
_input._keras_history[0].supports_masking = True
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
# encoded = encoder(_input)
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
# inputs = [_input]
# decoded = decoder(encoded)
# model = Model(inputs, decoded)
return encoder, decoder
def mctn_level2_model(input,
output_dim,
output_length,
batch_input_shape=None,
batch_size=None,
input_shape=None,
input_length=None,
input_dim=None,
hidden_dim=None,
depth=1,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.0):
"""
Level 2 MCTN used for translation between the joint embedded of
2 modalities to the third one. Due to the lack of ground truth, no
cycle phase happens
"""
if isinstance(depth, int):
depth = (depth, depth)
if batch_input_shape:
shape = batch_input_shape
elif input_shape:
shape = (batch_size, ) + input_shape
elif input_dim:
if input_length:
shape = (batch_size, ) + (input_length, ) + (input_dim, )
else:
shape = (batch_size, ) + (None, ) + (input_dim, )
else:
# TODO Proper error message
raise
if hidden_dim is None:
hidden_dim = output_dim
encoder = RecurrentSequential(unroll=unroll,
stateful=stateful,
return_sequences=True)
encoder.add(LSTMCell(hidden_dim, batch_input_shape=(shape[0], shape[2])))
for _ in range(1, depth[0]):
encoder.add(Dropout(dropout))
encoder.add(LSTMCell(hidden_dim))
if bidirectional:
encoder = Bidirectional(encoder, merge_mode='sum')
encoder.forward_layer.build(shape)
encoder.backward_layer.build(shape)
# patch
encoder.layer = encoder.forward_layer
encoded = encoder(input)
decoder = RecurrentSequential(decode=True,
output_length=output_length,
unroll=unroll,
stateful=stateful)
decoder.add(
Dropout(dropout, batch_input_shape=(shape[0], shape[1], hidden_dim)))
if depth[1] == 1:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
else:
decoder.add(
AttentionDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
for _ in range(depth[1] - 2):
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim))
decoder.add(Dropout(dropout))
decoder.add(
LSTMDecoderCell(output_dim=output_dim, hidden_dim=hidden_dim))
inputs = [input]
decoded = decoder(encoded)
return inputs, encoded, decoded
class Vae:
dec_input = Input(shape=(128, ))
decoder_input = Input(shape=(133, ))
h_in = Input(shape=(512, ))
c_in = Input(shape=(512, ))
readout_in = Input(shape=(133, ))
enc_1 = Bidirectional(LSTM(256, name='Enc_RNN'), name='BiDir')
enc_mean = Dense(128, name='mean')
enc_log_sigma = Dense(128, name='log_sigma')
h_init = Dense(1024, activation='tanh', name='state_init')
dec_1 = LSTM(512, return_sequences=True, name='Dec_RNN')
dec_3 = LSTMCell(512)
mdn = MixtureDensity(5, 20, name='mdn')
kl_tolerance = 0.2
kl_weight_start = 0.01
kl_weight = 0.5
learning_rate = 0.001
decay_rate = 0.9999
kl_decay_rate = 0.99995
min_learning_rate = 0.00001
def __init__(self, max_len=250):
self.curr_kl_weight = self.kl_weight_start
self.max_len = max_len
self.input = Input((
max_len,
5,
), name="stroke_batch")
self.output = Input((
max_len,
5,
), name="stroke_target")
self.build_model()
print(self.model.summary())
def tile(self, tensor):
return K.tile(tensor, [1, self.max_len, 1])
def build_model(self):
# ====================== ENCODER =============================
a = self.enc_1(self.output)
self.mean = self.enc_mean(a)
self.log_sigma = self.enc_log_sigma(a)
self.kl_loss = -0.5 * K.mean(
1 + self.log_sigma - K.square(self.mean) - K.exp(self.log_sigma),
axis=[0, 1])
self.kl_loss = K.maximum(self.kl_loss, self.kl_tolerance)
# encoded = concatenate([mean, log_sigma])
self.encoder = Model(self.output, self.mean, name='encoder')
# self.mean = Lambda(lambda x: x[:, :128])(encoded)
# self.log_sigma = Lambda(lambda x: x[:, 128:])(encoded)
self.z = Lambda(sampling)([self.mean, self.log_sigma])
# ====================== VAE ==============================
_h = self.h_init(self.z)
_h_1 = Lambda(lambda x: x[:, :512])(_h)
_c_1 = Lambda(lambda x: x[:, 512:])(_h)
z_ = Reshape((
1,
128,
))(self.z)
z_ = Lambda(self.tile)(z_)
z_ = concatenate([z_, self.input], axis=2)
out = self.dec_1(z_, initial_state=[_h_1, _c_1])
out = self.mdn(out)
self.model = Model([self.output, self.input], out)
# ====================== DECODER ===========================
_h_ = self.h_init(self.dec_input)
_h_1_ = Lambda(lambda x: x[:, :512])(_h_)
_c_1_ = Lambda(lambda x: x[:, 512:])(_h_)
_z_ = Reshape((
1,
128,
))(self.dec_input)
_z_ = Lambda(self.tile)(_z_)
_z_ = concatenate([_z_, self.input], axis=2)
out_ = self.dec_1(_z_, initial_state=[_h_1_, _c_1_])
out_ = self.mdn(out_)
self.decoder = Model([self.dec_input, self.input], out_)
def get_mixture_coef(output):
out_pi = output[:, :20]
out_mu_x = output[:, 20:40]
out_mu_y = output[:, 40:60]
out_sigma_x = output[:, 60:80]
out_sigma_y = output[:, 80:100]
out_ro = output[:, 100:120]
pen_logits = output[:, 120:123]
# use softmax to normalize pi and q into prob distribution
# max_pi = K.max(out_pi, axis = 1, keepdims=True)
# out_pi = out_pi - max_pi
# out_pi = K.exp(out_pi)
# normalize_pi = 1 / (K.sum(out_pi, axis = 1, keepdims = True))
# out_pi = normalize_pi * out_pi
out_pi = K.softmax(out_pi)
out_q = K.softmax(pen_logits)
# max_q = K.max(pen_logits, axis = 1, keepdims = True)
# out_q = pen_logits - max_q
# out_q = K.exp(pen_logits)
# normalize_q = 1 / (K.sum(out_q, axis = 1, keepdims = True))
# out_q = normalize_q * out_q
# sue tanh to normalize correlation coefficient
out_ro = K.tanh(out_ro)
# use exponential to make sure sigma is positive
out_sigma_x = K.exp(out_sigma_x)
out_sigma_y = K.exp(out_sigma_y)
return out_pi, out_mu_x, out_mu_y, out_sigma_x, out_sigma_y, out_ro, pen_logits, out_q
def tf_bi_normal(x, y, mu_x, mu_y, sigma_x, sigma_y, ro):
x_ = K.reshape(x, (-1, 1))
y_ = K.reshape(y, (-1, 1))
norm1 = x_ - mu_x
norm2 = y_ - mu_y
sigma = sigma_x * sigma_y
z = (K.square(norm1 / (sigma_x + 1e-8)) +
K.square(norm2 / (sigma_y + 1e-8)) -
(2 * ro * norm1 * norm2) / (sigma + 1e-8) + 1e-8)
ro_opp = 1 - K.square(ro)
result = K.exp(-z / (2 * ro_opp + 1e-8))
denom = 2 * np.pi * sigma * K.sqrt(ro_opp) + 1e-8
result = result / denom + 1e-8
return result
def get_lossfunc(out_pi, out_mu_x, out_mu_y, out_sigma_x, out_sigma_y,
out_ro, out_q, x, y, logits):
# L_r loss term calculation, L_s part
result = tf_bi_normal(x, y, out_mu_x, out_mu_y, out_sigma_x,
out_sigma_y, out_ro)
result = result * out_pi
result = K.sum(result, axis=1, keepdims=True)
result = -K.log(result + 1e-8)
fs = 1.0 - logits[:, 2]
fs = K.reshape(fs, (-1, 1))
result = result * fs
# L_r loss term, L_p part
result1 = K.categorical_crossentropy(out_q,
logits,
from_logits=True)
result1 = K.reshape(result1, (-1, 1))
result = result + result1
return K.mean(result, axis=[0, 1])
output = K.reshape(out, [-1, 123])
self.pi, self.mu_x, self.mu_y, self.sigma_x, self.sigma_y, self.ro, \
self.logits, self.q = get_mixture_coef(output)
target = K.reshape(self.output, [-1, 5])
val_x = target[:, 0]
val_y = target[:, 1]
pen = target[:, 2:]
self.rec_loss = get_lossfunc(self.pi, self.mu_x, self.mu_y,
self.sigma_x, self.sigma_y, self.ro,
self.logits, val_x, val_y, pen)
self.loss = self.rec_loss + self.curr_kl_weight * self.kl_loss
adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
self.model.add_loss(self.loss)
self.model.compile(optimizer=adam, metrics=['accuracy'])
# def build_decoder(self):
# dec_out, h, c = self.dec_3([self.decoder_input, self.h_in, self.c_in])
# dec_out = self.mdn(dec_out)
# rnn = RecurrentModel(input = self.decoder_input,
# initial_states = [self.h_in, self.c_in],
# output = dec_out, final_states = [h, c],
# return_sequences = True, name = 'Dec_RNN')
# rnn(z_, initial_state = [_h_1, _c_1])
def update_params(self, step):
curr_learning_rate = ((self.learning_rate - self.min_learning_rate) *
(self.decay_rate**step) + self.min_learning_rate)
self.curr_kl_weight = (self.kl_weight -
(self.kl_weight - self.kl_weight_start) *
(self.kl_decay_rate**step))
K.set_value(self.model.optimizer.lr, curr_learning_rate)
def _make_model(self):
"""Constructs ContraVAE model.
Returns:
Nothing.
"""
# Load embedding in Embedding layer
logger.info("Making model...")
embedding_matrix = self._load_embedding()
embedding_layer = Embedding(
self.num_words + 1,
self.config.embedding_dim,
weights=[embedding_matrix],
input_length=self.config.max_sequence_length,
trainable=False)
# Q(z|X,c) -- encoder
# Embedded sequence input
sequence_inputs = Input(batch_shape=(self.config.batch_size,
self.config.max_sequence_length),
dtype='int32')
embedded_sequence_inputs = embedding_layer(sequence_inputs)
# Merge with score inputs
score_inputs = Input(batch_shape=(self.config.batch_size, 1))
score_inputs_repeated = RepeatVector(
self.config.max_sequence_length)(score_inputs)
last_layer = concatenate(
[embedded_sequence_inputs, score_inputs_repeated], axis=2)
# LSTM layers
for dim in self.config.encoder_lstm_dims[:-1]:
last_layer = LSTM(dim, return_sequences=True)(last_layer)
last_layer = LSTM(self.config.encoder_lstm_dims[-1],
return_sequences=False)(last_layer)
# Mean and std of z
z_mean = Dense(self.config.latent_dim, activation='tanh')(last_layer)
z_log_sigma = Dense(self.config.latent_dim,
activation='tanh')(last_layer)
# Sample z ~ Q(z|X,c)
def sampling(args):
z_mean, z_log_sigma = args
epsilon = K.random_normal_variable(shape=(self.config.batch_size,
self.config.latent_dim),
mean=0.,
scale=1.)
return z_mean + K.exp(z_log_sigma / 2.) * epsilon
z = Lambda(sampling)([z_mean, z_log_sigma])
# Second score inputs - at training time this is simply equal to
# score_inputs; at sampling time this could vary.
score_inputs2 = Input(batch_shape=(self.config.batch_size, 1))
z_c = concatenate([z, score_inputs2], axis=1)
# Repeat z_c so every timestep has access to it
#z_c_repeated = RepeatVector(self.config.max_sequence_length)(z_c)
# P(X|z,c) -- decoder.
rnn = RecurrentSequential(
decode=True, output_length=self.config.max_sequence_length)
rnn.add(
LSTMCell(self.config.decoder_lstm_dims[0],
input_dim=self.config.latent_dim + 1))
for dim in self.config.decoder_lstm_dims[1:]:
rnn.add(LSTMCell(dim))
decoder_out = TimeDistributed(
Dense(self.num_words + 1, activation='tanh'))
# Decoder output
# x_decoded = rnn(z_c_repeated, ground_truth=sequence_inputs)
h_decoded = rnn(z_c)
x_decoded = decoder_out(h_decoded)
# Construct models
# VAE
vae = Model([sequence_inputs, score_inputs, score_inputs2], x_decoded)
# Encoder
encoder = Model([sequence_inputs, score_inputs], z_mean)
# Generator
generator_z_inputs = Input(batch_shape=(self.config.batch_size,
self.config.latent_dim))
generator_z_c = concatenate([generator_z_inputs, score_inputs2],
axis=1)
generator_h_decoded = rnn(generator_z_c)
generator_x_decoded = decoder_out(generator_h_decoded)
generator = Model([generator_z_inputs, score_inputs2],
generator_x_decoded)
# Define loss function
kl_weight = self.config.kl_weight
def recon_loss(y_true, y_pred):
"""E[log P(X|z,y)].
"""
recon = K.mean(K.sparse_categorical_crossentropy(output=y_pred,
target=y_true,
from_logits=True),
axis=1)
return recon
def kl_loss(y_true, y_pred):
"""D_KL(Q(z|X,y) || P(z|X)); calculate in closed form as both
dist. are Gaussian.
"""
kl = 0.5 * K.mean(
K.exp(z_log_sigma) + K.square(z_mean) - 1. - z_log_sigma,
axis=1)
kl = kl * kl_weight
return kl
def vae_loss(y_true, y_pred):
"""Calculate loss = reconstruction loss + KL loss.
"""
recon = recon_loss(y_true, y_pred)
kl = kl_loss(y_true, y_pred)
return recon + kl
# Compile model
vae.compile(loss=vae_loss,
optimizer=self.config.optimizer,
metrics=[recon_loss, kl_loss])
self.vae = vae
self.encoder = encoder
self.generator = generator
logger.info("Done making model.")