def test_Bidirectional_merged_value(merge_mode):
rnn = layers.LSTM
samples = 2
dim = 5
timesteps = 3
units = 3
X = [np.random.rand(samples, timesteps, dim)]
if merge_mode == 'sum':
merge_func = lambda y, y_rev: y + y_rev
elif merge_mode == 'mul':
merge_func = lambda y, y_rev: y * y_rev
elif merge_mode == 'ave':
merge_func = lambda y, y_rev: (y + y_rev) / 2
elif merge_mode == 'concat':
merge_func = lambda y, y_rev: np.concatenate((y, y_rev), axis=-1)
else:
merge_func = lambda y, y_rev: [y, y_rev]
# basic case
inputs = Input((timesteps, dim))
layer = wrappers.Bidirectional(rnn(units, return_sequences=True),
merge_mode=merge_mode)
f_merged = K.function([inputs], to_list(layer(inputs)))
f_forward = K.function([inputs], [layer.forward_layer.call(inputs)])
f_backward = K.function([inputs],
[K.reverse(layer.backward_layer.call(inputs), 1)])
y_merged = f_merged(X)
y_expected = to_list(merge_func(f_forward(X)[0], f_backward(X)[0]))
assert len(y_merged) == len(y_expected)
for x1, x2 in zip(y_merged, y_expected):
assert_allclose(x1, x2, atol=1e-5)
# test return_state
inputs = Input((timesteps, dim))
layer = wrappers.Bidirectional(rnn(units, return_state=True),
merge_mode=merge_mode)
f_merged = K.function([inputs], layer(inputs))
f_forward = K.function([inputs], layer.forward_layer.call(inputs))
f_backward = K.function([inputs], layer.backward_layer.call(inputs))
n_states = len(layer.layer.states)
y_merged = f_merged(X)
y_forward = f_forward(X)
y_backward = f_backward(X)
y_expected = to_list(merge_func(y_forward[0], y_backward[0]))
assert len(y_merged) == len(y_expected) + n_states * 2
for x1, x2 in zip(y_merged, y_expected):
assert_allclose(x1, x2, atol=1e-5)
# test if the state of a BiRNN is the concatenation of the underlying RNNs
y_merged = y_merged[-n_states * 2:]
y_forward = y_forward[-n_states:]
y_backward = y_backward[-n_states:]
for state_birnn, state_inner in zip(y_merged, y_forward + y_backward):
assert_allclose(state_birnn, state_inner, atol=1e-5)
def _loss_tensor(X, y_pred, y_true, CL, CR, tiles=2):
X = K.permute_dimensions(X, (1, 0, 2, 3))
Xcs = K.reshape(X, (X.shape[0], X.shape[1], X.shape[2] * X.shape[3])) # reshape x as n columns
X_pred = K.batch_dot(K.permute_dimensions(Xcs, (0, 2, 1)), y_pred) # replace column according to pred
X_pred = K.reshape(X_pred, (X.shape[0], X.shape[-2] * tiles, X.shape[-1] * tiles)) # reshape as matrix
X_flip_x = K.reverse(X_pred, axes=2)
X_flip_y = K.reverse(X_pred, axes=1)
LX_y = K.permute_dimensions(K.dot(CL, X_pred), (1, 0, 2))
RX_y = K.permute_dimensions(K.dot(CL, X_flip_y), (1, 0, 2))
LX_x = K.dot(X_pred, CR)
RX_x = K.dot(X_flip_x, CR)
dissimilarity = K.mean(K.mean(K.square(LX_x - RX_x), axis=2), axis=1) + K.mean(
K.mean(K.square(LX_y - RX_y), axis=2),
axis=1)
# return K.categorical_crossentropy(y_pred, y_true) + K.sum(dissimilarity)
return K.mean(dissimilarity)
def call(self, input):
forward_lstm = LSTM(self.units, dropout=self.dropout,
return_sequences=True, return_state=True)
backward_lstm = LSTM(self.units, dropout=self.dropout,
return_sequences=True, return_state=True, go_backwards=True)
fw_outputs, fw_output, fw_state = forward_lstm(input)
bw_outputs, bw_output, b_state = backward_lstm(input)
bw_outputs = K.reverse(bw_outputs, 1)
# bw_output = bw_state[0]
outputs = K.concatenate([fw_outputs, bw_outputs])
last_output = K.concatenate([fw_output, bw_output])
return [outputs, last_output]
def create_model(self, hyper_parameters):
"""
构建神经网络，行卷积加池化
:param hyper_parameters:json, hyper parameters of network
:return: tensor, moedl
"""
super().create_model(hyper_parameters)
embedding_output = self.word_embedding.output
# rnn layers
if self.rnn_units == "LSTM":
layer_cell = LSTM
else:
layer_cell = GRU
# 反向
x_backwords = layer_cell(units=self.rnn_units,
return_sequences=True,
kernel_regularizer=regularizers.l2(0.32 *
0.1),
recurrent_regularizer=regularizers.l2(0.32),
go_backwards=True)(embedding_output)
x_backwords_reverse = Lambda(lambda x: K.reverse(x, axes=1))(
x_backwords)
# 前向
x_fordwords = layer_cell(units=self.rnn_units,
return_sequences=True,
kernel_regularizer=regularizers.l2(0.32 *
0.1),
recurrent_regularizer=regularizers.l2(0.32),
go_backwards=False)(embedding_output)
# 拼接
x_feb = Concatenate(axis=2)(
[x_fordwords, embedding_output, x_backwords_reverse])
####使用多个卷积核##################################################
x_feb = Dropout(self.dropout)(x_feb)
# Concatenate后的embedding_size
dim_2 = K.int_shape(x_feb)[2]
x_feb_reshape = Reshape((self.len_max, dim_2, 1))(x_feb)
# 提取n-gram特征和最大池化， 一般不用平均池化
conv_pools = []
for filter in self.filters:
conv = Conv2D(
filters=self.filters_num,
kernel_size=(filter, dim_2),
padding='valid',
kernel_initializer='normal',
activation='relu',
)(x_feb_reshape)
pooled = MaxPooling2D(
pool_size=(self.len_max - filter + 1, 1),
strides=(1, 1),
padding='valid',
)(conv)
conv_pools.append(pooled)
# 拼接
x = Concatenate()(conv_pools)
x = Flatten()(x)
#########################################################################
output = Dense(units=self.label, activation=self.activate_classify)(x)
self.model = Model(inputs=self.word_embedding.input, outputs=output)
self.model.summary(120)
def reverse_sequence(kvar):
kvar2 = K.reverse(kvar, axes=1)
return kvar2
def get_Model(training):
input_shape = (img_w, img_h, 1) # (128, 64, 1)
# Make Networkw
inputs = Input(name='the_input', shape=input_shape,
dtype='float32') # (None, 128, 64, 1)
# Convolution layer (VGG)
n = 28
inner = Conv2D(n // 2, (3, 3),
padding='same',
name='conv1',
kernel_initializer='he_normal')(
inputs) # (None, 128, 64, 64)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max1')(inner) # (None,64, 32, 64)
inner = Conv2D(n, (3, 3),
padding='same',
name='conv2',
kernel_initializer='he_normal')(
inner) # (None, 64, 32, 128)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2),
name='max2')(inner) # (None, 32, 16, 128)
inner = Conv2D(n * 2, (3, 3),
padding='same',
name='conv3',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(n * 2, (3, 3),
padding='same',
name='conv4',
kernel_initializer='he_normal')(
inner) # (None, 32, 16, 256)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max3')(inner) # (None, 32, 8, 256)
inner = Conv2D(n * 4, (3, 3),
padding='same',
name='conv5',
kernel_initializer='he_normal')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(n * 4, (3, 3), padding='same',
name='conv6')(inner) # (None, 32, 8, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(1, 2),
name='max4')(inner) # (None, 32, 4, 512)
inner = Conv2D(n * 4, (2, 2),
padding='same',
kernel_initializer='he_normal',
name='con7')(inner) # (None, 32, 4, 512)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
# CNN to RNN
inner = Reshape(target_shape=((n // 4, n * 4)),
name='reshape')(inner) # (None, 32, 2048)
inner = Dense(n // 2,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(inner) # (None, 32, 64)
# RNN layer
lstm_1 = LSTM(n * 2,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm1')(inner) # (None, 32, 512)
lstm_1b = LSTM(n * 2,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm1_b')(inner)
reversed_lstm_1b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_1b)
lstm1_merged = add([lstm_1, reversed_lstm_1b]) # (None, 32, 512)
lstm1_merged = BatchNormalization()(lstm1_merged)
lstm_2 = LSTM(n * 2,
return_sequences=True,
kernel_initializer='he_normal',
name='lstm2')(lstm1_merged)
lstm_2b = LSTM(n * 2,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='lstm2_b')(lstm1_merged)
reversed_lstm_2b = Lambda(
lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_2b)
lstm2_merged = concatenate([lstm_2, reversed_lstm_2b]) # (None, 32, 1024)
lstm2_merged = BatchNormalization()(lstm2_merged)
# transforms RNN output to character activations:
inner = Dense(num_classes, kernel_initializer='he_normal',
name='dense2')(lstm2_merged) #(None, 32, 63)
y_pred = Activation('softmax', name='softmax')(inner)
labels = Input(name='the_labels', shape=[max_text_len],
dtype='float32') # (None ,8)
input_length = Input(name='input_length', shape=[1],
dtype='int64') # (None, 1)
label_length = Input(name='label_length', shape=[1],
dtype='int64') # (None, 1)
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length,
label_length]) #(None, 1)
if training:
return Model(inputs=[inputs, labels, input_length, label_length],
outputs=loss_out)
else:
return Model(inputs=[inputs], outputs=y_pred)
def call(self, inputs): # pylint: disable=arguments-differ
return K.reverse(inputs, self.axis)
from keras import models, optimizers, losses from keras import initializers import numpy as np from keras import backend as K x = [[1, 2, 3], [4, 5, 6]] xx = [[10, 20, 30], [40, 50, 60]] x = np.asarray(x) xx = np.asarray(xx) y = [0, 1] input1 = layers.Input(shape=(3, )) input2 = layers.Input(shape=(3, )) l = layers.Dense(1, kernel_initializer=initializers.RandomUniform(0, 1)) rel = layers.Lambda(lambda x: K.reverse(x, axes=0)) rel2 = layers.Lambda(lambda x: K.map_fn()) x1 = l(input1) x2 = l(input2) x3 = rel(x2) #model0 = models.Model(inputs = input1, outputs = x1) #model0.compile(loss = losses.binary_crossentropy , optimizer = optimizers.Adam()) #res = model0.predict(x) merged = layers.merge.Subtract()([x2, x1]) model1 = models.Model(inputs=[input1, input2], outputs=[x1, x2]) model1.compile(loss=losses.binary_crossentropy, optimizer=optimizers.Adam()) res1 = model1.predict([x, xx])
def create_model(self) -> Model:
if self.use_matrix:
context = Input(shape=(self.max_sequence_length,))
response = Input(shape=(self.max_sequence_length,))
emb_layer = self.embedding_layer()
emb_c = emb_layer(context)
emb_r = emb_layer(response)
else:
context = Input(shape=(self.max_sequence_length, self.embedding_dim,))
response = Input(shape=(self.max_sequence_length, self.embedding_dim,))
emb_c = context
emb_r = response
lstm_layer = self.create_lstm_layer_1()
lstm_a = lstm_layer(emb_c)
lstm_b = lstm_layer(emb_r)
f_layer_f = FullMatchingLayer(self.perspective_num)
f_layer_b = FullMatchingLayer(self.perspective_num)
f_a_forw = f_layer_f([lstm_a[0], lstm_b[0]])[0]
f_a_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_a[1]),
Lambda(lambda x: K.reverse(x, 1))(lstm_b[1])])[0]
f_a_back = Lambda(lambda x: K.reverse(x, 1))(f_a_back)
f_b_forw = f_layer_f([lstm_b[0], lstm_a[0]])[0]
f_b_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_b[1]),
Lambda(lambda x: K.reverse(x, 1))(lstm_a[1])])[0]
f_b_back = Lambda(lambda x: K.reverse(x, 1))(f_b_back)
mp_layer_f = MaxpoolingMatchingLayer(self.perspective_num)
mp_layer_b = MaxpoolingMatchingLayer(self.perspective_num)
mp_a_forw = mp_layer_f([lstm_a[0], lstm_b[0]])[0]
mp_a_back = mp_layer_b([lstm_a[1], lstm_b[1]])[0]
mp_b_forw = mp_layer_f([lstm_b[0], lstm_a[0]])[0]
mp_b_back = mp_layer_b([lstm_b[1], lstm_a[1]])[0]
at_layer_f = AttentiveMatchingLayer(self.perspective_num)
at_layer_b = AttentiveMatchingLayer(self.perspective_num)
at_a_forw = at_layer_f([lstm_a[0], lstm_b[0]])[0]
at_a_back = at_layer_b([lstm_a[1], lstm_b[1]])[0]
at_b_forw = at_layer_f([lstm_b[0], lstm_a[0]])[0]
at_b_back = at_layer_b([lstm_b[1], lstm_a[1]])[0]
ma_layer_f = MaxattentiveMatchingLayer(self.perspective_num)
ma_layer_b = MaxattentiveMatchingLayer(self.perspective_num)
ma_a_forw = ma_layer_f([lstm_a[0], lstm_b[0]])[0]
ma_a_back = ma_layer_b([lstm_a[1], lstm_b[1]])[0]
ma_b_forw = ma_layer_f([lstm_b[0], lstm_a[0]])[0]
ma_b_back = ma_layer_b([lstm_b[1], lstm_a[1]])[0]
concat_a = Lambda(lambda x: K.concatenate(x, axis=-1))([f_a_forw, f_a_back,
mp_a_forw, mp_a_back,
at_a_forw, at_a_back,
ma_a_forw, ma_a_back])
concat_b = Lambda(lambda x: K.concatenate(x, axis=-1))([f_b_forw, f_b_back,
mp_b_forw, mp_b_back,
at_b_forw, at_b_back,
ma_b_forw, ma_b_back])
concat_a = Dropout(self.ldrop_val)(concat_a)
concat_b = Dropout(self.ldrop_val)(concat_b)
lstm_layer_agg = self.create_lstm_layer_2()
agg_a = lstm_layer_agg(concat_a)
agg_b = lstm_layer_agg(concat_b)
agg_a = Dropout(self.dropout_val)(agg_a)
agg_b = Dropout(self.dropout_val)(agg_b)
reduced = Lambda(lambda x: K.concatenate(x, axis=-1))([agg_a, agg_b])
if self.triplet_mode:
dist = Lambda(self._pairwise_distances)([agg_a, agg_b])
else:
ker_in = glorot_uniform(seed=self.seed)
dense = Dense(self.dense_dim, kernel_initializer=ker_in)(reduced)
dist = Dense(1, activation='sigmoid', name="score_model")(dense)
model = Model([context, response], dist)
return model
def ResNet50_pre(imgs, scope):
return Lambda(
lambda x: K.reverse(x,
len(x.shape) - 1) - [103.939, 116.779, 123.68],
name=scope + 'resnet50_pre')(imgs)
def memLstm_custom_model(hparams, context, context_mask, utterances, kb,
kb_flag, kb_mask):
print("context_shape: ", context._keras_shape)
print("utterances_shape: ", utterances._keras_shape)
print("context_mask: ", context_mask._keras_shape)
print("kb_flag shape: ", kb_flag._keras_shape)
# Use embedding matrix pretrained by Gensim
embeddings_W = np.load(hparams.embedding_path)
print("embeddings_W: ", embeddings_W.shape)
################################## Define Regular Layers ##################################
# Utterances Embedding (Output shape: NUM_OPTIONS(100) x BATCH_SIZE(?) x LEN_SEQ(160) x EMBEDDING_DIM(300))
embedding_context_layer = Embedding(
input_dim=hparams.vocab_size,
output_dim=hparams.memn2n_embedding_dim,
weights=[embeddings_W],
input_length=hparams.max_context_len,
mask_zero=True,
trainable=False)
embedding_utterance_layer = Embedding(
input_dim=hparams.vocab_size,
output_dim=hparams.memn2n_embedding_dim,
weights=[embeddings_W],
input_length=hparams.max_utterance_len,
mask_zero=True,
trainable=False)
embedding_kb_layer = Embedding(input_dim=hparams.vocab_size,
output_dim=hparams.memn2n_embedding_dim,
weights=[embeddings_W],
input_length=hparams.max_kb_len,
mask_zero=True,
trainable=False)
# Define LSTM Context encoder 1
LSTM_A = LSTM(hparams.memn2n_rnn_dim,
input_shape=(hparams.max_context_len,
hparams.memn2n_embedding_dim),
use_bias=True,
unit_forget_bias=True,
return_state=True,
return_sequences=True)
# Define LSTM Utterances encoder
LSTM_B = LSTM(hparams.memn2n_rnn_dim,
input_shape=(hparams.max_utterance_len,
hparams.memn2n_embedding_dim),
use_bias=True,
unit_forget_bias=True,
return_state=False,
return_sequences=False)
# Define LSTM KBs
LSTM_K = LSTM(hparams.memn2n_rnn_dim,
input_shape=(hparams.max_kb_len,
hparams.memn2n_embedding_dim),
use_bias=True,
unit_forget_bias=True,
return_state=False,
return_sequences=False)
# Define Dense layer to transform utterances
Matrix_utterances = Dense(
hparams.memn2n_rnn_dim,
use_bias=False,
kernel_initializer=keras.initializers.TruncatedNormal(mean=0.0,
stddev=1.0,
seed=None),
input_shape=(hparams.memn2n_rnn_dim, ))
Matrix_kb = Dense(hparams.memn2n_rnn_dim,
use_bias=False,
kernel_initializer=keras.initializers.TruncatedNormal(
mean=0.0, stddev=1.0, seed=None),
input_shape=(hparams.memn2n_rnn_dim, ))
# Define Dense layer to do softmax
Dense_2 = Dense(1,
use_bias=False,
kernel_initializer=keras.initializers.TruncatedNormal(
mean=0.0, stddev=1.0, seed=None),
input_shape=(hparams.memn2n_rnn_dim, ))
Dense_3 = Dense(1,
use_bias=False,
kernel_initializer=keras.initializers.TruncatedNormal(
mean=0.0, stddev=1.0, seed=None),
input_shape=(2, ))
################################## Define Custom Layers ##################################
# Define repeat element layer
custom_repeat_layer = Lambda(
lambda x: K.repeat_elements(x, hparams.max_context_len, 1))
custom_repeat_layer2 = Lambda(lambda x: K.repeat_elements(
x, hparams.num_utterance_options + hparams.num_kb_options, 1))
# Expand dimension layer
expand_dim_layer = Lambda(lambda x: K.expand_dims(x, axis=1))
# Amplify layer
amplify_layer = Lambda(lambda x: x * hparams.amplify_val)
# Define Softmax layer
softmax_layer = Lambda(lambda x: K.softmax(Masking()(x), axis=-1))
softmax_layer2 = Lambda(lambda x: K.softmax(Masking()(x), axis=1))
# Define Stack & Concat layers
Stack = Lambda(lambda x: K.stack(x, axis=1))
# Naming tensors
kb_attention_layer = Lambda(lambda x: x, name='kb_attention')
responses_attention_layer = Lambda(lambda x: x, name='responses_attention')
context_attention_layer = Lambda(lambda x: x, name='context_attention')
# Concat = Lambda(lambda x: K.concatenate(x, axis=1))
# Sum up last dimension
Sum = Lambda(lambda x: K.sum(x, axis=-1))
Sum2 = Lambda(lambda x: K.sum(x, axis=1))
# Normalize layer
Normalize = Lambda(lambda x: K.l2_normalize(x, axis=-1))
# Define tensor slice layer
GetFirstHalfTensor = Lambda(lambda x: x[:, :, :hparams.memn2n_rnn_dim])
GetLastHalfTensor = Lambda(lambda x: x[:, :, hparams.memn2n_rnn_dim:])
GetFirstTensor = Lambda(lambda x: x[:, 0, :])
GetLastTensor = Lambda(lambda x: x[:, -1, :])
GetUtterancesTensor = Lambda(
lambda x: x[:, :hparams.num_utterance_options, :])
GetKbTensor = Lambda(lambda x: x[:, hparams.num_utterance_options:, :])
GetReverseTensor = Lambda(lambda x: K.reverse(x, axes=1))
################################## Apply layers ##################################
# Prepare Masks
utterances_mask = Reshape((1, hparams.max_context_len))(context_mask)
utterances_mask = custom_repeat_layer2(utterances_mask)
context_mask = Reshape((hparams.max_context_len, 1))(context_mask)
kb_mask = Reshape((1, hparams.num_kb_options))(kb_mask)
# Context Embedding: (BATCH_SIZE(?) x CONTEXT_LEN x EMBEDDING_DIM)
context_embedded = embedding_context_layer(context)
print("context_embedded: ", context_embedded._keras_shape)
print("context_embedded (history): ", context_embedded._keras_history,
'\n')
# Skip this?
# context_embedded = Concatenate(axis=-1)([context_embedded, context_speaker])
# Utterances Embedding: (BATCH_SIZE(?) x NUM_OPTIONS x UTTERANCE_LEN x EMBEDDING_DIM)
utterances_embedded = TimeDistributed(
embedding_utterance_layer,
input_shape=(hparams.num_utterance_options,
hparams.max_utterance_len))(utterances)
print("Utterances_embedded: ", utterances_embedded._keras_shape)
print("Utterances_embedded (history): ",
utterances_embedded._keras_history, '\n')
# KB embedding: (? x NUM_KB_OPTIONSS x MAX_KB_LEN x EMBEDING_DIM)
kb_embedded = TimeDistributed(embedding_kb_layer,
input_shape=(hparams.num_kb_options,
hparams.max_kb_len))(kb)
print("KB_embedded: ", kb_embedded._keras_shape)
print("KB_embedded: (history)", kb_embedded._keras_history, '\n')
# Encode context A: (BATCH_SIZE(?) x CONTEXT_LEN x RNN_DIM)
all_context_encoded_Forward,\
all_context_encoded_Forward_h,\
all_context_encoded_Forward_c = LSTM_A(context_embedded)
all_context_encoded_Backward,\
all_context_encoded_Backward_h,\
all_context_encoded_Backward_c = LSTM_A(Masking()(GetReverseTensor(context_embedded)))#,
#initial_state=[all_context_encoded_Forward_h, all_context_encoded_Forward_c])
all_context_encoded_Backward = Masking()(
GetReverseTensor(all_context_encoded_Backward))
# print("context_encoded_A: ", len(context_encoded_A))
print("all_context_encoded_Forward: ",
all_context_encoded_Forward._keras_shape)
print("all_context_encoded_Forward (history): ",
all_context_encoded_Forward._keras_history)
print("all_context_encoded_Backward: ",
all_context_encoded_Backward._keras_shape)
print("all_context_encoded_Backward (history): ",
all_context_encoded_Backward._keras_history, '\n')
all_context_encoded_Bidir_sum = Add()(
[all_context_encoded_Forward, all_context_encoded_Backward])
# Encode utterances B: (BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM)
all_utterances_encoded_B = TimeDistributed(
LSTM_B,
input_shape=(hparams.num_utterance_options, hparams.max_utterance_len,
hparams.memn2n_embedding_dim))(utterances_embedded)
all_utterances_encoded_B = TimeDistributed(
Matrix_utterances,
input_shape=(hparams.num_utterance_options,
hparams.memn2n_rnn_dim))(all_utterances_encoded_B)
print("all_utterances_encoded_B: ", all_utterances_encoded_B._keras_shape)
print("all_utterances_encoded_B: (history)",
all_utterances_encoded_B._keras_history, '\n')
# Encode utterances B: (BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM)
all_kb_encoded_K = TimeDistributed(
LSTM_K,
input_shape=(hparams.num_kb_options, hparams.max_kb_len,
hparams.memn2n_embedding_dim))(kb_embedded)
all_kb_encoded_K = TimeDistributed(
Matrix_kb,
input_shape=(hparams.num_kb_options,
hparams.memn2n_rnn_dim))(all_kb_encoded_K)
print("all_kb_encoded_K: ", all_kb_encoded_K._keras_shape)
print("all_kb_encoded_K: (history)", all_kb_encoded_K._keras_history, '\n')
# Stack all utterances and kb options: (? x (NUM_OPTIONS+NUM_KBs) x RNN_DIM)
all_utterances_kb_encoded = Concatenate(axis=1)(
[all_utterances_encoded_B, all_kb_encoded_K])
print("all_utterances_kb_encoded: ",
all_utterances_kb_encoded._keras_shape)
print("all_utterances_kb_encoded: (history)",
all_utterances_kb_encoded._keras_history, '\n')
responses_attention = []
kb_attention = []
for i in range(hparams.hops):
print(str(i + 1) + 'th hop:')
# 1st Attention & Weighted Sum
# between Utterances_B(NUM_OPTIONS x RNN_DIM) and Contexts_encoded_Forward(CONTEXT_LEN x RNN_DIM)
# and apply Softmax
# (Output shape: BATCH_SIZE(?) x (NUM_OPTIONS + NUM_KB) x CONTEXT_LEN)
attention_Forward = Dot(axes=[2, 2])([
all_utterances_kb_encoded,
# all_context_encoded_Forward])
all_context_encoded_Bidir_sum
])
attention_Forward = amplify_layer(attention_Forward)
attention_Forward = Add()([attention_Forward, utterances_mask])
attention_Forward = softmax_layer(attention_Forward)
print("attention_Forward: ", attention_Forward._keras_shape)
print("attention_Forward: (history)", attention_Forward._keras_history)
# between Attention(NUM_OPTIONS x CONTEXT_LEN) and Contexts_A(CONTEXT_LEN x RNN_DIM)
# equivalent to weighted sum of Contexts_A according to Attention
# (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM)
weighted_sum_Forward = Dot(axes=[2, 1])([
attention_Forward,
# all_context_encoded_Forward])
all_context_encoded_Bidir_sum
])
print("weighted_sum: ", weighted_sum_Forward._keras_shape)
print("weighted_sum: (history)", weighted_sum_Forward._keras_history,
'\n')
# (Output shape: ? x NUM_OPTIONS(100) x RNN_DIM)
all_utterances_kb_encoded = Add()(
[weighted_sum_Forward, all_utterances_kb_encoded])
# 2nd Attention & Weighted Sum
# between Utterances_B(NUM_OPTIONS x RNN_DIM) and Contexts_encoded_Backward(CONTEXT_LEN x RNN_DIM)
# and apply Softmax
# (Output shape: BATCH_SIZE(?) x (NUM_OPTIONS + NUM_KB) x CONTEXT_LEN)
attention_Backward = Dot(axes=[2, 2])(
[all_utterances_kb_encoded, all_context_encoded_Backward])
attention_Backward = amplify_layer(attention_Backward)
attention_Backward = Add()([attention_Backward, utterances_mask])
attention_Backward = softmax_layer(attention_Backward)
print("attention_Backward: ", attention_Backward._keras_shape)
print("attention_Backward: (history)",
attention_Backward._keras_history)
# between Attention(NUM_OPTIONS x CONTEXT_LEN) and Contexts_A(CONTEXT_LEN x RNN_DIM)
# equivalent to weighted sum of Contexts_A according to Attention
# (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM)
weighted_sum_Backward = Dot(axes=[2, 1])(
[attention_Backward, all_context_encoded_Backward])
print("weighted_sum_Backward: ", weighted_sum_Backward.shape)
print("weighted_sum_Backward: (history)",
weighted_sum_Backward._keras_history, '\n')
# (Output shape: ? x NUM_OPTIONS(100) x RNN_DIM)
all_utterances_kb_encoded = Add()(
[weighted_sum_Backward, all_utterances_kb_encoded])
att_responses_Forward = expand_dim_layer(
GetUtterancesTensor(attention_Forward))
att_responses_Backward = expand_dim_layer(
GetUtterancesTensor(attention_Backward))
att_kb_Forward = expand_dim_layer(GetKbTensor(attention_Forward))
att_kb_Backward = expand_dim_layer(GetKbTensor(attention_Backward))
merge_responses = Concatenate(axis=1)(
[att_responses_Forward, att_responses_Backward])
merge_kb = Concatenate(axis=1)([att_kb_Forward, att_kb_Backward])
responses_attention.append(merge_responses)
kb_attention.append(merge_kb)
print("repsonses_attention[i]:", merge_responses._keras_shape)
print("repsonses_attention[i]: (history)",
merge_responses._keras_history)
print("kb_attention[i]:", merge_kb._keras_shape)
print("kb_attention[i]: (history)", merge_kb._keras_history, '\n')
if i < hparams.hops - 1:
continue
'''
temp = all_context_encoded_Forward
all_context_encoded_Forward = all_context_encoded_Backward
all_context_encoded_Backward = temp
'''
else:
print("hop ended")
# split encoded utterances & kb
all_utterances_encoded_B = GetUtterancesTensor(
all_utterances_kb_encoded)
all_kb_encoded_K = GetKbTensor(all_utterances_kb_encoded)
print("all_utterances_encoded_B: ",
all_utterances_encoded_B._keras_shape)
print("all_utterances_encoded_B: (history)",
all_utterances_encoded_B._keras_history, '\n')
print("all_kb_encoded_K: ", all_utterances_encoded_B._keras_shape)
print("all_kb_encoded_K: (history)",
all_utterances_encoded_B._keras_history, '\n')
############# Attention to Context #############
# (Output shape: ? x MAX_CONTEXT_LEN x 1)
attention_Forward_wrt_context =\
TimeDistributed(Dense_2,
input_shape=(hparams.max_context_len,
hparams.memn2n_rnn_dim))(all_context_encoded_Forward)
attention_Forward_wrt_context = amplify_layer(
attention_Forward_wrt_context)
attention_Forward_wrt_context = Add()(
[attention_Forward_wrt_context, context_mask])
attention_Forward_wrt_context = softmax_layer2(
attention_Forward_wrt_context)
print("attention_Forward_wrt_context: ",
attention_Forward_wrt_context._keras_shape)
print("attention_Forward_wrt_context: (history)",
attention_Forward_wrt_context._keras_history)
# (Output shape: ? x 1 x RNN_DIM)
weighted_sum_Forward_wrt_context = Dot(axes=[1, 1])(
[attention_Forward_wrt_context, all_context_encoded_Bidir_sum])
print("weighted_sum_Forward_wrt_context: ",
weighted_sum_Forward_wrt_context._keras_shape)
print("weighted_sum_Forward_wrt_context: (history)",
weighted_sum_Forward_wrt_context._keras_history, '\n')
# (Output shape: ? x MAX_CONTEXT_LEN x 1)
attention_Backward_wrt_context =\
TimeDistributed(Dense_2,
input_shape=(hparams.max_context_len,
hparams.memn2n_rnn_dim))(all_context_encoded_Backward)
attention_Backward_wrt_context = amplify_layer(
attention_Backward_wrt_context)
attention_Backward_wrt_context = Add()(
[attention_Backward_wrt_context, context_mask])
attention_Backward_wrt_context = softmax_layer2(
attention_Backward_wrt_context)
print("attention_Backward_wrt_context: ",
attention_Backward_wrt_context._keras_shape)
print("attention_Backward_wrt_context: (history)",
attention_Backward_wrt_context._keras_history)
# (Output shape: ? x 1 x RNN_DIM)
weighted_sum_Backward_wrt_context = Dot(axes=[1, 1])([
attention_Backward_wrt_context, all_context_encoded_Bidir_sum
])
print("weighted_sum_Backward_wrt_context: ",
weighted_sum_Backward_wrt_context._keras_shape)
print("weighted_sum_Backward_wrt_context: (history)",
weighted_sum_Backward_wrt_context._keras_history, '\n')
att_Forward_wrt_context = Reshape(
(1, hparams.max_context_len))(attention_Forward_wrt_context)
att_Backward_wrt_context = Reshape(
(1, hparams.max_context_len))(attention_Backward_wrt_context)
context_attention = Concatenate(axis=1)(
[att_Forward_wrt_context, att_Backward_wrt_context])
context_encoded_AplusC = Add()([
weighted_sum_Forward_wrt_context,
weighted_sum_Backward_wrt_context
])
# context_encoded_AplusC = Reshape((1,hparams.memn2n_rnn_dim))(context_encoded_AplusC)
print("context_encoded_AplusC: ", context_encoded_AplusC.shape)
print("context_encoded_AplusC: (history)",
context_encoded_AplusC._keras_history, '\n')
# (output shape: ? x 1 x NUM_KB_OPTIONS)
kb_score = Dot(axes=[2, 2])(
[context_encoded_AplusC, all_kb_encoded_K])
kb_score = amplify_layer(kb_score)
kb_score = Add()([kb_score, kb_mask])
kb_score = softmax_layer(kb_score)
print("kb_score: ", kb_score._keras_shape)
print("kb_score: (history)", kb_score._keras_history)
# (output shape: ? x 1 x RNN_DIM)
kb_weighted_sum = Dot(axes=[2, 1])([kb_score, all_kb_encoded_K])
print("kb_weighted_sum: ", kb_weighted_sum._keras_shape)
print("kb_weighted_sum: (history)", kb_weighted_sum._keras_history,
'\n')
########## Normal Sum or Wighted Sum between context and external knowledge ##########
### Normal Sum ###
# context_encoded_AplusCplusKB = Add()([context_encoded_AplusC,
# kb_weighted_sum])
### Weighted Sum ###
context_encoded_AplusCplusKB = Concatenate(axis=1)(
[context_encoded_AplusC, kb_weighted_sum])
context_encoded_AplusCplusKB = Permute(
(2, 1), input_shape=(
2, hparams.memn2n_rnn_dim))(context_encoded_AplusCplusKB)
print("context_encoded_AplusCplusKB: ",
context_encoded_AplusCplusKB.shape)
print("context_encoded_AplusCplusKB: (history)",
context_encoded_AplusCplusKB._keras_history, '\n')
context_encoded_AplusCplusKB = TimeDistributed(
Dense_3, input_shape=(hparams.memn2n_rnn_dim,
2))(context_encoded_AplusCplusKB)
context_encoded_AplusCplusKB = Permute(
(2, 1), input_shape=(hparams.memn2n_rnn_dim,
1))(context_encoded_AplusCplusKB)
print("context_encoded_AplusCplusKB: ",
context_encoded_AplusCplusKB.shape)
print("context_encoded_AplusCplusKB: (history)",
context_encoded_AplusCplusKB._keras_history, '\n')
# (Output shape: ? x 1 x NUM_OPTIONS(100))
logits = Dot(axes=[2, 2])(
[context_encoded_AplusCplusKB, all_utterances_encoded_B])
logits = Reshape((hparams.num_utterance_options, ))(logits)
print("logits: ", logits.shape)
print("logits: (history)", logits._keras_history, '\n')
# Softmax layer for probability of each of Dot products in previous layer
# Softmaxing logits (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100))
probs = Activation('softmax', name='probs')(logits)
print("probs: ", probs.shape)
print("final History: ", probs._keras_history, '\n')
# Return probabilities(likelihoods) of each of utterances
# Those will be used to calculate the loss ('sparse_categorical_crossentropy')
if hparams.hops == 1:
responses_attention = expand_dim_layer(responses_attention[0])
kb_attention = expand_dim_layer(kb_attention[0])
else:
responses_attention = Stack(responses_attention)
kb_attention = Stack(kb_attention)
context_attention = context_attention_layer(context_attention)
responses_attention = responses_attention_layer(responses_attention)
kb_attention = kb_attention_layer(kb_attention)
print("context_attention:", context_attention._keras_shape)
print("repsonses_attention:", responses_attention._keras_shape)
print("kb_attention:", kb_attention._keras_shape)
return probs, context_attention, responses_attention, kb_attention
def getModel(training):
inputShape = (128, 32, 1)
kernelVals = [5, 5, 3, 3, 3]
convFilters = [32, 64, 128, 128, 256]
strideVals = [(2, 2), (2, 2), (1, 2), (1, 2), (1, 2)]
rnnUnits = 256
maxStringLen = 32
inputs = Input(name='inputX', shape=inputShape, dtype='float32')
inner = inputs
for i in range(len(kernelVals)):
inner = Conv2D(convFilters[i],(kernelVals[i],kernelVals[i]),padding = 'same',\
name = 'conv'+str(i), kernel_initializer = 'he_normal')(inner)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=strideVals[i],
name='max' + str(i + 1))(inner)
inner = Reshape(target_shape=(maxStringLen, rnnUnits),
name='reshape')(inner)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM1F')(inner)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM1B')(inner)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS1 = average([LSF, LSB])
LS1 = BatchNormalization()(LS1)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM2F')(LS1)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM2B')(LS1)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS2 = concatenate([LSF, LSB])
LS2 = BatchNormalization()(LS2)
yPred = Dense(len(unicodes) + 1,
kernel_initializer='he_normal',
name='dense2')(LS2)
yPred = Activation('softmax', name='softmax')(yPred)
#Model(inputs = inputs,outputs = yPred).summary()
labels = Input(name='label', shape=[32], dtype='float32')
inputLength = Input(name='inputLen', shape=[1], dtype='int64') # (None, 1)
labelLength = Input(name='labelLen', shape=[1], dtype='int64')
lossOut = Lambda(ctcLambdaFunc, output_shape=(1, ),
name='ctc')([yPred, labels, inputLength, labelLength])
if training:
return Model(inputs=[inputs, labels, inputLength, labelLength],
outputs=[lossOut, yPred])
return Model(inputs=[inputs], outputs=yPred)
def build_train(self):
# Input Lauer
# T: 50, L: 10, h:64, w:64, c:3
X_input = keras.layers.Input(shape=self.x_shape,
dtype='float32',
name='x_input')
E_input = keras.layers.Input(shape=self.e_shape,
dtype='float32',
name='e_input')
R_input = keras.layers.Input(shape=self.r_shape,
dtype='float32',
name='r_input')
for t in range(self.T):
""" === E unit === """
if t == 0:
E = keras.layers.Lambda(lambda x: x)(E_input)
def E_to_Et(E):
# E_t: (None, L, h, w, 2c)
E_t = K.permute_dimensions(E, [1, 0, 2, 3, 4, 5])
E_t = K.gather(E_t, [t])
E_t = K.permute_dimensions(E_t, [1, 0, 2, 3, 4, 5])
E_t = K.squeeze(E_t, axis=1)
return E_t
E_t = keras.layers.Lambda(E_to_Et)(E)
""" === R unit === """
if t == 0:
def R_to_Rt(R_input):
R_t = K.permute_dimensions(R_input, [1, 0, 2, 3, 4, 5])
R_t = K.gather(R_t, [t])
R_t = K.permute_dimensions(R_t, [1, 0, 2, 3, 4, 5])
R_t = K.squeeze(R_t, axis=1)
return R_t
R_t = keras.layers.Lambda(R_to_Rt)(R_input)
else:
# R_t: (None, L, h, w, 3)
E_t_rev = keras.layers.Lambda(lambda x: K.reverse(x, axes=1))(
E_t)
R_t, state_h_t, state_c_t = keras.layers.ConvLSTM2D(
3, (3, 3),
padding='same',
activation='tanh',
return_sequences=True,
return_state=True)(E_t_rev)
for l in range(self.L):
""" === R_tl === """
def Rt_to_Rtl(R_t):
R_tl = K.permute_dimensions(R_t, [1, 0, 2, 3, 4])
R_tl = K.gather(R_tl, [l])
R_tl = K.permute_dimensions(R_tl, [1, 0, 2, 3, 4])
R_tl = K.squeeze(R_tl, axis=1)
return R_tl
# R_tl: (None, h, w, 3)
R_tl = keras.layers.Lambda(Rt_to_Rtl)(R_t)
print("t:", t, "l:", l, "R_tl", R_tl)
""" === Ahat_tl === """
# Ahat_tl: (None, h, w, 3)
Ahat_tl = keras.layers.Conv2D(3, (3, 3), padding='same')(R_tl)
Ahat_tl = keras.layers.Activation('relu')(Ahat_tl)
print("t", t, "l,", l, "Ahat_tl", Ahat_tl)
if l == 0:
def X_to_Atl(x_input):
# (None, T, 1, h, w, c) --> (?, h, w, c)
A_tl = K.squeeze(x_input,
axis=2) # (None, T, 64, 64, 3)
A_tl = K.permute_dimensions(
A_tl, [1, 0, 2, 3, 4]) # (T, None, 64, 64, 3)
A_tl = K.gather(A_tl, [t]) # (1, None, 64, 64, 3)
A_tl = K.squeeze(A_tl, axis=0) # (None, 64, 64, 3)
return A_tl
# A_tl: (None, h, w, 3)
A_tl = keras.layers.Lambda(X_to_Atl)(X_input)
# E_tl: (None, h, w, c)
err0_tl = keras.layers.Subtract()([A_tl, Ahat_tl])
err0_tl = keras.layers.Activation('relu')(err0_tl)
err1_tl = keras.layers.Subtract()([Ahat_tl, A_tl])
err1_tl = keras.layers.Activation('relu')(err1_tl)
E_tl = keras.layers.Concatenate(axis=-1)([err0_tl, err1_tl])
# A_tl: (None, h, w, 3)
if l < self.L - 1:
def Et_to_Etl(E_t):
E_tl = K.permute_dimensions(E_t, [1, 0, 2, 3, 4])
E_tl = K.gather(E_tl, [l])
E_tl = K.permute_dimensions(E_tl, [1, 0, 2, 3, 4])
E_tl = K.squeeze(E_tl, axis=1)
return E_tl
# E_tl: (None, h, w, c)
E_tl = keras.layers.Lambda(Et_to_Etl)(E_t)
# A_tl
A_tl = keras.layers.Conv2D(3, (3, 3), padding='same')(E_tl)
if l == 0:
# E_l: (None, 1, h, w, c)
E_t = keras.layers.Lambda(
lambda x: K.expand_dims(x, axis=1))(E_tl)
else:
# E_tl_: (None, 1, h, w, c)
E_tl_ = keras.layers.Lambda(
lambda x: K.expand_dims(x, axis=1))(E_tl)
# E_l: (None, T, h, w, c)
E_t = keras.layers.Concatenate(axis=1)([E_t, E_tl_])
if t == 0:
# E: (None, 1, T, h, w, c)
E = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=1))(
E_t)
else:
# E_l_: (None, 1, T, h, w, c)
E_t_ = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=1))(
E_t)
# E: (None, L, T, h, w, c)
E = keras.layers.Concatenate(axis=1)([E, E_t_])
print("t:", t, "l:", l, "E:", E)
# print(E) # 50, 10, 64, 64, 6
model_train = keras.models.Model(inputs=[X_input, E_input, R_input],
outputs=[E])
return model_train
def build_model_morph_rnn(num_classes, embed_matrix, word_length, num_chars):
words_indices_input = Input(shape=(None, ), name='words_indices_input')
word_embeddings_out = Embedding(
embed_matrix.shape[0],
embed_matrix.shape[1],
weights=[embed_matrix],
trainable=False,
name='word_embeddings')(words_indices_input)
casings_input = Input(shape=(None, 6), name='casings_input')
chars_input = Input(shape=(None, word_length), name='chars_input')
char_embeddings_out = \
TimeDistributed(Embedding(num_chars + 1,
32,
embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)),
name='char_embeddings')(chars_input)
char_dropout_out = Dropout(0.5, name='char_dropout')(char_embeddings_out)
char_conv1d_out = TimeDistributed(
Conv1D(kernel_size=3,
filters=32,
padding='same',
activation='tanh',
strides=1,
name='char_conv1d'))(char_dropout_out)
char_maxpool_out = TimeDistributed(MaxPooling1D(word_length),
name='char_maxpool')(char_conv1d_out)
char_flat_out = TimeDistributed(Flatten(),
name='char_flat')(char_maxpool_out)
char_flat_dropout_out = Dropout(0.5,
name='char_flat_dropout')(char_flat_out)
concatenated_out = concatenate(
[word_embeddings_out, casings_input, char_flat_dropout_out],
name='concat')
forward_lstm_out = LSTM(128,
dropout=0.5,
recurrent_dropout=0.25,
return_sequences=True)(concatenated_out)
backward_lstm_out = LSTM(128,
dropout=0.5,
recurrent_dropout=0.25,
return_sequences=True,
go_backwards=True)(concatenated_out)
reverse = Lambda(lambda x: K.reverse(x, axes=1))
bilstm_out = concatenate([forward_lstm_out, reverse(backward_lstm_out)])
# Предсказываем текущую метку по выходам bilstm
bilstm_2_out = Bidirectional(
LSTM(128, dropout=0.5, recurrent_dropout=0.25,
return_sequences=True))(bilstm_out)
fc_out = TimeDistributed(Dense(128, activation='relu'),
name='fc_1')(bilstm_2_out)
bn_out = TimeDistributed(BatchNormalization(axis=1))(fc_out)
fc_2_out = TimeDistributed(Dense(num_classes, activation='softmax'),
name='fc_2')(bn_out)
# forward lstm предсказывает следующую метку справа
fc_fw_out = TimeDistributed(Dense(128, activation='relu'),
name='fc_fw_1')(forward_lstm_out)
fc_fw_2_out = TimeDistributed(
Dense(num_classes, activation='softmax', name='fc_fw_2'))(fc_fw_out)
# backward lstm предсказывает следующую метку слева
fc_bw_out = TimeDistributed(Dense(128, activation='relu'),
name='fc_bw_1')(backward_lstm_out)
fc_bw_2_out = TimeDistributed(
Dense(num_classes, activation='softmax', name='fc_bw_2'))(fc_bw_out)
model = Model([words_indices_input, casings_input, chars_input],
[fc_2_out, fc_bw_2_out, fc_fw_2_out])
model.compile(optimizer='nadam', loss='sparse_categorical_crossentropy')
model.name = 'morph_rnn'
return model
embedder = Embedding(MAX_TOKENS + 1,
embedding_dim,
weights=[embeddings],
trainable=False)
doc_embedding = embedder(document)
l_embedding = embedder(left_context)
r_embedding = embedder(right_context)
# I use LSTM RNNs instead of vanilla RNNs as described in the paper.
forward = LSTM(hidden_dim_1,
return_sequences=True)(l_embedding) # See equation (1).
backward = LSTM(hidden_dim_1, return_sequences=True,
go_backwards=True)(r_embedding) # See equation (2).
# Keras returns the output sequences in reverse order.
backward = Lambda(lambda x: backend.reverse(x, axes=1))(backward)
together = concatenate([forward, doc_embedding, backward],
axis=2) # See equation (3).
semantic = TimeDistributed(Dense(hidden_dim_2, activation="tanh"))(
together) # See equation (4).
# Keras provides its own max-pooling layers, but they cannot handle variable length input
# (as far as I can tell). As a result, I define my own max-pooling layer here.
pool_rnn = Lambda(lambda x: backend.max(x, axis=1),
output_shape=(hidden_dim_2, ))(semantic) # See equation (5).
output = Dense(NUM_CLASSES, input_dim=hidden_dim_2,
activation="softmax")(pool_rnn) # See equations (6) and (7).
model = Model(inputs=[document, left_context, right_context], outputs=output)
def create_model_cls(self, hyper_parameters):
"""
构建神经网络, col, 论文中maxpooling使用的是列池化, 不过实验效果似乎不佳，而且训练速度超级慢
:param hyper_parameters:json, hyper parameters of network
:return: tensor, moedl
"""
super().create_model(hyper_parameters)
embedding_output = self.word_embedding.output
# rnn layers
if self.rnn_units == "LSTM":
layer_cell = LSTM
else:
layer_cell = GRU
# 反向
x_backwords = layer_cell(units=self.rnn_units,
return_sequences=True,
kernel_regularizer=regularizers.l2(0.32 *
0.1),
recurrent_regularizer=regularizers.l2(0.32),
go_backwards=True)(embedding_output)
x_backwords_reverse = Lambda(lambda x: K.reverse(x, axes=1))(
x_backwords)
# 前向
x_fordwords = layer_cell(units=self.rnn_units,
return_sequences=True,
kernel_regularizer=regularizers.l2(0.32 *
0.1),
recurrent_regularizer=regularizers.l2(0.32),
go_backwards=False)(embedding_output)
# 拼接
x_feb = Concatenate(axis=2)(
[x_fordwords, embedding_output, x_backwords_reverse])
####列池化##################################################
x_feb = Dropout(self.dropout)(x_feb)
dim_2 = K.int_shape(x_feb)[2]
x_feb_reshape = Reshape((dim_2, self.len_max))(x_feb)
conv_pools = []
for filter in self.filters:
conv = Conv1D(
filters=self.filters_num, # filter=300
kernel_size=filter,
padding='valid',
kernel_initializer='normal',
activation='relu',
)(x_feb_reshape)
pooled = MaxPooling1D(
padding='valid',
pool_size=32,
)(conv)
conv_pools.append(pooled)
x = Concatenate(axis=1)(conv_pools)
# x = MaxPooling1D(padding = 'VALID',)(x_feb_reshape)
x = Flatten()(x)
x = Dropout(self.dropout)(x)
#########################################################################
output = Dense(units=self.label, activation=self.activate_classify)(x)
self.model = Model(inputs=self.word_embedding.input, outputs=output)
self.model.summary(120)
embedder = Embedding(vocab_length + 3, emb_dim)
embedded_inputs = embedder(inputs)
h = Bidirectional(SimpleRNN(20), merge_mode='concat')(embedded_inputs)
z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim)(h)
z = Lambda(sample_z, output_shape=(latent_dim, ))([z_mean, z_log_var])
encoder = Model(inputs, [z_mean, z_log_var, z])
forward_in = Input(shape=(None, ))
f_emb = embedder(forward_in)
f_rnn = SimpleRNN(latent_dim,
return_sequences=True,
return_state=True,
go_backwards=True)
h, _ = f_rnn(f_emb, initial_state=z)
h = Lambda(lambda x: K.reverse(x, axes=1))(h)
f_bn = BatchNormalization()
h = f_bn(h)
soft_dense = Dense(vocab_length + 1, activation='softmax')
decoded = TimeDistributed(soft_dense)(h)
#decoded = soft_dense(h)
vae = Model([inputs, forward_in], decoded, name='forward_vae')
vae.compile(optimizer='adam', loss=vae_loss, metrics=['acc'])
test_gen = backward_autoencoder_shaper(test_harness1)(
vocab_length=vocab_length)
vae.fit_generator(test_gen, steps_per_epoch=1000, epochs=5)
p = next(test_gen)
use_bias=0,
name='conv2d',
trainable=True)
tensor = conv2d(input0)
# 设定卷积网络的初值
init_cnn_weight = []
init_cnn_kernel = np.array([[[[-1]], [[1]], [[0]], [[0]]]]) #一阶差分初始权重
init_cnn_weight.append(init_cnn_kernel)
conv2d.set_weights(init_cnn_weight)
tensor = Lambda(lambda x: K.squeeze(x, axis=-1), name='squeeze_0')(
tensor) # 4D 数据压缩为 3D 数据,因为channels=1
tensor = Lambda(lambda x: tf.nn.top_k(x, x.shape[2])[0],
name='sequential_pooling_drop')(
tensor) # 在第2个维度上对数据降序排列,即各滑动窗内数据
tensor = Lambda(lambda x: K.reverse(x, axes=2),
name='sequential_pooling_asend')(tensor) # 升序排列
tensor = LSTM(units=units, name='lstm')(tensor) # 多对一LSTM
tensor = Dense(20, activation='tanh', name='FC0')(tensor)
prediction = Dense(1,
activation='sigmoid',
name='output',
use_bias='False')(tensor)
model = Model(inputs=input0, outputs=prediction)
# 中间输出
# layer_name = 'sequential_pooling_asend'
# intermediate_layer_model = Model(input=model.input,
# output=model.get_layer(layer_name).output)
# intermediate_output = intermediate_layer_model.predict(X_train)
# intermediate_output = np.squeeze(intermediate_output)
# 训练模型
def recursion(self,
input_energy,
mask=None,
go_backwards=False,
return_sequences=True,
return_logZ=True,
input_length=None):
"""Forward (alpha) or backward (beta) recursion
If `return_logZ = True`, compute the logZ, the normalization constant:
\[ Z = \sum_{y1, y2, y3} exp(-E) # energy
= \sum_{y1, y2, y3} exp(-(u1' y1 + y1' W y2 + u2' y2 + y2' W y3 + u3' y3))
= sum_{y2, y3} (exp(-(u2' y2 + y2' W y3 + u3' y3))
sum_{y1} exp(-(u1' y1' + y1' W y2))) \]
Denote:
\[ S(y2) := sum_{y1} exp(-(u1' y1 + y1' W y2)), \]
\[ Z = sum_{y2, y3} exp(log S(y2) - (u2' y2 + y2' W y3 + u3' y3)) \]
\[ logS(y2) = log S(y2) = log_sum_exp(-(u1' y1' + y1' W y2)) \]
Note that:
yi's are one-hot vectors
u1, u3: boundary energies have been merged
If `return_logZ = False`, compute the Viterbi's best path lookup table.
"""
chain_energy = self.chain_kernel
# shape=(1, F, F): F=num of output features. 1st F is for t-1, 2nd F for t
chain_energy = K.expand_dims(chain_energy, 0)
# shape=(B, F), dtype=float32
prev_target_val = K.zeros_like(input_energy[:, 0, :])
if go_backwards:
input_energy = K.reverse(input_energy, 1)
if mask is not None:
mask = K.reverse(mask, 1)
initial_states = [
prev_target_val,
K.zeros_like(prev_target_val[:, :1])
]
constants = [chain_energy]
if mask is not None:
mask = K.cast(mask, K.floatx())
mask2 = K.cast(
K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1),
K.floatx())
constants.append(mask2)
def _step(input_energy_i, states):
return self.step(input_energy_i, states, return_logZ)
target_val_last, target_val_seq, _ = K.rnn(_step,
input_energy,
initial_states,
constants=constants,
input_length=input_length,
unroll=self.unroll)
if return_sequences:
if go_backwards:
target_val_seq = K.reverse(target_val_seq, 1)
return target_val_seq
else:
return target_val_last
def reverse(inputs, axes=1):
return K.reverse(inputs, axes=axes)
def labels_to_image_model(im_shape,
n_channels,
crop_shape,
label_list,
n_neutral_labels,
vox2ras,
nonlin_shape_factor=0.0625,
crop_channel2=None,
output_div_by_n=None,
flipping=True):
# get shapes
n_dims, _ = utils.get_dims(im_shape)
crop_shape = get_shapes(crop_shape, im_shape, output_div_by_n)
deformation_field_size = utils.get_resample_shape(im_shape,
nonlin_shape_factor,
len(im_shape))
# create new_label_list and corresponding LUT to make sure that labels go from 0 to N-1
new_label_list, lut = utils.rearrange_label_list(label_list)
# define mandatory inputs
image_input = KL.Input(shape=im_shape + [n_channels], name='image_input')
labels_input = KL.Input(shape=im_shape + [1], name='labels_input')
aff_in = KL.Input(shape=(n_dims + 1, n_dims + 1), name='aff_input')
nonlin_field_in = KL.Input(shape=deformation_field_size,
name='nonlin_input')
list_inputs = [image_input, labels_input, aff_in, nonlin_field_in]
# convert labels to new_label_list
labels = KL.Lambda(lambda x: tf.gather(
tf.convert_to_tensor(lut, dtype='int32'), tf.cast(x, dtype='int32')))(
labels_input)
# deform labels
image_input._keras_shape = tuple(image_input.get_shape().as_list())
labels._keras_shape = tuple(labels.get_shape().as_list())
labels = KL.Lambda(lambda x: tf.cast(x, dtype='float'))(labels)
resize_shape = [
max(int(im_shape[i] / 2), deformation_field_size[i])
for i in range(len(im_shape))
]
nonlin_field = nrn_layers.Resize(size=resize_shape,
interp_method='linear')(nonlin_field_in)
nonlin_field = nrn_layers.VecInt()(nonlin_field)
nonlin_field = nrn_layers.Resize(size=im_shape,
interp_method='linear')(nonlin_field)
image = nrn_layers.SpatialTransformer(interp_method='linear')(
[image_input, aff_in, nonlin_field])
labels = nrn_layers.SpatialTransformer(interp_method='nearest')(
[labels, aff_in, nonlin_field])
labels = KL.Lambda(lambda x: tf.cast(x, dtype='int32'))(labels)
# cropping
if crop_shape is not None:
image, crop_idx = l2i_sa.random_cropping(image, crop_shape, n_dims)
labels = KL.Lambda(
lambda x: tf.slice(x[0],
begin=tf.cast(x[1], dtype='int32'),
size=tf.convert_to_tensor(
[-1] + crop_shape + [-1], dtype='int32')))(
[labels, crop_idx])
else:
crop_shape = im_shape
# flipping
if flipping:
labels, flip = l2i_sa.label_map_random_flipping(
labels, label_list, n_neutral_labels, vox2ras, n_dims)
ras_axes = edit_volumes.get_ras_axes(vox2ras, n_dims)
flip_axis = [ras_axes[0] + 1]
image = KL.Lambda(lambda y: K.switch(
y[0],
KL.Lambda(lambda x: K.reverse(x, axes=flip_axis))(y[1]), y[1]))(
[flip, image])
# convert labels back to original values
labels = KL.Lambda(
lambda x: tf.gather(tf.convert_to_tensor(label_list, dtype='int32'),
tf.cast(x, dtype='int32')),
name='labels_out')(labels)
# intensity augmentation
image = KL.Lambda(lambda x: K.clip(x, 0, 300), name='clipping')(image)
# loop over channels
if n_channels > 1:
split = KL.Lambda(lambda x: tf.split(x, [1] * n_channels, axis=-1))(
image)
else:
split = [image]
processed_channels = list()
for i, channel in enumerate(split):
# normalise and shift intensities
image = l2i_ia.min_max_normalisation(image)
image = KL.Lambda(lambda x: K.random_uniform(
(1, ), .85, 1.1) * x + K.random_uniform((1, ), -.3, .3))(image)
image = KL.Lambda(lambda x: K.clip(x, 0, 1))(image)
image = l2i_ia.gamma_augmentation(image)
# randomly crop sides of second channel
if (crop_channel2 is not None) & (channel == 1):
image = l2i_sa.restrict_tensor(image, crop_channel2, n_dims)
# concatenate all channels back, and clip output (include labels to keep it when plugging to other models)
if n_channels > 1:
image = KL.concatenate(processed_channels)
else:
image = processed_channels[0]
image = KL.Lambda(lambda x: K.clip(x[0], 0, 1),
name='image_out')([image, labels])
# build model
brain_model = Model(inputs=list_inputs, outputs=[image, labels])
# shape of returned images
output_shape = image.get_shape().as_list()[1:]
return brain_model, output_shape
def call(self, x, mask=None):
# 'RGB'->'BGR'
x = K.reverse(x, axes=-1)
# Zero-center by mean pixel
x = x - self.MEAN_VALUE_TENSOR
return x