def test_Bidirectional_merged_value(merge_mode): rnn = layers.LSTM samples = 2 dim = 5 timesteps = 3 units = 3 X = [np.random.rand(samples, timesteps, dim)] if merge_mode == 'sum': merge_func = lambda y, y_rev: y + y_rev elif merge_mode == 'mul': merge_func = lambda y, y_rev: y * y_rev elif merge_mode == 'ave': merge_func = lambda y, y_rev: (y + y_rev) / 2 elif merge_mode == 'concat': merge_func = lambda y, y_rev: np.concatenate((y, y_rev), axis=-1) else: merge_func = lambda y, y_rev: [y, y_rev] # basic case inputs = Input((timesteps, dim)) layer = wrappers.Bidirectional(rnn(units, return_sequences=True), merge_mode=merge_mode) f_merged = K.function([inputs], to_list(layer(inputs))) f_forward = K.function([inputs], [layer.forward_layer.call(inputs)]) f_backward = K.function([inputs], [K.reverse(layer.backward_layer.call(inputs), 1)]) y_merged = f_merged(X) y_expected = to_list(merge_func(f_forward(X)[0], f_backward(X)[0])) assert len(y_merged) == len(y_expected) for x1, x2 in zip(y_merged, y_expected): assert_allclose(x1, x2, atol=1e-5) # test return_state inputs = Input((timesteps, dim)) layer = wrappers.Bidirectional(rnn(units, return_state=True), merge_mode=merge_mode) f_merged = K.function([inputs], layer(inputs)) f_forward = K.function([inputs], layer.forward_layer.call(inputs)) f_backward = K.function([inputs], layer.backward_layer.call(inputs)) n_states = len(layer.layer.states) y_merged = f_merged(X) y_forward = f_forward(X) y_backward = f_backward(X) y_expected = to_list(merge_func(y_forward[0], y_backward[0])) assert len(y_merged) == len(y_expected) + n_states * 2 for x1, x2 in zip(y_merged, y_expected): assert_allclose(x1, x2, atol=1e-5) # test if the state of a BiRNN is the concatenation of the underlying RNNs y_merged = y_merged[-n_states * 2:] y_forward = y_forward[-n_states:] y_backward = y_backward[-n_states:] for state_birnn, state_inner in zip(y_merged, y_forward + y_backward): assert_allclose(state_birnn, state_inner, atol=1e-5)
def _loss_tensor(X, y_pred, y_true, CL, CR, tiles=2): X = K.permute_dimensions(X, (1, 0, 2, 3)) Xcs = K.reshape(X, (X.shape[0], X.shape[1], X.shape[2] * X.shape[3])) # reshape x as n columns X_pred = K.batch_dot(K.permute_dimensions(Xcs, (0, 2, 1)), y_pred) # replace column according to pred X_pred = K.reshape(X_pred, (X.shape[0], X.shape[-2] * tiles, X.shape[-1] * tiles)) # reshape as matrix X_flip_x = K.reverse(X_pred, axes=2) X_flip_y = K.reverse(X_pred, axes=1) LX_y = K.permute_dimensions(K.dot(CL, X_pred), (1, 0, 2)) RX_y = K.permute_dimensions(K.dot(CL, X_flip_y), (1, 0, 2)) LX_x = K.dot(X_pred, CR) RX_x = K.dot(X_flip_x, CR) dissimilarity = K.mean(K.mean(K.square(LX_x - RX_x), axis=2), axis=1) + K.mean( K.mean(K.square(LX_y - RX_y), axis=2), axis=1) # return K.categorical_crossentropy(y_pred, y_true) + K.sum(dissimilarity) return K.mean(dissimilarity)
def call(self, input): forward_lstm = LSTM(self.units, dropout=self.dropout, return_sequences=True, return_state=True) backward_lstm = LSTM(self.units, dropout=self.dropout, return_sequences=True, return_state=True, go_backwards=True) fw_outputs, fw_output, fw_state = forward_lstm(input) bw_outputs, bw_output, b_state = backward_lstm(input) bw_outputs = K.reverse(bw_outputs, 1) # bw_output = bw_state[0] outputs = K.concatenate([fw_outputs, bw_outputs]) last_output = K.concatenate([fw_output, bw_output]) return [outputs, last_output]
def create_model(self, hyper_parameters): """ 构建神经网络,行卷积加池化 :param hyper_parameters:json, hyper parameters of network :return: tensor, moedl """ super().create_model(hyper_parameters) embedding_output = self.word_embedding.output # rnn layers if self.rnn_units == "LSTM": layer_cell = LSTM else: layer_cell = GRU # 反向 x_backwords = layer_cell(units=self.rnn_units, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=True)(embedding_output) x_backwords_reverse = Lambda(lambda x: K.reverse(x, axes=1))( x_backwords) # 前向 x_fordwords = layer_cell(units=self.rnn_units, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=False)(embedding_output) # 拼接 x_feb = Concatenate(axis=2)( [x_fordwords, embedding_output, x_backwords_reverse]) ####使用多个卷积核################################################## x_feb = Dropout(self.dropout)(x_feb) # Concatenate后的embedding_size dim_2 = K.int_shape(x_feb)[2] x_feb_reshape = Reshape((self.len_max, dim_2, 1))(x_feb) # 提取n-gram特征和最大池化, 一般不用平均池化 conv_pools = [] for filter in self.filters: conv = Conv2D( filters=self.filters_num, kernel_size=(filter, dim_2), padding='valid', kernel_initializer='normal', activation='relu', )(x_feb_reshape) pooled = MaxPooling2D( pool_size=(self.len_max - filter + 1, 1), strides=(1, 1), padding='valid', )(conv) conv_pools.append(pooled) # 拼接 x = Concatenate()(conv_pools) x = Flatten()(x) ######################################################################### output = Dense(units=self.label, activation=self.activate_classify)(x) self.model = Model(inputs=self.word_embedding.input, outputs=output) self.model.summary(120)
def reverse_sequence(kvar): kvar2 = K.reverse(kvar, axes=1) return kvar2
def get_Model(training): input_shape = (img_w, img_h, 1) # (128, 64, 1) # Make Networkw inputs = Input(name='the_input', shape=input_shape, dtype='float32') # (None, 128, 64, 1) # Convolution layer (VGG) n = 28 inner = Conv2D(n // 2, (3, 3), padding='same', name='conv1', kernel_initializer='he_normal')( inputs) # (None, 128, 64, 64) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(2, 2), name='max1')(inner) # (None,64, 32, 64) inner = Conv2D(n, (3, 3), padding='same', name='conv2', kernel_initializer='he_normal')( inner) # (None, 64, 32, 128) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(2, 2), name='max2')(inner) # (None, 32, 16, 128) inner = Conv2D(n * 2, (3, 3), padding='same', name='conv3', kernel_initializer='he_normal')( inner) # (None, 32, 16, 256) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = Conv2D(n * 2, (3, 3), padding='same', name='conv4', kernel_initializer='he_normal')( inner) # (None, 32, 16, 256) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(1, 2), name='max3')(inner) # (None, 32, 8, 256) inner = Conv2D(n * 4, (3, 3), padding='same', name='conv5', kernel_initializer='he_normal')(inner) # (None, 32, 8, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = Conv2D(n * 4, (3, 3), padding='same', name='conv6')(inner) # (None, 32, 8, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=(1, 2), name='max4')(inner) # (None, 32, 4, 512) inner = Conv2D(n * 4, (2, 2), padding='same', kernel_initializer='he_normal', name='con7')(inner) # (None, 32, 4, 512) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) # CNN to RNN inner = Reshape(target_shape=((n // 4, n * 4)), name='reshape')(inner) # (None, 32, 2048) inner = Dense(n // 2, activation='relu', kernel_initializer='he_normal', name='dense1')(inner) # (None, 32, 64) # RNN layer lstm_1 = LSTM(n * 2, return_sequences=True, kernel_initializer='he_normal', name='lstm1')(inner) # (None, 32, 512) lstm_1b = LSTM(n * 2, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm1_b')(inner) reversed_lstm_1b = Lambda( lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_1b) lstm1_merged = add([lstm_1, reversed_lstm_1b]) # (None, 32, 512) lstm1_merged = BatchNormalization()(lstm1_merged) lstm_2 = LSTM(n * 2, return_sequences=True, kernel_initializer='he_normal', name='lstm2')(lstm1_merged) lstm_2b = LSTM(n * 2, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='lstm2_b')(lstm1_merged) reversed_lstm_2b = Lambda( lambda inputTensor: K.reverse(inputTensor, axes=1))(lstm_2b) lstm2_merged = concatenate([lstm_2, reversed_lstm_2b]) # (None, 32, 1024) lstm2_merged = BatchNormalization()(lstm2_merged) # transforms RNN output to character activations: inner = Dense(num_classes, kernel_initializer='he_normal', name='dense2')(lstm2_merged) #(None, 32, 63) y_pred = Activation('softmax', name='softmax')(inner) labels = Input(name='the_labels', shape=[max_text_len], dtype='float32') # (None ,8) input_length = Input(name='input_length', shape=[1], dtype='int64') # (None, 1) label_length = Input(name='label_length', shape=[1], dtype='int64') # (None, 1) # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([y_pred, labels, input_length, label_length]) #(None, 1) if training: return Model(inputs=[inputs, labels, input_length, label_length], outputs=loss_out) else: return Model(inputs=[inputs], outputs=y_pred)
def call(self, inputs): # pylint: disable=arguments-differ return K.reverse(inputs, self.axis)
from keras import models, optimizers, losses from keras import initializers import numpy as np from keras import backend as K x = [[1, 2, 3], [4, 5, 6]] xx = [[10, 20, 30], [40, 50, 60]] x = np.asarray(x) xx = np.asarray(xx) y = [0, 1] input1 = layers.Input(shape=(3, )) input2 = layers.Input(shape=(3, )) l = layers.Dense(1, kernel_initializer=initializers.RandomUniform(0, 1)) rel = layers.Lambda(lambda x: K.reverse(x, axes=0)) rel2 = layers.Lambda(lambda x: K.map_fn()) x1 = l(input1) x2 = l(input2) x3 = rel(x2) #model0 = models.Model(inputs = input1, outputs = x1) #model0.compile(loss = losses.binary_crossentropy , optimizer = optimizers.Adam()) #res = model0.predict(x) merged = layers.merge.Subtract()([x2, x1]) model1 = models.Model(inputs=[input1, input2], outputs=[x1, x2]) model1.compile(loss=losses.binary_crossentropy, optimizer=optimizers.Adam()) res1 = model1.predict([x, xx])
def create_model(self) -> Model: if self.use_matrix: context = Input(shape=(self.max_sequence_length,)) response = Input(shape=(self.max_sequence_length,)) emb_layer = self.embedding_layer() emb_c = emb_layer(context) emb_r = emb_layer(response) else: context = Input(shape=(self.max_sequence_length, self.embedding_dim,)) response = Input(shape=(self.max_sequence_length, self.embedding_dim,)) emb_c = context emb_r = response lstm_layer = self.create_lstm_layer_1() lstm_a = lstm_layer(emb_c) lstm_b = lstm_layer(emb_r) f_layer_f = FullMatchingLayer(self.perspective_num) f_layer_b = FullMatchingLayer(self.perspective_num) f_a_forw = f_layer_f([lstm_a[0], lstm_b[0]])[0] f_a_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_a[1]), Lambda(lambda x: K.reverse(x, 1))(lstm_b[1])])[0] f_a_back = Lambda(lambda x: K.reverse(x, 1))(f_a_back) f_b_forw = f_layer_f([lstm_b[0], lstm_a[0]])[0] f_b_back = f_layer_b([Lambda(lambda x: K.reverse(x, 1))(lstm_b[1]), Lambda(lambda x: K.reverse(x, 1))(lstm_a[1])])[0] f_b_back = Lambda(lambda x: K.reverse(x, 1))(f_b_back) mp_layer_f = MaxpoolingMatchingLayer(self.perspective_num) mp_layer_b = MaxpoolingMatchingLayer(self.perspective_num) mp_a_forw = mp_layer_f([lstm_a[0], lstm_b[0]])[0] mp_a_back = mp_layer_b([lstm_a[1], lstm_b[1]])[0] mp_b_forw = mp_layer_f([lstm_b[0], lstm_a[0]])[0] mp_b_back = mp_layer_b([lstm_b[1], lstm_a[1]])[0] at_layer_f = AttentiveMatchingLayer(self.perspective_num) at_layer_b = AttentiveMatchingLayer(self.perspective_num) at_a_forw = at_layer_f([lstm_a[0], lstm_b[0]])[0] at_a_back = at_layer_b([lstm_a[1], lstm_b[1]])[0] at_b_forw = at_layer_f([lstm_b[0], lstm_a[0]])[0] at_b_back = at_layer_b([lstm_b[1], lstm_a[1]])[0] ma_layer_f = MaxattentiveMatchingLayer(self.perspective_num) ma_layer_b = MaxattentiveMatchingLayer(self.perspective_num) ma_a_forw = ma_layer_f([lstm_a[0], lstm_b[0]])[0] ma_a_back = ma_layer_b([lstm_a[1], lstm_b[1]])[0] ma_b_forw = ma_layer_f([lstm_b[0], lstm_a[0]])[0] ma_b_back = ma_layer_b([lstm_b[1], lstm_a[1]])[0] concat_a = Lambda(lambda x: K.concatenate(x, axis=-1))([f_a_forw, f_a_back, mp_a_forw, mp_a_back, at_a_forw, at_a_back, ma_a_forw, ma_a_back]) concat_b = Lambda(lambda x: K.concatenate(x, axis=-1))([f_b_forw, f_b_back, mp_b_forw, mp_b_back, at_b_forw, at_b_back, ma_b_forw, ma_b_back]) concat_a = Dropout(self.ldrop_val)(concat_a) concat_b = Dropout(self.ldrop_val)(concat_b) lstm_layer_agg = self.create_lstm_layer_2() agg_a = lstm_layer_agg(concat_a) agg_b = lstm_layer_agg(concat_b) agg_a = Dropout(self.dropout_val)(agg_a) agg_b = Dropout(self.dropout_val)(agg_b) reduced = Lambda(lambda x: K.concatenate(x, axis=-1))([agg_a, agg_b]) if self.triplet_mode: dist = Lambda(self._pairwise_distances)([agg_a, agg_b]) else: ker_in = glorot_uniform(seed=self.seed) dense = Dense(self.dense_dim, kernel_initializer=ker_in)(reduced) dist = Dense(1, activation='sigmoid', name="score_model")(dense) model = Model([context, response], dist) return model
def ResNet50_pre(imgs, scope): return Lambda( lambda x: K.reverse(x, len(x.shape) - 1) - [103.939, 116.779, 123.68], name=scope + 'resnet50_pre')(imgs)
def memLstm_custom_model(hparams, context, context_mask, utterances, kb, kb_flag, kb_mask): print("context_shape: ", context._keras_shape) print("utterances_shape: ", utterances._keras_shape) print("context_mask: ", context_mask._keras_shape) print("kb_flag shape: ", kb_flag._keras_shape) # Use embedding matrix pretrained by Gensim embeddings_W = np.load(hparams.embedding_path) print("embeddings_W: ", embeddings_W.shape) ################################## Define Regular Layers ################################## # Utterances Embedding (Output shape: NUM_OPTIONS(100) x BATCH_SIZE(?) x LEN_SEQ(160) x EMBEDDING_DIM(300)) embedding_context_layer = Embedding( input_dim=hparams.vocab_size, output_dim=hparams.memn2n_embedding_dim, weights=[embeddings_W], input_length=hparams.max_context_len, mask_zero=True, trainable=False) embedding_utterance_layer = Embedding( input_dim=hparams.vocab_size, output_dim=hparams.memn2n_embedding_dim, weights=[embeddings_W], input_length=hparams.max_utterance_len, mask_zero=True, trainable=False) embedding_kb_layer = Embedding(input_dim=hparams.vocab_size, output_dim=hparams.memn2n_embedding_dim, weights=[embeddings_W], input_length=hparams.max_kb_len, mask_zero=True, trainable=False) # Define LSTM Context encoder 1 LSTM_A = LSTM(hparams.memn2n_rnn_dim, input_shape=(hparams.max_context_len, hparams.memn2n_embedding_dim), use_bias=True, unit_forget_bias=True, return_state=True, return_sequences=True) # Define LSTM Utterances encoder LSTM_B = LSTM(hparams.memn2n_rnn_dim, input_shape=(hparams.max_utterance_len, hparams.memn2n_embedding_dim), use_bias=True, unit_forget_bias=True, return_state=False, return_sequences=False) # Define LSTM KBs LSTM_K = LSTM(hparams.memn2n_rnn_dim, input_shape=(hparams.max_kb_len, hparams.memn2n_embedding_dim), use_bias=True, unit_forget_bias=True, return_state=False, return_sequences=False) # Define Dense layer to transform utterances Matrix_utterances = Dense( hparams.memn2n_rnn_dim, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal(mean=0.0, stddev=1.0, seed=None), input_shape=(hparams.memn2n_rnn_dim, )) Matrix_kb = Dense(hparams.memn2n_rnn_dim, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal( mean=0.0, stddev=1.0, seed=None), input_shape=(hparams.memn2n_rnn_dim, )) # Define Dense layer to do softmax Dense_2 = Dense(1, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal( mean=0.0, stddev=1.0, seed=None), input_shape=(hparams.memn2n_rnn_dim, )) Dense_3 = Dense(1, use_bias=False, kernel_initializer=keras.initializers.TruncatedNormal( mean=0.0, stddev=1.0, seed=None), input_shape=(2, )) ################################## Define Custom Layers ################################## # Define repeat element layer custom_repeat_layer = Lambda( lambda x: K.repeat_elements(x, hparams.max_context_len, 1)) custom_repeat_layer2 = Lambda(lambda x: K.repeat_elements( x, hparams.num_utterance_options + hparams.num_kb_options, 1)) # Expand dimension layer expand_dim_layer = Lambda(lambda x: K.expand_dims(x, axis=1)) # Amplify layer amplify_layer = Lambda(lambda x: x * hparams.amplify_val) # Define Softmax layer softmax_layer = Lambda(lambda x: K.softmax(Masking()(x), axis=-1)) softmax_layer2 = Lambda(lambda x: K.softmax(Masking()(x), axis=1)) # Define Stack & Concat layers Stack = Lambda(lambda x: K.stack(x, axis=1)) # Naming tensors kb_attention_layer = Lambda(lambda x: x, name='kb_attention') responses_attention_layer = Lambda(lambda x: x, name='responses_attention') context_attention_layer = Lambda(lambda x: x, name='context_attention') # Concat = Lambda(lambda x: K.concatenate(x, axis=1)) # Sum up last dimension Sum = Lambda(lambda x: K.sum(x, axis=-1)) Sum2 = Lambda(lambda x: K.sum(x, axis=1)) # Normalize layer Normalize = Lambda(lambda x: K.l2_normalize(x, axis=-1)) # Define tensor slice layer GetFirstHalfTensor = Lambda(lambda x: x[:, :, :hparams.memn2n_rnn_dim]) GetLastHalfTensor = Lambda(lambda x: x[:, :, hparams.memn2n_rnn_dim:]) GetFirstTensor = Lambda(lambda x: x[:, 0, :]) GetLastTensor = Lambda(lambda x: x[:, -1, :]) GetUtterancesTensor = Lambda( lambda x: x[:, :hparams.num_utterance_options, :]) GetKbTensor = Lambda(lambda x: x[:, hparams.num_utterance_options:, :]) GetReverseTensor = Lambda(lambda x: K.reverse(x, axes=1)) ################################## Apply layers ################################## # Prepare Masks utterances_mask = Reshape((1, hparams.max_context_len))(context_mask) utterances_mask = custom_repeat_layer2(utterances_mask) context_mask = Reshape((hparams.max_context_len, 1))(context_mask) kb_mask = Reshape((1, hparams.num_kb_options))(kb_mask) # Context Embedding: (BATCH_SIZE(?) x CONTEXT_LEN x EMBEDDING_DIM) context_embedded = embedding_context_layer(context) print("context_embedded: ", context_embedded._keras_shape) print("context_embedded (history): ", context_embedded._keras_history, '\n') # Skip this? # context_embedded = Concatenate(axis=-1)([context_embedded, context_speaker]) # Utterances Embedding: (BATCH_SIZE(?) x NUM_OPTIONS x UTTERANCE_LEN x EMBEDDING_DIM) utterances_embedded = TimeDistributed( embedding_utterance_layer, input_shape=(hparams.num_utterance_options, hparams.max_utterance_len))(utterances) print("Utterances_embedded: ", utterances_embedded._keras_shape) print("Utterances_embedded (history): ", utterances_embedded._keras_history, '\n') # KB embedding: (? x NUM_KB_OPTIONSS x MAX_KB_LEN x EMBEDING_DIM) kb_embedded = TimeDistributed(embedding_kb_layer, input_shape=(hparams.num_kb_options, hparams.max_kb_len))(kb) print("KB_embedded: ", kb_embedded._keras_shape) print("KB_embedded: (history)", kb_embedded._keras_history, '\n') # Encode context A: (BATCH_SIZE(?) x CONTEXT_LEN x RNN_DIM) all_context_encoded_Forward,\ all_context_encoded_Forward_h,\ all_context_encoded_Forward_c = LSTM_A(context_embedded) all_context_encoded_Backward,\ all_context_encoded_Backward_h,\ all_context_encoded_Backward_c = LSTM_A(Masking()(GetReverseTensor(context_embedded)))#, #initial_state=[all_context_encoded_Forward_h, all_context_encoded_Forward_c]) all_context_encoded_Backward = Masking()( GetReverseTensor(all_context_encoded_Backward)) # print("context_encoded_A: ", len(context_encoded_A)) print("all_context_encoded_Forward: ", all_context_encoded_Forward._keras_shape) print("all_context_encoded_Forward (history): ", all_context_encoded_Forward._keras_history) print("all_context_encoded_Backward: ", all_context_encoded_Backward._keras_shape) print("all_context_encoded_Backward (history): ", all_context_encoded_Backward._keras_history, '\n') all_context_encoded_Bidir_sum = Add()( [all_context_encoded_Forward, all_context_encoded_Backward]) # Encode utterances B: (BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM) all_utterances_encoded_B = TimeDistributed( LSTM_B, input_shape=(hparams.num_utterance_options, hparams.max_utterance_len, hparams.memn2n_embedding_dim))(utterances_embedded) all_utterances_encoded_B = TimeDistributed( Matrix_utterances, input_shape=(hparams.num_utterance_options, hparams.memn2n_rnn_dim))(all_utterances_encoded_B) print("all_utterances_encoded_B: ", all_utterances_encoded_B._keras_shape) print("all_utterances_encoded_B: (history)", all_utterances_encoded_B._keras_history, '\n') # Encode utterances B: (BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM) all_kb_encoded_K = TimeDistributed( LSTM_K, input_shape=(hparams.num_kb_options, hparams.max_kb_len, hparams.memn2n_embedding_dim))(kb_embedded) all_kb_encoded_K = TimeDistributed( Matrix_kb, input_shape=(hparams.num_kb_options, hparams.memn2n_rnn_dim))(all_kb_encoded_K) print("all_kb_encoded_K: ", all_kb_encoded_K._keras_shape) print("all_kb_encoded_K: (history)", all_kb_encoded_K._keras_history, '\n') # Stack all utterances and kb options: (? x (NUM_OPTIONS+NUM_KBs) x RNN_DIM) all_utterances_kb_encoded = Concatenate(axis=1)( [all_utterances_encoded_B, all_kb_encoded_K]) print("all_utterances_kb_encoded: ", all_utterances_kb_encoded._keras_shape) print("all_utterances_kb_encoded: (history)", all_utterances_kb_encoded._keras_history, '\n') responses_attention = [] kb_attention = [] for i in range(hparams.hops): print(str(i + 1) + 'th hop:') # 1st Attention & Weighted Sum # between Utterances_B(NUM_OPTIONS x RNN_DIM) and Contexts_encoded_Forward(CONTEXT_LEN x RNN_DIM) # and apply Softmax # (Output shape: BATCH_SIZE(?) x (NUM_OPTIONS + NUM_KB) x CONTEXT_LEN) attention_Forward = Dot(axes=[2, 2])([ all_utterances_kb_encoded, # all_context_encoded_Forward]) all_context_encoded_Bidir_sum ]) attention_Forward = amplify_layer(attention_Forward) attention_Forward = Add()([attention_Forward, utterances_mask]) attention_Forward = softmax_layer(attention_Forward) print("attention_Forward: ", attention_Forward._keras_shape) print("attention_Forward: (history)", attention_Forward._keras_history) # between Attention(NUM_OPTIONS x CONTEXT_LEN) and Contexts_A(CONTEXT_LEN x RNN_DIM) # equivalent to weighted sum of Contexts_A according to Attention # (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM) weighted_sum_Forward = Dot(axes=[2, 1])([ attention_Forward, # all_context_encoded_Forward]) all_context_encoded_Bidir_sum ]) print("weighted_sum: ", weighted_sum_Forward._keras_shape) print("weighted_sum: (history)", weighted_sum_Forward._keras_history, '\n') # (Output shape: ? x NUM_OPTIONS(100) x RNN_DIM) all_utterances_kb_encoded = Add()( [weighted_sum_Forward, all_utterances_kb_encoded]) # 2nd Attention & Weighted Sum # between Utterances_B(NUM_OPTIONS x RNN_DIM) and Contexts_encoded_Backward(CONTEXT_LEN x RNN_DIM) # and apply Softmax # (Output shape: BATCH_SIZE(?) x (NUM_OPTIONS + NUM_KB) x CONTEXT_LEN) attention_Backward = Dot(axes=[2, 2])( [all_utterances_kb_encoded, all_context_encoded_Backward]) attention_Backward = amplify_layer(attention_Backward) attention_Backward = Add()([attention_Backward, utterances_mask]) attention_Backward = softmax_layer(attention_Backward) print("attention_Backward: ", attention_Backward._keras_shape) print("attention_Backward: (history)", attention_Backward._keras_history) # between Attention(NUM_OPTIONS x CONTEXT_LEN) and Contexts_A(CONTEXT_LEN x RNN_DIM) # equivalent to weighted sum of Contexts_A according to Attention # (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100) x RNN_DIM) weighted_sum_Backward = Dot(axes=[2, 1])( [attention_Backward, all_context_encoded_Backward]) print("weighted_sum_Backward: ", weighted_sum_Backward.shape) print("weighted_sum_Backward: (history)", weighted_sum_Backward._keras_history, '\n') # (Output shape: ? x NUM_OPTIONS(100) x RNN_DIM) all_utterances_kb_encoded = Add()( [weighted_sum_Backward, all_utterances_kb_encoded]) att_responses_Forward = expand_dim_layer( GetUtterancesTensor(attention_Forward)) att_responses_Backward = expand_dim_layer( GetUtterancesTensor(attention_Backward)) att_kb_Forward = expand_dim_layer(GetKbTensor(attention_Forward)) att_kb_Backward = expand_dim_layer(GetKbTensor(attention_Backward)) merge_responses = Concatenate(axis=1)( [att_responses_Forward, att_responses_Backward]) merge_kb = Concatenate(axis=1)([att_kb_Forward, att_kb_Backward]) responses_attention.append(merge_responses) kb_attention.append(merge_kb) print("repsonses_attention[i]:", merge_responses._keras_shape) print("repsonses_attention[i]: (history)", merge_responses._keras_history) print("kb_attention[i]:", merge_kb._keras_shape) print("kb_attention[i]: (history)", merge_kb._keras_history, '\n') if i < hparams.hops - 1: continue ''' temp = all_context_encoded_Forward all_context_encoded_Forward = all_context_encoded_Backward all_context_encoded_Backward = temp ''' else: print("hop ended") # split encoded utterances & kb all_utterances_encoded_B = GetUtterancesTensor( all_utterances_kb_encoded) all_kb_encoded_K = GetKbTensor(all_utterances_kb_encoded) print("all_utterances_encoded_B: ", all_utterances_encoded_B._keras_shape) print("all_utterances_encoded_B: (history)", all_utterances_encoded_B._keras_history, '\n') print("all_kb_encoded_K: ", all_utterances_encoded_B._keras_shape) print("all_kb_encoded_K: (history)", all_utterances_encoded_B._keras_history, '\n') ############# Attention to Context ############# # (Output shape: ? x MAX_CONTEXT_LEN x 1) attention_Forward_wrt_context =\ TimeDistributed(Dense_2, input_shape=(hparams.max_context_len, hparams.memn2n_rnn_dim))(all_context_encoded_Forward) attention_Forward_wrt_context = amplify_layer( attention_Forward_wrt_context) attention_Forward_wrt_context = Add()( [attention_Forward_wrt_context, context_mask]) attention_Forward_wrt_context = softmax_layer2( attention_Forward_wrt_context) print("attention_Forward_wrt_context: ", attention_Forward_wrt_context._keras_shape) print("attention_Forward_wrt_context: (history)", attention_Forward_wrt_context._keras_history) # (Output shape: ? x 1 x RNN_DIM) weighted_sum_Forward_wrt_context = Dot(axes=[1, 1])( [attention_Forward_wrt_context, all_context_encoded_Bidir_sum]) print("weighted_sum_Forward_wrt_context: ", weighted_sum_Forward_wrt_context._keras_shape) print("weighted_sum_Forward_wrt_context: (history)", weighted_sum_Forward_wrt_context._keras_history, '\n') # (Output shape: ? x MAX_CONTEXT_LEN x 1) attention_Backward_wrt_context =\ TimeDistributed(Dense_2, input_shape=(hparams.max_context_len, hparams.memn2n_rnn_dim))(all_context_encoded_Backward) attention_Backward_wrt_context = amplify_layer( attention_Backward_wrt_context) attention_Backward_wrt_context = Add()( [attention_Backward_wrt_context, context_mask]) attention_Backward_wrt_context = softmax_layer2( attention_Backward_wrt_context) print("attention_Backward_wrt_context: ", attention_Backward_wrt_context._keras_shape) print("attention_Backward_wrt_context: (history)", attention_Backward_wrt_context._keras_history) # (Output shape: ? x 1 x RNN_DIM) weighted_sum_Backward_wrt_context = Dot(axes=[1, 1])([ attention_Backward_wrt_context, all_context_encoded_Bidir_sum ]) print("weighted_sum_Backward_wrt_context: ", weighted_sum_Backward_wrt_context._keras_shape) print("weighted_sum_Backward_wrt_context: (history)", weighted_sum_Backward_wrt_context._keras_history, '\n') att_Forward_wrt_context = Reshape( (1, hparams.max_context_len))(attention_Forward_wrt_context) att_Backward_wrt_context = Reshape( (1, hparams.max_context_len))(attention_Backward_wrt_context) context_attention = Concatenate(axis=1)( [att_Forward_wrt_context, att_Backward_wrt_context]) context_encoded_AplusC = Add()([ weighted_sum_Forward_wrt_context, weighted_sum_Backward_wrt_context ]) # context_encoded_AplusC = Reshape((1,hparams.memn2n_rnn_dim))(context_encoded_AplusC) print("context_encoded_AplusC: ", context_encoded_AplusC.shape) print("context_encoded_AplusC: (history)", context_encoded_AplusC._keras_history, '\n') # (output shape: ? x 1 x NUM_KB_OPTIONS) kb_score = Dot(axes=[2, 2])( [context_encoded_AplusC, all_kb_encoded_K]) kb_score = amplify_layer(kb_score) kb_score = Add()([kb_score, kb_mask]) kb_score = softmax_layer(kb_score) print("kb_score: ", kb_score._keras_shape) print("kb_score: (history)", kb_score._keras_history) # (output shape: ? x 1 x RNN_DIM) kb_weighted_sum = Dot(axes=[2, 1])([kb_score, all_kb_encoded_K]) print("kb_weighted_sum: ", kb_weighted_sum._keras_shape) print("kb_weighted_sum: (history)", kb_weighted_sum._keras_history, '\n') ########## Normal Sum or Wighted Sum between context and external knowledge ########## ### Normal Sum ### # context_encoded_AplusCplusKB = Add()([context_encoded_AplusC, # kb_weighted_sum]) ### Weighted Sum ### context_encoded_AplusCplusKB = Concatenate(axis=1)( [context_encoded_AplusC, kb_weighted_sum]) context_encoded_AplusCplusKB = Permute( (2, 1), input_shape=( 2, hparams.memn2n_rnn_dim))(context_encoded_AplusCplusKB) print("context_encoded_AplusCplusKB: ", context_encoded_AplusCplusKB.shape) print("context_encoded_AplusCplusKB: (history)", context_encoded_AplusCplusKB._keras_history, '\n') context_encoded_AplusCplusKB = TimeDistributed( Dense_3, input_shape=(hparams.memn2n_rnn_dim, 2))(context_encoded_AplusCplusKB) context_encoded_AplusCplusKB = Permute( (2, 1), input_shape=(hparams.memn2n_rnn_dim, 1))(context_encoded_AplusCplusKB) print("context_encoded_AplusCplusKB: ", context_encoded_AplusCplusKB.shape) print("context_encoded_AplusCplusKB: (history)", context_encoded_AplusCplusKB._keras_history, '\n') # (Output shape: ? x 1 x NUM_OPTIONS(100)) logits = Dot(axes=[2, 2])( [context_encoded_AplusCplusKB, all_utterances_encoded_B]) logits = Reshape((hparams.num_utterance_options, ))(logits) print("logits: ", logits.shape) print("logits: (history)", logits._keras_history, '\n') # Softmax layer for probability of each of Dot products in previous layer # Softmaxing logits (Output shape: BATCH_SIZE(?) x NUM_OPTIONS(100)) probs = Activation('softmax', name='probs')(logits) print("probs: ", probs.shape) print("final History: ", probs._keras_history, '\n') # Return probabilities(likelihoods) of each of utterances # Those will be used to calculate the loss ('sparse_categorical_crossentropy') if hparams.hops == 1: responses_attention = expand_dim_layer(responses_attention[0]) kb_attention = expand_dim_layer(kb_attention[0]) else: responses_attention = Stack(responses_attention) kb_attention = Stack(kb_attention) context_attention = context_attention_layer(context_attention) responses_attention = responses_attention_layer(responses_attention) kb_attention = kb_attention_layer(kb_attention) print("context_attention:", context_attention._keras_shape) print("repsonses_attention:", responses_attention._keras_shape) print("kb_attention:", kb_attention._keras_shape) return probs, context_attention, responses_attention, kb_attention
def getModel(training): inputShape = (128, 32, 1) kernelVals = [5, 5, 3, 3, 3] convFilters = [32, 64, 128, 128, 256] strideVals = [(2, 2), (2, 2), (1, 2), (1, 2), (1, 2)] rnnUnits = 256 maxStringLen = 32 inputs = Input(name='inputX', shape=inputShape, dtype='float32') inner = inputs for i in range(len(kernelVals)): inner = Conv2D(convFilters[i],(kernelVals[i],kernelVals[i]),padding = 'same',\ name = 'conv'+str(i), kernel_initializer = 'he_normal')(inner) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=strideVals[i], name='max' + str(i + 1))(inner) inner = Reshape(target_shape=(maxStringLen, rnnUnits), name='reshape')(inner) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM1F')(inner) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM1B')(inner) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS1 = average([LSF, LSB]) LS1 = BatchNormalization()(LS1) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM2F')(LS1) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM2B')(LS1) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS2 = concatenate([LSF, LSB]) LS2 = BatchNormalization()(LS2) yPred = Dense(len(unicodes) + 1, kernel_initializer='he_normal', name='dense2')(LS2) yPred = Activation('softmax', name='softmax')(yPred) #Model(inputs = inputs,outputs = yPred).summary() labels = Input(name='label', shape=[32], dtype='float32') inputLength = Input(name='inputLen', shape=[1], dtype='int64') # (None, 1) labelLength = Input(name='labelLen', shape=[1], dtype='int64') lossOut = Lambda(ctcLambdaFunc, output_shape=(1, ), name='ctc')([yPred, labels, inputLength, labelLength]) if training: return Model(inputs=[inputs, labels, inputLength, labelLength], outputs=[lossOut, yPred]) return Model(inputs=[inputs], outputs=yPred)
def build_train(self): # Input Lauer # T: 50, L: 10, h:64, w:64, c:3 X_input = keras.layers.Input(shape=self.x_shape, dtype='float32', name='x_input') E_input = keras.layers.Input(shape=self.e_shape, dtype='float32', name='e_input') R_input = keras.layers.Input(shape=self.r_shape, dtype='float32', name='r_input') for t in range(self.T): """ === E unit === """ if t == 0: E = keras.layers.Lambda(lambda x: x)(E_input) def E_to_Et(E): # E_t: (None, L, h, w, 2c) E_t = K.permute_dimensions(E, [1, 0, 2, 3, 4, 5]) E_t = K.gather(E_t, [t]) E_t = K.permute_dimensions(E_t, [1, 0, 2, 3, 4, 5]) E_t = K.squeeze(E_t, axis=1) return E_t E_t = keras.layers.Lambda(E_to_Et)(E) """ === R unit === """ if t == 0: def R_to_Rt(R_input): R_t = K.permute_dimensions(R_input, [1, 0, 2, 3, 4, 5]) R_t = K.gather(R_t, [t]) R_t = K.permute_dimensions(R_t, [1, 0, 2, 3, 4, 5]) R_t = K.squeeze(R_t, axis=1) return R_t R_t = keras.layers.Lambda(R_to_Rt)(R_input) else: # R_t: (None, L, h, w, 3) E_t_rev = keras.layers.Lambda(lambda x: K.reverse(x, axes=1))( E_t) R_t, state_h_t, state_c_t = keras.layers.ConvLSTM2D( 3, (3, 3), padding='same', activation='tanh', return_sequences=True, return_state=True)(E_t_rev) for l in range(self.L): """ === R_tl === """ def Rt_to_Rtl(R_t): R_tl = K.permute_dimensions(R_t, [1, 0, 2, 3, 4]) R_tl = K.gather(R_tl, [l]) R_tl = K.permute_dimensions(R_tl, [1, 0, 2, 3, 4]) R_tl = K.squeeze(R_tl, axis=1) return R_tl # R_tl: (None, h, w, 3) R_tl = keras.layers.Lambda(Rt_to_Rtl)(R_t) print("t:", t, "l:", l, "R_tl", R_tl) """ === Ahat_tl === """ # Ahat_tl: (None, h, w, 3) Ahat_tl = keras.layers.Conv2D(3, (3, 3), padding='same')(R_tl) Ahat_tl = keras.layers.Activation('relu')(Ahat_tl) print("t", t, "l,", l, "Ahat_tl", Ahat_tl) if l == 0: def X_to_Atl(x_input): # (None, T, 1, h, w, c) --> (?, h, w, c) A_tl = K.squeeze(x_input, axis=2) # (None, T, 64, 64, 3) A_tl = K.permute_dimensions( A_tl, [1, 0, 2, 3, 4]) # (T, None, 64, 64, 3) A_tl = K.gather(A_tl, [t]) # (1, None, 64, 64, 3) A_tl = K.squeeze(A_tl, axis=0) # (None, 64, 64, 3) return A_tl # A_tl: (None, h, w, 3) A_tl = keras.layers.Lambda(X_to_Atl)(X_input) # E_tl: (None, h, w, c) err0_tl = keras.layers.Subtract()([A_tl, Ahat_tl]) err0_tl = keras.layers.Activation('relu')(err0_tl) err1_tl = keras.layers.Subtract()([Ahat_tl, A_tl]) err1_tl = keras.layers.Activation('relu')(err1_tl) E_tl = keras.layers.Concatenate(axis=-1)([err0_tl, err1_tl]) # A_tl: (None, h, w, 3) if l < self.L - 1: def Et_to_Etl(E_t): E_tl = K.permute_dimensions(E_t, [1, 0, 2, 3, 4]) E_tl = K.gather(E_tl, [l]) E_tl = K.permute_dimensions(E_tl, [1, 0, 2, 3, 4]) E_tl = K.squeeze(E_tl, axis=1) return E_tl # E_tl: (None, h, w, c) E_tl = keras.layers.Lambda(Et_to_Etl)(E_t) # A_tl A_tl = keras.layers.Conv2D(3, (3, 3), padding='same')(E_tl) if l == 0: # E_l: (None, 1, h, w, c) E_t = keras.layers.Lambda( lambda x: K.expand_dims(x, axis=1))(E_tl) else: # E_tl_: (None, 1, h, w, c) E_tl_ = keras.layers.Lambda( lambda x: K.expand_dims(x, axis=1))(E_tl) # E_l: (None, T, h, w, c) E_t = keras.layers.Concatenate(axis=1)([E_t, E_tl_]) if t == 0: # E: (None, 1, T, h, w, c) E = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=1))( E_t) else: # E_l_: (None, 1, T, h, w, c) E_t_ = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=1))( E_t) # E: (None, L, T, h, w, c) E = keras.layers.Concatenate(axis=1)([E, E_t_]) print("t:", t, "l:", l, "E:", E) # print(E) # 50, 10, 64, 64, 6 model_train = keras.models.Model(inputs=[X_input, E_input, R_input], outputs=[E]) return model_train
def build_model_morph_rnn(num_classes, embed_matrix, word_length, num_chars): words_indices_input = Input(shape=(None, ), name='words_indices_input') word_embeddings_out = Embedding( embed_matrix.shape[0], embed_matrix.shape[1], weights=[embed_matrix], trainable=False, name='word_embeddings')(words_indices_input) casings_input = Input(shape=(None, 6), name='casings_input') chars_input = Input(shape=(None, word_length), name='chars_input') char_embeddings_out = \ TimeDistributed(Embedding(num_chars + 1, 32, embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)), name='char_embeddings')(chars_input) char_dropout_out = Dropout(0.5, name='char_dropout')(char_embeddings_out) char_conv1d_out = TimeDistributed( Conv1D(kernel_size=3, filters=32, padding='same', activation='tanh', strides=1, name='char_conv1d'))(char_dropout_out) char_maxpool_out = TimeDistributed(MaxPooling1D(word_length), name='char_maxpool')(char_conv1d_out) char_flat_out = TimeDistributed(Flatten(), name='char_flat')(char_maxpool_out) char_flat_dropout_out = Dropout(0.5, name='char_flat_dropout')(char_flat_out) concatenated_out = concatenate( [word_embeddings_out, casings_input, char_flat_dropout_out], name='concat') forward_lstm_out = LSTM(128, dropout=0.5, recurrent_dropout=0.25, return_sequences=True)(concatenated_out) backward_lstm_out = LSTM(128, dropout=0.5, recurrent_dropout=0.25, return_sequences=True, go_backwards=True)(concatenated_out) reverse = Lambda(lambda x: K.reverse(x, axes=1)) bilstm_out = concatenate([forward_lstm_out, reverse(backward_lstm_out)]) # Предсказываем текущую метку по выходам bilstm bilstm_2_out = Bidirectional( LSTM(128, dropout=0.5, recurrent_dropout=0.25, return_sequences=True))(bilstm_out) fc_out = TimeDistributed(Dense(128, activation='relu'), name='fc_1')(bilstm_2_out) bn_out = TimeDistributed(BatchNormalization(axis=1))(fc_out) fc_2_out = TimeDistributed(Dense(num_classes, activation='softmax'), name='fc_2')(bn_out) # forward lstm предсказывает следующую метку справа fc_fw_out = TimeDistributed(Dense(128, activation='relu'), name='fc_fw_1')(forward_lstm_out) fc_fw_2_out = TimeDistributed( Dense(num_classes, activation='softmax', name='fc_fw_2'))(fc_fw_out) # backward lstm предсказывает следующую метку слева fc_bw_out = TimeDistributed(Dense(128, activation='relu'), name='fc_bw_1')(backward_lstm_out) fc_bw_2_out = TimeDistributed( Dense(num_classes, activation='softmax', name='fc_bw_2'))(fc_bw_out) model = Model([words_indices_input, casings_input, chars_input], [fc_2_out, fc_bw_2_out, fc_fw_2_out]) model.compile(optimizer='nadam', loss='sparse_categorical_crossentropy') model.name = 'morph_rnn' return model
embedder = Embedding(MAX_TOKENS + 1, embedding_dim, weights=[embeddings], trainable=False) doc_embedding = embedder(document) l_embedding = embedder(left_context) r_embedding = embedder(right_context) # I use LSTM RNNs instead of vanilla RNNs as described in the paper. forward = LSTM(hidden_dim_1, return_sequences=True)(l_embedding) # See equation (1). backward = LSTM(hidden_dim_1, return_sequences=True, go_backwards=True)(r_embedding) # See equation (2). # Keras returns the output sequences in reverse order. backward = Lambda(lambda x: backend.reverse(x, axes=1))(backward) together = concatenate([forward, doc_embedding, backward], axis=2) # See equation (3). semantic = TimeDistributed(Dense(hidden_dim_2, activation="tanh"))( together) # See equation (4). # Keras provides its own max-pooling layers, but they cannot handle variable length input # (as far as I can tell). As a result, I define my own max-pooling layer here. pool_rnn = Lambda(lambda x: backend.max(x, axis=1), output_shape=(hidden_dim_2, ))(semantic) # See equation (5). output = Dense(NUM_CLASSES, input_dim=hidden_dim_2, activation="softmax")(pool_rnn) # See equations (6) and (7). model = Model(inputs=[document, left_context, right_context], outputs=output)
def create_model_cls(self, hyper_parameters): """ 构建神经网络, col, 论文中maxpooling使用的是列池化, 不过实验效果似乎不佳,而且训练速度超级慢 :param hyper_parameters:json, hyper parameters of network :return: tensor, moedl """ super().create_model(hyper_parameters) embedding_output = self.word_embedding.output # rnn layers if self.rnn_units == "LSTM": layer_cell = LSTM else: layer_cell = GRU # 反向 x_backwords = layer_cell(units=self.rnn_units, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=True)(embedding_output) x_backwords_reverse = Lambda(lambda x: K.reverse(x, axes=1))( x_backwords) # 前向 x_fordwords = layer_cell(units=self.rnn_units, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=False)(embedding_output) # 拼接 x_feb = Concatenate(axis=2)( [x_fordwords, embedding_output, x_backwords_reverse]) ####列池化################################################## x_feb = Dropout(self.dropout)(x_feb) dim_2 = K.int_shape(x_feb)[2] x_feb_reshape = Reshape((dim_2, self.len_max))(x_feb) conv_pools = [] for filter in self.filters: conv = Conv1D( filters=self.filters_num, # filter=300 kernel_size=filter, padding='valid', kernel_initializer='normal', activation='relu', )(x_feb_reshape) pooled = MaxPooling1D( padding='valid', pool_size=32, )(conv) conv_pools.append(pooled) x = Concatenate(axis=1)(conv_pools) # x = MaxPooling1D(padding = 'VALID',)(x_feb_reshape) x = Flatten()(x) x = Dropout(self.dropout)(x) ######################################################################### output = Dense(units=self.label, activation=self.activate_classify)(x) self.model = Model(inputs=self.word_embedding.input, outputs=output) self.model.summary(120)
embedder = Embedding(vocab_length + 3, emb_dim) embedded_inputs = embedder(inputs) h = Bidirectional(SimpleRNN(20), merge_mode='concat')(embedded_inputs) z_mean = Dense(latent_dim)(h) z_log_var = Dense(latent_dim)(h) z = Lambda(sample_z, output_shape=(latent_dim, ))([z_mean, z_log_var]) encoder = Model(inputs, [z_mean, z_log_var, z]) forward_in = Input(shape=(None, )) f_emb = embedder(forward_in) f_rnn = SimpleRNN(latent_dim, return_sequences=True, return_state=True, go_backwards=True) h, _ = f_rnn(f_emb, initial_state=z) h = Lambda(lambda x: K.reverse(x, axes=1))(h) f_bn = BatchNormalization() h = f_bn(h) soft_dense = Dense(vocab_length + 1, activation='softmax') decoded = TimeDistributed(soft_dense)(h) #decoded = soft_dense(h) vae = Model([inputs, forward_in], decoded, name='forward_vae') vae.compile(optimizer='adam', loss=vae_loss, metrics=['acc']) test_gen = backward_autoencoder_shaper(test_harness1)( vocab_length=vocab_length) vae.fit_generator(test_gen, steps_per_epoch=1000, epochs=5) p = next(test_gen)
use_bias=0, name='conv2d', trainable=True) tensor = conv2d(input0) # 设定卷积网络的初值 init_cnn_weight = [] init_cnn_kernel = np.array([[[[-1]], [[1]], [[0]], [[0]]]]) #一阶差分初始权重 init_cnn_weight.append(init_cnn_kernel) conv2d.set_weights(init_cnn_weight) tensor = Lambda(lambda x: K.squeeze(x, axis=-1), name='squeeze_0')( tensor) # 4D 数据压缩为 3D 数据,因为channels=1 tensor = Lambda(lambda x: tf.nn.top_k(x, x.shape[2])[0], name='sequential_pooling_drop')( tensor) # 在第2个维度上对数据降序排列,即各滑动窗内数据 tensor = Lambda(lambda x: K.reverse(x, axes=2), name='sequential_pooling_asend')(tensor) # 升序排列 tensor = LSTM(units=units, name='lstm')(tensor) # 多对一LSTM tensor = Dense(20, activation='tanh', name='FC0')(tensor) prediction = Dense(1, activation='sigmoid', name='output', use_bias='False')(tensor) model = Model(inputs=input0, outputs=prediction) # 中间输出 # layer_name = 'sequential_pooling_asend' # intermediate_layer_model = Model(input=model.input, # output=model.get_layer(layer_name).output) # intermediate_output = intermediate_layer_model.predict(X_train) # intermediate_output = np.squeeze(intermediate_output) # 训练模型
def recursion(self, input_energy, mask=None, go_backwards=False, return_sequences=True, return_logZ=True, input_length=None): """Forward (alpha) or backward (beta) recursion If `return_logZ = True`, compute the logZ, the normalization constant: \[ Z = \sum_{y1, y2, y3} exp(-E) # energy = \sum_{y1, y2, y3} exp(-(u1' y1 + y1' W y2 + u2' y2 + y2' W y3 + u3' y3)) = sum_{y2, y3} (exp(-(u2' y2 + y2' W y3 + u3' y3)) sum_{y1} exp(-(u1' y1' + y1' W y2))) \] Denote: \[ S(y2) := sum_{y1} exp(-(u1' y1 + y1' W y2)), \] \[ Z = sum_{y2, y3} exp(log S(y2) - (u2' y2 + y2' W y3 + u3' y3)) \] \[ logS(y2) = log S(y2) = log_sum_exp(-(u1' y1' + y1' W y2)) \] Note that: yi's are one-hot vectors u1, u3: boundary energies have been merged If `return_logZ = False`, compute the Viterbi's best path lookup table. """ chain_energy = self.chain_kernel # shape=(1, F, F): F=num of output features. 1st F is for t-1, 2nd F for t chain_energy = K.expand_dims(chain_energy, 0) # shape=(B, F), dtype=float32 prev_target_val = K.zeros_like(input_energy[:, 0, :]) if go_backwards: input_energy = K.reverse(input_energy, 1) if mask is not None: mask = K.reverse(mask, 1) initial_states = [ prev_target_val, K.zeros_like(prev_target_val[:, :1]) ] constants = [chain_energy] if mask is not None: mask = K.cast(mask, K.floatx()) mask2 = K.cast( K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1), K.floatx()) constants.append(mask2) def _step(input_energy_i, states): return self.step(input_energy_i, states, return_logZ) target_val_last, target_val_seq, _ = K.rnn(_step, input_energy, initial_states, constants=constants, input_length=input_length, unroll=self.unroll) if return_sequences: if go_backwards: target_val_seq = K.reverse(target_val_seq, 1) return target_val_seq else: return target_val_last
def reverse(inputs, axes=1): return K.reverse(inputs, axes=axes)
def labels_to_image_model(im_shape, n_channels, crop_shape, label_list, n_neutral_labels, vox2ras, nonlin_shape_factor=0.0625, crop_channel2=None, output_div_by_n=None, flipping=True): # get shapes n_dims, _ = utils.get_dims(im_shape) crop_shape = get_shapes(crop_shape, im_shape, output_div_by_n) deformation_field_size = utils.get_resample_shape(im_shape, nonlin_shape_factor, len(im_shape)) # create new_label_list and corresponding LUT to make sure that labels go from 0 to N-1 new_label_list, lut = utils.rearrange_label_list(label_list) # define mandatory inputs image_input = KL.Input(shape=im_shape + [n_channels], name='image_input') labels_input = KL.Input(shape=im_shape + [1], name='labels_input') aff_in = KL.Input(shape=(n_dims + 1, n_dims + 1), name='aff_input') nonlin_field_in = KL.Input(shape=deformation_field_size, name='nonlin_input') list_inputs = [image_input, labels_input, aff_in, nonlin_field_in] # convert labels to new_label_list labels = KL.Lambda(lambda x: tf.gather( tf.convert_to_tensor(lut, dtype='int32'), tf.cast(x, dtype='int32')))( labels_input) # deform labels image_input._keras_shape = tuple(image_input.get_shape().as_list()) labels._keras_shape = tuple(labels.get_shape().as_list()) labels = KL.Lambda(lambda x: tf.cast(x, dtype='float'))(labels) resize_shape = [ max(int(im_shape[i] / 2), deformation_field_size[i]) for i in range(len(im_shape)) ] nonlin_field = nrn_layers.Resize(size=resize_shape, interp_method='linear')(nonlin_field_in) nonlin_field = nrn_layers.VecInt()(nonlin_field) nonlin_field = nrn_layers.Resize(size=im_shape, interp_method='linear')(nonlin_field) image = nrn_layers.SpatialTransformer(interp_method='linear')( [image_input, aff_in, nonlin_field]) labels = nrn_layers.SpatialTransformer(interp_method='nearest')( [labels, aff_in, nonlin_field]) labels = KL.Lambda(lambda x: tf.cast(x, dtype='int32'))(labels) # cropping if crop_shape is not None: image, crop_idx = l2i_sa.random_cropping(image, crop_shape, n_dims) labels = KL.Lambda( lambda x: tf.slice(x[0], begin=tf.cast(x[1], dtype='int32'), size=tf.convert_to_tensor( [-1] + crop_shape + [-1], dtype='int32')))( [labels, crop_idx]) else: crop_shape = im_shape # flipping if flipping: labels, flip = l2i_sa.label_map_random_flipping( labels, label_list, n_neutral_labels, vox2ras, n_dims) ras_axes = edit_volumes.get_ras_axes(vox2ras, n_dims) flip_axis = [ras_axes[0] + 1] image = KL.Lambda(lambda y: K.switch( y[0], KL.Lambda(lambda x: K.reverse(x, axes=flip_axis))(y[1]), y[1]))( [flip, image]) # convert labels back to original values labels = KL.Lambda( lambda x: tf.gather(tf.convert_to_tensor(label_list, dtype='int32'), tf.cast(x, dtype='int32')), name='labels_out')(labels) # intensity augmentation image = KL.Lambda(lambda x: K.clip(x, 0, 300), name='clipping')(image) # loop over channels if n_channels > 1: split = KL.Lambda(lambda x: tf.split(x, [1] * n_channels, axis=-1))( image) else: split = [image] processed_channels = list() for i, channel in enumerate(split): # normalise and shift intensities image = l2i_ia.min_max_normalisation(image) image = KL.Lambda(lambda x: K.random_uniform( (1, ), .85, 1.1) * x + K.random_uniform((1, ), -.3, .3))(image) image = KL.Lambda(lambda x: K.clip(x, 0, 1))(image) image = l2i_ia.gamma_augmentation(image) # randomly crop sides of second channel if (crop_channel2 is not None) & (channel == 1): image = l2i_sa.restrict_tensor(image, crop_channel2, n_dims) # concatenate all channels back, and clip output (include labels to keep it when plugging to other models) if n_channels > 1: image = KL.concatenate(processed_channels) else: image = processed_channels[0] image = KL.Lambda(lambda x: K.clip(x[0], 0, 1), name='image_out')([image, labels]) # build model brain_model = Model(inputs=list_inputs, outputs=[image, labels]) # shape of returned images output_shape = image.get_shape().as_list()[1:] return brain_model, output_shape
def call(self, x, mask=None): # 'RGB'->'BGR' x = K.reverse(x, axes=-1) # Zero-center by mean pixel x = x - self.MEAN_VALUE_TENSOR return x