def func(input): # Get original number of filters channel_axis = 1 if K.image_data_format() == "channels_first" else -1 orig_filters = K.int_shape(input)[channel_axis] if num_final_layers is None: final_filters = orig_filters else: final_filters = num_final_layers # Perform spatial dimension reduction squeeze = GlobalAveragePooling2D()(input) squeeze = Reshape((1, 1, orig_filters))(squeeze) # Pass through Dense, ReLU, Dense, Sigmoid sequence (for attention vector) excite = Dense(int(final_filters // reduction_ratio), activation='relu', kernel_initializer='he_normal', use_bias=False)(squeeze) excite = Dense(final_filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(excite) # Multiply each spatial channel by the attention value if only_excite_vec: return excite else: return multiply([input, excite])
def square_diff(sequence_1_input, sequence_2_input): Max_num = 200000 embedding_layer = Embedding( Max_num, embedding_matrix.shape[1], weights=[embedding_matrix], #input_length=MAX_SEQUENCE_LENGTH, trainable=False) #lstm_layer = LSTM(75, recurrent_dropout=0.2) lstm_layer = Bidirectional(CuDNNLSTM(40)) embedded_sequences_1 = embedding_layer(sequence_1_input) x1 = lstm_layer(embedded_sequences_1) embedded_sequences_2 = embedding_layer(sequence_2_input) y1 = lstm_layer(embedded_sequences_2) addition = add([x1, y1]) minus_y1 = Lambda(lambda x: -x)(y1) merged = add([x1, minus_y1]) merged = multiply([merged, merged]) merged = concatenate([merged, addition]) merged = Dropout(0.4)(merged) return merged
def create_model(self): user_input = Input(shape=(1, self.read_hist, self.words, 300)) article_input = Input(shape=(1, self.words, 300)) conv3d = Conv3D(64, kernel_size=(3, 3, 3), activation='relu')(user_input) maxpooling3d = MaxPooling3D(pool_size=(2, 2, 2))(conv3d) conv3d2 = Conv3D(32, kernel_size=(3, 3, 3), activation='relu')(maxpooling3d) maxpooling3d2 = MaxPooling3D(pool_size=(1, 2, 2))(conv3d2) flatten3d = Flatten()(maxpooling3d2) dense3d = Dense(128, activation='relu')(flatten3d) #dropout3d = Dropout(0.3)(dense3d) conv2d = Conv2D(64, kernel_size=(3, 3), activation='relu')(article_input) maxpooling2d = MaxPooling2D(pool_size=(2, 2))(conv2d) conv2d2 = Conv2D(32, kernel_size=(3, 3), activation='relu')(maxpooling2d) maxpooling2d2 = MaxPooling2D(pool_size=(2, 2))(conv2d2) flatten2d = Flatten()(maxpooling2d2) dense2d = Dense(128, activation='relu')(flatten2d) #dropout2d = Dropout(0.5)(dense2d) output1 = multiply([dense3d, dense2d]) output2 = Dense(1, activation='sigmoid')(output1) self.model = Model(inputs=[user_input, article_input], outputs=output2) self.model.compile(optimizer='adadelta', loss='binary_crossentropy', metrics=['accuracy'])
def build_model_resnet50_with_mask(): img_input = Input(INPUT_SHAPE, name='data') mask_model = model_unet(INPUT_SHAPE) mask_model.load_weights( '../output/checkpoints/fish_mask_unet/model_fish_unet2/checkpoint-best-064-0.0476.hdf5' ) mask = mask_model(img_input) mask3 = concatenate([mask, mask, mask], axis=3) masked_image = multiply([img_input, mask3]) base_model = ResNet50(input_shape=INPUT_SHAPE, include_top=False, pooling='avg') base_model_output = base_model(masked_image) species_dense = Dense(len(SPECIES_CLASSES), activation='softmax', name='cat_species')(base_model_output) cover_dense = Dense(len(COVER_CLASSES), activation='softmax', name='cat_cover')(base_model_output) model = Model(inputs=img_input, outputs=[species_dense, cover_dense]) sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) return model
def create_model(self): user_input = Input(shape=(1, self.read_hist, self.nwords, self.embed_size)) article_input = Input(shape=(1, self.nwords, self.embed_size)) conv3d = Conv3D(64, kernel_size=(3, 3, 3), activation='relu')(user_input) maxpooling3d = MaxPooling3D(pool_size=(2, 2, 2))(conv3d) conv3d2 = Conv3D(32, kernel_size=(3, 3, 3), activation='relu')(maxpooling3d) maxpooling3d2 = MaxPooling3D(pool_size=(2, 2, 2))(conv3d2) flatten3d = Flatten()(maxpooling3d2) conv2d = Conv2D(64, kernel_size=(3, 3), activation='relu')(article_input) maxpooling2d = MaxPooling2D(pool_size=(2, 2))(conv2d) conv2d2 = Conv2D(32, kernel_size=(3, 3), activation='relu')(maxpooling2d) maxpooling2d2 = MaxPooling2D(pool_size=(2, 2))(conv2d2) flatten2d = Flatten()(maxpooling2d2) element_wise_product = multiply([flatten3d, flatten2d]) fc = Dense(128, activation='relu')(element_wise_product) output = Dense(1, activation='sigmoid')(fc) self.model = Model(inputs=[user_input, article_input], outputs=output) self.model.compile(optimizer='adadelta', loss='binary_crossentropy', metrics=['accuracy'])
def AttnGatingBlock(x, g, inter_shape): shape_x = K.int_shape(x) shape_g = K.int_shape(g) # Getting the gating signal to the same number of filters as the inter_shape phi_g = Conv3D(filters=inter_shape, kernel_size=1, strides=1, padding='same')(g) # Getting the x signal to the same shape as the gating signal theta_x = Conv3D(filters=inter_shape, kernel_size=3, strides=(shape_x[1] // shape_g[1], shape_x[2] // shape_g[2], shape_x[3] // shape_g[3]), padding='same')(x) # Element-wise addition of the gating and x signals add_xg = add([phi_g, theta_x]) add_xg = Activation('relu')(add_xg) # 1x1x1 convolution psi = Conv3D(filters=1, kernel_size=1, padding='same')(add_xg) psi = Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(psi) # Upsampling psi back to the original dimensions of x signal upsample_sigmoid_xg = UpSampling3D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2], shape_x[3] // shape_sigmoid[3]))(psi) # Expanding the filter axis to the number of filters in the original x signal upsample_sigmoid_xg = expend_as(upsample_sigmoid_xg, shape_x[4]) # Element-wise multiplication of attention coefficients back onto original x signal attn_coefficients = multiply([upsample_sigmoid_xg, x]) # Final 1x1x1 convolution to consolidate attention signal to original x dimensions output = Conv3D(filters=shape_x[4], kernel_size=1, strides=1, padding='same')(attn_coefficients) output = BatchNormalization()(output) return output
def build_generator(self): """ 这是构建生成器网络的函数 :return:返回生成器模型generotor_model """ model = Sequential() model.add(Dense(256, input_dim=self.config.generator_noise_input_dim)) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(BatchNormalization(momentum=self.config.batchnormalization_momentum)) model.add(Dense(512)) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(BatchNormalization(momentum=self.config.batchnormalization_momentum)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(BatchNormalization(momentum=self.config.batchnormalization_momentum)) model.add(Dense(np.prod(self.config.discriminator_image_input_dim), activation='tanh')) model.add(Reshape(self.config.discriminator_image_input_dim)) model.summary() noise = Input(shape=(self.config.generator_noise_input_dim,)) label = Input(shape=(1,), dtype='int32') label_embedding = Flatten()(Embedding(self.config.condational_label_num, self.config.generator_noise_input_dim)(label)) model_input = multiply([noise, label_embedding]) img = model(model_input) return Model([noise, label], img)
def build_model(): seq_input1 = Input(shape=(seq_size, dim), name='seq1') seq_input2 = Input(shape=(seq_size, dim), name='seq2') l1 = Conv1D(hidden_dim, 3) l2 = Conv1D(hidden_dim, 3) l3 = Conv1D(hidden_dim, 3) l4 = Conv1D(hidden_dim, 3) l5 = Conv1D(hidden_dim, 3) l6 = Conv1D(hidden_dim, 3) s1 = GlobalAveragePooling1D()(l6( MaxPooling1D(2)(l5( MaxPooling1D(2)(l4( MaxPooling1D(2)(l3( MaxPooling1D(2)(l2(MaxPooling1D(2)( l1(seq_input1)))))))))))) s2 = GlobalAveragePooling1D()(l6( MaxPooling1D(2)(l5( MaxPooling1D(2)(l4( MaxPooling1D(2)(l3( MaxPooling1D(2)(l2(MaxPooling1D(2)( l1(seq_input2)))))))))))) merge_text = multiply([s1, s2]) x = Dense(hidden_dim, activation='linear')(merge_text) x = keras.layers.LeakyReLU(alpha=0.3)(x) x = Dense(int((hidden_dim + 7) / 2), activation='linear')(x) x = keras.layers.LeakyReLU(alpha=0.3)(x) main_output = Dense(2, activation='softmax')(x) merge_model = Model(inputs=[seq_input1, seq_input2], outputs=[main_output]) return merge_model
def shared_model(_input): len_embeddings = 11715 embedded = Embedding(len_embeddings, embedding_dim, weights=[embeddings], input_shape=(max_seq_length, ), trainable=False)(_input) # Bi-LSTM activations = Bidirectional(LSTM(n_hidden, return_sequences=True), merge_mode='concat')(embedded) activations = Bidirectional(LSTM(n_hidden, return_sequences=True), merge_mode='concat')(activations) # dropout activations = Dropout(0.5)(activations) # Attention attention = TimeDistributed(Dense(1, activation='tanh'))(activations) attention = Flatten()(attention) attention = Activation('softmax')(attention) attention = RepeatVector(n_hidden * 2)(attention) attention = Permute([2, 1])(attention) sent_representation = multiply([activations, attention]) sent_representation = Lambda(lambda x_lambda: K.sum(x_lambda, axis=1))( sent_representation) # DropOut sent_representation = Dropout(0.1)(sent_representation) return sent_representation
def weighted_categorical_crossentropy(X): y_pred, weights, y_true = X loss = K.categorical_crossentropy(y_pred, y_true) loss = multiply([loss, weights]) #loss = K.mean(loss,axis=-1) return loss
def build_train_on_batch(self): ## first layer for the input (x_t) x = Input(batch_shape=(None, None, self.input_dim), name='x') masked = (Masking(mask_value=-1, input_shape=(None, None, self.input_dim)))(x) lstm_out = SimpleRNN(self.hidden_layer_size, input_shape=(None, None, self.input_dim), return_sequences=True)(masked) dense_out = Dense(self.input_dim_order, input_shape=(None, None, self.hidden_layer_size), activation='sigmoid')(lstm_out) y_order = Input(batch_shape=(None, None, self.input_dim_order), name='y_order') merged = multiply([dense_out, y_order]) def reduce_dim(x): x = K.max(x, axis=2, keepdims=True) return x def reduce_dim_shape(input_shape): shape = list(input_shape) shape[-1] = 1 print("reduced_shape", shape) return tuple(shape) earlyStopping = EarlyStopping(monitor='val_loss', patience=2, verbose=0, mode='auto') reduced = Lambda(reduce_dim, output_shape=reduce_dim_shape)(merged) self.model = Model(inputs=[x, y_order], outputs=reduced) self.model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
def mlp_ptscorer(inputs, Ddim, N, l2reg, pfx='out', oact='sigmoid', extra_inp=[]): """ Element-wise features from the pair fed to an MLP. """ sum_vec = add(inputs) mul_vec = multiply(inputs) mlp_input = concatenate([sum_vec, mul_vec]) # Ddim may be either 0 (no hidden layer), scalar (single hidden layer) or # list (multiple hidden layers) if Ddim == 0: Ddim = [] elif not isinstance(Ddim, list): Ddim = [Ddim] if Ddim: for i, D in enumerate(Ddim): shared_dense = Dense(int(N * D), kernel_regularizer=l2(l2reg), activation='linear', name=pfx + 'hdn%d' % (i)) mlp_input = Activation('tanh')(shared_dense(mlp_input)) shared_dense = Dense(1, kernel_regularizer=l2(l2reg), activation=oact, name=pfx + 'mlp') mlp_out = shared_dense(mlp_input) return mlp_out
def weighted_categorical_crossentropy(X): import keras.backend as K import keras.layers.merge as merge y_pred, weights, y_true = X loss = K.categorical_crossentropy(y_pred, y_true) loss = multiply([loss, weights]) return loss
def AttnGatingBlock(x, g, inter_shape): shape_x = K.int_shape(x) shape_g = K.int_shape(g) theta_x = Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same')(x) shape_theta_x = K.int_shape(theta_x) phi_g = Conv2D(inter_shape, (1, 1), padding='same')(g) upsample_g = Conv2DTranspose(inter_shape, (3, 3), strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), padding='same')(phi_g) concat_xg = add([upsample_g, theta_x]) act_xg = Activation('relu')(concat_xg) psi = Conv2D(1, (1, 1), padding='same')(act_xg) sigmoid_xg = Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(sigmoid_xg) upsample_psi = UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]))(sigmoid_xg) upsample_psi = expend_as(upsample_psi, shape_x[3]) y = multiply([upsample_psi, x]) result = Conv2D(shape_x[3], (1, 1), padding='same')(y) result_bn = BatchNormalization()(result) return result_bn
def build_discriminator_model(self): """ 这是搭建生成器模型的函数 :return: """ model = Sequential() model.add(Dense(512, input_dim=np.prod(self.config.discriminator_image_input_dim))) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(Dense(512)) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(Dropout(self.config.LeakyReLU_alpha)) model.add(Dense(512)) model.add(LeakyReLU(alpha=self.config.LeakyReLU_alpha)) model.add(Dropout(self.config.LeakyReLU_alpha)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.config.discriminator_image_input_dim) label = Input(shape=(1,), dtype='int32') label_embedding = Flatten()(Embedding(self.config.condational_label_num, np.prod(self.config.discriminator_image_input_dim))(label)) flat_img = Flatten()(img) model_input = multiply([flat_img, label_embedding]) validity = model(model_input) return Model([img, label], validity)
def shared_lstm(_input): # Embedded version of the inputs embedded = Embedding(len(embeddings), embedding_dim, weights=[embeddings], input_shape=(max_seq_length, ), trainable=False)(_input) # Multilayer Bi-LSTM activations = Bidirectional(LSTM(n_hidden, return_sequences=True), merge_mode='concat')(embedded) activations = Bidirectional(LSTM(n_hidden, return_sequences=True), merge_mode='concat')(activations) # dropout # activations = Dropout(0.5)(activations) # Attention Mechanism attention = TimeDistributed(Dense(1, activation='tanh'))(activations) attention = Flatten()(attention) attention = Activation('softmax')(attention) attention = RepeatVector(n_hidden * 2)(attention) attention = Permute([2, 1])(attention) sent_representation = multiply([activations, attention]) sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))( sent_representation) # dropout # sent_representation = Dropout(0.1)(sent_representation) return sent_representation
def dense_SE(input, size, group1_num, group2_num, ratio): input_size = K.int_shape(input)[-1] group1_input_size = input_size // group1_num group2_output_size = size // group1_num group1 = [] for i in range(group1_num): group = Lambda(lambda z: z[:, i * group1_input_size:i * group1_input_size + group1_input_size])( input) group1.append( Dense(group2_output_size, activation='relu', kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(group)) output = Concatenate()(group1) output = add_common_layer(output) output = Reshape([size // group2_num, group2_num])(output) se = GlobalAveragePooling1D()(output) se = Reshape([1, group2_num])(se) se = Dense(group2_num // ratio, activation='relu', kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay), use_bias=False)(se) se = Dense(group2_num, activation='sigmoid', kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay), use_bias=False)(se) output = multiply([output, se]) return Reshape([size])(output)
def gatExpertLayer(inputGate, inputExpert, expert_num, nb_class): # The Inputs input_vector1 = Input(shape=(inputGate.shape[1:])) input_vector2 = Input(shape=(inputExpert.shape[1:])) # The Gate gate = Dense(expert_num * nb_class, activation='softmax')(input_vector1) gate = Reshape((nb_class, expert_num))(gate) gate = Lambda(lambda x: tf.split(x, num_or_size_splits=1, axis=1))(gate) # The Expert expert = DenseMoE(nb_class * expert_num, 3, expert_activation="softmax", gating_activation='softmax')(input_vector2) #expert = Dense(nb_class*expert_num, activation='tanh')(input_vector2) expert = Reshape((nb_class, expert_num))(expert) # The Output output = multiply([gate, expert]) output = Lambda(reduce, output_shape=(nb_class, ), arguments={'axis': 2})(output) #The model model = Model(inputs=[input_vector1, input_vector2], outputs=output) model.compile(loss='mean_squared_error', metrics=['mse'], optimizer='adam') return model
def get_model_3(max_work, max_user): dim_embedddings = 30 bias = 1 # inputs w_inputs = Input(shape=(1,), dtype='int32') w = Embedding(max_work+1, dim_embedddings, name="work")(w_inputs) w_bis = Embedding(max_work + 1, bias, name="workbias")(w_inputs) num_dense = 5 num_node = 100 # context u_inputs = Input(shape=(1,), dtype='int32') u = Embedding(max_user+1, dim_embedddings, name="user")(u_inputs) u_bis = Embedding(max_user + 1, bias, name="userbias")(u_inputs) o = multiply([w, u]) o = Dropout(0.5)(o) o = concatenate([o, u_bis, w_bis]) o = Flatten()(o) o = modifiedDense(num_node, activation="relu")(o) #o = Dense(num_node, activation="relu")(o) #o = Dense(num_node, activation="relu")(o) # for i in range(num_dense-1): # o = Dense(num_node, activation="relu")(o) #o = modifiedDense(num_node, activation="relu")(o) o = Dense(1)(o) rec_model = Model(inputs=[w_inputs, u_inputs], outputs=o) #rec_model.summary() adam = optimizers.Adam(lr=0.001, decay=0.0) rec_model.compile(loss='mae', optimizer=adam, metrics=["mse"]) return rec_model
def AttnGatingBlock(self, x, g, inter_shape): shape_x = K.int_shape(x) # 32 shape_g = K.int_shape(g) # 16 theta_x = Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same')(x) # 16 shape_theta_x = K.int_shape(theta_x) phi_g = Conv2D(inter_shape, (1, 1), padding='same')(g) upsample_g = Conv2DTranspose(inter_shape, (3, 3), strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), padding='same')(phi_g) # 16 concat_xg = add([upsample_g, theta_x]) act_xg = Activation('relu')(concat_xg) psi = Conv2D(1, (1, 1), padding='same')(act_xg) sigmoid_xg = Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(sigmoid_xg) upsample_psi = UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]))( sigmoid_xg) # 32 # my_repeat=Lambda(lambda xinput:K.repeat_elements(xinput[0],shape_x[1],axis=1)) # upsample_psi=my_repeat([upsample_psi]) upsample_psi = self.expend_as(upsample_psi, shape_x[3]) y = multiply([upsample_psi, x]) # print(K.is_keras_tensor(upsample_psi)) result = Conv2D(shape_x[3], (1, 1), padding='same')(y) result_bn = BatchNormalization()(result) return result_bn
def get_model(params): inputs = Input(shape = (201, 4,)) cnn_out = Convolution1D(int(params['filter']), int(params['window_size']), kernel_initializer=params['kernel_initializer'], kernel_regularizer=regularizers.l2(params['l2_reg']), activation="relu")(inputs) pooling_out = MaxPooling1D(pool_size=int(params['pool_size']), strides=int(params['pool_size']))(cnn_out) dropout1 = Dropout(params['drop_out_cnn'])(pooling_out) lstm_out = Bidirectional(LSTM(int(params['lstm_unit']), return_sequences=True, kernel_initializer=params['kernel_initializer'], kernel_regularizer=regularizers.l2(params['l2_reg'])), merge_mode = 'concat')(dropout1) a = Permute((2, 1))(lstm_out) a = Dense(lstm_out._keras_shape[1], activation='softmax')(a) a_probs = Permute((2, 1), name='attention_vec')(a) attention_out = multiply([lstm_out, a_probs]) attention_out = Lambda(lambda x: K.sum(x, axis=1))(attention_out) dropout2 = Dropout(params['drop_out_lstm'])(attention_out) dense_out = Dense(int(params['dense_unit']), activation='relu', kernel_initializer=params['kernel_initializer'], kernel_regularizer=regularizers.l2(params['l2_reg']))(dropout2) output = Dense(1, activation='sigmoid')(dense_out) model = Model(input=[inputs], output=output) adam = Adam(lr=params['learning_rate'],epsilon=10**-8) model.compile(loss='binary_crossentropy', optimizer=adam, metrics=[roc_auc]) return model
def build_network(self): features = Input(shape=(self.n_features, ), name="features") action_picked = Input(shape=(self.n_actions, ), name="action_picked") hidden_1 = Dense(20, activation='relu', name="dense_1", kernel_initializer=RandomNormal(mean=0., stddev=.1), bias_initializer=Constant(0.1))(features) act_prob = Dense(self.n_actions, activation='softmax', name="output_layer", kernel_initializer=RandomNormal(mean=0., stddev=.1), bias_initializer=Constant(0.1))(hidden_1) selected_act_prob = multiply([act_prob, action_picked]) selected_act_prob = Lambda(lambda x: K.sum(x, axis=-1, keepdims=True), output_shape=(1, ))(selected_act_prob) # actor model = Model(inputs=[features, action_picked], outputs=[act_prob, selected_act_prob]) opt = Adam(lr=self.lr) model.compile(loss=['mse', loss], loss_weights=[0.0, 1.0], optimizer=opt) model.summary() return model
def ecoder_decoder_model(ref_words_size): """ """ X = Input(shape=(Tx, embeddings_weights_shape)) #Pre-attention LSTM (encoder) encoder = Bidirectional(LSTM(n_a, dropout=0.2, return_sequences=True))(X) #Attention model flat_layer = Flatten()(encoder) attention_outputs = [] for t in range(Ty): A = Dense(Tx, activation='softmax')(flat_layer) B = Permute([2, 1])(RepeatVector(n_a * 2)(A)) C = multiply([encoder, B]) D = Lambda(lambda x: K.sum(x, axis=-2))(C) attention_outputs.append(Reshape((1, n_a * 2))(D)) attention_out = concatenate(attention_outputs, axis=-2) # Post-attention LSTM (decoder) decoder = LSTM(n_s, return_sequences=True)(attention_out) #Dense layer decoder = Dense(ref_words_size, activation='softmax')(decoder) model = Model(inputs=X, outputs=decoder) return model
def dynamic_attention(input_layer, prefix, mask=None): n_hidden = input_layer.shape[-1].value h1 = TimeDistributed(Dense(n_hidden, activation='tanh'), name='%s_att_TDdense1' % prefix)(input_layer) h2 = TimeDistributed(Dense(1, use_bias=False), name='%s_att_TDdense2' % prefix)( h1) # [n_sample, n_steps, 1] # calculate an importance "value" for each step flat = Lambda(lambda x: K.squeeze(x, axis=-1), name='%s_att_flat' % prefix)(h2) # convert the importance values to probabilities with the softmax attention = Activation('softmax', name='%s_att_softmax' % prefix)( flat) # [n_sample, n_steps] if mask is not None: # zero out the masked attention masked_attention = multiply([attention, mask], name='%s_masked_attention' % prefix) # renormalize the attention to a probability attention = Lambda(lambda x: x / K.sum(x, axis=1, keepdims=True), name='%s_masked_normed_attention' % prefix)(masked_attention) # weight each step by it's probabilistic importance and sum them together attended_texts = Lambda(K_dot, output_shape=get_K_dot_shape, name='%s_summarized_attention' % prefix)([attention, input_layer]) return attended_texts
def _shared_model(input, embeddings): embedded = Embedding(len(embeddings), EMBEDDING_DIM, weights=[embeddings], input_shape=(MAX_SEQ_LENGTH, ), trainable=False)(input) activations = Bidirectional(LSTM(HIDDEN_LAYERS, return_sequences=True), merge_mode="concat")(embedded) activations = Bidirectional(LSTM(HIDDEN_LAYERS, return_sequences=True), merge_mode="concat")(activations) activations = Dropout(0.5)(activations) attention = TimeDistributed(Dense(1, activation="tanh"))(activations) attention = Flatten()(attention) attention = Activation("softmax")(attention) attention = RepeatVector(HIDDEN_LAYERS * 2)(attention) attention = Permute([2, 1])(attention) sent_representation = multiply([activations, attention]) sent_representation = Lambda(lambda xin: backend.sum(xin, axis=1))( sent_representation) sent_representation = Dropout(0.1)(sent_representation) return sent_representation
def build_model_densenet121_with_mask(): img_input = Input(INPUT_SHAPE, name='data') mask_model = model_unet(INPUT_SHAPE) mask_model.load_weights( '../output/checkpoints/fish_mask_unet/model_fish_unet2/checkpoint-best-064-0.0476.hdf5' ) mask = mask_model(img_input) mask3 = concatenate([mask, mask, mask], axis=3) masked_image = multiply([img_input, mask3]) base_model = densenet121.DenseNet( img_input=img_input, reduction=0.5, weights_path='../input/densenet121_weights_tf.h5', classes=1000) base_model.layers.pop() base_model.layers.pop() base_model_output = base_model(masked_image) species_dense = Dense(len(SPECIES_CLASSES), activation='softmax', name='cat_species')(base_model_output) cover_dense = Dense(len(COVER_CLASSES), activation='softmax', name='cat_cover')(base_model_output) model = Model(inputs=img_input, outputs=[species_dense, cover_dense]) sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) return model
def get_model(num_users, num_items, latent_dim, regs=[0, 0]): # Input variables user_input = Input(shape=(1,), dtype='int32', name='user_input') item_input = Input(shape=(1,), dtype='int32', name='item_input') MF_Embedding_User = Embedding(input_dim=num_users, output_dim=latent_dim, name='user_embedding', init=init_normal(), W_regularizer=l2(regs[0]), input_length=1) MF_Embedding_Item = Embedding(input_dim=num_items, output_dim=latent_dim, name='item_embedding', init=init_normal(), W_regularizer=l2(regs[1]), input_length=1) # Crucial to flatten an embedding vector! user_latent = Flatten()(MF_Embedding_User(user_input)) item_latent = Flatten()(MF_Embedding_Item(item_input)) # Element-wise product of user and item embeddings # predict_vector = merge([user_latent, item_latent], mode = 'mul') predict_vector = merge.multiply([user_latent, item_latent]) # Final prediction layer # prediction = Lambda(lambda x: K.sigmoid(K.sum(x)), output_shape=(1,))(predict_vector) prediction = Dense(1, activation='sigmoid', init='lecun_uniform', name='prediction')(predict_vector) model = Model(input=[user_input, item_input], output=prediction) return model
def create_regression_model(n_hidden, input, l2_factor): """ RNN, single layer, for regression score, with Attention Args: n_hidden: number of hidden neurons for this RNN input: taken from Seq2Seq model l2_factor: regularization factor Returns: regression float score """ activations = LSTM(n_hidden, name='lstm_layer', trainable=True, return_sequences=True)(input) attention = TimeDistributed(Dense(1, activation='tanh'))(activations) attention = Flatten()(attention) attention = Activation('softmax')(attention) attention = RepeatVector(n_hidden)(attention) attention = Permute([2, 1])(attention) # apply the attention sent_representation = multiply([activations, attention]) sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))( sent_representation) regression_score = Dense( 1, activation='sigmoid', name='regression_output', kernel_regularizer=l2(l2_factor))(sent_representation) return regression_score
def build(self): """ Go through train on batch function above""" ## first layer for the input (x_t) x = Input(batch_shape=(None, None, self.input_dim), name='x') masked = Masking(mask_value=-1, input_shape=(None, None, self.input_dim))(x) lstm_out = LSTM(self.hidden_layer_size, return_sequences=True)(masked) dense_out = Dense(self.input_dim_order, activation='sigmoid')(lstm_out) y_order = Input(batch_shape=(None, None, self.input_dim_order), name='y_order') merged = multiply([dense_out, y_order]) def reduce_dim(x): x = K.max(x, axis=2, keepdims=True) return x def reduce_dim_shape(input_shape): shape = list(input_shape) shape[-1] = 1 return tuple(shape) reduced = Lambda(reduce_dim, output_shape=reduce_dim_shape)(merged) self.model = Model(inputs=[x, y_order], outputs=reduced) self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy', metrics=['accuracy']) print('Summary of the model') self.model.summary()
def get_model(num_users, num_items, item_features, latent_dim, regs=[1e-6, 1e-6]): ### define placeholder. user_id_input = Input(shape=[1], name='user') item_id_input = Input(shape=[1], name='item') item_feature_input = Input(shape=[item_features], name='itemfeature') ### define embedding size and layers. user_embedding = Embedding(output_dim=latent_dim, input_dim=num_users, input_length=1, name='user_embedding', embeddings_regularizer=l2( regs[0]))(user_id_input) item_embedding = Embedding(output_dim=latent_dim, input_dim=num_items, input_length=1, name='item_embedding', embeddings_regularizer=l2( regs[1]))(item_id_input) user_vecs = Reshape([latent_dim])(user_embedding) item_vecs = Reshape([latent_dim])(item_embedding) mf = multiply([user_vecs, item_vecs]) concat = Concatenate()([mf, item_feature_input]) concat_dropout = keras.layers.Dropout(0.25)(concat) out = Dense(num_items, activation='softmax')(concat_dropout) model = Model(inputs=[user_id_input, item_id_input, item_feature_input], outputs=out) return model
K.set_floatx('float64') in_vals = Input((poscount, 1), name='vals', dtype='float64') normd = BatchNormalization( axis=1, gamma_constraint=min_max_norm(), beta_constraint=min_max_norm())(in_vals) in_locs = Input((poscount, ), name='locs', dtype='uint64') embed_locs = Embedding( locidx.watermark, embedding_dim, input_length=poscount)(in_locs) merged = concatenate([embed_locs, normd]) dense_list = [] for i in range(dense_count): dense_list.append( Dropout(dropout_prob)(Dense(1, activation='sigmoid')(Flatten()( merged)))) mult = multiply(dense_list) ml = Model(inputs=[in_locs, in_vals], outputs=mult) ml.compile(optr, metrics=['acc'], loss=mse) locs, vals, labels = rfutils.read_data(gl, poscount, locidx) def fit( eps=int(sys.argv[1]) if len(sys.argv) > 1 else 1, # Large batches tend to cause NaNs in batch normalization. bsz=int(sys.argv[2]) if len(sys.argv) > 2 else 50): ml.fit([locs, vals], labels, batch_size=bsz, epochs=eps) fit()