def build_model(max_features, maxlen): ''' Derived CNN model from Keegan Hines' Snowman https://github.com/keeganhines/snowman/ ''' text_input = Input(shape=(maxlen, ), name='text_input') x = Embedding(input_dim=max_features, input_length=maxlen, output_dim=128)(text_input) conv_a = Conv1D(15, 2, activation='relu')(x) conv_b = Conv1D(15, 3, activation='relu')(x) conv_c = Conv1D(15, 4, activation='relu')(x) conv_d = Conv1D(15, 5, activation='relu')(x) conv_e = Conv1D(15, 6, activation='relu')(x) pool_a = GlobalMaxPooling1D()(conv_a) pool_b = GlobalMaxPooling1D()(conv_b) pool_c = GlobalMaxPooling1D()(conv_c) pool_d = GlobalMaxPooling1D()(conv_d) pool_e = GlobalMaxPooling1D()(conv_e) flattened = concatenate([pool_a, pool_b, pool_c, pool_d, pool_e]) drop = Dropout(.2)(flattened) dense = Dense(1)(drop) out = Activation("sigmoid")(dense) model = Model(inputs=text_input, outputs=out) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def GRUdeep(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(0.1, name='embed_drop')(x) if params['bidirectional']: x = Bidirectional(CuDNNGRU(params['lstm_units'], return_sequences=True))(x) x = Bidirectional(CuDNNGRU(params['lstm_units'], return_sequences=True))(x) x = GlobalMaxPooling1D()(x) else: x = CuDNNGRU(params['lstm_units'], return_sequences=True)(x) x = CuDNNGRU(params['lstm_units'], return_sequences=True)(x) x = GlobalMaxPooling1D()(x) x = Dropout(params['dropout_rate'])(x) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def text_cnn(max_seq_index, max_seq_length): text_input = Input(shape = (max_seq_length,), name='text_input') x = Embedding(output_dim=15, input_dim=max_seq_index, input_length=max_seq_length)(text_input) conv_a = Conv1D(15,2, activation='relu')(x) conv_b = Conv1D(15,4, activation='relu')(x) conv_c = Conv1D(15,6, activation='relu')(x) pool_a = GlobalMaxPooling1D()(conv_a) pool_b = GlobalMaxPooling1D()(conv_b) pool_c = GlobalMaxPooling1D()(conv_c) flattened = concatenate( [pool_a, pool_b, pool_c]) drop = Dropout(.2)(flattened) dense = Dense(1)(drop) out = Activation("sigmoid")(dense) model = Model(inputs=text_input, outputs=out) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def build_model(self): input1 = Input(shape=(self.config.max_sent_len, self.config.word_emb_dim)) input2 = Input(shape=(self.config.max_sent_len, self.config.word_emb_dim)) lstm1 = LSTM(300, dropout=0.3, activity_regularizer=l1_l2(0.01), return_sequences=True, activation='relu')(input1) lstm2 = LSTM(300, dropout=0.3, activity_regularizer=l1_l2(0.01), return_sequences=True, activation='relu')(input2) pool1 = GlobalMaxPooling1D()(lstm1) pool2 = GlobalMaxPooling1D()(lstm2) diffvec = merge([pool1, pool2], mode=abs_diff, output_shape=(300, )) product = Dot(-1)([pool1, pool2]) features = merge([pool1, pool2, diffvec, product], mode='concat', concat_axis=-1) out = Dense(2, kernel_regularizer=regularizers.l1_l2(0.01), activity_regularizer=regularizers.l1_l2(0.01), activation='softmax')(features) model = Model(inputs=[input1, input2], outputs=[out]) return model
def GRUHierarchical(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) l_lstm = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_) l_dense = TimeDistributed(Dense(256))(l_lstm) l_att = GlobalMaxPooling1D()(l_dense) sentEncoder = Model(input_, l_att) review_input = Input(shape=(1, params['sequence_length'])) review_encoder = TimeDistributed(sentEncoder)(review_input) l_lstm_sent = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(review_encoder) l_dense_sent = TimeDistributed(Dense(256))(l_lstm_sent) l_att = GlobalMaxPooling1D()(l_dense_sent) x = Dense(6, activation='sigmoid')(l_att) model = Model(inputs=review_input, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def __call__(self, a, b): a_max = GlobalMaxPooling1D()(a) a_avg = GlobalAveragePooling1D()(a) b_max = GlobalMaxPooling1D()(b) b_avg = GlobalAveragePooling1D()(b) return self.model(merge([a_avg, a_max, b_avg, b_max], mode='concat')) # shape=(batch_size, 4 * units)
def r_c_nn(train_lexical_data, train_sentence_data, train_label, test_lexical_data, test_sentence_data, test_label): train_label = np_utils.to_categorical(train_label, label_count) test_label_categorical = np_utils.to_categorical(test_label, label_count) lexical_branch = Sequential() lexical_branch.add( Bidirectional(SimpleRNN(500, return_sequences=True), merge_mode='concat', input_shape=(lexical_length, embedding_length))) lexical_branch.add(Activation('tanh')) lexical_branch.add(GlobalMaxPooling1D()) lexical_branch.add(Dense(200, activation='tanh')) sentence_branch = Sequential() sentence_branch.add( Convolution1D(500, 3, border_mode='same', input_shape=(timestep, sentence_embedding_length), W_regularizer=l2(0.01))) sentence_branch.add(GlobalMaxPooling1D()) sentence_branch.add(Dropout(0.6)) sentence_branch.add(Dense(200, activation='tanh', W_regularizer=l2(0.01))) merged = Merge([lexical_branch, sentence_branch], mode='concat') final_model = Sequential() final_model.add(merged) final_model.add(Dense(50, activation='tanh', W_regularizer=l2(0.01))) final_model.add(Dense(label_count, activation='softmax', name='output')) sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True) final_model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) epoch = 50 for i in range(epoch): print i, '/', epoch final_model.fit( [train_lexical_data, train_sentence_data], train_label, batch_size=64, nb_epoch=1, shuffle=True, verbose=1, validation_data=([test_lexical_data, test_sentence_data], test_label_categorical)) # 带权重的类别 # model.fit([train_lexical_data, train_sentence_data],train_label, batch_size=64, # nb_epoch=10, class_weight=classWeight, shuffle=True,verbose=1) # loss_and_metrics = model.evaluate([test_lexical_data,test_sentence_data], test_label_categorical, batch_size=32) # print loss_and_metrics test_classes = final_model.predict_classes( [test_lexical_data, test_sentence_data], batch_size=64) putOut(test_classes, test_label)
def cnn_word_model_ner(text_size, n_classes, tag_size, embedding_size, filters_cnn_emb, filters_cnn_tag, kernel_sizes, coef_reg_cnn_emb, coef_reg_cnn_tag, coef_reg_den, dropout_rate, dense_size): inp_emb = Input(shape=(text_size, embedding_size)) inp_tag = Input(shape=(text_size, tag_size)) outputs_emb = [] for i in range(len(kernel_sizes)): output_i = Conv1D(filters_cnn_emb, kernel_size=kernel_sizes[i], activation=None, kernel_regularizer=l2(coef_reg_cnn_emb), padding='same')(inp_emb) output_i = BatchNormalization()(output_i) output_i = Activation('relu')(output_i) output_i = GlobalMaxPooling1D()(output_i) outputs_emb.append(output_i) outputs_tag = [] for i in range(len(kernel_sizes)): output_i = Conv1D(filters_cnn_tag, kernel_size=kernel_sizes[i], activation=None, kernel_regularizer=l2(coef_reg_cnn_tag), padding='same')(inp_tag) output_i = BatchNormalization()(output_i) output_i = Activation('relu')(output_i) output_i = GlobalMaxPooling1D()(output_i) outputs_tag.append(output_i) output_emb = concatenate(outputs_emb, axis=1) print('Concatenate emb shape:', output_emb.shape) output_tag = concatenate(outputs_tag, axis=1) print('Concatenate tag shape:', output_tag.shape) output = concatenate([output_emb, output_tag], axis=1) print('Concatenate shape:', output.shape) output = Dropout(rate=dropout_rate)(output) output = Dense(dense_size, activation=None, kernel_regularizer=l2(coef_reg_den))(output) output = BatchNormalization()(output) output = Activation('relu')(output) output = Dropout(rate=dropout_rate)(output) output = Dense(n_classes, activation=None, kernel_regularizer=l2(coef_reg_den))(output) output = BatchNormalization()(output) act_output = Activation('softmax')(output) model = Model(inputs=[inp_emb, inp_tag], outputs=act_output) return model
def cnnModel(): embedding_layer = Embedding(MAX_NB_WORDS+1,EMBEDDING_DIM, embeddings_initializer=initializers.he_uniform(20), #weights=[embedding_matrix], input_length=INPUT_LEN, trainable=True) # #trainable=False)) model1 = Sequential() model1.add(embedding_layer) model1.add(Convolution1D(128, 4,padding='same',init='he_normal')) model1.add(BatchNormalization()) model1.add(Activation('relu')) model1.add(Convolution1D(128, 4, padding='same',activation='relu',init='he_normal')) model1.add(GlobalMaxPooling1D()) model2= Sequential() model2.add(embedding_layer) model2.add(Convolution1D(128, 3,padding='same',init='he_normal')) model2.add(BatchNormalization()) model2.add(Activation('relu')) model2.add(Convolution1D(128, 3, padding='same',activation='relu',init='he_normal')) model2.add(GlobalMaxPooling1D()) model3 = Sequential() model3.add(embedding_layer) model3.add(Convolution1D(128, 5,padding='same',init='he_normal')) model3.add(BatchNormalization()) model3.add(Activation('relu')) model3.add(Convolution1D(128, 5, padding='same',activation='relu',init='he_normal')) model3.add(GlobalMaxPooling1D()) model4 = Sequential() model4.add(embedding_layer) model4.add(Convolution1D(128, 7,padding='same',init='he_normal')) model4.add(BatchNormalization()) model4.add(Activation('relu')) model4.add(Convolution1D(128, 7, padding='same',activation='relu',init='he_normal')) model4.add(GlobalMaxPooling1D()) model = Sequential() model.add(Merge([model1,model2,model3,model4],mode='concat',concat_axis=1)) #model.add(GRU(128, dropout=0.2, recurrent_dropout=0.1)) model.add(Dropout(0.3)) model.add(Dense(128,activation='relu',init='he_normal')) model.add(Dense(LAW_COUNT,activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adamax', metrics=[Jaccard_Sim]) model.summary() return model
def Model1(dim, max_ques_len, max_ans_len, vocab_lim, embedding): inp_q = Input(shape=(max_ques_len, )) embedding_q = Embedding(vocab_lim, dim, input_length=max_ques_len, weights=[embedding], trainable=False)(inp_q) conv_q = Convolution1D(100, 5, border_mode='same', activation='relu')(embedding_q) conv_q = Dropout(0.25)(conv_q) pool_q = GlobalMaxPooling1D()(conv_q) inp_a = Input(shape=(max_ans_len, )) embedding_a = Embedding(vocab_lim, dim, input_length=max_ans_len, weights=[embedding], trainable=False)(inp_a) conv_a = Convolution1D(100, 5, border_mode='same', activation='relu')(embedding_a) conv_a = Dropout(0.25)(conv_a) pool_a = GlobalMaxPooling1D()(conv_a) #sim = SimLayer(1)([pool_q, pool_a]) sim = merge([Dense(100, bias=False)(pool_q), pool_a], mode='dot') # print pool_a, pool_q # model1 = merge([pool_q, pool_a, sim], mode='concat') # model = Model(input=[inp_q, inp_a], output=[model1]) # model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # print model.summary() # return model model_sim = merge([pool_q, pool_a, sim], mode='concat') print model_sim # #model_final = Flatten()(model_sim) model_final = Dropout(0.5)(model_sim) model_final = Dense(201)(model_final) model_final = Dropout(0.5)(model_final) model_final = Dense(1, activation='sigmoid')(model_final) model = Model(input=[inp_q, inp_a], output=[model_final]) print(model.output_shape) model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy']) print model.summary() return model
def LSTMdeep2(params): embed_dropout_rate = 0.1 Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x) if params['bidirectional']: x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x1 = GlobalMaxPooling1D()(x) x2 = GlobalAveragePooling1D()(x) x3 = AttentionWithContext()(x) else: x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = GlobalMaxPooling1D()(x) merge_layer = concatenate([x1, x2, x3]) x = Dropout(params['dropout_rate'])(merge_layer) x = Dense(256)(x) x = PReLU()(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def bilstm_self_mult_attention_model(self, params): """ Method builds uncompiled model of BiLSTM with self multiplicative attention Args: params: disctionary of parameters for NN Returns: Uncompiled model """ inp = Input(shape=(params['text_size'], params['embedding_size'])) output = Bidirectional(LSTM(params['units_lstm'], activation='tanh', return_sequences=True, kernel_regularizer=l2(params['coef_reg_lstm']), dropout=params['dropout_rate'], recurrent_dropout=params['rec_dropout_rate']))(inp) output = MaxPooling1D(pool_size=2, strides=3)(output) output = multiplicative_self_attention(output, n_hidden=params.get("self_att_hid"), n_output_features=params.get("self_att_out")) output = GlobalMaxPooling1D()(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(self.n_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) return model
def bigru_model(params): inp = Input(shape=(params['text_size'], params['embedding_size'])) output = Bidirectional( GRU(params['units_lstm'], activation='tanh', return_sequences=True, kernel_regularizer=l2(params['coef_reg_lstm']), dropout=params['dropout_rate'], recurrent_dropout=params['rec_dropout_rate']))(inp) output = GlobalMaxPooling1D()(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(nb_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) model.compile( optimizer="Adam", loss="binary_crossentropy", metrics=['accuracy'], #loss_weights=loss_weights, #sample_weight_mode=sample_weight_mode, # weighted_metrics=weighted_metrics, # target_tensors=target_tensors ) return model
def build_model(self): data = self.data inputs = Input(shape=(data.seq_length, )) embeddings = Embedding( data.max_feature, data.embed_dim, weights=[data.embed_matrix], trainable=False)(inputs) x = SpatialDropout1D(self.dropout)(embeddings) x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x) x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x) ave_pool = GlobalAveragePooling1D()(x) max_pool = GlobalMaxPooling1D()(x) conc = concatenate([ave_pool, max_pool]) # bn = BatchNormalization()(pool) x = Dense(self.dense_size, activation="relu")(conc) # drop = Dropout(self.dropout)(x) outputs = Dense(6, activation="sigmoid")(x) model = Model(inputs=inputs, outputs=outputs) optimizer = self.get_optimizer(self.lr, self.optim_name) model.compile( loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.model = model
def bilstm_model(self, params): """ Build un-compiled BiLSTM Args: params: dictionary of parameters for NN Returns: Un-compiled model """ inp = Input(shape=(params['text_size'], params['embedding_size'])) output = Bidirectional(LSTM(params['units_lstm'], activation='tanh', return_sequences=True, kernel_regularizer=l2(params['coef_reg_lstm']), dropout=params['dropout_rate'], recurrent_dropout=params['rec_dropout_rate']))(inp) output = GlobalMaxPooling1D()(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(self.n_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) return model
def build_model(ft=False): # CNN Model Hyperparameters embedding_dim = 300 vocabsize = len(vocabulary_inv) dropout_prob = 0.5 nb_filter = 100 hidden_dims = 100 filter_len = [1, 2, 3, 4] if ft == False: graph_input1 = Input(shape=(sequence_length, embedding_dim)) embed1 = Embedding(vocabsize, embedding_dim, input_length=sequence_length, name="embedding")(graph_input1) convs1 = [] for fsz in filter_len: conv1 = Conv1D(filters=nb_filter, kernel_size=fsz, activation='relu')(graph_input1) pool1 = GlobalMaxPooling1D()(conv1) convs1.append(pool1) y1 = Concatenate()(convs1) if len(convs1) > 1 else convs1[0] z1 = Dropout(dropout_prob)(y1) z1 = Dense(hidden_dims, kernel_constraint=maxnorm(2))(z1) z1 = BatchNormalization()(z1) z1 = Activation('tanh')(z1) z1 = Dropout(dropout_prob)(z1) z1 = Dense(nb_classes)(z1) z1 = BatchNormalization()(z1) model_output1 = Activation('softmax')(z1) cnn_model = Model(inputs=graph_input1, outputs=model_output1) return cnn_model
def architect_model(input_shape, n_classes): inp = Input(shape=input_shape, dtype="float32") x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(inp) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(x) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(x) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Concatenate()([GlobalAveragePooling1D()(x), GlobalMaxPooling1D()(x)]) # Fully connected x = Dense(2048, activation="relu")(x) x = Dense(2048, activation="relu")(x) out = Dense(n_classes, activation="softmax")(x) return Model(inputs=inp, outputs=out)
def cnn_model(input_shape): inputs = Input(input_shape) x = inputs levels = 64 for level in range(3): x = Conv1D(levels, 3, activation='relu')(x) x = BatchNormalization()(x) x = MaxPooling1D(pool_size=2, strides=2)(x) levels *= 2 # Global Layers x = GlobalMaxPooling1D()(x) for fc in range(2): x = Dense(256, activation='relu')(x) x = Dropout(0.5)(x) labels = Dense(10, activation='softmax')(x) model = Model(inputs=[inputs], outputs=[labels]) sgd = keras.optimizers.SGD(lr=0.0003, momentum=0.9, decay=1e-5, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model
def cnn_model(params): inp = Input(shape=(params['text_size'], params['embedding_size'])) outputs = [] for i in range(len(params['kernel_sizes_cnn'])): output_i = Conv1D(params['filters_cnn'], kernel_size=params['kernel_sizes_cnn'][i], activation=None, kernel_regularizer=l2(params['coef_reg_cnn']), padding='same')(inp) output_i = BatchNormalization()(output_i) output_i = Activation('relu')(output_i) output_i = GlobalMaxPooling1D()(output_i) outputs.append(output_i) output = concatenate(outputs, axis=1) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = BatchNormalization()(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(nb_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = BatchNormalization()(output) act_output = Activation("softmax")(output) model = Model(inputs=inp, outputs=act_output) model.compile(optimizer="Adam", loss="binary_crossentropy", metrics=['accuracy'], #loss_weights=loss_weights, #sample_weight_mode=sample_weight_mode, # weighted_metrics=weighted_metrics, # target_tensors=target_tensors ) return model
def __call__(self, sent, align, **kwargs): result = self.model(merge([sent, align], mode='concat')) # Shape: (i, n) avged = GlobalAveragePooling1D()(result, mask=self.words) maxed = GlobalMaxPooling1D()(result, mask=self.words) merged = merge([avged, maxed]) result = BatchNormalization()(merged) return result
def GRUconcat3(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) if params['bidirectional']: x = Bidirectional(CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_) else: x = CuDNNGRU(params['lstm_units'], return_sequences=True)(embed_input_) x1 = GlobalMaxPooling1D()(x) x2 = GlobalAveragePooling1D()(x) x3 = AttentionWithContext()(x) x4 = Attention(params['sequence_length'])(x) merge_layer = concatenate([x1, x2, x3, x4]) x = Dropout(params['dropout_rate'])(merge_layer) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def simple_cnn_model(self): model = Sequential() # model.add(Masking(mask_value=0., input_shape=(self.SEQ_LEN, self.FEATURES_DIM))) model.add(Conv1D(input_shape=(self.SEQ_LEN, self.FEATURES_DIM), filters=32, kernel_size=14, padding='same', name='conv1')) model.add(Activation(activation='relu', name='act1')) model.add(MaxPooling1D(pool_size=3)) model.add(GlobalMaxPooling1D()) model.add(Dense(1024)) # model.add(BatchNormalization()) model.add(Activation(activation='relu')) model.add(Dense(self.n_classes)) model.add(Activation('softmax')) optimizer = Adam(lr=0.002) gpu_model = multi_gpu_model(model, gpus=self.n_gpus) gpu_model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=['categorical_accuracy']) return (gpu_model, model)
def bilstm_bilstm_model(self, units_lstm_1: int, units_lstm_2: int, dense_size: int, coef_reg_lstm: float = 0., coef_reg_den: float = 0., dropout_rate: float = 0., rec_dropout_rate: float = 0., **kwargs) -> Model: """ Build un-compiled two-layers BiLSTM. Args: units_lstm_1: number of units for the first LSTM layer. units_lstm_2: number of units for the second LSTM layer. dense_size: number of units for dense layer. coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``. coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``. dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``. rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``. kwargs: other non-used parameters Returns: keras.models.Model: uncompiled instance of Keras Model """ inp = Input(shape=(self.opt['text_size'], self.opt['embedding_size'])) output = Bidirectional( LSTM(units_lstm_1, activation='tanh', return_sequences=True, kernel_regularizer=l2(coef_reg_lstm), dropout=dropout_rate, recurrent_dropout=rec_dropout_rate))(inp) output = Dropout(rate=dropout_rate)(output) output = Bidirectional( LSTM(units_lstm_2, activation='tanh', return_sequences=True, kernel_regularizer=l2(coef_reg_lstm), dropout=dropout_rate, recurrent_dropout=rec_dropout_rate))(output) output = GlobalMaxPooling1D()(output) output = Dropout(rate=dropout_rate)(output) output = Dense(dense_size, activation=None, kernel_regularizer=l2(coef_reg_den))(output) output = Activation('relu')(output) output = Dropout(rate=dropout_rate)(output) output = Dense(self.n_classes, activation=None, kernel_regularizer=l2(coef_reg_den))(output) act_output = Activation( self.opt.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) return model
def get_raw_model(self): branches = [] tweet_input = Input(shape=(self.sentence_length, self.word_vectors_container.dimension), dtype='float32') x = Dropout(0.2)(tweet_input) for size, filters_count in [(2, 10), (3, 10), (4, 10), (5, 10)]: for i in range(filters_count): branch = Conv1D(filters=1, kernel_size=size, padding='valid', activation='relu')(x) branch = GlobalMaxPooling1D()(branch) branches.append(branch) x = concatenate(branches, axis=1) x = Dropout(0.2)(x) x = Dense(30, activation='relu')(x) x = Dense(len(self.corpus.label_encoder.classes_))(x) output = Activation('softmax')(x) model = Model(inputs=[tweet_input], outputs=[output]) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['categorical_accuracy']) return model
def build_and_load_model(path): DIM = 200 NUM = 100000 embedding_matrix = np.zeros((NUM, DIM)) tweet_input = Input(shape=(SENTENCE_LENGTH, ), dtype='int32') tweet_encoder = Embedding(NUM, DIM, input_length=SENTENCE_LENGTH, weights=[embedding_matrix], trainable=False)(tweet_input) branches = [] x = Dropout(0.2)(tweet_encoder) for size, filters_count in [(2, 10), (3, 10), (4, 10), (5, 10)]: for i in range(filters_count): branch = Conv1D(filters=1, kernel_size=size, padding='valid', activation='relu')(x) branch = GlobalMaxPooling1D()(branch) branches.append(branch) x = concatenate(branches, axis=1) x = Dropout(0.2)(x) x = Dense(30, activation='relu')(x) x = Dense(1)(x) output = Activation('sigmoid')(x) model = Model(inputs=[tweet_input], outputs=[output]) model.load_weights(path) import gc gc.collect() return model
def lstmmodel(): inp = Input(shape=(maxlen, )) x = Embedding(embedding_matrix_1.shape[0], embedding_matrix_1.shape[1], weights=[embedding_matrix_1], trainable=False)(inp) x = SpatialDropout1D(.1)(x) x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x) #x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x) x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x) x1 = Attention(maxlen)(x) x2 = GlobalAveragePooling1D()(x) x3 = GlobalMaxPooling1D()(x) x = concatenate([x1, x2, x3]) x = Dropout(.1)(x) x = Dense(100, activation="relu")(x) x = BatchNormalization()(x) x = Dropout(.1)(x) x = Dense(100, activation="relu")(x) x = BatchNormalization()(x) x = Dropout(.1)(x) x = Dense(1, activation="sigmoid")(x) model = Model(inputs=[inp], outputs=x) model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3, clipnorm=0.85), metrics=['accuracy']) return model
def build_local(input_set_size, vector_size, maxlen, max_sentence_nb, mul_label_set_size): sentence_input = Input(shape=( max_sentence_nb, maxlen, )) sentence_model = build_sentence_model(input_set_size, vector_size, maxlen) paragraph = TimeDistributed(sentence_model)(sentence_input) #gru = Bidirectional(GRU(128,return_sequences=False), merge_mode='sum')(paragraph) #drop5 = Dropout(0.5)(gru) outs = [] for i in range(0, 4): if i == 0: merged = paragraph else: merged = merge([outs[i - 1], paragraph], mode='concat') dense = TimeDistributed(Dense(256, activation='relu'))(merged) p = TimeDistributed(Dense(mul_label_set_size[i], activation='sigmoid'))(dense) #p = Dense(mul_label_set_size[i], activation='sigmoid')(merged) outs.append(p) # out outs = [GlobalMaxPooling1D()(outs[i]) for i in range(4)] model = Model(input=[sentence_input], output=outs) model.summary() model.compile( loss='binary_crossentropy', optimizer='adam', ) return model
def TextCNN(config, model_view=True): inputs = Input(shape=(config['INPUT_LEN'], ), dtype='int32', name='input') embedding_cnn = Embedding(config['MAX_NB_WORDS'] + 1, config['EMBEDDING_DIM'], input_length=config['INPUT_LEN'])(inputs) conv_out = [] for filter in [1, 2, 3, 4, 5, 6]: conv1 = Convolution1D(256, filter, padding='same')(embedding_cnn) conv1 = Activation('relu')(conv1) x_conv1 = GlobalMaxPooling1D()(conv1) conv_out.append(x_conv1) x = concatenate(conv_out) x = Dense(128)(x) x = Dropout(0.5)(x) x = Activation('relu')(x) x = Dense(452)(x) outputs = Activation('sigmoid', name='outputs')(x) model = Model(inputs=inputs, outputs=outputs) model.compile(loss='binary_crossentropy', optimizer='adamax', metrics=[Jaccard_Sim]) if model_view: model.summary() return model
def bigru_model(self, params): """ Method builds uncompiled model BiGRU Args: params: disctionary of parameters for NN Returns: Uncompiled model """ inp = Input(shape=(params['text_size'], params['embedding_size'])) output = Bidirectional( GRU(params['units_lstm'], activation='tanh', return_sequences=True, kernel_regularizer=l2(params['coef_reg_lstm']), dropout=params['dropout_rate'], recurrent_dropout=params['rec_dropout_rate']))(inp) output = GlobalMaxPooling1D()(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(self.n_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) act_output = Activation('sigmoid')(output) model = Model(inputs=inp, outputs=act_output) return model
def __call__(self, inputs): x = self.model(inputs) avg_x = GlobalAveragePooling1D()(x) max_x = GlobalMaxPooling1D()(x) x = concatenate([avg_x, max_x]) x = BatchNormalization()(x) return x