def bigru_with_max_aver_pool_model(self, units_gru: int, dense_size: int, coef_reg_gru: float = 0., coef_reg_den: float = 0., dropout_rate: float = 0., rec_dropout_rate: float = 0., **kwargs) -> Model: """ Method builds uncompiled model Bidirectional GRU with concatenation of max and average pooling after BiGRU. Args: units_gru: number of units for GRU. dense_size: number of units for dense layer. coef_reg_gru: l2-regularization coefficient for GRU. Default: ``0.0``. coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``. dropout_rate: dropout rate to be used after BiGRU and between dense layers. Default: ``0.0``. rec_dropout_rate: dropout rate for GRU. Default: ``0.0``. kwargs: other non-used parameters Returns: keras.models.Model: uncompiled instance of Keras Model """ inp = Input(shape=(self.opt['text_size'], self.opt['embedding_size'])) output = Dropout(rate=dropout_rate)(inp) output, state1, state2 = Bidirectional( GRU(units_gru, activation='tanh', return_sequences=True, return_state=True, kernel_regularizer=l2(coef_reg_gru), dropout=dropout_rate, recurrent_dropout=rec_dropout_rate))(output) output1 = GlobalMaxPooling1D()(output) output2 = GlobalAveragePooling1D()(output) output = Concatenate()([output1, output2, state1, state2]) output = Dropout(rate=dropout_rate)(output) output = Dense(dense_size, activation=None, kernel_regularizer=l2(coef_reg_den))(output) output = Activation('relu')(output) output = Dropout(rate=dropout_rate)(output) output = Dense(self.n_classes, activation=None, kernel_regularizer=l2(coef_reg_den))(output) act_output = Activation( self.opt.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) return model
def GRUConvdeep3(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(0.1, name='embed_drop')(x) if params['bidirectional']: x_g1 = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(x) x_l1 = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True))(x_g1) x_g2 = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(x_l1) x1 = GlobalMaxPooling1D()(x_g2) x2 = GlobalAveragePooling1D()(x_g2) merge_layer = concatenate([x_g1, x_l1, x_g2]) x_conv = Conv1D(64, kernel_size=3, padding='valid', kernel_initializer='glorot_uniform')(merge_layer) x1_conv = GlobalMaxPooling1D()(x_conv) x2_conv = GlobalAveragePooling1D()(x_conv) merge_layer2 = concatenate([x1_conv, x2_conv]) x = Dropout(params['dropout_rate'])(merge_layer2) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def model_conv_glob(data, paramdims): ''' Conv1D: {} x {}, relu GlobPool ''' input = Input(shape=data['input_1'].shape, name='input_1') layer = Conv1D(paramdims[0], kernel_size=(paramdims[1]), activation='relu')(input) output = GlobalAveragePooling1D()(layer) return input, output
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]): assert len(layers) == len(reg_layers) num_layer = len(layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(44, ), name='user_input') item_input = Input(shape=(16, ), name='item_input') vector = concatenate([user_input, item_input]) ''' 15d: vector = Embedding(input_dim=vocabulary_lenth + 1, output_dim=100)(vector) print('1:', vector.shape) vector = Reshape((15, 4, 100))(vector) print('2:', vector.shape) vector = Lambda(lambda x: K.mean(x, axis=2))(vector) print('3:', vector.shape) O_seq = Attention(5, 120)([vector, vector, vector]) print('5', O_seq.shape) O_seq = GlobalAveragePooling1D()(O_seq) print('6', O_seq.shape) O_seq = Dropout(0.5)(O_seq) outputs = Dense(1, activation='sigmoid')(O_seq) print('7', outputs.shape) ''' vector = Embedding(input_dim=vocabulary_lenth + 1, output_dim=500)(vector) print(vector.shape) out_u = Lambda(lambda x: x[:, 0:44])(vector) out_i = Lambda(lambda x: x[:, 44:60])(vector) uilist = [] for vector, size in ((out_u, 11), (out_i, 4)): print('0:', vector.shape) vector = Reshape((size, 4, 500))(vector) print('1:', vector.shape) vector = Lambda(lambda x: K.mean(x, axis=1))(vector) # vector = GlobalAveragePooling1D()(vector) print('2:', vector.shape) uilist.append(vector) vector = Concatenate(axis=1)(uilist) print('3:', vector.shape) O_seq = Attention(5, 240)([vector, vector, vector]) print('4', O_seq.shape) O_seq = GlobalAveragePooling1D()(O_seq) print('5', O_seq.shape) O_seq = Dropout(0.5)(O_seq) outputs = Dense(1, activation='sigmoid')(O_seq) print('6', outputs.shape) model = Model(inputs=[user_input, item_input], outputs=outputs) return model
def _createModel(self, outDim): modelLeft = Sequential() #modelLeft.add(Masking(mask_value=0., input_shape=(self._phraseLen, self._vecLen))) modelLeft.add(LSTM(outDim, input_shape=(self._phraseLen, self._vecLen), dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)) modelLeft.add(LSTM(outDim, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)) modelLeft.add(GlobalAveragePooling1D()) modelRight = Sequential() #modelRight.add(Masking(mask_value=0., input_shape=(self._phraseLen, self._vecLen))) modelRight.add(LSTM(outDim, input_shape=(self._phraseLen, self._vecLen), dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)) modelRight.add(LSTM(outDim, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True)) modelRight.add(GlobalAveragePooling1D()) model = Sequential() model.add(Merge([modelLeft, modelRight], mode='concat')) model.add(Dense(outDim, activation='relu')) model.add(Dense(outDim, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def build_model(): model = Sequential() model.add(TimeDistributed(Conv2D(num_filters, kernel_size, padding=padding_mode, data_format="channels_last"), input_shape=(NUM_FRAMES, IMG_HEIGHT, IMG_WIDTH, 3), name='Conv')) # shape = (frames, width, height, channel) model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2), name='MaxPooling'))) model.add(TimeDistributed(Flatten(), name='Flatten_1')) model.add(LSTM(20, return_sequences=True, name='LSTM')) model.add(TimeDistributed(Dense(NUM_CLASSES, activation='linear'), name='Dense')) model.add(GlobalAveragePooling1D(name='average')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def FastText(): input_q = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64') q = embedding_raw()(input_q) q = GlobalAveragePooling1D()(q) q = Dropout(DROPOUT_RATE)(q) input_a = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64') a = embedding_raw()(input_a) a = GlobalAveragePooling1D()(a) a = Dropout(DROPOUT_RATE)(a) merged = concatenate([q, a]) merged = Dense(64)(merged) merged = BatchNormalization()(merged) merged = Activation('relu')(merged) merged = Dropout(DROPOUT_RATE)(merged) merged = Dense(1, activation="sigmoid")(merged) model = Model([input_q, input_a], [merged]) return model
def build_model(): desc_input = Input(shape=(length, dim), name='title1') x = CuDNNGRU(3 * dim, return_sequences=True)(desc_input) x = CuDNNGRU(3 * dim, return_sequences=True)(x) x = CuDNNGRU(3 * dim, return_sequences=True)(x) x = CuDNNGRU(3 * dim, return_sequences=True)(x) x = CuDNNGRU(3 * dim, return_sequences=True)(x) x = CuDNNGRU(2 * dim, return_sequences=True)(x) h = Dense(2 * dim, activation='tanh')(x) s = Reshape((1, 2 * dim))(GlobalAveragePooling1D()(x)) a = Dot(axes=-1)([x, s]) a = Reshape((length, 1))(Activation('softmax')(a)) x = Multiply()([x, a]) x = Lambda(lambda z: z * length)(x) # AVG x = GlobalAveragePooling1D()(x) x = Dense(dim, activation=None)(x) output = Dense(dim, activation=None)(x) model = Model(inputs=[desc_input], outputs=[output]) return model
def model_lstm(data, paramdims): ''' LSTM: {}, relu GlobPool ''' input = Input(shape=data['input_1'].shape, name='input_1') layer = LSTM(paramdims[0], return_sequences=True)(input) #layer = Conv1D(paramdims[0], kernel_size=(paramdims[1]), activation = 'relu')(input) output = GlobalAveragePooling1D()(layer) return input, output
def get_capsule_model(): maxlen = 100 learning_rate = 0.00001 input1 = Input(shape=(maxlen, )) embed_layer = Embedding(len(word2index) + 1, EMBEDDING_DIM, weights=[embedding_matrix], input_length=maxlen, trainable=True)(input1) embed_layer = SpatialDropout1D(rate_drop_dense)(embed_layer) x = Bidirectional( GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(embed_layer) # capsule是一个卷积网络 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings, share_weights=True)(x) # output_capsule = Lambda(lambda x: K.sqrt(K.sum(K.square(x), 2)))(capsule) avg_pool = GlobalAveragePooling1D()(capsule) max_pool = GlobalMaxPooling1D()(capsule) capsule_concat = Concatenate()([avg_pool, max_pool]) capsule = Flatten()(capsule) caspule = BatchNormalization()(capsule_concat) capsule = Activation('relu')(capsule) capsule = Dropout(dropout_p)(capsule) output = Dense(30, activation='sigmoid')(capsule) model = Model(inputs=input1, outputs=output) # print(capsule.shape, output.shape) adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0) sgd = SGD(lr=learning_rate, momentum=0.0, decay=0.0, nesterov=False) rmsprop = RMSprop(lr=learning_rate, rho=0.9, epsilon=None, decay=0.0) nadam = keras.optimizers.Nadam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004) model.compile(loss='binary_crossentropy', optimizer=adam, metrics=[f1_score]) return model
def build_bidirectional_model(opts, vocab_size=0, maxnum=50, maxlen=50, embedd_dim=50, embedding_weights=None, verbose=False, init_mean_value=None): N = maxnum L = maxlen logger = get_logger("Build bidirectional model") logger.info("Model parameters: max_sentnum = %d, max_sentlen = %d, embedding dim = %s, lstm_units = %s, drop rate = %s, l2 = %s" % (N, L, embedd_dim, opts.lstm_units, opts.dropout, opts.l2_value)) word_input = Input(shape=(N*L,), dtype='int32', name='word_input') x = Embedding(output_dim=embedd_dim, input_dim=vocab_size, input_length=N*L, weights=embedding_weights, name='x')(word_input) drop_x = Dropout(opts.dropout, name='drop_x')(x) resh_W = Reshape((N, L, embedd_dim), name='resh_W')(drop_x) z_fwd = TimeDistributed(LSTM(opts.lstm_units, return_sequences=True), name='z_fwd')(resh_W) z_bwd = TimeDistributed(LSTM(opts.lstm_units, return_sequences=True, go_backwards=True), name='z_bwd')(resh_W) z_merged = merge([z_fwd, z_bwd], mode='concat', name='z_merged') avg_z = TimeDistributed(GlobalAveragePooling1D(), name='avg_z')(z_merged) hz_fwd = LSTM(opts.lstm_units, return_sequences=True, name='hz_fwd')(avg_z) hz_bwd = LSTM(opts.lstm_units, return_sequences=True, go_backwards=True, name='hz_bwd')(avg_z) hz_merged = merge([hz_fwd, hz_bwd], mode='concat', name='hz_merged') # avg_h = MeanOverTime(mask_zero=True, name='avg_h')(hz) avg_hz = GlobalAveragePooling1D(name='avg_hz')(hz_merged) y = Dense(output_dim=1, activation='sigmoid', name='output')(avg_hz) model = Model(input=word_input, output=y) if opts.init_bias and init_mean_value: logger.info("Initialise output layer bias with log(y_mean/1-y_mean)") bias_value = (np.log(init_mean_value) - np.log(1 - init_mean_value)).astype(K.floatx()) model.layers[-1].bias = bias_value if verbose: model.summary() start_time = time.time() model.compile(loss='mse', optimizer='rmsprop') total_time = time.time() - start_time logger.info("Model compiled in %.4f s" % total_time) return model
def build_model(maxlen, vector_dim, dropout=0.4): """build model Build the model according to the arguments. # Arguments maxlen: The max length of a sample vector_dim: The size of token's dim dropout : the rate of dropout winlen : the length of average pooling windows # Returns model : model just build """ print('Build model...') inputs = Input(shape=(maxlen, vector_dim)) mask_1 = Masking(mask_value=0.0, name='mask_1')(inputs) bgru_1 = Bidirectional(GRU(units=512, activation='tanh', recurrent_activation='hard_sigmoid', return_sequences=True), name='bgru_1')(mask_1) dropout_1 = Dropout(dropout, name='dropout_1')(bgru_1) bgru_2 = Bidirectional(GRU(units=512, activation='tanh', recurrent_activation='hard_sigmoid', return_sequences=True), name='bgru_2')(dropout_1) dropout_2 = Dropout(dropout, name='dropout_2')(bgru_2) dense_1 = TimeDistributed(Dense(1), name='dense1')(dropout_2) activation_1 = Activation('sigmoid', name='activation_1')(dense_1) unmask_1 = NonMasking()(activation_1) vulner_mask_input = Input(shape=(maxlen, maxlen), name='vulner_mask_input') multiply_1 = Multiply(name='multiply_1')([vulner_mask_input, unmask_1]) reshape_1 = Reshape((1, maxlen**2))(multiply_1) k_max_1 = KMaxPooling(k=1, name='k_max_1')(reshape_1) average_1 = GlobalAveragePooling1D(name='average_1')(k_max_1) print("begin compile") model = Model(inputs=[inputs, vulner_mask_input], outputs=average_1) model.compile(loss='binary_crossentropy', optimizer='adamax', metrics=[ 'TP_count', 'FP_count', 'FN_count', 'precision', 'recall', 'fbeta_score' ]) model.summary() return model
def build_model(self): data = self.data inputs = Input(shape=(data.seq_length, )) embeddings = Embedding( data.max_feature, data.embed_dim, weights=[data.embed_matrix], trainable=False)(inputs) x = SpatialDropout1D(self.dropout)(embeddings) x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x) x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x) # x = Bidirectional(CuDNNGRU(data.embed_dim, return_sequences=True))(x) x = TimeDistributed(Dense(100, activation="relu"))(x) attention = AttLayer()(x) avg_pool = GlobalAveragePooling1D()(x) max_pool = GlobalMaxPooling1D()(x) conc = concatenate([avg_pool, max_pool, attention]) encoder = Model(inputs=inputs, outputs=conc) inputs2 = Input(shape=(self.max_sent, data.seq_length, )) x2 = TimeDistributed(encoder)(inputs2) x2 = Bidirectional(CuDNNGRU(100, return_sequences=True))(x2) x2 = Bidirectional(CuDNNGRU(100, return_sequences=True))(x2) # x2 = Bidirectional(CuDNNGRU(data.embed_dim, return_sequences=True))(x2) x2 = TimeDistributed(Dense(100, activation="relu"))(x2) attention2 = AttLayer()(x2) avg_pool2 = GlobalAveragePooling1D()(x2) max_pool2 = GlobalMaxPooling1D()(x2) conc2 = concatenate([avg_pool2, max_pool2, attention2]) x2 = Dense(self.dense_size, activation="relu")(conc2) outputs2 = Dense(6, activation="sigmoid")(x2) model = Model(inputs=inputs2, outputs=outputs2) optimizer = self.get_optimizer(self.lr, self.optim_name) model.compile( loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.model = model
def get_cnn_model(num_amino_acids, max_sequence_size, max_num_functions, target_function): logging.info('Building CNN model using functional API ...') logging.debug("Embedding dims = " + str(embedding_dims)) # no need to mention the num_amino_acids when using functional API input = Input(shape=(max_sequence_size,)) embedding = Embedding(num_amino_acids, embedding_dims, input_length=max_sequence_size)(input) x = Convolution1D(250, 15, activation='relu', subsample_length=1)(embedding) x = Dropout(0.3)(x) x = Convolution1D(100, 15, activation='relu', subsample_length=1)(x) x = Dropout(0.3)(x) # x = ZeroPadding1D((0,20))(x) # # z = Convolution1D(100, 5, subsample_length=1)(embedding) # z = BatchNormalization()(z) # z = Activation('sigmoid')(z) # z = Dropout(0.5)(z) # # z = Convolution1D(50, 5, subsample_length=1)(z) # z = BatchNormalization()(z) # z = Activation('sigmoid')(z) # z = Dropout(0.5)(z) # residual connection # x = merge([x, z], mode='concat') x = GlobalAveragePooling1D()(x) # x = Flatten()(x) if target_function != '': output_nodes = max_num_functions# 2 output_activation = 'sigmoid' # 'softmax' else: output_nodes = max_num_functions output_activation = 'sigmoid' x = Dense(output_nodes)(x) x = BatchNormalization()(x) # can also try to do this after the activation (didn't work) output = Activation(output_activation)(x) model = Model([input], output) model.summary() return model
def BiGRU(X_train, y_train, X_test, y_test, gru_units, dense_units, input_shape, \ batch_size, epochs, drop_out, patience): model = Sequential() reg = L1L2(l1=0.2, l2=0.2) model.add( Bidirectional(GRU(units=gru_units, dropout=drop_out, activation='relu', recurrent_regularizer=reg, return_sequences=True), input_shape=input_shape, merge_mode="concat")) model.add(BatchNormalization()) model.add(TimeDistributed(Dense(dense_units, activation='relu'))) model.add(BatchNormalization()) model.add( Bidirectional(GRU(units=gru_units, dropout=drop_out, activation='relu', recurrent_regularizer=reg, return_sequences=True), merge_mode="concat")) model.add(BatchNormalization()) model.add(Dense(units=1)) model.add(GlobalAveragePooling1D()) print(model.summary()) early_stopping = EarlyStopping(monitor="val_loss", patience=patience) model.compile(loss='mse', optimizer='adam') #date_time = time.strftime('%Y-%m-%d-%H', time.localtime(time.time())) #tensorboard = TensorBoard(log_dir="logs/{}".format(data_time)) #history_callback = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, \ # verbose=2, callbacks=[tensorboard], validation_data=[X_test, y_test], shuffle=True) history_callback = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, \ callbacks=[early_stopping], validation_data=[X_test, y_test], shuffle=True) return model, history_callback
def cnn_melspect_urban_sound(input_shape): kernel_size = 4 # activation_func = LeakyReLU() activation_func = Activation('relu') inputs = Input(input_shape) # Convolutional block_1 conv1 = Conv1D(8, kernel_size)(inputs) act1 = activation_func(conv1) bn1 = BatchNormalization()(act1) pool1 = MaxPooling1D(pool_size=2, strides=2)(bn1) # Convolutional block_2 conv2 = Conv1D(16, kernel_size)(pool1) act2 = activation_func(conv2) bn2 = BatchNormalization()(act2) pool2 = MaxPooling1D(pool_size=2, strides=2)(bn2) # Convolutional block_3 conv3 = Conv1D(32, kernel_size)(pool2) act3 = activation_func(conv3) pool3 = BatchNormalization()(act3) # Convolutional block_4 conv4 = Conv1D(64, kernel_size)(pool3) act4 = activation_func(conv4) bn4 = BatchNormalization()(act4) # Global Layers gmaxpl = GlobalMaxPooling1D()(bn4) gmeanpl = GlobalAveragePooling1D()(bn4) mergedlayer = concatenate([gmaxpl, gmeanpl], axis=1) # Regular MLP dense1 = Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')(mergedlayer) actmlp = activation_func(dense1) reg = Dropout(0.5)(actmlp) dense2 = Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')(reg) actmlp = activation_func(dense2) reg = Dropout(0.5)(actmlp) dense2 = Dense(10, activation='softmax')(reg) model = Model(inputs=[inputs], outputs=[dense2]) return model, 'Kernel size 4 - 8/16/32/64'
def _createModel(self, outDim, vecLen): model = Sequential() #model.add(Masking(mask_value=0., input_shape=(self._phraseLen, vecLen))) model.add(Embedding(vecLen, self._embeddingSize, input_length=self._phraseLen)) model.add(Bidirectional(LSTM(self._lstmCells, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))) model.add(Bidirectional(LSTM(self._lstmCells, dropout=self._dropout, recurrent_dropout=self._dropout, return_sequences=True))) model.add(GlobalAveragePooling1D()) model.add(Dense(outDim, activation='relu')) model.add(Dense(outDim, activation='relu')) model.add(Dense(outDim, activation='softmax')) opt = RMSprop(lr = self._learningRate) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) return model
def Conv1DQuora(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) conv1 = Conv1D(filters=128, kernel_size=1, padding='same', activation='relu') conv2 = Conv1D(filters=128, kernel_size=2, padding='same', activation='relu') conv3 = Conv1D(filters=128, kernel_size=3, padding='same', activation='relu') conv4 = Conv1D(filters=128, kernel_size=4, padding='same', activation='relu') conv5 = Conv1D(filters=32, kernel_size=5, padding='same', activation='relu') conv6 = Conv1D(filters=32, kernel_size=6, padding='same', activation='relu') conv1a = conv1(embed_input_) glob1a = GlobalAveragePooling1D()(conv1a) conv2a = conv2(embed_input_) glob2a = GlobalAveragePooling1D()(conv2a) conv3a = conv3(embed_input_) glob3a = GlobalAveragePooling1D()(conv3a) conv4a = conv4(embed_input_) glob4a = GlobalAveragePooling1D()(conv4a) conv5a = conv5(embed_input_) glob5a = GlobalAveragePooling1D()(conv5a) conv6a = conv6(embed_input_) glob6a = GlobalAveragePooling1D()(conv6a) merge_layer = concatenate([glob1a, glob2a, glob3a, glob4a, glob5a, glob6a]) x = Dropout(0.2)(merge_layer) x = BatchNormalization()(x) x = Dense(256)(x) x = PReLU()(x) x = Dropout(0.2)(x) x = BatchNormalization()(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss='binary_crossentropy', optimizer=params['optimizer'], metrics=['accuracy']) return model
def BuildModelKerasMn(self): start = time.time() model = Sequential() model.add( Bidirectional(LSTM(256, return_sequences=True), input_shape=(self.n_steps, self.n_input))) model.add(GlobalAveragePooling1D()) model.add(Dense(512, activation='relu')) model.add(Dropout(0.3)) model.add(Dense(10, activation='softmax')) model.summary() # compile model.compile(optimizer=Adam(lr=self.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0), loss='categorical_crossentropy', metrics=['accuracy']) # data no_of_samples = np.shape(self.X)[0] step = 1 Loss = [] Elapsed = [] model.fit(self.X, self.Y, epochs=self.training_iters, batch_size=self.batch_size, shuffle=True, validation_data=(self.X, self.Y)) # Calculate accuracy for test data score = model.evaluate(self.testX, self.testY) # Save the model file_suffix = str(self.n_steps) + '_segments_' + str(self.n_input) model_name = 'model_' + file_suffix + '.h5' model.save(model_name) print("Testing Loss:", score[0]) print("Testing Accuracy:", score[1]) elapsed = time.time() - start """ if self.deg_of_logsig==0: plt.plot(Elapsed, Loss, label='Time series') else: plt.plot(Elapsed, Loss, label='degree %d' %self.deg_of_logsig) """ return {'Accuracy': score[1], 'Time': elapsed, 'NStep': step}
def create_conv1d_model(X_train): """ """ model = Sequential() # example here: https://gist.github.com/jkleint/1d878d0401b28b281eb75016ed29f2ee model.add( Conv1D(64, 30, strides=1, padding='valid', kernel_initializer='glorot_normal', activation=None, input_shape=(X_train.shape[1], 1))) model.add(BatchNormalization()) model.add(LeakyReLU()) # https://github.com/fchollet/keras/issues/4403 note on TimeDistributed model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid')) model.add( Conv1D(256, 30, strides=1, padding='valid', kernel_initializer='glorot_normal', activation=None, input_shape=(X_train.shape[1], 1))) model.add(BatchNormalization()) model.add(LeakyReLU()) # https://github.com/fchollet/keras/issues/4403 note on TimeDistributed model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid')) # model.add(Flatten()) # dimensions were too big with this model.add(GlobalAveragePooling1D()) model.add(Dense(256, kernel_initializer='glorot_normal')) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dropout(0.5)) model.add(Dense(128, kernel_initializer='glorot_normal')) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dropout(0.5)) model.add(Dense(1)) # build model using keras documentation recommended optimizer initialization optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) # compile the model model.compile(loss='mean_squared_error', optimizer=optimizer) return model
def build_model(): seq_input1 = Input(shape=(seq_size, dim), name='seq1') seq_input2 = Input(shape=(seq_size, dim), name='seq2') l1 = Conv1D(hidden_dim, 3) r1 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True)) l2 = Conv1D(hidden_dim, 3) r2 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True)) l3 = Conv1D(hidden_dim, 3) r3 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True)) l4 = Conv1D(hidden_dim, 3) r4 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True)) l5 = Conv1D(hidden_dim, 3) r5 = Bidirectional(CuDNNGRU(hidden_dim, return_sequences=True)) l6 = Conv1D(hidden_dim, 3) s1 = MaxPooling1D(2)(l1(seq_input1)) s1 = concatenate([r1(s1), s1]) s1 = MaxPooling1D(2)(l2(s1)) s1 = concatenate([r2(s1), s1]) s1 = MaxPooling1D(3)(l3(s1)) s1 = concatenate([r3(s1), s1]) s1 = l6(s1) s1 = GlobalAveragePooling1D()(s1) s2 = MaxPooling1D(2)(l1(seq_input2)) s2 = concatenate([r1(s2), s2]) s2 = MaxPooling1D(2)(l2(s2)) s2 = concatenate([r2(s2), s2]) s2 = MaxPooling1D(3)(l3(s2)) s2 = concatenate([r3(s2), s2]) s2 = l6(s2) s2 = GlobalAveragePooling1D()(s2) merge_text = multiply([s1, s2]) x = Dense(100, activation='linear')(merge_text) x = keras.layers.LeakyReLU(alpha=0.3)(x) main_output = Dense(1, activation='sigmoid')(x) merge_model = Model(inputs=[seq_input1, seq_input2], outputs=[main_output]) return merge_model
def conv_filt_v2(input_a, no_filters, filt_width): ''' base module for text data Inputs: input_a -- input tensor no_filters -- number of filter maps used in conv layer (details) filt_width -- convolution window width outputs: output tensor of base module ''' # 20000 is the vocabulary size. It should be greater than or equal to your tokenizer vocabulary size embed = Embedding(vocab_size, embed_size)(input_a) conv = Conv1D(no_filters, filt_width, strides=1)(embed) conv = Activation('relu')(conv) conv = AveragePooling1D(pool_size=2)(conv) conv = Dropout(0.5)(conv) conv = GlobalAveragePooling1D()(conv) return conv
def initialize_net(train_params): act = "relu" model = Sequential() # Layer 1 model.add(Conv1D(nb_filter=5, filter_length=10, init='glorot_uniform', border_mode='same', input_shape=(train_params['max_size'], 3), bias=True)) model.add(Activation(act)) model.add(MaxPooling1D(pool_size=2)) # Layer 2 model.add(Conv1D(nb_filter=10, filter_length=20, init='glorot_uniform', border_mode='same', bias=True)) model.add(Activation(act)) model.add(MaxPooling1D(pool_size=2)) # Layer 3 model.add(Conv1D(nb_filter=20, filter_length=20, init='glorot_uniform', border_mode='same', bias=True)) model.add(Activation(act)) model.add(MaxPooling1D(pool_size=2)) model.add(GlobalAveragePooling1D(input_shape=model.output_shape[1:])) model.add(Dense(input_dim=20, output_dim=2, init='glorot_uniform')) model.add(Activation(act)) model.add(Dropout(0.3)) model.add(Dense(input_dim=2, output_dim=2, init='glorot_uniform')) model.add(Activation('softmax')) return model
def LSTMdeep2(params): embed_dropout_rate = 0.1 Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x) if params['bidirectional']: x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x1 = GlobalMaxPooling1D()(x) x2 = GlobalAveragePooling1D()(x) x3 = AttentionWithContext()(x) else: x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = GlobalMaxPooling1D()(x) merge_layer = concatenate([x1, x2, x3]) x = Dropout(params['dropout_rate'])(merge_layer) x = Dense(256)(x) x = PReLU()(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def build_model(opts, overall_maxlen, vocab_size=0, embedd_dim=50, embedding_weights=None, verbose=True, init_mean_value=None): word_input = Input(shape=(overall_maxlen, ), dtype='int32', name='word_input') x = Embedding(output_dim=embedd_dim, input_dim=vocab_size, input_length=overall_maxlen, weights=[embedding_weights], mask_zero=True, name='x')(word_input) x_maskedout = ZeroMaskedEntries(name='x_maskedout')(x) drop_x = Dropout(opts.dropout, name='drop_x')(x_maskedout) position_embedding = Position_Embedding(name='position_embedding')(drop_x) self_att = MutilHeadAttention(nb_head=8, size_per_head=16, name='self_att')(position_embedding) lstm = LSTM(opts.rnn_dim, return_sequences=True, name='lstm')(self_att) avg_pooling = GlobalAveragePooling1D(name='avg_pooling')(lstm) y = Dense(output_dim=1, activation='sigmoid', name='y', W_regularizer=l2(opts.l2_value))(avg_pooling) model = Model(input=[word_input], output=y) if opts.init_bias and init_mean_value: bias_value = (np.log(init_mean_value) - np.log(1 - init_mean_value)).astype(K.floatx()) model.layers[-1].b.set_value(bias_value) if verbose: model.summary() optimize = optimizers.rmsprop(lr=0.001) model.compile(loss='mse', optimizer=optimize) return model
def fGlove_avg(MAX_NB_WORDS, MAX_WORDS, MAX_SENTS, EMBEDDING_DIM, WORDGRU, embedding_matrix, DROPOUTPER): wordInputs = Input(shape=(MAX_SENTS+1, MAX_WORDS), name="wordInputs", dtype='float32') wordInp = Flatten()(wordInputs) wordEmbedding = Embedding(MAX_NB_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], mask_zero=False, trainable=True, name='wordEmbedding')(wordInp) head = GlobalAveragePooling1D()(wordEmbedding) v6 = Dense(1, activation="sigmoid", name="dense")(head) model = Model(inputs=[wordInputs] , outputs=[v6]) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def build_model(opts, vocab_size=0, maxnum=50, maxlen=50, embedd_dim=50, embedding_weights=None, verbose=False, init_mean_value=None): N = maxnum L = maxlen p = Input(shape=(4, 2048), dtype='float32', name='p') # img_vector = Dense(name='img_vector', units=128)(p) word_input = Input(shape=(N * L,), dtype='int32', name='word_input') x = Embedding(output_dim=embedd_dim, input_dim=vocab_size, input_length=N * L, weights=embedding_weights, mask_zero=True, trainable=False, name='x')(word_input) x_maskedout = ZeroMaskedEntries(name='x_maskedout')(x) drop_x = Dropout(opts.dropout, name='drop_x')(x_maskedout) resh_W = Reshape((N, L, embedd_dim), name='resh_W')(drop_x) cnn_e = TimeDistributed(Conv1D(opts.nbfilters, opts.filter1_len, border_mode='valid'), name='cnn_e')(resh_W) att_cnn_e = TimeDistributed(Attention(), name='att_cnn_e')(cnn_e) lstm_e = LSTM(opts.lstm_units, return_sequences=True, name='lstm_e')(att_cnn_e) G = CoAttention(name='essay')([lstm_e, p]) avg = GlobalAveragePooling1D()(G) final_vec_drop = Dropout(rate=0.5, name='final_vec_drop')(avg) if opts.l2_value: logger.info("Use l2 regularizers, l2 value = %s" % opts.l2_value) y = Dense(units=1, activation='sigmoid', name='output', W_regularizer=l2(opts.l2_value))(final_vec_drop) else: y = Dense(units=1, activation='sigmoid', name='output')(final_vec_drop) model = Model(input=[word_input, p], output=y) if opts.init_bias and init_mean_value: logger.info("Initialise output layer bias with log(y_mean/1-y_mean)") bias_value = (np.log(init_mean_value) - np.log(1 - init_mean_value)).astype(K.floatx()) model.layers[-1].b.set_value(bias_value ) if verbose: model.summary() start_time = time.time() model.compile(loss='mse', optimizer='adam') total_time = time.time() - start_time logger.info("Model compiled in %.4f s" % total_time) return model
def _build_network(self, vocab_size, maxlen, embedding_dimension=256, hidden_units=256, trainable=False): print('Build model...') model = Sequential() model.add( Embedding(vocab_size, embedding_dimension, input_length=maxlen, embeddings_initializer='glorot_normal')) model.add( Convolution1D(hidden_units, 2, kernel_initializer='he_normal', padding='valid', activation='sigmoid')) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.25)) model.add( LSTM(hidden_units, kernel_initializer='he_normal', activation='sigmoid', dropout=0.5, recurrent_activation=0.5, unroll=True, return_sequences=True)) model.add(GlobalAveragePooling1D()) model.add(Dropout(0.5)) model.add(Dense(2)) model.add(Activation('softmax')) adam = Adam(lr=0.001) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) print('No of parameter:', model.count_params()) print(model.summary()) return model
def cnn_model_max_and_aver_pool(params): if type(params['kernel_sizes_cnn']) is str: params['kernel_sizes_cnn'] = [ int(x) for x in params['kernel_sizes_cnn'].split(' ') ] inp = Input(shape=(params['text_size'], params['embedding_size'])) outputs = [] for i in range(len(params['kernel_sizes_cnn'])): output_i = Conv1D(params['filters_cnn'], kernel_size=params['kernel_sizes_cnn'][i], activation=None, kernel_regularizer=l2(params['coef_reg_cnn']), padding='same')(inp) output_i = BatchNormalization()(output_i) output_i = Activation('relu')(output_i) output_i_0 = GlobalMaxPooling1D()(output_i) output_i_1 = GlobalAveragePooling1D()(output_i) output_i = Concatenate()([output_i_0, output_i_1]) outputs.append(output_i) output = concatenate(outputs, axis=1) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(params['dense_size'], activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = BatchNormalization()(output) output = Activation('relu')(output) output = Dropout(rate=params['dropout_rate'])(output) output = Dense(nb_classes, activation=None, kernel_regularizer=l2(params['coef_reg_den']))(output) output = BatchNormalization()(output) act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output) model = Model(inputs=inp, outputs=act_output) model.compile( optimizer="Adam", loss="binary_crossentropy", metrics=['accuracy'], #loss_weights=loss_weights, #sample_weight_mode=sample_weight_mode, # weighted_metrics=weighted_metrics, # target_tensors=target_tensors ) return model
def get_model(num_users, num_items, layers=[20, 10], reg_layers=[0, 0]): assert len(layers) == len(reg_layers) num_layer = len(layers) # Number of layers in the MLP # Input variables user_input = Input(shape=(1, ), dtype='int32', name='user_input') item_input = Input(shape=(1, ), dtype='int32', name='item_input') MLP_Embedding_User = Embedding( input_dim=num_users, output_dim=50, name='user_embedding', embeddings_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None), embeddings_regularizer=l2(reg_layers[0]), input_length=1) MLP_Embedding_Item = Embedding( input_dim=num_items, output_dim=50, name='item_embedding', embeddings_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None), embeddings_regularizer=l2(reg_layers[0]), input_length=1) # Crucial to flatten an embedding vector! # user_latent = Flatten()(MLP_Embedding_User(user_input)) # item_latent = Flatten()(MLP_Embedding_Item(item_input)) user_latent = MLP_Embedding_User(user_input) item_latent = MLP_Embedding_Item(item_input) user_latent = Reshape((10, 5))(user_latent) item_latent = Reshape((10, 5))(item_latent) # The 0-th layer is the concatenation of embedding layers vector = concatenate([user_latent, item_latent], axis=1) #vector = K.permute_dimensions(vector, (0, 2, 1)) O_seq = Attention(32, 32)([vector, vector, vector]) # O_seq = Dense(20,activation='Relu')(O_seq) O_seq = GlobalAveragePooling1D()(O_seq) O_seq = Dropout(0.5)(O_seq) outputs = Dense(1, activation='sigmoid')(O_seq) model = Model(inputs=[user_input, item_input], outputs=outputs) return model