def build_cnn_layers(layers, lstm_units, ksize, stride_1, filter_num, cnn_layers, single_lstm_layer): conv_stride = 2 padding = "same" dropout = 0.5 pool_size = 2 branch = Sequential() branch.add(Bidirectional(LSTM( lstm_units, return_sequences=True), input_shape = (layers[1], layers[0]))) branch.add(Dropout(dropout/2)) branch.add(Bidirectional(LSTM( lstm_units, return_sequences=True))) branch.add(Dropout(dropout)) branch.add(Conv1D( input_shape = (layers[1], layers[0]), filters = filter_num, kernel_size = ksize, strides = stride_1, padding = padding, activation = None)) BatchNormalization(axis=-1) branch.add(Activation('relu')) branch.add(MaxPooling1D( pool_size = pool_size)) for x in range(1, cnn_layers): branch.add(Conv1D( filters = filter_num*int(math.pow(2,x)), kernel_size = ksize, strides = conv_stride, padding = padding, activation = None)) BatchNormalization(axis=-1) branch.add(Activation('relu')) branch.add(MaxPooling1D( pool_size = pool_size)) if (not single_lstm_layer): branch.add(LSTM( lstm_units, return_sequences = True)) branch.add(Dropout(dropout/2)) branch.add(LSTM( lstm_units, return_sequences = False)) branch.add(Dropout(dropout)) branch.add(Dense( output_dim = 1)) branch.add(Activation('linear')) return branch
def get_model(): inp = Input(shape=(maxlen, )) x_3 = Embedding(max_features, EMBEDDING_DIM)(inp) cnn1 = Conv1D(128, 2, padding='same', strides=1, activation='relu')(x_3) cnn2 = Conv1D(128, 3, padding='same', strides=1, activation='relu')(x_3) cnn = keras.layers.concatenate([cnn1, cnn2], axis=-1) cnn1 = Conv1D(64, 2, padding='same', strides=1, activation='relu')(cnn) cnn1 = MaxPooling1D(pool_size=100)(cnn1) cnn2 = Conv1D(64, 3, padding='same', strides=1, activation='relu')(cnn) cnn2 = MaxPooling1D(pool_size=100)(cnn2) cnn = keras.layers.concatenate([cnn1, cnn2], axis=-1) flat = Flatten()(cnn) drop = Dropout(0.2)(flat) x = Dense(400, kernel_initializer='he_normal')(drop) x = PReLU()(x) x = BatchNormalization()(x) x = Dropout(0.2)(x) x = Dense(6, activation="sigmoid")(x) adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) sgd = keras.optimizers.SGD(lr=0.001) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy', 'binary_crossentropy']) return model
def get_siamese_model(input_shape): # Define the tensors for the two input images left_input = Input(input_shape) right_input = Input(input_shape) # Convolutional Neural Network model = Sequential() model.add( Conv1D(filters=256, kernel_size=50, strides=50, activation='relu', weights=wghts_pretrained[0], padding='same', input_shape=input_shape)) # model.add(MaxPooling1D(pool_size=2)) model.add( Conv1D(filters=128, kernel_size=10, strides=1, activation='relu', weights=wghts_pretrained[1], padding='same')) model.add(MaxPooling1D(pool_size=2)) model.add(Conv1D(filters=128, kernel_size=5, strides=1, activation='sigmoid', weights=wghts_pretrained[2], padding='same')) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) # model.add(Dense((512),activation='sigmoid',kernel_regularizer=l2(1e-3), # kernel_initializer=initialize_weights, bias_initializer=initialize_bias)) # model.add(Dropout(0.25)) encoded_l = model(left_input) encoded_r = model(right_input) # Connect the inputs with the outputs siamese_net = Model(inputs=[left_input, right_input], outputs=last_layer(encoded_l, encoded_r, lyr_name='L2')) ## prediction and cosine_similarity return siamese_net
def make_cnn(loss='categorical_crossentropy', optimizer='adam'): model = Sequential() model.add( Convolution1D(1, 19, subsample_length=19, border_mode='same', input_shape=(n_components, 1))) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Convolution1D(32, 3, border_mode='same', input_shape=(32, 1))) model.add(Activation('relu')) model.add(Convolution1D(32, 3)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Dropout(0.25)) model.add(Convolution1D(64, 3, border_mode='same')) model.add(Activation('relu')) model.add(Convolution1D(64, 3)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(n_cl)) model.add(Activation('softmax')) model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) return model
def design_network(model_name): input1 = Input(shape=(input_length, 768)) if model_name == 'CNN': lcov1 = Conv1D(filters=units, kernel_size=1, activation=relu)(input1) out = MaxPooling1D(pool_size=1)(lcov1) if model_name == 'BiLSTM': out = Bidirectional(LSTM(units))(input1) if 'Bert' in model_name or 'Xlnet' in model_name: convs = [] for fsz in kernel_size: l_conv = Conv1D(filters=units, kernel_size=fsz, activation=relu)(input1) lpool = MaxPooling1D(input_length - fsz + 1)(l_conv) convs.append(lpool) merge = concatenate(convs, axis=1) # reshape = Reshape((units,3))(merge) permute = Permute((2, 1))(merge) if 'Att' in model_name: out = Bidirectional(LSTM(units, return_sequences=True))(permute) out = AttentionLayer(step_dim=units)(out) else: out = Bidirectional(LSTM(units))(permute) out = Dropout(keep_prob)(out) output = Dense(class_nums, activation=softmax)(out) model = Model(input1, output) model.compile(loss=losses.categorical_crossentropy, optimizer=optimizers.Adam(lr=learning_rate), metrics=['accuracy']) model.summary() return model
class Model9(): model = Sequential() model.add( Conv1D(1, 5, padding="same", activation="relu", input_shape=(1024, 1))) model.add(Conv1D(16, 5, padding="same", activation="relu")) model.add(MaxPooling1D(pool_size=5, padding="same")) model.add(Conv1D(16, 5, padding="same", activation="relu")) model.add(Conv1D(32, 5, padding="same", activation="relu")) model.add(MaxPooling1D(pool_size=5, padding="same")) model.add(Conv1D(32, 5, padding="same", activation="relu")) model.add(Conv1D(64, 5, padding="same", activation="relu")) model.add(MaxPooling1D(pool_size=5, padding="same")) model.add(Conv1D(64, 5, padding="same", activation="relu")) model.add(Conv1D(128, 5, padding="same", activation="relu")) model.add(MaxPooling1D(pool_size=5, padding="same")) model.add(Conv1D(128, 5, padding="same", activation="relu")) model.add(Conv1D(256, 5, padding="same", activation="relu")) model.add(MaxPooling1D(pool_size=5, padding="same")) model.add(Dense(256, activation="relu", input_dim=1024)) model.add(Dense(128, activation="relu", input_dim=256)) model.add(Dense(64, activation="relu", input_dim=128)) model.add(Dense(1, activation="sigmoid", input_dim=64)) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
def model_conv_3l_glob(data, paramdims): ''' Conv1D: {} x {}, relu {} pooling {} x {} {} pooling {} Dense GlobPool ''' input = Input(shape=data['input_1'].shape, name='input_1') layer = Conv1D(paramdims[0], kernel_size=(paramdims[1]), activation='relu')(input) layer = BatchNormalization()(layer) layer = MaxPooling1D(pool_size=paramdims[2])(layer) layer = Conv1D(paramdims[3], kernel_size=(paramdims[4]), activation='relu')(layer) layer = BatchNormalization()(layer) layer = MaxPooling1D(pool_size=paramdims[5])(layer) layer = Dense(paramdims[6])(layer) layer = Dropout(0.5)(layer) output = GlobalAveragePooling1D()(layer) return input, output
def make_cnn(loss='mean_absolute_error',optimizer='adam'): model = Sequential() model.add(Convolution1D(1,19,subsample_length=19,border_mode='same',input_shape=(n_components,1))) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Convolution1D(32, 3, border_mode='same',input_shape=(1,32))) model.add(Activation('relu')) model.add(Convolution1D(32, 3)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Dropout(0.25)) model.add(Convolution1D(64, 3, border_mode='same')) model.add(Activation('relu')) model.add(Convolution1D(64, 3)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1)) model.add(Activation('relu')) model.compile(loss=loss, optimizer=optimizer, metrics=['mean_absolute_error']) return model
def architect_model(input_shape, n_classes): inp = Input(shape=input_shape, dtype="float32") x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(inp) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(x) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Conv1D(input_shape[1] * 2, kernel_size=4, padding="causal", activation="relu")(x) x = MaxPooling1D(pool_size=4)(x) x = Dropout(0.25)(x) x = Concatenate()([GlobalAveragePooling1D()(x), GlobalMaxPooling1D()(x)]) # Fully connected x = Dense(2048, activation="relu")(x) x = Dense(2048, activation="relu")(x) out = Dense(n_classes, activation="softmax")(x) return Model(inputs=inp, outputs=out)
def __call__(self, window_size, n_channels=4, regression=False, dense=False): model = Sequential() model.add( Conv1D(64, 7, padding='valid', input_shape=(window_size, n_channels), kernel_regularizer=l2(0.0001))) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Conv1D(64, 3, padding='valid', kernel_regularizer=l2(0.0001))) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Conv1D(64, 3, padding='valid', kernel_regularizer=l2(0.0001))) model.add(Activation('relu')) model.add(MaxPooling1D(2)) model.add(Flatten()) model.add(Dense(64, kernel_regularizer=l2(0.0001))) model.add(Activation('relu')) output_size = window_size if dense else 1 model.add(Dense(output_size, kernel_regularizer=l2(0.0001))) if not regression: model.add(Activation('sigmoid')) return model
def build_model(): seq_input1 = Input(shape=(seq_size, dim), name='seq1') seq_input2 = Input(shape=(seq_size, dim), name='seq2') l1 = Conv1D(hidden_dim, 3) l2 = Conv1D(hidden_dim, 3) l3 = Conv1D(hidden_dim, 3) l4 = Conv1D(hidden_dim, 3) l5 = Conv1D(hidden_dim, 3) l6 = Conv1D(hidden_dim, 3) s1 = GlobalAveragePooling1D()(l6( MaxPooling1D(2)(l5( MaxPooling1D(2)(l4( MaxPooling1D(2)(l3( MaxPooling1D(2)(l2(MaxPooling1D(2)( l1(seq_input1)))))))))))) s2 = GlobalAveragePooling1D()(l6( MaxPooling1D(2)(l5( MaxPooling1D(2)(l4( MaxPooling1D(2)(l3( MaxPooling1D(2)(l2(MaxPooling1D(2)( l1(seq_input2)))))))))))) merge_text = multiply([s1, s2]) x = Dense(hidden_dim, activation='linear')(merge_text) x = keras.layers.LeakyReLU(alpha=0.3)(x) x = Dense(int((hidden_dim + 7) / 2), activation='linear')(x) x = keras.layers.LeakyReLU(alpha=0.3)(x) main_output = Dense(2, activation='softmax')(x) merge_model = Model(inputs=[seq_input1, seq_input2], outputs=[main_output]) return merge_model
def build_model(layers, cnn_layers, lstm_units=200, kernel_size=5, stride_1=2, filter_num=128): # parameters obtained from stock_model.py in Convolutional Neural Stock Market Technical Analyser dropout = 0.5 conv_stride = 2 ksize = kernel_size pool_size = 2 padding = "same" model = Sequential() model.add( Conv1D( input_shape=(layers[1], layers[0]), # (50, 1) filters=filter_num, kernel_size=ksize, strides=conv_stride, padding=padding, activation=None)) BatchNormalization(axis=-1) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=pool_size)) for x in range(1, cnn_layers): model.add( Conv1D(filters=filter_num * 2 * x, kernel_size=ksize, strides=conv_stride, padding=padding, activation=None)) BatchNormalization(axis=-1) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=pool_size)) model.add( GRU(lstm_units, input_shape=(layers[1], layers[0]), activation='tanh', return_sequences=True)) model.add(Dropout(dropout / 2)) model.add(GRU(lstm_units, activation='tanh', return_sequences=False)) model.add(Dropout(dropout)) model.add(Dense(output_dim=1)) # Linear output model.add(Activation("linear")) print(model.summary()) start = time.time() model.compile(loss="mse", optimizer="adadelta") print("> Compilation Time : ", time.time() - start) return model
def build_model(layers, cnn_layers): # parameters obtained from stock_model.py in Convolutional Neural Stock Market Technical Analyser dropout = 0.5 conv_size = 9 conv_stride = 1 ksize = 2 pool_stride = 2 filter_num = 64 padding = "same" model = Sequential() model.add( Conv1D( input_shape=(layers[1], layers[0]), # (50, 1) filters=filter_num, kernel_size=ksize, strides=conv_stride, padding=padding, activation=None)) BatchNormalization(axis=-1) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=2)) for x in range(1, cnn_layers): model.add( Conv1D(filters=filter_num * 2 * x, kernel_size=ksize, strides=conv_stride, padding=padding, activation=None)) BatchNormalization(axis=-1) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=2)) model.add(LSTM( #32, output_dim=layers[1], # 50 return_sequences=True)) model.add(Dropout(dropout / 2)) model.add( LSTM( #16, #layers[2], # 100 layers[1] * 2, # 100 return_sequences=False)) model.add(Dropout(dropout)) model.add(Dense(output_dim=layers[3])) # 1 model.add(Activation("linear")) start = time.time() model.compile(loss="mse", optimizer="adadelta") print("> Compilation Time : ", time.time() - start) return model
def model2(maxlen, batch_size, num_epochs, w2v, traindf, cvdf): def encoder(b): encoded_x, encoded_y = data_init.encode_w2v(df=b, w2v=w2v, maxlen=maxlen) return [[encoded_x, encoded_x, encoded_x], encoded_y] train_gen = batch_generator(df=traindf, encoder=encoder, batch_size=batch_size, force_batch_size=True) cv_gen = batch_generator(df=cvdf, encoder=encoder, batch_size=batch_size) # creates the neural network consisting in 3 convolutional layers going to an LSTM layer model1 = Sequential() model1.add( Convolution1D(50, 6, border_mode='valid', activation='relu', input_shape=(maxlen, 300))) model1.add(MaxPooling1D(pool_length=3, border_mode='valid')) model1.add(Dropout(0.5)) model2 = Sequential() model2.add( Convolution1D(50, 5, border_mode='valid', activation='relu', input_shape=(maxlen, 300))) model2.add(MaxPooling1D(pool_length=3, border_mode='valid')) model2.add(Dropout(0.5)) model3 = Sequential() model3.add( Convolution1D(50, 4, border_mode='valid', activation='relu', input_shape=(maxlen, 300))) model3.add(MaxPooling1D(pool_length=3, border_mode='valid')) model3.add(Dropout(0.5)) model = Sequential() model.add(Merge([model1, model2, model3], mode='concat', concat_axis=1)) model.add(LSTM(60, return_sequences=True)) model.add(Dropout(0.5)) model.add(LSTM(60)) model.add(Dropout(0.5)) model.add(Dense(1, activation='sigmoid')) # compiles the model model.compile('adam', 'binary_crossentropy', metrics=['accuracy', 'mse']) nb_val_samples = len(cvdf) return model, train_gen, cv_gen, nb_val_samples
def cnnModel(inputShape, CNNType): # model construction if CNNType == '1D': inputs = inputShape kernelSize = 3 activFunc = 'relu' #activFunc = LeakyReLU() model = Sequential() model.add(Conv1D(32, kernelSize, input_shape=inputs)) model.add(Activation(activFunc)) model.add(BatchNormalization()) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(64, kernelSize)) model.add(Activation(activFunc)) model.add(BatchNormalization()) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(128, kernelSize)) model.add(Activation(activFunc)) model.add(BatchNormalization()) model.add(GlobalMaxPooling1D()) model.add( Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')) model.add(Activation(activFunc)) model.add(Dropout(0.5)) model.add( Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')) model.add(Activation(activFunc)) model.add(Dropout(0.5)) model.add(Dense(10, activation=tf.nn.softmax)) loss = keras.losses.categorical_crossentropy # loss=keras.losses.binary_crossentropy # Model optimizers and compilers #adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-5) sgd = keras.optimizers.SGD(lr=0.001, decay=1e-5, momentum=0.9, nesterov=True) model.compile(loss=loss, optimizer=sgd, metrics=['accuracy']) cnn = model earlystop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=0, mode='auto') return cnn, earlystop
def buildSplitModel(numClasses, seqLen, vecDim, contextDim, eventMap): """ Builds a GRU model on top of word embeddings, and doc2vec """ hidden = 512 rnnHidden = 128 denseDim = 256 cl2 = .001 drop = .5 convSize = 2 maxSize = 2 shape = (seqLen, vecDim) left = Sequential() #left.add(GRU(rnnHidden, W_regularizer=l2(cl2), U_regularizer=l2(cl2), dropout_W=drop, dropout_U=drop, input_shape=shape, return_sequences=True)) left.add(Conv1D(denseDim, convSize, W_regularizer=l2(cl2), input_shape=shape)) left.add(MaxPooling1D(maxSize)) #left.add(Conv1D(denseDim, convSize, W_regularizer=l2(cl2))) #left.add(MaxPooling1D(convSize)) left.add(GRU(rnnHidden, W_regularizer=l2(cl2), U_regularizer=l2(cl2), dropout_W=drop, dropout_U=drop)) right = Sequential() #right.add(GRU(rnnHidden, W_regularizer=l2(cl2), U_regularizer=l2(cl2), dropout_W=drop, dropout_U=drop, input_shape=shape, return_sequences=True)) right.add(Conv1D(denseDim, convSize, W_regularizer=l2(cl2), input_shape=shape)) right.add(MaxPooling1D(maxSize)) #right.add(Conv1D(denseDim, convSize, W_regularizer=l2(cl2))) #right.add(MaxPooling1D(convSize)) right.add(GRU(rnnHidden, W_regularizer=l2(cl2), U_regularizer=l2(cl2), dropout_W=drop, dropout_U=drop)) context = Sequential() context.add(Dense(denseDim, input_shape=(contextDim,))) context.add(LeakyReLU(.01)) context.add(Dropout(drop)) #do nothing #context.add(Reshape((contextDim,), input_shape=(contextDim,))) model = Sequential() model.add(Merge([left, right, context], mode="concat")) model.add(Dense(denseDim, W_regularizer=l2(cl2))) model.add(LeakyReLU(.01)) #model.add(MaxoutDense(denseDim, W_regularizer=l2(cl2))) model.add(Dropout(drop)) model.add(Dense(numClasses)) model.add(Activation("softmax")) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', microF1(eventMap)]) return model
def create_model2(num_task, input_len_l, input_len_r): K.clear_session() tf.set_random_seed(5005) left_dim = 4 right_dim = 4 num_units = 50 input_l = input_len_l input_r = input_len_r nb_f_l = [90, 100] f_len_l = [7, 7] p_len_l = [4, 10] s_l = [2, 5] nb_f_r = [90, 100] f_len_r = [7, 7] p_len_r = [10, 10] s_r = [5, 5] left_input = Input(shape=(input_l, left_dim), name="left_input") right_input = Input(shape=(input_r, right_dim), name="right_input") left_conv1 = Conv1D(filters=nb_f_l[0], kernel_size=f_len_l[0], padding='valid', activation="relu", name="left_conv1")(left_input) left_pool1 = MaxPooling1D(pool_size=p_len_l[0], strides=s_l[0], name="left_pool1")(left_conv1) left_drop1 = Dropout(0.25, name="left_drop1")(left_pool1) right_conv1 = Conv1D(filters=nb_f_r[0], kernel_size=f_len_r[0], padding='valid', activation="relu", name="right_conv1")(right_input) right_pool1 = MaxPooling1D(pool_size=p_len_r[0], strides=s_r[0], name="right_pool1")(right_conv1) right_drop1 = Dropout(0.25, name="right_drop1")(right_pool1) merge = concatenate([left_drop1, right_drop1], name="merge", axis=-2) conv_merged = Conv1D(filters=100, kernel_size=5, padding='valid', activation="relu", name="conv_merged")(merge) #merged_pool = MaxPooling1D(pool_size=4, strides=2)(conv_merged) merged_pool = MaxPooling1D(pool_size=10, strides=5)(conv_merged) merged_drop = Dropout(0.25)(merged_pool) merged_flat = Flatten()(merged_drop) hidden1 = Dense(250, activation='relu', name="hidden1")(merged_flat) output = Dense(num_task, activation='sigmoid', name="output")(hidden1) model = Model(inputs=[left_input, right_input], outputs=output) print(model.summary()) return model
def get_model(numLabels, numConvLayers, numConvFilters, preLastLayerUnits, poolingDropout, learningRate, momentum, length): model = Sequential() l1_reg = 0.00001 l2_reg = 0.00001 l3_reg = 0.00001 l4_reg = 0.00001 dropout = 0.2 #0.2 filter1 = 500 filter2 = 250 filter3 = 100 conv1_layer = Conv1D(filters=filter1, kernel_size=8, input_shape=(length, 4), padding="valid", activation="relu", use_bias=True, kernel_regularizer=l2(l1_reg)) # use_bias=True) model.add(conv1_layer) model.add(MaxPooling1D(pool_size=4)) model.add(Dropout(dropout)) convn_layer = Conv1D(padding="valid", activation="relu", kernel_size=4, filters=filter2, use_bias=True, kernel_regularizer=l2(l2_reg)) # use_bias=True) model.add(convn_layer) model.add(MaxPooling1D(pool_size=4)) model.add(Dropout(dropout)) convn_layer = Conv1D(padding="valid", activation="relu", kernel_size=4, filters=filter3, use_bias=True, kernel_regularizer=l2(l3_reg)) # use_bias=True) model.add(convn_layer) model.add(MaxPooling1D(pool_size=4)) model.add(Dropout(dropout)) model.add(Flatten()) model.add( Dense(units=numLabels, activation='relu', use_bias=True, kernel_regularizer=l2(l4_reg))) print("Regularization values: ", l1_reg, l2_reg, l3_reg, l4_reg) print("Dropout values: ", dropout) print("Filter values: ", filter1, filter2, filter3) return model
def createModel(label_num=3000): beishu = 2 input_data = Input(name='features', shape=(None, 39)) x1 = Conv1D(40 * beishu, 1, padding='same', activation='relu')(input_data) x1 = Conv1D(40 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(40 * beishu, 3, padding='same')(x1) x1 = BatchNormalization()(x1) x1 = keras.layers.ReLU()(x1) x1 = MaxPooling1D(2)(x1) x1 = Conv1D(80 * beishu, 1, padding='same', activation='relu')(x1) x1 = Conv1D(80 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(80 * beishu, 3, padding='same')(x1) x1 = BatchNormalization()(x1) x1 = keras.layers.ReLU()(x1) x1 = MaxPooling1D(2)(x1) x1 = Conv1D(160 * beishu, 1, padding='same', activation='relu')(x1) x1 = Conv1D(160 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(160 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(160 * beishu, 3, padding='same')(x1) # x1 = MaxPooling1D(1)(x1) x1 = BatchNormalization()(x1) x1 = keras.layers.ReLU()(x1) x1 = MaxPooling1D(2)(x1) x1 = Conv1D(320 * beishu, 1, padding='same', activation='relu')(x1) x1 = Conv1D(320 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(320 * beishu, 3, padding='same', activation='relu')(x1) x1 = Conv1D(320 * beishu, 3, padding='same')(x1) # x1 = MaxPooling1D(1)(x1) x1 = BatchNormalization()(x1) x1 = keras.layers.ReLU()(x1) # # with tf.device('/cpu'): # x1 = GRU(100, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', dropout=0.3)(x1) # x2 = GRU(100, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', dropout=0.3)(x1) # x3 = add([x1, x2]) x4 = Dense(1024, activation='relu')(x1) x4 = Dropout(0.3)(x4) preds = Dense(label_num)(x4) preds = Activation('softmax', name='Activation0')(preds) model_data = Model(input_data, outputs=preds) # ctc labels = Input(name='labels', shape=[None], dtype='float32') input_len = Input(name='input_len', shape=[1], dtype='int64') label_len = Input(name='label_len', shape=[1], dtype='int64') loss_out = Lambda(ctc_lambda, output_shape=(1, ), name='ctc')([labels, preds, input_len, label_len]) model = Model(inputs=[input_data, labels, input_len, label_len], outputs=loss_out) model.summary() adam = Adam(lr=0.01) model.compile(loss={'ctc': lambda y_true, output: output}, optimizer=adam) print('model compile over') return model, model_data
def create_model(tokenizer=None): ## with tfidf input feat_nr = len(tokenizer.word_counts)+1 S, F = 30, 300 x1_input = Input(shape=(max_features,), dtype='int32', name='x1_input') x2_input = Input(shape=(max_features,), dtype='int32', name='x2_input') x1 = Embedding(input_dim=feat_nr, output_dim=64, trainable=False)(x1_input) x2 = Embedding(input_dim=feat_nr, output_dim=64, trainable=False)(x2_input) x1 = BatchNormalization()(x1) x1 = Conv1D(128, 3, activation='relu')(x1) x1 = MaxPooling1D(3)(x1) x2 = BatchNormalization()(x2) x2 = Conv1D(128, 3, activation='relu')(x2) x2 = MaxPooling1D(3)(x2) x = dot([x1, x2], -1, normalize=True) # x = Conv1D(128, 3, activation='relu')(x) # x = MaxPooling1D(3)(x) # x = Conv1D(128, 3, activation='relu')(x) # x = MaxPooling1D(3)(x) # x = Conv1D(128, 3, activation='relu')(x) # x = MaxPooling1D(3)(x) x = LSTM(128)(x) # x = Flatten()(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, kernel_initializer='uniform', activation='relu')(x) x = Dropout(0.3)(x) y = Dense(2, kernel_initializer='uniform', activation='softmax', name='output')(x) model = Model(inputs=[x1_input, x2_input], outputs=[y], name='final') # sgd = SGD(lr=learning_rate) opt = RMSprop(lr=learning_rate, rho=0.9, epsilon=1e-08, decay=0.00000000001) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['acc','sparse_categorical_accuracy', 'binary_accuracy']) model.summary() plot_model(model, to_file='{}.png'.format(model_path), show_shapes=True, show_layer_names=True) print('model_path:{} steps:{} epochs:{}/{} batch_size:{} max_features:{}'.format( model_path, steps_per_epoch, init_epoch, total_epochs, batch_size, max_features)) return model
def create_model4(num_task, input_len_l, input_len_r): K.clear_session() tf.set_random_seed(5005) left_dim = 4 right_dim = 4 num_units = 60 input_l = input_len_l input_r = input_len_r nb_f_l = [90, 100] f_len_l = [7, 7] p_len_l = [4, 10] s_l = [2, 5] nb_f_r = [90, 100] f_len_r = [7, 7] p_len_r = [10, 10] s_r = [5, 5] left_input = Input(shape=(input_l, left_dim), name="left_input") right_input = Input(shape=(input_r, right_dim), name="right_input") left_conv1 = Conv1D(filters=nb_f_l[0], kernel_size=f_len_l[0], padding='valid', activation="relu", name="left_conv1")(left_input) left_pool1 = MaxPooling1D(pool_size=p_len_l[0], strides=s_l[0], name="left_pool1")(left_conv1) left_drop1 = Dropout(0.25, name="left_drop1")(left_pool1) right_conv1 = Conv1D(filters=nb_f_r[0], kernel_size=f_len_r[0], padding='valid', activation="relu", name="right_conv1")(right_input) right_pool1 = MaxPooling1D(pool_size=p_len_r[0], strides=s_r[0], name="right_pool1")(right_conv1) right_drop1 = Dropout(0.25, name="right_drop1")(right_pool1) merge = concatenate([left_drop1, right_drop1], name="merge", axis=-2) gru = Bidirectional(GRU(num_units, return_sequences=True), name="gru")(merge) #gru = Bidirectional(GRU(num_units),return_sequences=True,name="gru")(merged) flat = Flatten(name="flat")(gru) hidden1 = Dense(250, activation='relu', name="hidden1")(flat) output = Dense(num_task, activation='sigmoid', name="output")(hidden1) model = Model(inputs=[left_input, right_input], outputs=output) print(model.summary()) return model
def Model(): model = Sequential() model.add(Conv1D(128, 5, activation='relu', input_shape=(300, 100))) model.add(MaxPooling1D(2)) model.add(Conv1D(256, 5, activation='relu')) model.add(MaxPooling1D(2)) model.add(Conv1D(512, 5, activation='relu')) model.add(LSTM(512)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc', f1score]) return model
def make_model_cnn(dir_name): mhc_in = Input(shape=(34, 20)) mhc_branch = Conv1D(32, 5, kernel_initializer="he_uniform")(mhc_in) mhc_branch = PReLU()(mhc_branch) mhc_branch = Conv1D(32, 3, kernel_initializer="he_uniform")(mhc_branch) mhc_branch = PReLU()(mhc_branch) mhc_branch = MaxPooling1D(pool_size=POOL_SIZE)(mhc_branch) mhc_branch = Dropout(.2)(mhc_branch) pep_in = Input(shape=(9, 20)) pep_branch = Conv1D(32, 5, kernel_initializer="he_uniform")(pep_in) pep_branch = PReLU()(pep_branch) pep_branch = Conv1D(32, 3, kernel_initializer="he_uniform")(pep_branch) pep_branch = PReLU()(pep_branch) pep_branch = MaxPooling1D(pool_size=POOL_SIZE)(pep_branch) pep_branch = Dropout(.2)(pep_branch) mhc_branch = Flatten()(mhc_branch) pep_branch = Flatten()(pep_branch) merged = concatenate([pep_branch, mhc_branch]) merged = Dense(128, kernel_initializer="he_uniform")(merged) merged = Dropout(.3)(merged) merged = PReLU()(merged) merged = Dense(64, kernel_initializer="he_uniform")(merged) merged = Dropout(.3)(merged) merged = PReLU()(merged) merged = Dense(16, kernel_initializer="he_uniform")(merged) merged = Dropout(.3)(merged) merged = PReLU()(merged) merged = Dense(8, kernel_initializer="he_uniform")(merged) merged = Dropout(.3)(merged) merged = PReLU()(merged) merged = Dense(1)(merged) pred = PReLU()(merged) model = Model([mhc_in, pep_in], pred) model.compile(loss='mse', optimizer="nadam") with open(dir_name + "model.json", "w") as outf: outf.write(model.to_json()) return model
def regression_model(input_dim, emb_mat, seq_len, conv_layers=1, conv_filters=32, filter_size=3, lstm_dim=32, fc_layers=1, fc_units=64, dropout=0.0, metrics=[]): """ Compiles a model that learns representations from convolutional and recurrent layers. These representations are combined with auxiliary input, informing about the tweet context. """ seqs = Input(shape=(seq_len, ), dtype='int32', name='text_input') emb = Embedding(emb_mat.shape[0], emb_mat.shape[1], weights=[emb_mat], input_length=seq_len, trainable=True, name='word_embedding')(seqs) lstm = LSTM(lstm_dim, name='lstm_1')(emb) x = ZeroPadding1D(name='pad_1')(emb) x = Conv1D(conv_filters, filter_size, activation='relu', name='conv_1')(x) x = MaxPooling1D(name='pool_1')(x) for i in range(2, conv_layers + 1): pad_name = 'pad_' + str(i) conv_name = 'conv_' + str(i) pool_name = 'pool_' + str(i) x = ZeroPadding1D(name=pad_name)(x) x = Conv1D(conv_filters, filter_size, activation='relu', name=conv_name)(x) x = MaxPooling1D(name=pool_name)(x) flatten = Flatten(name='flatten')(x) aux_input = Input(shape=(input_dim, ), name='aux_input') norm_inputs = BatchNormalization(name='bn_aux')(aux_input) x = concatenate([flatten, lstm, norm_inputs], name='comb_input') x = Dropout(dropout, name='dropout_1')(x) for i in range(1, fc_layers + 1): fc_name = 'fc_' + str(i) bn_name = 'bn_' + str(i) x = Dense(fc_units, activation='relu', name=fc_name)(x) x = BatchNormalization(name=bn_name)(x) output = Dense(1, activation='relu', name='output')(x) model = Model(inputs=[seqs, aux_input], outputs=[output]) model.compile(optimizer='Adam', loss='mean_squared_error', metrics=metrics) return model
def cnn_melspect_urban_sound(input_shape): kernel_size = 4 # activation_func = LeakyReLU() activation_func = Activation('relu') inputs = Input(input_shape) # Convolutional block_1 conv1 = Conv1D(8, kernel_size)(inputs) act1 = activation_func(conv1) bn1 = BatchNormalization()(act1) pool1 = MaxPooling1D(pool_size=2, strides=2)(bn1) # Convolutional block_2 conv2 = Conv1D(16, kernel_size)(pool1) act2 = activation_func(conv2) bn2 = BatchNormalization()(act2) pool2 = MaxPooling1D(pool_size=2, strides=2)(bn2) # Convolutional block_3 conv3 = Conv1D(32, kernel_size)(pool2) act3 = activation_func(conv3) pool3 = BatchNormalization()(act3) # Convolutional block_4 conv4 = Conv1D(64, kernel_size)(pool3) act4 = activation_func(conv4) bn4 = BatchNormalization()(act4) # Global Layers gmaxpl = GlobalMaxPooling1D()(bn4) gmeanpl = GlobalAveragePooling1D()(bn4) mergedlayer = concatenate([gmaxpl, gmeanpl], axis=1) # Regular MLP dense1 = Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')(mergedlayer) actmlp = activation_func(dense1) reg = Dropout(0.5)(actmlp) dense2 = Dense(512, kernel_initializer='glorot_normal', bias_initializer='glorot_normal')(reg) actmlp = activation_func(dense2) reg = Dropout(0.5)(actmlp) dense2 = Dense(10, activation='softmax')(reg) model = Model(inputs=[inputs], outputs=[dense2]) return model, 'Kernel size 4 - 8/16/32/64'
def create_conv1d_model(X_train): """ """ model = Sequential() # example here: https://gist.github.com/jkleint/1d878d0401b28b281eb75016ed29f2ee model.add( Conv1D(64, 30, strides=1, padding='valid', kernel_initializer='glorot_normal', activation=None, input_shape=(X_train.shape[1], 1))) model.add(BatchNormalization()) model.add(LeakyReLU()) # https://github.com/fchollet/keras/issues/4403 note on TimeDistributed model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid')) model.add( Conv1D(256, 30, strides=1, padding='valid', kernel_initializer='glorot_normal', activation=None, input_shape=(X_train.shape[1], 1))) model.add(BatchNormalization()) model.add(LeakyReLU()) # https://github.com/fchollet/keras/issues/4403 note on TimeDistributed model.add(MaxPooling1D(pool_size=2, strides=2, padding='valid')) # model.add(Flatten()) # dimensions were too big with this model.add(GlobalAveragePooling1D()) model.add(Dense(256, kernel_initializer='glorot_normal')) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dropout(0.5)) model.add(Dense(128, kernel_initializer='glorot_normal')) model.add(BatchNormalization()) model.add(LeakyReLU()) model.add(Dropout(0.5)) model.add(Dense(1)) # build model using keras documentation recommended optimizer initialization optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) # compile the model model.compile(loss='mean_squared_error', optimizer=optimizer) return model
def create_cnn_model(l_seq): inputs = Input(shape=(l_seq, in_out_neurons)) x = Conv1D(32, 3, activation='relu', padding='valid')(inputs) x = Conv1D(32, 3, activation='relu', padding='valid')(x) x = MaxPooling1D(pool_size=2)(x) x = Conv1D(64, 3, activation='relu', padding='valid')(inputs) x = Conv1D(64, 3, activation='relu', padding='valid')(x) x = MaxPooling1D(pool_size=2)(x) x = Flatten()(x) x = Dense(128, activation='relu')(x) x = Dense(64, activation='relu')(x) predictions = Dense(in_out_neurons, activation='linear')(x) model = Model(input=inputs, output=predictions) model.compile(loss="mean_squared_error", optimizer="adam") return model
def model(self): input_tensor = Input(shape=self.get_input_shape(), name='input') cnn_layer_1 = Conv1D(kernel_initializer=self.initializer, activation=self.activation, padding="same", filters=self.num_filters[0], kernel_size=self.filter_len, kernel_regularizer=l2(self.l2), bias_regularizer=l2(self.l2))(input_tensor) cnn_layer_2 = Conv1D(kernel_initializer=self.initializer, activation=self.activation, padding="same", filters=self.num_filters[1], kernel_size=self.filter_len, kernel_regularizer=l2(self.l2), bias_regularizer=l2(self.l2))(cnn_layer_1) maxpool_1 = MaxPooling1D(pool_size=self.pool_length)(cnn_layer_2) flattener = Flatten()(maxpool_1) layer_3 = Dense(units=self.num_hidden[0], kernel_initializer=self.initializer, activation=self.activation, kernel_regularizer=l2(self.l2), bias_regularizer=l2(self.l2))(flattener) layer_4 = Dense(units=self.num_hidden[1], kernel_initializer=self.initializer, activation=self.activation, kernel_regularizer=l2(self.l2), bias_regularizer=l2(self.l2))(layer_3) mu, sigma = GaussianLayer(len(self.targetname), name='main_output')(layer_4) model = Model(inputs=input_tensor, outputs=mu) return model, sigma
def VDCNN_model(input_shape, num_classes, num_words, emb_size, emb_matrix, num_filters=num_filters_default, top_k=8, emb_trainable=False): inputs = Input(shape=(input_shape,), dtype='int32', name='inputs') embedded_sent = Embedding(num_words, emb_size, weights=[emb_matrix], trainable=emb_trainable, name='embs')(inputs) conv = Conv1D(filters=64, kernel_size=3, strides=2, padding="same")( embedded_sent) for i in range(len(num_filters)): conv = ConvBlockVDCNN(conv.get_shape().as_list()[1:], num_filters[i])( conv) conv = MaxPooling1D(pool_size=3, strides=2, padding="same")(conv) def k_max_pooling(x): x = tf.transpose(x, [0, 2, 1]) k_max = tf.nn.top_k(x, k=top_k) return tf.reshape(k_max[0], (-1, num_filters[-1] * top_k)) k_max = Lambda(k_max_pooling, output_shape=(num_filters[-1] * top_k,))( conv) # fully-connected layers fc1 = Dropout(0.2)( Dense(4096, activation='relu', kernel_initializer='he_normal')(k_max)) fc2 = Dropout(0.2)( Dense(2048, activation='relu', kernel_initializer='he_normal')(fc1)) fc3 = Dense(num_classes, activation='softmax')(fc2) model = Model(inputs=inputs, outputs=fc3) return model
def build_sentence_model(input_set_size, vector_size, maxlen): word_inputs = Input(shape=(maxlen, ), dtype='int32') embed = Embedding(input_set_size, vector_size, input_length=maxlen)(word_inputs) # 1 conv conv1_1 = Convolution1D(128, 3, border_mode='same', activation='relu')(embed) bn1 = BatchNormalization(mode=1)(conv1_1) pool1 = MaxPooling1D(pool_length=2)(bn1) drop1 = Dropout(0.3)(pool1) ''' # 2 conv conv2_1 = Convolution1D(128, 3, border_mode='same', activation='relu')(drop1) bn2 = BatchNormalization(mode=1)(conv2_1) pool2 = MaxPooling1D(pool_length=2)(bn2) drop2 = Dropout(0.2)(bn2) ''' #gru 256 bgru = Bidirectional(GRU(192, return_sequences=False), merge_mode='sum')(drop1) drop = Dropout(0.5)(bgru) drop_3d = Reshape((192, 1))(drop) att = TimeDistributed(Dense(1))(drop_3d) att = Activation(activation="softmax")(att) merg1 = Flatten()(merge([drop_3d, att], mode='mul')) sentence_model = Model(input=[word_inputs], output=merg1) sentence_model.summary() return sentence_model