def create_LSTM(input_dim, output_dim, time_steps=10, embedding_matrix=[]): batch_size = 1 # inputs.shape = (batch_size, time_steps, input_dim) inputs = Input(shape=(batch_size, time_steps, input_dim)) if embedding_matrix != []: embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], input_shape=(input_dim, ), trainable=False) x = embedding_layer(inputs) x = Reshape([embedding_matrix.shape[1], input_dim])(x) else: x = Reshape([time_steps, input_dim])(inputs) # x = LSTM(200,return_sequences=True)(x) x = Bidirectional( LSTM(100, return_sequences=True, kernel_regularizer=regularizers.l2(0.01)))(x) x = GaussianDropout(0.1)(x) #https://arxiv.org/pdf/1611.07004v1.pdf # x = GaussianNoise(0.05)(x) #https://arxiv.org/pdf/1611.07004v1.pdf x = BatchNormalization()(x) x = Activation('relu')(x) # x = LSTM(200,return_sequences=True)(x) # x = Bidirectional(LSTM(100, return_sequences=True))(x) # x = BatchNormalization()(x) # x = Activation('relu')(x) x = attention_3d_block(x, input_dim=200) # x = GaussianDropout(0.1)(x) #https://arxiv.org/pdf/1611.07004v1.pdf # x = GaussianNoise(0.05)(x) #https://arxiv.org/pdf/1611.07004v1.pdf x = BatchNormalization()(x) x = Activation('relu')(x) # x = LSTM(200,return_sequences=True)(x) # x = Bidirectional(LSTM(100, return_sequences=True))(x) # x = BatchNormalization()(x) # x = Activation('relu')(x) #LSTM OUT # x = LSTM(200)(x) x = Bidirectional(LSTM(150, kernel_regularizer=regularizers.l2(0.01)))(x) x = GaussianDropout(0.1)(x) #https://arxiv.org/pdf/1611.07004v1.pdf # x = GaussianNoise(0.05)(x) #https://arxiv.org/pdf/1611.07004v1.pdf x = BatchNormalization()(x) x = Activation('relu')(x) #NN OUT # x = Flatten()(x) # x = BatchNormalization()(x) # x = Dense(256, activation='tanh')(x) x = Dense(output_dim, activation='tanh', kernel_regularizer=regularizers.l2(0.01))(x) model = Model(input=inputs, output=x) print(model.summary()) return model
def fine_dense(MODEL, preprocess, height, freeze_till, lr, batch, nb_epoch, weights=None): x = Input(shape=(height, height, 3)) x = Lambda(preprocess)(x) base_model = MODEL(include_top=False, input_tensor=x, weights='imagenet') for layer in base_model.layers: layer.trainable = True for layer in base_model.layers[:freeze_till]: layer.trainable = False y = GaussianDropout(0.4)(base_model.output) y = GlobalAveragePooling2D()(y) y = Dense(128, activation='selu', kernel_initializer='he_normal')(y) y = GaussianDropout(0.4)(y) y = Dense(1, activation='sigmoid', kernel_initializer='he_normal')(y) model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning') sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) # print('Trainable: %d, Non-Trainable: %d' % get_params_count(model)) # Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard. log_name = '/Differentiation-EP{epoch:02d}-LOSS{val_loss:.4f}.h5' log_dir = datetime.now().strftime('transfer_model_%Y%m%d_%H%M') if not os.path.exists(log_dir): os.mkdir(log_dir) es = EarlyStopping(monitor='val_loss', patience=20) mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True) tb = TensorBoard(log_dir=log_dir) history = model.fit(x=X_train, y=y_train, batch_size=batch, epochs=nb_epoch, verbose=2, validation_data=(X_val, y_val), callbacks=[es, mc, tb]) with open('log_txt', 'w') as f: f.write(str(history.history)) plt_score(history) trainscore = model.evaluate(x=train, y=trainlabel) testscore = model.evaluate(x=test, y=testlabel) # train_pre = model.predict(x=train) # test_pre = model.predict(x=test) # pd.DataFrame(test_pre).to_excel(output_path) print(trainscore, testscore)
def create_G(input_dim=(100, ), output_dim=(9, 20)): G = Sequential() G.add(Dense(input_shape=input_dim, \ units= 128, \ kernel_initializer=initializers.random_normal(stddev=0.02))) G.add(BatchNormalization()) #G.add(Conv2DTranspose(32, 5, strides=(2,1), activation=Activation('relu'), padding='same',kernel_initializer='glorot_uniform')) # G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf # G.add(GaussianNoise(0.05)) G.add(Activation('relu')) G.add(Dense(128)) G.add(BatchNormalization()) G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf G.add(GaussianNoise(0.05)) G.add(Activation('relu')) G.add( Dense(np.prod(output_dim), kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l2(0.01))) G.add(BatchNormalization()) # G.add(GaussianDropout(0.25)) #https://arxiv.org/pdf/1611.07004v1.pdf # G.add(GaussianNoise(0.05)) G.add(Activation('sigmoid')) G.add(Reshape(output_dim)) return (G)
def sumModelSimple(Inputs,nclasses,nregressions,dropoutRate=0.05,momentum=0.6): x=Inputs[1] globals=Inputs[0] x=BatchNormalization(momentum=momentum)(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(12,kernel_size=(3,3,3),strides=(1,1,1),padding='valid', activation='relu',kernel_initializer='lecun_uniform',name='conv3D_0a')(x) x=Convolution3D(12,kernel_size=(3,3,3),strides=(2,2,2),padding='valid', activation='relu',kernel_initializer='lecun_uniform',name='conv3D_0')(x) x=Convolution3D(16,kernel_size=(5,5,5),strides=(2,2,2),padding='same', activation='relu',kernel_initializer='lecun_uniform',name='conv3D_1')(x) x=BatchNormalization(momentum=momentum)(x) x=Convolution3D(4,kernel_size=(3,3,5),strides=(2,2,3),padding='same', activation='relu',kernel_initializer='lecun_uniform',name='conv3D_2')(x) x=BatchNormalization(momentum=momentum)(x) x=Flatten()(x) x=Concatenate()( [globals,x]) x=Dense(128, activation='relu',kernel_initializer='lecun_uniform')(x) predictID=Dense(nclasses, activation='softmax',kernel_initializer='lecun_uniform',name='ID_pred')(x) predictE=Dense(1, activation='linear',kernel_initializer='lecun_uniform',name='pre_Epred')(x) predictions = [predictID,predictE] model = Model(inputs=Inputs, outputs=predictions) return model
def SynapticNeuronUnit(dendrites, filter_size, kernel_size, CRP, d_rate, use_STR): if CRP[1] == 'UpSampling': dendrites = UpSampling2D(interpolation='bilinear')(dendrites) # Synaptic Transmission Regulator, STR, calculates weight and bias for each channel of input tensor if use_STR: neuro_potential = SynapticTransmissionRegulator()(dendrites) else: neuro_potential = dendrites # Main neural potential if CRP[0] == 'Normal': neuro_potential = Conv2D(filters=filter_size, kernel_size=kernel_size, padding=CRP[2], kernel_initializer='he_uniform', use_bias=False)(neuro_potential) elif CRP[0] == 'Transpose': neuro_potential = Conv2DTranspose(filters=filter_size, kernel_size=kernel_size, padding=CRP[2], kernel_initializer='he_uniform', use_bias=False)(neuro_potential) elif CRP[0] == 'Separable': neuro_potential = SeparableConv2D(filters=filter_size, kernel_size=kernel_size, padding=CRP[2], depthwise_initializer='he_uniform', pointwise_initializer='he_uniform', use_bias=False)(neuro_potential) elif CRP[0] == 'Atrous': neuro_potential = Conv2D(filters=filter_size, kernel_size=kernel_size, strides=2, padding=CRP[2], kernel_initializer='he_uniform', use_bias=False)(neuro_potential) neuro_potential = ZeroPadding2D(padding=((1, 0), (1, 0)))(neuro_potential) else: neuro_potential = None # Will be error neuro_potential = BatchNormalization(momentum=0.95)(neuro_potential) neuro_potential = ParametricSwish()(neuro_potential) # Output potential to axons if CRP[1] == 'MaxPooling': neuro_potential = MaxPooling2D()(neuro_potential) if d_rate[0] > 0.0: neuro_potential = GaussianDropout(rate=d_rate[0])(neuro_potential) if d_rate[1] > 0.0: neuro_potential = SpatialDropout2D(rate=d_rate[1])(neuro_potential) return neuro_potential
def bestModel(Inputs,nclasses,nregressions,dropoutRate=0.05,momentum=0.6): x=Inputs[1] globals=Inputs[0] x=BatchNormalization(momentum=momentum)(x) x=Convolution3D(16,kernel_size=(3,3,3),strides=(1,1,1), activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization(momentum=momentum)(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(16,kernel_size=(3,3,6),strides=(1,1,2), activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization(momentum=momentum)(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(32,kernel_size=(8,8,12),strides=(2,2,2), activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization(momentum=momentum)(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(3,kernel_size=(1,1,1), activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization(momentum=momentum)(x) x = GaussianDropout(dropoutRate)(x) x = Flatten()(x) merged=Concatenate()( [globals,x]) #add the inputs again in case some don't like the multiplications x = Dense(300, activation='relu',kernel_initializer='lecun_uniform')(merged) x=BatchNormalization(momentum=momentum)(x) x = Dropout(dropoutRate)(x) x = Dense(200, activation='relu',kernel_initializer='lecun_uniform')(merged) x=BatchNormalization(momentum=momentum)(x) x = Dropout(dropoutRate)(x) x = Dense(100, activation='relu',kernel_initializer='lecun_uniform')(merged) x=BatchNormalization(momentum=momentum)(x) x = Dropout(dropoutRate)(x) x = Dense(100, activation='relu',kernel_initializer='lecun_uniform')(merged) x=BatchNormalization(momentum=momentum)(x) x = Dropout(dropoutRate)(x) predictID=Dense(nclasses, activation='softmax',kernel_initializer='lecun_uniform',name='ID_pred')(x) predictE=Dense(1, activation='linear',kernel_initializer='zeros',name='E_pred_E')(x) predictions = [predictID,predictE] model = Model(inputs=Inputs, outputs=predictions) return model
def build_model2(): inp1 = Input(shape=(59,14,)) inp2 = Input(shape=(1792,)) out1 = LSTM( # input_dim=14, output_dim=128, init='glorot_uniform', inner_init = 'orthogonal', forget_bias_init='one', # W_regularizer=l2(0.0001), return_sequences=True)(inp1) out1 = GaussianDropout(0.5)(out1) out1_2 = BatchNormalization()(inp1) out1 = merge([out1, out1_2], mode='concat') out1 = LSTM( output_dim=256, init='glorot_uniform', inner_init='orthogonal', forget_bias_init='one', # W_regularizer=l2(0.0001), return_sequences=False)(out1) out1 = GaussianDropout(0.5)(out1) out1 = BatchNormalization()(out1) out2 = BatchNormalization()(inp2) # out2 = Dropout(0.8)(out2) # out2 = Dense(256, W_regularizer=l1(0.01), activity_regularizer=activity_l2(0.01)) out = merge([out1, out2], mode='concat') # out = Dense(256, activation='linear', W_regularizer=l1(0.001))(out) out = BatchNormalization()(out) out = GaussianDropout(0.5)(out) out = Dense(512, activation='relu')(out) out = BatchNormalization()(out) out = Dense(3, activation='softmax')(out) model = Model(input=[inp1, inp2], output=out) start = time.time() model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy','categorical_accuracy','recall']) print "Compilation Time : ", time.time() - start return model
def makecnn(in_shape, K): model = Sequential() model.add( Convolution2D(32, 3, 3, border_mode='same', input_shape=in_shape[1:])) model.add(SReLU()) model.add(MaxPooling2D(pool_size=(2, 2), dim_ordering='tf')) model.add(GaussianNoise(1)) model.add(GaussianDropout(0.4)) model.add(Convolution2D(32, 3, 3, border_mode='same')) model.add(SReLU()) model.add(MaxPooling2D(pool_size=(2, 2), dim_ordering='tf')) model.add(GaussianNoise(1)) model.add(GaussianDropout(0.4)) model.add(Flatten()) model.add(Dense(64)) model.add(SReLU()) model.add(Dense(64)) # model.add(SReLU()) model.add(Dense(1)) model.add(Activation('linear')) return model
def complexModel(Inputs,nclasses,nregressions,dropoutRate=0.05): x=Inputs[1] globals=Inputs[0] x=BatchNormalization()(x) x=Convolution3D(16,kernel_size=(3,3,8),strides=(1,1,2),padding='same', activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(16,kernel_size=(9,9,9),strides=(3,3,3),padding='same', activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(4,kernel_size=(3,3,3),padding='same', activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = GaussianDropout(dropoutRate)(x) x=Convolution3D(4,kernel_size=(1,1,1),padding='same', activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = GaussianDropout(dropoutRate)(x) x = Flatten()(x) merged=Concatenate()( [globals,x]) x = Dense(128, activation='relu',kernel_initializer='lecun_uniform')(merged) x=BatchNormalization()(x) x = Dropout(dropoutRate)(x) x = Dense(128, activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = Dropout(dropoutRate)(x) x = Dense(64, activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = Dropout(dropoutRate)(x) x = Dense(64, activation='relu',kernel_initializer='lecun_uniform')(x) x=BatchNormalization()(x) x = Dropout(dropoutRate)(x) predictID=Dense(nclasses, activation='softmax',kernel_initializer='lecun_uniform',name='ID_pred')(x) predictE=Dense(1, activation='linear',kernel_initializer='lecun_uniform',name='E_pred_E')(x) predictions = [predictID,predictE] model = Model(inputs=Inputs, outputs=predictions) return model
def AutoEncoder(self, dims=[739999, 500, 1000, 20], act='relu', init='uniform', drop_rate=0.3): #TODO should fix """ Fully connected auto-encoder model, symmetric. Arguments: dims: list of number of units in each layer of encoder. dims[0] is input dim, dims[-1] is units in hidden layer. The decoder is symmetric with encoder. So number of layers of the auto-encoder is 2*len(dims)-1 act: activation, not applied to Input, Hidden and Output layers return: (ae_model, encoder_model), Model of autoencoder and model of encoder """ n_stacks = len(dims) - 1 # input input_img = Input(shape=(dims[0], ), name='input') x = input_img x = GaussianDropout(drop_rate)(x) # internal layers in encoder for i in range(n_stacks - 1): x = Dense(dims[i + 1], activation=act, kernel_initializer=init, name='encoder_%d' % i)(x) # hidden layer encoded = Dense( dims[-1], kernel_initializer=init, name='encoder_%d' % (n_stacks - 1))( x) # hidden layer, features are extracted from here x = encoded # internal layers in decoder for i in range(n_stacks - 1, 0, -1): x = Dense(dims[i], activation=act, kernel_initializer=init, name='decoder_%d' % i)(x) # output x = Dense(dims[0], kernel_initializer=init, activation='sigmoid', name='decoder_0')(x) decoded = x self.autoencoder = Model(inputs=input_img, outputs=decoded, name='AE') print(input_img) self.encoder = Model(inputs=input_img, outputs=encoded, name='encoder') self.ae_layer_dims = dims
def dumbestModelEver(Inputs,nclasses,nregressions,dropoutRate=0.05,momentum=0.6): x=Inputs[1] globals=Inputs[0] x=Flatten()(x) x=Concatenate()([globals,x]) x = GaussianDropout(dropoutRate)(x) predictE=Dense(1, activation='linear',kernel_initializer='ones',name='pre_Epred')(x) predictID=Dense(nclasses, activation='softmax',kernel_initializer='lecun_uniform',name='ID_pred')(predictE) predictions = [predictID,predictE] model = Model(inputs=Inputs, outputs=predictions) return model
def rnn(self): start_cr_a_fit_net = time.time() self.split_dataset_rnn() rnn_model = Sequential() # RNN层设计 rnn_model.add( SimpleRNN(15, input_shape=(None, self.look_back), return_sequences=True)) rnn_model.add( SimpleRNN(10, input_shape=(None, self.look_back), return_sequences=True)) # SN层 if self.isdropout: rnn_model.add(SwitchNormalization(axis=-1)) rnn_model.add( SimpleRNN(15, input_shape=(None, self.look_back), return_sequences=True)) rnn_model.add(SimpleRNN(10, input_shape=(None, self.look_back))) rnn_model.add(Dense(1)) # dropout层 if self.isdropout: rnn_model.add(GaussianDropout(0.2)) rnn_model.summary() rnn_model.compile(loss='mean_squared_error', optimizer='adam') rnn_model.fit(self.x_train, self.y_train, epochs=self.epochs, batch_size=self.batch_size, verbose=1) end_cr_a_fit_net = time.time() - start_cr_a_fit_net print( 'Running time of creating and fitting the RNN network: %.2f Seconds' % (end_cr_a_fit_net)) # LSTM prediction/LSTM进行预测 trainPredict = rnn_model.predict( self.x_train) # Predict by training data set/训练集预测 testPredict = rnn_model.predict( self.x_test) # Predict by test data set/测试集预测 return trainPredict, testPredict, self.y_train, self.y_test
def lstm(): model = kr.Sequential() model.add(BatchNormalization(input_shape=(1, 465))) model.add( LSTM(64, return_sequences=True, kernel_initializer='he_normal', use_bias=True, bias_initializer=kr.initializers.one(), unit_forget_bias=True, kernel_regularizer=kr.regularizers.l1_l2(0.001, 0.0001))) model.add(LeakyReLU()) model.add( LSTM(64, return_sequences=False, go_backwards=True, kernel_initializer='he_normal')) model.add(Highway()) model.add(GaussianDropout(0.5)) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Dense(32)) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Highway()) model.add(Dense(64)) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Highway()) model.add(Dense(128)) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Highway()) model.add(Dense(256)) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Dense(1)) sgd = kr.optimizers.sgd(lr=0.1, momentum=0.1, decay=0.001, nesterov=True, clipnorm=3) model.compile(loss='mape', optimizer=sgd, metrics=['mae', 'mse']) return model
def conv_block(x_input, num_filters, pool=True, norm=False, drop_rate=0.0): x1 = Convolution3D(num_filters, 3, 3, 3, border_mode='same', W_regularizer=l2(1e-4))(x_input) if norm: x1 = BatchNormalization(axis=1)(x1) #x1 = Lambda(relu_norm)(x1) if drop_rate > 0.0: x1 = GaussianDropout(drop_rate)(x1) x1 = LeakyReLU(.1)(x1) if pool: x1 = MaxPooling3D()(x1) x_out = x1 return x_out
# vectors using Spark. reshape_transformer = ReshapeTransformer("features_normalized", "matrix", (64, 64, 1)) dataset_train = reshape_transformer.transform(dataset_train) dataset_test = reshape_transformer.transform(dataset_test) ## modle model = Sequential() model.add( Conv2D(filters=32, kernel_size=(2, 2), padding='same', input_shape=(64, 64, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(GaussianDropout(0.3)) model.add( Conv2D(filters=64, kernel_size=(2, 2), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(GaussianDropout(0.3)) model.add( Conv2D(filters=128, kernel_size=(2, 2), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(GaussianDropout(0.3)) model.add( Conv2D(filters=256, kernel_size=(2, 2), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(GaussianDropout(0.3))
ReLU_layers = [ num for num, l in enumerate(model.layers) if l.name.split('_')[0] == Activation('relu').name.split('_')[0] ] for i in ReLU_layers: model.layers[i] = LeakyReLU(alpha=0.1) if i_iter == 5: Dropout_layers = [ num for num, l in enumerate(model.layers) if l.name.split('_')[0] == Dropout(0.25).name.split('_')[0] ] Dropout_amounts = np.array([0.5, 0.5, 0.75]) i_amount = 0 for i in Dropout_layers: model.layers[i] = GaussianDropout(Dropout_amounts[i_amount]) i_amount += 1 # pdb.set_trace() # let's train the model using SGD + momentum (how original). sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) if i_iter == 3: model.compile(loss='categorical_crossentropy', optimizer=adagrad, metrics=["accuracy"]) else: model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=["accuracy"])
def LTSM(df, step=1): tf.Session(config=tf.ConfigProto(log_device_placement=True)) y = df[['Ticket1', 'Ticket2']].as_matrix() x = df.drop(['Ticket1', 'Ticket2'], axis=1).as_matrix() # x= MinMaxScaler((0,1)).fit_transform(x) x = PCA().fit_transform(x) # x = PolynomialFeatures(2).fit_transform(x) x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) x_train = x_train.reshape( (x_train.shape[0], step, int(x_train.shape[1] / step))) x_test = x_test.reshape( (x_test.shape[0], step, int(x_test.shape[1] / step))) print(x_train.shape, y_train.shape, x_test.shape, y_test.shape) ES = EarlyStopping(monitor='loss', patience=5000, verbose=1, mode='min') RS = ReduceLROnPlateau(monitor='loss', patience=50, verbose=1, factor=0.5) model = kr.Sequential() model.add( BatchNormalization(input_shape=(x_train.shape[1], x_train.shape[2]))) model.add(GaussianNoise(0.5)) model.add( LSTM(24, return_sequences=True, kernel_initializer='he_normal', use_bias=True, bias_initializer=kr.initializers.one(), unit_forget_bias=True, kernel_regularizer=kr.regularizers.l1_l2(0.001, 0.0001))) model.add(LeakyReLU()) model.add( LSTM(24, return_sequences=False, go_backwards=True, kernel_initializer='he_normal')) # model.add(LSTM(16, return_sequences=False, go_backwards=True,kernel_initializer='he_normal')) model.add(GaussianDropout(0.5)) # model.add(LeakyReLU()) # model.add(BatchNormalization()) # model.add(Dense(32)) # model.add(LeakyReLU()) # model.add(BatchNormalization()) # model.add(Highway()) # model.add(Dense(64)) # model.add(LeakyReLU()) # model.add(BatchNormalization()) # model.add(Highway()) # model.add(Dense(128)) # model.add(LeakyReLU()) # model.add(BatchNormalization()) # model.add(Highway()) # model.add(Dense(256)) model.add(LeakyReLU()) # model.add(BatchNormalization()) model.add(Dense(2)) sgd = kr.optimizers.sgd(lr=0.1, momentum=0.01, decay=0.001, nesterov=True, clipnorm=3) model.compile(loss='mape', optimizer=sgd, metrics=['mae', 'mse']) his = model.fit(x=x_train, y=y_train, epochs=50000, batch_size=3000, validation_split=0.3, callbacks=[RS, ES], shuffle=True) y_pre = model.predict(x_test, batch_size=3000) plt.plot(his.history['loss'], label='train') plt.plot(his.history['val_loss'], label='test') plt.title('LSTM Performance') plt.xlabel('Epochs') plt.ylabel('MAPE') plt.legend() plt.show() scores = model.evaluate(x_test, y_test, verbose=0, batch_size=3000) r2 = r2_score(y_test, y_pre) mse = mean_squared_error(y_test, y_pre) print(scores) print("LTSM MAPE: %.2f%%" % (scores[2])) print('R2 Score is ', r2) print('MSE is ', mse) return y_pre
conv = Convolution1D(64, 3, activation='relu', init='he_normal') maxpooling = MaxPooling1D() model.add_node(conv, name=conv_name, input=input_name) model.add_node(maxpooling, name=maxpooling_name, input=conv_name) input_name = maxpooling_name for i in range(num_gru): gru_forward_name = 'fw_gru' + str(i) gru_backward_name = 'bw_gru' + str(i) dropout_name = 'dropout' + str(i) return_sequences = i < num_gru - 1 forward = GRU(128, activation='relu', inner_activation='sigmoid', init='he_normal', return_sequences=return_sequences) backward = GRU(128, activation='relu', inner_activation='sigmoid', init='he_normal', return_sequences=return_sequences, go_backwards=True) model.add_node(forward, name=gru_forward_name, input=input_name) model.add_node(backward, name=gru_backward_name, input=input_name) model.add_node(GaussianDropout(0.4), name=dropout_name, inputs=[gru_forward_name,gru_backward_name], merge_mode='sum') input_name = dropout_name for i in range(num_dense): dense_name = 'dense' + str(i) dense = Dense(128, activation='relu', init='he_normal') model.add_node(dense, name=dense_name, input=input_name) input_name = dense_name model.add_node(Dense(1, activation='sigmoid', init='he_normal'), name='dense', input=input_name) model.add_output(name='output', input='dense') model.compile(loss={'output': 'binary_crossentropy'}, optimizer=optimizer) else: for i in range(num_conv):
def fine_dense(MODEL, preprocess, height, freeze_till, lr, batch, nb_epoch, weights=None): x = Input(shape=(height, height, 3)) x = Lambda(preprocess)(x) base_model = MODEL(include_top=False, input_tensor=x, weights='imagenet') for layer in base_model.layers: layer.trainable = True for layer in base_model.layers[:freeze_till]: layer.trainable = False y = GaussianDropout(0.5)(base_model.output) y = GlobalAveragePooling2D()(y) y = Dense(256, activation='relu')( y) # , kernel_initializer='he_normal' , 'selu' y = GaussianDropout(0.5)(y) y = Dense(1, activation='sigmoid')(y) # , kernel_initializer='he_normal' train_datagen = ImageDataGenerator(rotation_range=40, width_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode='nearest', height_shift_range=0.2) validation_datagen = ImageDataGenerator( rotation_range=40, width_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) # rescale=1./255, # test_datagen = ImageDataGenerator(rotation_range=40, width_shift_range=0.2) train_generator = train_datagen.flow(x=X_train, y=y_train, batch_size=batch, shuffle=True) # save_to_dir=r'G:\Radiomics\Py_program\generator_png', save_prefix='train', save_format='png') validation_generator = validation_datagen.flow(x=X_val, y=y_val, batch_size=batch, shuffle=True) # test_generator = test_datagen.flow(x=test, y=np.array(testlabel), batch_size=batch, shuffle=False) model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning') sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) # print('Trainable: %d, Non-Trainable: %d' % get_params_count(model)) model.summary() if weights is not None: model.load_weights(weights) # Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard. log_name = '/REICST-EP{epoch:02d}-LOSS{val_loss:.4f}-AUC{roc_auc_val:.4f}.h5' # log_dir = datetime.now().strftime('transfer_resnet50_model_%Y%m%d_%H%M') if not os.path.exists(log_dir): os.mkdir(log_dir) rc = roc_auc_callback(training_data=(X_train, y_train), validation_data=(X_val, y_val)) es = EarlyStopping(monitor='val_loss', patience=20, verbose=0, mode='min') cl = CSVLogger('keras-5fold-run-01-v1-epochs.log', separator=',', append=False) mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True, verbose=0, mode='min') # val_loss tb = TensorBoard(log_dir=log_dir) history = model.fit_generator(generator=train_generator, epochs=nb_epoch, verbose=2, class_weight='auto', validation_data=validation_generator, callbacks=[rc, es, cl, mc, tb]) with open('log_txt', 'w') as f: f.write(str(history.history)) plt_score(history) plt_aucScore(history) files = os.listdir(log_dir) model.load_weights(os.path.join(log_dir, files[-1])) if print_txt == True: f_result = open('./clf_log.txt', 'a') sys.stdout = f_result print('\n', os.path.join(log_dir, files[-1])) X_trainacc = model.evaluate(x=X_train, y=y_train, verbose=0) X_valacc = model.evaluate(x=X_val, y=y_val, verbose=0) X_trainauc = roc_auc_score(y_train, model.predict(X_train)) X_valauc = roc_auc_score(y_val, model.predict(X_val)) trainacc = model.evaluate(x=train, y=trainlabel, verbose=0) testacc = model.evaluate(x=test, y=testlabel, verbose=0) # testacc = model.evaluate_generator(test_generator, steps=len(test_generator) * 10) trainscore = roc_auc_score(trainlabel, model.predict(train)) testscore = roc_auc_score(testlabel, model.predict(test)) # test_output = model.predict_generator(test_generator, steps=len(test_generator) * 10) # test_output = test_output.reshape((test.shape[0], 10)) # test_pre = np.mean(test_output, axis=1) # testscore = roc_auc_score(testlabel, test_pre) train_pre = model.predict(x=train, batch_size=batch) test_pre = model.predict(x=test, batch_size=batch) print('X_trainauc: %.4f, X_valauc: %.4f' % (X_trainauc, X_valauc)) print('X_trainacc: %.4f, X_valacc: %.4f' % (X_trainacc[1], X_valacc[1])) print('trainauc: %.4f, testauc: %.4f' % (trainscore, testscore)) print('trainacc: %.4f, testacc: %.4f' % (trainacc[1], testacc[1])) return train_pre, test_pre
model.add(Conv2D(24, (5, 5), padding="valid", strides=(2, 2), activation="relu", kernel_initializer="glorot_normal")) # 24x36x158 model.add(Conv2D(36, (5, 5), padding="valid", strides=(2, 2), activation="relu", kernel_initializer="glorot_normal")) # 36x16x77 model.add(Conv2D(48, (5, 5), padding="valid", strides=(2, 2), activation="relu", kernel_initializer="glorot_normal")) # 48x6x37 # since the origianl image was slightly bigger than the one used by nividia # I changed the filter size of layer 4 tom atch the features height at layer 5 model.add(Conv2D(64, (4, 4), padding="valid", activation="relu", kernel_initializer="glorot_normal")) # 64x3x34 model.add(Conv2D(64, (3, 3), padding="valid", activation="relu", kernel_initializer="glorot_normal")) # 64x1x32 model.add(Flatten()) # multiplicative normal noise with mean 1 and STD around 0.5 to prevent overfitting model.add(GaussianDropout(0.25)) model.add(Dense(100, activation="relu", kernel_initializer="glorot_normal")) model.add(Dense(50, activation="relu", kernel_initializer="glorot_normal")) model.add(Dense(10, activation="relu", kernel_initializer="glorot_normal")) model.add(Dense(1, kernel_initializer="glorot_normal")) model.compile(loss='mse', optimizer='adam') history_object = model.fit_generator(train_generator, steps_per_epoch=len(train_samples) / 64, validation_data=validation_generator, validation_steps=len(validation_samples) / 64, epochs=25, verbose=1)
def get_deeplearning_model(reshaped_embed_data, num_classes): """ Get the deep-learning model for scientific articles Parameters ---------- reshaped_embed_data: the reshaped embedding data of raw texts (train or text) num_classes: the number of classes """ print('\nBuilding Deeplearning Model...\n') # Load Shape & Input title_shape = reshaped_embed_data[0][0].shape abstract_shape = reshaped_embed_data[1][0].shape keywords_shape = reshaped_embed_data[2][0].shape # title_input = reshaped_embed_data[0] # abstract_input = reshaped_embed_data[1] # keywords_input = reshaped_embed_data[2] title_input = Input(title_shape, dtype='float32', name='title_input') abstract_input = Input(abstract_shape, dtype='float32', name='abstract_input') keywords_input = Input(keywords_shape, dtype='float32', name='keywords_input') # Title Part title_input_bn = BatchNormalization()(title_input) title_conv_1_1 = Convolution2D(input_shape=title_shape, filters=10, kernel_size=(2, 1), strides=(1, 1), activation='relu', data_format='channels_last')(title_input_bn) title_pool_1_1 = GlobalMaxPooling2D()(title_conv_1_1) title_conv_1_2 = Convolution2D(input_shape=title_shape, filters=10, kernel_size=(3, 1), strides=(1, 1), activation='relu', data_format='channels_last')(title_input) title_pool_1_2 = GlobalMaxPooling2D()(title_conv_1_2) title_merge = keras.layers.concatenate([title_pool_1_1, title_pool_1_2]) title_merge_bn = BatchNormalization()(title_merge) title_output = Dense(64, activation='sigmoid', kernel_regularizer=l2(0.01))(title_merge_bn) # Abstract Part abstract_input_bn = BatchNormalization()(abstract_input) abstract_conv_1_1 = Convolution2D(input_shape=abstract_shape, filters=20, kernel_size=(20, 1), strides=(1, 1), activation='relu', data_format='channels_last')(abstract_input_bn) abstract_pool_1_1 = MaxPooling2D(pool_size=(2, 1))(abstract_conv_1_1) abstract_conv_2_1 = Convolution2D(filters=10, kernel_size=(10, 1), strides=(1, 1), activation='relu', data_format='channels_last')(abstract_pool_1_1) abstract_pool_2_1 = MaxPooling2D(pool_size=(3, 1))(abstract_conv_2_1) abstract_conv_3_1 = Convolution2D(filters=10, kernel_size=(3, 1), strides=(2, 1), activation='relu', data_format='channels_last')(abstract_pool_2_1) abstract_pool_3_1 = GlobalMaxPooling2D()(abstract_conv_3_1) abstract_conv_3_2 = Convolution2D(filters=10, kernel_size=(4, 1), strides=(2, 1), activation='relu', data_format='channels_last')(abstract_pool_2_1) abstract_pool_3_2 = GlobalMaxPooling2D()(abstract_conv_3_2) abstract_conv_3_3 = Convolution2D(filters=10, kernel_size=(5, 1), strides=(2, 1), activation='relu', data_format='channels_last')(abstract_pool_2_1) abstract_pool_3_3 = GlobalMaxPooling2D()(abstract_conv_3_3) abstract_conv_3_4 = Convolution2D(filters=10, kernel_size=(6, 1), strides=(2, 1), activation='relu', data_format='channels_last')(abstract_pool_2_1) abstract_pool_3_4 = GlobalMaxPooling2D()(abstract_conv_3_4) abstract_merge = keras.layers.concatenate([abstract_pool_3_1, abstract_pool_3_2, abstract_pool_3_3, abstract_pool_3_4]) abstract_merge_bn = BatchNormalization()(abstract_merge) abstract_output = Dense(512, activation='sigmoid', kernel_regularizer=l2(0.01))(abstract_merge_bn) # Keywords Part keywords_input_bn = BatchNormalization()(keywords_input) keywords_flatten = Flatten()(keywords_input_bn) keywords_output = Dense(64, activation='sigmoid', kernel_regularizer=l2(0.01))(keywords_flatten) # Merge and Output merge_layer = keras.layers.concatenate([title_output, abstract_output, keywords_output]) merge_layer_bn = BatchNormalization()(merge_layer) dense_1 = Dense(256, activation='sigmoid', kernel_regularizer=l2(0.01))(merge_layer_bn) dropout_1 = GaussianDropout(0.25)(dense_1) dense_2 = Dense(32, activation='sigmoid', kernel_regularizer=l2(0.01))(dropout_1) dropout_2 = GaussianDropout(0.25)(dense_2) total_output = Dense(num_classes, activation='softmax', kernel_regularizer=l2(0.01), name='total_output')(dropout_2) # Print Model Summary dl_model = Model(inputs=[title_input, abstract_input, keywords_input], outputs=[total_output]) print(dl_model.summary()) print('\nDeeplearning Model Building Completed...\n') return dl_model