def MCNN(trainX1,trainX2,trainY1,valX1,valX2,valY1,input_1,input_2, i,class_weights): onehot_secstr = conv.Conv1D(5, 10, kernel_initializer='glorot_normal',kernel_regularizer=l2(0.04), padding='valid', name='0_secstr')(input_1) onehot_secstr = Dropout(0.6)(onehot_secstr) onehot_secstr = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_secstr) onehot_secstr = core.Flatten()(onehot_secstr) onehot_secstr2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal',kernel_regularizer=l2(0.02), padding='valid', name='1_secstr')(input_1) onehot_secstr2 = Dropout(0.4)(onehot_secstr2) onehot_secstr2 = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_secstr2) onehot_secstr2 = core.Flatten()(onehot_secstr2) output_onehot_sec = concatenate([onehot_secstr, onehot_secstr2], axis=-1) onehot_x = conv.Conv1D(5, 10, kernel_initializer='glorot_normal',kernel_regularizer=l2(0.04), padding='valid', name='0')(input_2) onehot_x = Dropout(0.6)(onehot_x) onehot_x = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_x) onehot_x = core.Flatten()(onehot_x) onehot_x2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal',kernel_regularizer=l2(0.02), padding='valid', name='1')(input_2) onehot_x2 = Dropout(0.4)(onehot_x2) onehot_x2 = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_x2) onehot_x2 = core.Flatten()(onehot_x2) output_onehot_seq = concatenate([onehot_x, onehot_x2], axis=-1) final_output = concatenate([output_onehot_sec, output_onehot_seq]) dense_out = Dense(100, kernel_initializer='glorot_normal', activation='softplus', name='dense_concat')(final_output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal', name='6')(dense_out) ########## Set Net ########## cnn = Model(inputs=[input_1,input_2], outputs=out) cnn.summary() nadam = Nadam(lr=0.001) #early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=20, verbose=1, mode='auto') cnn.compile(loss='binary_crossentropy', optimizer=nadam, metrics=[keras.metrics.binary_accuracy]) # Nadam early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint(filepath='%d-secstr_seq_denseconcat_60perc.h5' % i, verbose=1,save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1,trainX2], trainY1, batch_size=256, nb_epoch=500,validation_data=([valX1,valX2], valY1),class_weight=class_weights,callbacks=[checkpointer,early_stopping]) history_dict = fitHistory.history myjson_file = "myhist_" +"dict_" + "secstr_seq_denseconcat_60perc_" +str(i) json.dump(history_dict, open(myjson_file, 'w')) return cnn, fitHistory
def create_simple_model(num_classes, layer1_filters=32, layer2_filters=64): epochs = 5 n_conv = 2 model = models.Sequential() # First layer model.add(conv.ZeroPadding2D( (1, 1), input_shape=(1, IMG_COLS, IMG_ROWS), )) model.add( conv.Convolution2D(layer1_filters, n_conv, n_conv, activation="relu")) model.add(conv.MaxPooling2D(strides=(2, 2))) # Second layer model.add(conv.ZeroPadding2D((1, 1))) model.add( conv.Convolution2D(layer2_filters, n_conv, n_conv, activation="relu")) model.add(conv.MaxPooling2D(strides=(2, 2))) model.add(core.Flatten()) model.add(core.Dropout(0.2)) model.add(core.Dense(128, activation="relu")) model.add(core.Dense(num_classes, activation="softmax")) model.summary() model.compile(loss="categorical_crossentropy", optimizer="adadelta", metrics=["accuracy"]) return model, epochs
def __init__(self, img_size, nb_classes): batch_size = 128 img_rows, img_cols = img_size nb_filters_1 = 32 # 64 nb_filters_2 = 64 # 128 nb_filters_3 = 128 # 256 nb_conv = 3 cnn = models.Sequential() cnn.add(conv.Convolution2D(nb_filters_1, nb_conv, nb_conv, activation="relu", input_shape=(img_rows, img_cols, 1), border_mode='same')) cnn.add(conv.Convolution2D(nb_filters_1, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.MaxPooling2D(strides=(2, 2))) cnn.add(conv.Convolution2D(nb_filters_2, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.Convolution2D(nb_filters_2, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.MaxPooling2D(strides=(2, 2))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.MaxPooling2D(strides=(2,2))) cnn.add(core.Flatten()) cnn.add(core.Dropout(0.2)) cnn.add(core.Dense(128, activation="relu")) # 4096 cnn.add(core.Dense(nb_classes, activation="softmax")) cnn.summary() cnn.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) self.cnn = cnn
def Dave_v3(input_tensor=None, load_weights=False): model = models.Sequential() model.add( convolutional.Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(32, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(64, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(500, activation='relu')) #model.add(core.Dropout(.5)) model.add(core.Dense(100, activation='relu')) #model.add(core.Dropout(.25)) model.add(core.Dense(20, activation='relu')) model.add(core.Dense(1)) model.add( Lambda(One_to_radius, output_shape=atan_layer_shape, name="prediction")) if load_weights: model.load_weights('./models/dave3/dave3.h5') model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') return model
def convnet_simple_lion_keras(image_dims): model = keras.models.Sequential() model.add(core.Lambda(lambda x: (x / 255.0) - 0.5, input_shape=image_dims)) model.add( convolutional.Conv2D(32, (3, 3), activation='relu', padding='same')) model.add(convolutional.MaxPooling2D(pool_size=(2, 2))) model.add( convolutional.Conv2D(64, (3, 3), activation='relu', padding='same')) model.add(convolutional.MaxPooling2D(pool_size=(2, 2))) model.add( convolutional.Conv2D(128, (3, 3), activation='relu', padding='same')) model.add(convolutional.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(512, activation='relu')) model.add(core.Dropout(0.5)) model.add(core.Dense(1024, activation='relu')) model.add(core.Dropout(0.5)) model.add(core.Dense(6, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) return model
def model(input): x = conv.Conv1D(100, 3, activation=config.get('first layer', 'activation'), kernel_initializer=config.get('first layer', 'kernel_initializer'), kernel_regularizer=l2( config.getfloat('first layer', 'kernel_regularizer')), padding=config.get('first layer', 'padding'), name='Conv1')(input) x = Dropout(config.getfloat('first layer', 'dropout'), name='drop1')(x) x = BatchNormalization()(x) x = MaxPooling1D(pool_size=2, strides=None, padding='valid')(x) x = conv.Conv1D(200, 7, activation=config.get('second layer', 'activation'), kernel_initializer=config.get('second layer', 'kernel_initializer'), kernel_regularizer=l2( config.getfloat('second layer', 'kernel_regularizer')), padding=config.get('second layer', 'padding'), name='Conv2')(x) x = Dropout(config.getfloat('second layer', 'dropout'), name='drop2')(x) x = BatchNormalization()(x) x = MaxPooling1D(pool_size=2, strides=None, padding='valid')(x) x = GRU(units=64, return_sequences=True)(x) x = Dropout(0.2)(x) x = GRU(units=64, return_sequences=True)(x) x = Dropout(0.2)(x) output = core.Flatten()(x) output = BatchNormalization()(output) output = Dropout(config.getint('flatten layer', 'dropout'), name='dropo3')(output) output = Dense(config.getint('first dense layer', 'units'), kernel_initializer=config.get('first dense layer', 'kernel_initializer'), activation=config.get('first dense layer', 'activation'), name='Denseo1')(output) output = Dropout(config.getfloat('first dense layer', 'dropout'), name='dropo4')(output) output = BatchNormalization()(output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal', name='Denseo2')(output) # ########## Set Cnn ########## cnn = Model(inputs=input, outputs=out) cnn.summary() adam = Adam(lr=0.0005) cnn.compile(loss='binary_crossentropy', optimizer=adam, metrics=[keras.metrics.binary_accuracy]) # Nadam return cnn
def main(n_filters, conv_size, pool_size, dropout, patch_size, n_astro=7, out_path=None): # Imports must be in the function, or whenever we import this module, Keras # will dump to stdout. import keras.layers.core as core from keras.layers import Input, Dense, Concatenate import keras.layers.convolutional as conv import keras.layers.merge from keras.models import Model im_in = Input(shape=(1, patch_size, patch_size)) astro_in = Input(shape=(n_astro, )) # 1 x 32 x 32 conv1 = conv.Convolution2D(filters=n_filters, kernel_size=(conv_size, conv_size), border_mode='valid', activation='relu', data_format='channels_first')(im_in) # 32 x 28 x 28 pool1 = conv.MaxPooling2D(pool_size=(pool_size, pool_size), data_format='channels_first')(conv1) # 32 x 14 x 14 conv2 = conv.Convolution2D(filters=n_filters, kernel_size=(conv_size, conv_size), border_mode='valid', activation='relu', data_format='channels_first')(pool1) # 32 x 10 x 10 pool2 = conv.MaxPooling2D(pool_size=(pool_size, pool_size), data_format='channels_first')(conv2) # 32 x 5 x 5 conv3 = conv.Convolution2D(filters=n_filters, kernel_size=(conv_size, conv_size), border_mode='valid', activation='relu', data_format='channels_first')(pool2) # 32 x 1 x 1 dropout = core.Dropout(dropout)(conv3) flatten = core.Flatten()(dropout) conc = Concatenate()([astro_in, flatten]) lr = Dense(1, activation='sigmoid')(conc) model = Model(inputs=[astro_in, im_in], outputs=[lr]) model.compile(loss='binary_crossentropy', optimizer='adadelta') model_json = model.to_json() if out_path is not None: with open(out_path, 'w') as f: f.write(model_json) return model_json
def Simple_Convo(train, nb_classes): batch_size = 128 img_rows, img_cols = 56, 56 nb_filters_1 = 32 # 64 nb_filters_2 = 64 # 128 nb_filters_3 = 128 # 256 nb_conv = 3 # train = np.concatenate([train, train], axis=1) trainX = train[:, 1:].reshape(train.shape[0], 28, 28, 1) trainX = trainX.astype(float) trainX /= 255.0 trainX = np.concatenate([trainX, np.roll(trainX, 14, axis=1)], axis=1) trainX = np.concatenate([trainX, np.fliplr(np.roll(trainX, 7, axis=2))], axis=2) print(trainX.shape) cnn = models.Sequential() cnn.add(conv.Convolution2D(nb_filters_1, nb_conv, nb_conv, activation="relu", input_shape=(img_rows, img_cols, 1), border_mode='same')) cnn.add(conv.Convolution2D(nb_filters_1, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.MaxPooling2D(strides=(2, 2))) cnn.add(conv.Convolution2D(nb_filters_2, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.Convolution2D(nb_filters_2, nb_conv, nb_conv, activation="relu", border_mode='same')) cnn.add(conv.MaxPooling2D(strides=(2, 2))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same')) # cnn.add(conv.MaxPooling2D(strides=(2,2))) cnn.add(core.Flatten()) cnn.add(core.Dropout(0.2)) cnn.add(core.Dense(128, activation="relu")) # 4096 cnn.add(core.Dense(nb_classes, activation="softmax")) cnn.summary() return cnn
def mnist_model(input_shape): """Creates a MNIST model.""" model = sequential_model_lib.Sequential() # Adding custom pass-through layer to visualize input images. model.add(LayerForImageSummary()) model.add( conv_layer_lib.Conv2D( 32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(conv_layer_lib.Conv2D(64, (3, 3), activation='relu')) model.add(pool_layer_lib.MaxPooling2D(pool_size=(2, 2))) model.add(layer_lib.Dropout(0.25)) model.add(layer_lib.Flatten()) model.add(layer_lib.Dense(128, activation='relu')) model.add(layer_lib.Dropout(0.5)) model.add(layer_lib.Dense(NUM_CLASSES, activation='softmax')) # Adding custom pass-through layer for summary recording. model.add(LayerForHistogramSummary()) return model
def model(): model1 = Sequential() model1.add( Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model1.add(MaxPooling2D(pool_size=(2, 2))) model1.add(Convolution2D(32, 3, 3, activation='relu')) model1.add(MaxPooling2D(pool_size=(2, 2))) model1.add(Convolution2D(64, 3, 3, activation='relu')) model1.add(MaxPooling2D(pool_size=(2, 2))) model1.add(core.Flatten()) model1.add(core.Dropout(0.5)) model1.add(core.Dense(200)) model1.add(Activation('relu')) model1.add(core.Dropout(0.25)) model1.add(core.Dense(50)) model1.add(Activation('relu')) model1.add(core.Dropout(0.25)) model1.add(core.Dense(10)) model1.add(Activation('relu')) model1.add(core.Dense(1)) return model1
def testIgnoreSaveCounter(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") with self.cached_session() as session: # Create and save a model using Saver() before using a Checkpoint. This # generates a snapshot without the Checkpoint's `save_counter`. model = sequential.Sequential() model.add(core.Flatten(input_shape=(1,))) model.add(core.Dense(1)) name_saver = tf.compat.v1.train.Saver(model.trainable_variables) save_path = name_saver.save( sess=session, save_path=checkpoint_prefix, global_step=1) # Checkpoint.restore must successfully load that checkpoint. ckpt = tf.train.Checkpoint(model=model) status = ckpt.restore(save_path) status.assert_existing_objects_matched() # It should, however, refuse to load a checkpoint where an unrelated # `save_counter` variable is missing. model.layers[1].var = tf.Variable(0., name="save_counter") status = ckpt.restore(save_path) with self.assertRaises(AssertionError): status.assert_existing_objects_matched()
def cpg_layers(params): layers = [] if params.drop_in: layer = kcore.Dropout(params.drop_in) layers.append(('xd', layer)) nb_layer = len(params.nb_filter) w_reg = kr.WeightRegularizer(l1=params.l1, l2=params.l2) for l in range(nb_layer): layer = kconv.Convolution2D(nb_filter=params.nb_filter[l], nb_row=1, nb_col=params.filter_len[l], activation=params.activation, init='glorot_uniform', W_regularizer=w_reg, border_mode='same') layers.append(('c%d' % (l + 1), layer)) layer = kconv.MaxPooling2D(pool_size=(1, params.pool_len[l])) layers.append(('p%d' % (l + 1), layer)) layer = kcore.Flatten() layers.append(('f1', layer)) if params.drop_out: layer = kcore.Dropout(params.drop_out) layers.append(('f1d', layer)) if params.nb_hidden: layer = kcore.Dense(params.nb_hidden, activation='linear', init='glorot_uniform') layers.append(('h1', layer)) if params.batch_norm: layer = knorm.BatchNormalization() layers.append(('h1b', layer)) layer = kcore.Activation(params.activation) layers.append(('h1a', layer)) if params.drop_out: layer = kcore.Dropout(params.drop_out) layers.append(('h1d', layer)) return layers
def model(): inputs = ks.layers.Input(shape=(150, 150, 3)) #conv2d x0 = ks.layers.Conv2D(128, (3, 3), padding='same')(inputs) x0 = b_n()(x0) x00 = ks.layers.core.Activation('relu')(x0) #residual x1 = ks.layers.Conv2D(128, (3, 3), padding='same')(inputs) x2 = b_n()(x1) x3 = ks.layers.core.Activation('relu')(x2) x4 = ks.layers.Conv2D(128, (3, 3), padding='same')(x3) x5 = b_n()(x4) x6 = ks.layers.merge([x00, x5], mode='sum') x7 = ks.layers.core.Activation('relu')(x6) #residual end # residual x1 = ks.layers.Conv2D(128, (3, 3), padding='same')(x7) x2 = b_n()(x1) x3 = ks.layers.core.Activation('relu')(x2) x4 = ks.layers.Conv2D(128, (3, 3), padding='same')(x3) x5 = b_n()(x4) x6 = ks.layers.merge([x00, x5], mode='sum') x7 = ks.layers.core.Activation('relu')(x6) # residual end # residual x1 = ks.layers.Conv2D(128, (3, 3), padding='same')(x7) x2 = b_n()(x1) x3 = ks.layers.core.Activation('relu')(x2) x4 = ks.layers.Conv2D(128, (3, 3), padding='same')(x3) x5 = b_n()(x4) x6 = ks.layers.merge([x00, x5], mode='sum') x7 = ks.layers.core.Activation('relu')(x6) # residual end flat = core.Flatten()(x7) dense1 = Dense(1000, activation='relu')(flat) out = Dense(2)(dense1) model = ks.models.Model(inputs=inputs, outputs=out) return model
def model(input): # ######### First Network ########## x = GRU(units=64, return_sequences=True)(input) x = Dropout(0.2)(x) x = GRU(units=64, return_sequences=True)(x) x = Dropout(0.2)(x) output = core.Flatten()(x) output = BatchNormalization()(output) out = Dense(64, activation="relu", kernel_initializer='glorot_normal')(output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal')(out) # ########## Set Cnn ########## cnn = Model(inputs=input, outputs=out) cnn.summary() adam = keras.optimizers.Adam() cnn.compile(loss='binary_crossentropy', optimizer=adam, metrics=[keras.metrics.binary_accuracy]) # Nadam return cnn
if __name__ == '__main__': # Read splitted data df_train = pd.read_csv('train.csv') df_valid = pd.read_csv('test.csv') # CNN Model Architecture model = models.Sequential() model.add(convolutional.Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(32, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(64, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(500, activation='relu')) model.add(core.Dropout(.5)) model.add(core.Dense(100, activation='relu')) model.add(core.Dropout(.25)) model.add(core.Dense(20, activation='relu')) model.add(core.Dense(1)) model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') # load the exist model model.load_weights("model.h5") history = model.fit_generator(# continue training model for 17 epochs generate_samples(df_train, ''), samples_per_epoch=df_train.shape[0], nb_epoch=17,#0.016
def test_flatten(self): layer = core.Flatten() self._runner(layer)
cnn.add(kpool.MaxPooling2D(strides=(2, 2))) conv_dim_2 = 3 cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kcore.Flatten()) cnn.add(kcore.Dropout(0.5)) cnn.add(kcore.Dense(128, activation="relu")) # 4096 cnn.add(kcore.Dense(num_class, activation="softmax")) cnn.summary() myadadelta = kopt.Adadelta(lr=0.65, rho=0.95, epsilon=1e-08, decay=0.001) cnn.compile(loss="categorical_crossentropy", optimizer=myadadelta, metrics=["accuracy"]) cnn.fit(train_X, train_Y, batch_size=128, nb_epoch=180, verbose=1) # test_X = test_data_raw.reshape(test_data_raw.shape[0], 1, 28, 28) test_X = test_X.astype(float) test_X /= 255.0 yPred = cnn.predict_classes(test_X)
def HydrophobicityNetwork(trainX, trainY, valX, valY, physical_H_input, folds, train_time=None): if (train_time == 0): x = core.Flatten()(physical_H_input) x = BatchNormalization()(x) x = Dense(1024, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.4)(x) x = Dense(512, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.3)(x) x = Dense(256, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.2)(x) x = Dense(128, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.1)(x) physical_H_output = Dense(2, init='glorot_normal', activation='softmax', W_regularizer=l2(0.001))(x) HydrophobicityNetwork = Model(physical_H_input, physical_H_output) # optimization = SGD(lr=0.01, momentum=0.9, nesterov= True) optimization = 'Nadam' HydrophobicityNetwork.compile(loss='binary_crossentropy', optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) else: HydrophobicityNetwork = load_model('model/' + str(folds) + '/model/' + str(train_time - 1) + 'HydrophobicityNetwork.h5') if (trainY is not None): weight_checkpointer = ModelCheckpoint( filepath='./model/' + str(folds) + '/weight/' + str(train_time) + 'Hydrophobicityweight.h5', verbose=1, save_best_only=True, monitor='val_loss', mode='min', save_weights_only=True) early_stopping = EarlyStopping(monitor='val_loss', mode='min', patience=50) loss_checkpointer = LossModelCheckpoint( model_file_path='model/' + str(folds) + '/model/' + str(train_time) + 'HydrophobicityNetwork.h5', monitor_file_path='model/' + str(folds) + '/loss/' + str(train_time) + 'Hydrophobicityloss.json', verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = HydrophobicityNetwork.fit( trainX, trainY, batch_size=512, epochs=5000, verbose=2, validation_data=(valX, valY), shuffle=True, class_weight='auto', callbacks=[early_stopping, loss_checkpointer, weight_checkpointer]) return HydrophobicityNetwork
def OOKCNN(trainX, trainY, nb_epoch, earlystop=None, compiletimes=0, compilemodels=None, batch_size=2048, class_weights={ 0: 1, 1: 1 }, predict=False): #Set Oneofkey Network Size and Data input_row = trainX.shape[2] input_col = trainX.shape[3] trainX_t = trainX # Early_stop if (earlystop is not None): early_stopping = EarlyStopping(monitor='val_loss', mode='min', patience=earlystop) # set to a very big value since earlystop used nb_epoch = nb_epoch # TrainX_t For Shape trainX_t.shape = (trainX_t.shape[0], input_row, input_col) input = Input(shape=(input_row, input_col)) # params = {'dropout1': 0.09055921027754717, 'dropout2': 0.6391239298866936, 'dropout3': 0.4494981811340072, # 'dropout4': 0.13858850326177857, 'dropout5': 0.37168935754516325, 'layer1_node': 21.380001953812567, # 'layer1_size': 1, 'layer2_node': 42.3937544103545, 'layer2_size': 16, 'layer3_node': 184.87943202539697, # 'layer4_node': 61.85302597240724, 'layer5_node': 415.9952475249118, 'nb_epoch': 178, 'windowSize': 16} # layer1_node = int(params["layer1_node"]) # layer2_node = int(params["layer2_node"]) # layer3_node = int(params["layer3_node"]) # layer4_node = int(params["layer4_node"]) # layer5_node = int(params["layer5_node"]) # layer1_size = params["layer1_size"] # layer2_size = params["layer2_size"] # dropout1 = params["dropout1"] # dropout2 = params["dropout2"] # dropout3 = params["dropout3"] # dropout4 = params["dropout4"] # dropout5 = params["dropout5"] if compiletimes == 0: # Total Set Classes nb_classes = 2 # Total Set Batch_size batch_size = 8192 # Total Set Optimizer # optimizer = SGD(lr=0.0001, momentum=0.9, nesterov= True) optimization = 'Nadam' #begin of Oneofkey Network # x = conv.Conv1D(layer1_node, layer1_size, name="layer1", kernel_initializer="glorot_normal", # kernel_regularizer=l2(0), padding="same")(input) # x = Dropout(dropout1)(input) # x = Activation('softsign')(x) # # x = conv.Conv1D(layer2_node, layer2_size, name="layer2", kernel_initializer="glorot_normal", # kernel_regularizer=l2(0), padding="same")(x) # x = Dropout(dropout2)(x) # x = Activation('softsign')(x) # # output_x = core.Flatten()(x) # output = BatchNormalization()(output_x) # output = Dropout(dropout3)(output) # # # attention_probs = Dense(1155, activation='softmax', name='attention_probs')(output) # # attention_mul = Multiply()([output, attention_probs]) # # output = Dense(layer3_node, kernel_initializer='glorot_normal', activation='relu', name='layer3')(output) # output = Dropout(dropout4)(output) # output = Dense(layer4_node, kernel_initializer='glorot_normal', activation="relu", name='layer4')(output) # output = Dropout(dropout5)(output) # output = Dense(layer5_node, kernel_initializer='glorot_normal', activation="relu", name='layer5')(output) # End of Oneofkey Network # out = Dense(nb_classes, kernel_initializer='glorot_normal', activation='softmax', kernel_regularizer=l2(0.001), # name='7')(output) # # cnn = Model(input, out) # cnn.compile(loss=keras.losses.binary_crossentropy, optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) x = conv.Conv1D(51, 2, name="0", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(input) x = Dropout(0.3)(x) x = Activation('softsign')(x) x = conv.Conv1D(21, 3, name="1", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(x) x = Dropout(0.4)(x) x = Activation('softsign')(x) # x = conv.Conv1D(21, 5, name="2", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(x) # x = Dropout(0.4)(x) # x = Activation('softsign')(x) # # x = conv.Conv1D(101, 7, name="3", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(x) # x = Activation('softsign')(x) # # x_reshape = core.Reshape((x._keras_shape[2], x._keras_shape[1]))(x) # x = Dropout(0.4)(x) output_x = core.Flatten()(x) output = BatchNormalization()(output_x) output = Dropout(0.3)(output) # attention_probs = Dense(1155, activation='softmax', name='attention_probs')(output) # attention_mul = Multiply()([output, attention_probs]) output = Dense(128, kernel_initializer='glorot_normal', activation='relu', name='4')(output) output = Dropout(0.2)(output) output = Dense(64, kernel_initializer='glorot_normal', activation="relu", name='5')(output) output = Dropout(0.2)(output) output = Dense(415, kernel_initializer='glorot_normal', activation="relu", name='6')(output) # End of Oneofkey Network out = Dense(nb_classes, kernel_initializer='glorot_normal', activation='softmax', kernel_regularizer=l2(0.001), name='7')(output) cnn = Model(input, out) cnn.compile(loss=keras.losses.binary_crossentropy, optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) else: cnn = compilemodels oneofkclass_weights = class_weights if (predict is False): if (trainY is not None): if (earlystop is None): fitHistory = cnn.fit(trainX_t, trainY, batch_size=batch_size, nb_epoch=nb_epoch) else: # checkpointer = ModelCheckpoint(filepath='oneofk.h5', verbose=1, save_best_only=True) weight_checkpointer = ModelCheckpoint( filepath='oneofkweight9.h5', verbose=0, save_best_only=True, monitor='val_binary_accuracy', mode='max', save_weights_only=True) fitHistory = cnn.fit( trainX_t, trainY, batch_size=batch_size, epochs=nb_epoch, shuffle=True, validation_split=0.2, callbacks=[early_stopping, weight_checkpointer], class_weight=oneofkclass_weights, verbose=0) else: fitHistory = cnn.fit(trainX_t, trainY, batch_size=batch_size, nb_epoch=nb_epoch) return cnn
cnn.add(conv.Convolution2D(64, 3, 3, activation="relu")) cnn.add(conv.ZeroPadding2D((1, 1))) cnn.add(conv.Convolution2D(64, 3, 3, activation="relu")) cnn.add(conv.MaxPooling2D(strides=(2, 2))) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.MaxPooling2D(strides=(2,2))) cnn.add(core.Flatten()) cnn.add(core.Dropout(0.2)) cnn.add(core.Dense(128, activation="relu")) # 4096 cnn.add(core.Dense(nb_classes, activation="softmax")) cnn.summary() cnn.compile(loss="categorical_crossentropy", optimizer="adadelta", metrics=["accuracy"]) cnn.fit(trainX, trainY, batch_size=128, nb_epoch=1, verbose=1) testX = test.reshape(test.shape[0], 1, 48, 48) testX = testX.astype(float) testX /= 255.0
def OtherNetwork(trainX, trainY, valX, valY, physical_O_input, folds, train_time=None): if (train_time == 0): x = core.Flatten()(physical_O_input) x = BatchNormalization()(x) x = Dense(256, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.2)(x) x = Dense(128, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.2)(x) x = Dense(64, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.1)(x) x = Dense(32, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) physical_O_output = Dense(2, init='glorot_normal', activation='softmax', W_regularizer=l2(0.001))(x) OtherNetwork = Model(physical_O_input, physical_O_output) optimization = Nadam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) OtherNetwork.compile(loss='binary_crossentropy', optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) else: OtherNetwork = load_model('model/' + str(folds) + '/model/' + str(train_time - 1) + 'OtherNetwork.h5') if (trainY is not None): weight_checkpointer = ModelCheckpoint( filepath='./model/' + str(folds) + '/weight/' + str(train_time) + 'Otherweight.h5', verbose=1, save_best_only=True, monitor='val_loss', mode='min', save_weights_only=True) early_stopping = EarlyStopping(monitor='val_loss', mode='min', patience=200) loss_checkpointer = LossModelCheckpoint( model_file_path='model/' + str(folds) + '/model/' + str(train_time) + 'OtherNetwork.h5', monitor_file_path='model/' + str(folds) + '/loss/' + str(train_time) + 'Otherloss.json', verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = OtherNetwork.fit( trainX, trainY, batch_size=512, epochs=5000, verbose=2, validation_data=(valX, valY), shuffle=True, class_weight='auto', callbacks=[early_stopping, loss_checkpointer, weight_checkpointer]) return OtherNetwork
def MCNN_best(trainX1, trainX2, trainY1, valX1, valX2, valY1, input_1, input_2, i, class_weights, t): if (t == 0): print("####################################bootstrap iteration ", t, "#############fold iteration ", i, "\n") onehot_secstr = conv.Conv1D( 5, 10, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.0010949235793883667), padding='valid', name='0_secstr')(input_1) onehot_secstr = Dropout(0.745150134528914)(onehot_secstr) onehot_secstr = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_secstr) onehot_secstr = core.Flatten()(onehot_secstr) onehot_secstr2 = conv.Conv1D( 9, 4, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.03405758144816304), padding='valid', name='1_secstr')(input_1) onehot_secstr2 = Dropout(0.36965944352568686)(onehot_secstr2) onehot_secstr2 = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_secstr2) onehot_secstr2 = core.Flatten()(onehot_secstr2) output_onehot_sec = concatenate([onehot_secstr, onehot_secstr2], axis=-1) onehot_x = conv.Conv1D(5, 10, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.03217477728270726), padding='valid', name='0')(input_2) onehot_x = Dropout(0.6653716368558287)(onehot_x) onehot_x = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_x) onehot_x = core.Flatten()(onehot_x) onehot_x2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.01608962762003551), padding='valid', name='1')(input_2) onehot_x2 = Dropout(0.038045356303735206)(onehot_x2) onehot_x2 = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_x2) onehot_x2 = core.Flatten()(onehot_x2) output_onehot_seq = concatenate([onehot_x, onehot_x2], axis=-1) final_output = concatenate([output_onehot_sec, output_onehot_seq]) dense_out = Dense(512, kernel_initializer='glorot_normal', activation='softplus', name='dense_concat')(final_output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal', name='6')(dense_out) ########## Set Net ########## cnn = Model(inputs=[input_1, input_2], outputs=out) cnn.load_weights('weightsfile_hyperasbest.h5') cnn.summary() adam = Adam(lr=0.08582474007227135) nadam = Nadam(lr=0.0014045290291504406) rmsprop = RMSprop(lr=0.037289952982092284) sgd = SGD(lr=0.01373965388919854) optim = sgd ## choiceval = {{choice(['adam', 'sgd', 'rmsprop','nadam'])}} ## if choiceval == 'adam': ## optim = adam ## elif choiceval == 'rmsprop': ## optim = rmsprop ## elif choiceval=='nadam': ## optim = nadam ## else: ## optim = sgd cnn.compile(loss='binary_crossentropy', optimizer=optim, metrics=[keras.metrics.binary_accuracy]) # Nadam early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint( filepath='%d-secstr_seq_denseconcat_60perc_best.h5' % i, verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1, trainX2], trainY1, batch_size=32, nb_epoch=500, validation_data=([valX1, valX2], valY1), callbacks=[checkpointer, early_stopping], class_weight=class_weights) myjson_file = "myhist_" + "dict_" + "secstr_seq_denseconcat_60perc_best_" + str( i) json.dump(fitHistory.history, open(myjson_file, 'w')) return cnn, fitHistory else: print("####################################bootstrap iteration ", t, "#############fold iteration ", i, "\n") cnn = models.load_model('%d-secstr_seq_denseconcat_60perc_best.h5' % i) early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint( filepath='%d-secstr_seq_denseconcat_60perc_best.h5' % i, verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1, trainX2], trainY1, batch_size=32, nb_epoch=500, validation_data=([valX1, valX2], valY1), class_weight=class_weights, callbacks=[checkpointer, early_stopping]) myjson_file = "myhist_" + "dict_" + "secstr_seq_denseconcat_60perc_best_" + str( i) json.dump(fitHistory.history, open(myjson_file, 'a')) return cnn, fitHistory
def mixallDNNmodel(trainX, trainY, valX, valY, physical_D_input, folds, train_time=None): if (train_time == 0): x = core.Flatten()(physical_D_input) x = BatchNormalization()(x) x = Dense(2048, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.4)(x) x = Dense(512, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.4)(x) x = Dense(128, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.4)(x) x = Dense(64, init='glorot_normal', activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.1)(x) physical_D_output = Dense(2, init='glorot_normal', activation='softmax', W_regularizer=l2(0.001))(x) mixallDNNmodel = Model(physical_D_input, physical_D_output) optimization = 'Nadam' mixallDNNmodel.compile(loss='binary_crossentropy', optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) else: mixallDNNmodel = load_model('model/' + str(folds) + '/model/' + str(train_time - 1) + 'DNNNetwork.h5') if (trainY is not None): weight_checkpointer = ModelCheckpoint( filepath='./model/' + str(folds) + '/weight/' + str(train_time) + 'DNNweight.h5', verbose=1, save_best_only=True, monitor='val_loss', mode='min', save_weights_only=True) loss_checkpointer = LossModelCheckpoint( model_file_path='model/' + str(folds) + '/model/' + str(train_time) + 'DNNNetwork.h5', monitor_file_path='model/' + str(folds) + '/loss/' + str(train_time) + 'DNNloss.json', verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = mixallDNNmodel.fit( trainX, trainY, batch_size=4096, nb_epoch=50, shuffle=True, callbacks=[early_stopping, loss_checkpointer, weight_checkpointer], class_weight='auto', validation_data=(valX, valY)) return mixallDNNmodel
def MultiCNN(train_ook_X, trainAAIndexX, train_ook_Y, nb_epoch, earlystop=None, compiletimes=0, batch_size=2048, predict=False, compileModel=None, class_weight={ 0: 0.5, 1: 0.5 }, verbose=1, model_id=0): # Set Oneofkey Data ook_row = train_ook_X.shape[2] ook_col = train_ook_X.shape[3] ook_x_t = train_ook_X ook_x_t.shape = (ook_x_t.shape[0], ook_row, ook_col) ook_input = Input(shape=(ook_row, ook_col)) # AAindex aaindex_x_t = trainAAIndexX aaindex_row = trainAAIndexX.shape[2] aaindex_col = trainAAIndexX.shape[3] aaindex_x_t.shape = (trainAAIndexX.shape[0], aaindex_row, aaindex_col) aaindex_input = Input(shape=(aaindex_row, aaindex_col)) if (earlystop is not None): early_stopping = EarlyStopping(monitor='val_loss', mode='min', patience=earlystop) nb_epoch = nb_epoch # TrainX_t For Shape if compiletimes == 0: # Total Set Classes nb_classes = 2 # Total Set Batch_size batch_size = 8192 # Total Set Optimizer # optimizer = SGD(lr=0.0001, momentum=0.9, nesterov= True) optimization = 'Nadam' # begin of Oneofkey Network ook_x = conv.Conv1D(51, 2, name="0", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(ook_input) ook_x = Dropout(0.3)(ook_x) ook_x = Activation('softsign')(ook_x) ook_x = conv.Conv1D(21, 3, name="1", kernel_initializer="glorot_normal", kernel_regularizer=l2(0), padding="same")(ook_x) ook_x = Dropout(0.4)(ook_x) ook_x = Activation('softsign')(ook_x) output_ook_x = core.Flatten()(ook_x) output_ook_x = BatchNormalization()(output_ook_x) output_ook_x = Dropout(0.3)(output_ook_x) output_ook_x = Dense(128, kernel_initializer='glorot_normal', activation='relu', name='2')(output_ook_x) output_ook_x = Dropout(0.2)(output_ook_x) output_ook_x = Dense(64, kernel_initializer='glorot_normal', activation="relu", name='3')(output_ook_x) output_ook_x = Dropout(0.2)(output_ook_x) # below modified output_ook_x = Dense(415, kernel_initializer='glorot_normal', activation="relu", name='4')(output_ook_x) # output_ook_x = Dense(nb_classes, kernel_initializer='glorot_normal', activation='softmax', kernel_regularizer=l2(0.001), # name='7')(output_ook_x) # End of Oneofkey Network # start with AAindex Dnn aaindex_x = core.Flatten()(aaindex_input) attention_probs = Dense(aaindex_row * aaindex_col, activation='softmax', name='5')(aaindex_x) aaindex_x = Multiply()([aaindex_x, attention_probs]) aaindex_x = BatchNormalization()(aaindex_x) aaindex_x = Dense(256, kernel_initializer='he_uniform', activation='relu', name='6')(aaindex_x) aaindex_x = Dropout(0.6)(aaindex_x) aaindex_x = Dense(128, kernel_initializer='he_uniform', activation='softplus', name='7')(aaindex_x) # aaindex_x = BatchNormalization()(aaindex_x) aaindex_x = Dropout(0.55)(aaindex_x) aaindex_x = GaussianNoise(10)(aaindex_x) output_aaindex_x = Dense(64, kernel_initializer='glorot_normal', activation='relu', name='8')(aaindex_x) # output_aaindex_x = Dense(nb_classes, kernel_initializer='glorot_normal', activation='softmax', # kernel_regularizer=l2(0.001), name='19')(output_aaindex_x) # output = Maximum()([output_ook_x, output_aaindex_x]) output = Concatenate()([output_ook_x, output_aaindex_x]) output = BatchNormalization()(output) # output = Dense(64, activation="relu", kernel_initializer="he_normal", kernel_regularizer=l2(0.001), name="21")(output) # output = BatchNormalization()(output) output = Dense(128, activation="relu", kernel_initializer="he_normal", kernel_regularizer=l2(0.001), name="9")(output) output = Dropout(0.6)(output) output = Dense(64, activation="relu", kernel_initializer="he_normal", kernel_regularizer=l2(0.001), name="10")(output) output = Dropout(0.5)(output) output = Dense(16, activation="relu", kernel_initializer="he_normal", kernel_regularizer=l2(0.001), name="11")(output) out = Dense(nb_classes, kernel_initializer='glorot_normal', activation='softmax', kernel_regularizer=l2(0.001), name='12')(output) multinn = Model([ook_input, aaindex_input], out) multinn.compile(loss=keras.losses.binary_crossentropy, optimizer=optimization, metrics=[keras.metrics.binary_accuracy]) else: multinn = compileModel oneofkclass_weights = class_weight if (earlystop is None): fitHistory = multinn.fit([ook_x_t, aaindex_x_t], train_ook_Y, batch_size=batch_size, nb_epoch=nb_epoch) else: weight_checkpointer = ModelCheckpoint(filepath='temp/temp.h5', verbose=verbose, save_best_only=True, monitor='val_binary_accuracy', mode='auto', save_weights_only=True) fitHistory = multinn.fit( [ook_x_t, aaindex_x_t], train_ook_Y, batch_size=batch_size, epochs=nb_epoch, shuffle=True, validation_split=0.2, callbacks=[early_stopping, weight_checkpointer], class_weight=oneofkclass_weights, verbose=verbose) return multinn
strides=(1, 1), padding='same', activation='relu')(pool_layer2) pool_layer3 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer4) conv_layer5 = convolutional.Conv2D(32, (3, 3), strides=(1, 1), padding='same', activation='relu')(conv_layer3) pool_layer4 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer5) flatten_layer = core.Flatten()(pool_layer4) hidden1 = core.Dense(64, activation='relu')(flatten_layer) inp_2 = Input(shape=(img_width, img_height, 3)) conv_layer1_2 = convolutional.Conv2D(8, (3, 3), strides=(1, 1), padding='same', activation='relu')(inp_2) conv_layer2_2 = convolutional.Conv2D(8, (3, 3), strides=(1, 1), padding='same', activation='relu')(conv_layer1_2) pool_layer1_2 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer2_2)
def MCNN(trainX1,trainX2,trainY1,valX1,valX2,valY1,testX1,testX2,testY): import pickle from sklearn.metrics import precision_score, recall_score, f1_score import pandas as pd #import pirna_kmer as pk from pandas import DataFrame from sklearn.model_selection import train_test_split import numpy as np #import phy_net as pn from keras.layers import Input import keras.utils.np_utils as kutils #import threading import time from keras.utils import np_utils from keras.utils.np_utils import to_categorical from keras.models import Sequential from keras.layers import Dense, Dropout, BatchNormalization, Activation, Flatten from keras.optimizers import Adam from keras.wrappers.scikit_learn import KerasClassifier from keras.models import load_model from sklearn.model_selection import cross_val_score from sklearn.preprocessing import LabelEncoder from sklearn.model_selection import StratifiedKFold from sklearn.preprocessing import StandardScaler from sklearn.pipeline import Pipeline from sklearn.utils.class_weight import compute_class_weight from keras.layers import Convolution2D as Conv2D from keras.layers import MaxPooling2D from keras.callbacks import EarlyStopping import json #from sklearn.metrics import matthews_corrcoef from keras.models import Model import tensorflow as tf ## from tensorflow.keras.callbacks import TensorBoard import os import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt from sklearn.metrics import precision_recall_curve, roc_curve, auc, average_precision_score, matthews_corrcoef from sklearn.metrics import precision_score, recall_score, f1_score #from sklearn.metrics import accuracy_score, recall_score remark = '' # The mark written in the result file. import time import numpy as np import matplotlib import pickle matplotlib.use('Agg') import keras.layers.core as core import keras.layers.convolutional as conv import keras.models as models from keras.models import Model from keras.layers.merge import concatenate from keras.callbacks import EarlyStopping, ModelCheckpoint, Callback, LearningRateScheduler, History from keras.layers import Dense, Dropout, Activation, Flatten, Input from keras.layers.normalization import BatchNormalization from keras.regularizers import l1, l2, l1_l2 import keras.metrics import matplotlib.pyplot as plt from keras.optimizers import Nadam,Adam,RMSprop,SGD from sklearn.metrics import precision_recall_curve, roc_curve, auc, average_precision_score, matthews_corrcoef import os from sklearn import svm from sklearn.manifold import TSNE from matplotlib import offsetbox from sklearn.metrics import accuracy_score, recall_score import random ## from tensorflow.keras.callbacks import TensorBoard row1,col1 = trainX1[0].shape input_1 = Input(shape=(row1,col1)) row2,col2 = trainX2[0].shape input_2 = Input(shape=(row2,col2)) NAME = "combined_secstr_seq_CNN_model_emboss-{}".format(int(time.time())) tensorboard = TensorBoard(log_dir="logs/{}".format(NAME)) onehot_secstr = conv.Conv1D(5, 10, kernel_initializer='glorot_normal',kernel_regularizer=l2({{uniform(0.0001, 0.1)}}), padding='valid', name='0_secstr')(input_1) onehot_secstr = Dropout({{uniform(0, 1)}})(onehot_secstr) onehot_secstr = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_secstr) onehot_secstr = core.Flatten()(onehot_secstr) onehot_secstr2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal',kernel_regularizer=l2({{uniform(0.0001, 0.1)}}), padding='valid', name='1_secstr')(input_1) onehot_secstr2 = Dropout({{uniform(0, 1)}})(onehot_secstr2) onehot_secstr2 = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_secstr2) onehot_secstr2 = core.Flatten()(onehot_secstr2) output_onehot_sec = concatenate([onehot_secstr, onehot_secstr2], axis=-1) onehot_x = conv.Conv1D(5, 10, kernel_initializer='glorot_normal',kernel_regularizer=l2({{uniform(0.0001, 0.1)}}), padding='valid', name='0')(input_2) onehot_x = Dropout({{uniform(0, 1)}})(onehot_x) onehot_x = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_x) onehot_x = core.Flatten()(onehot_x) onehot_x2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal',kernel_regularizer=l2({{uniform(0.0001, 0.1)}}), padding='valid', name='1')(input_2) onehot_x2 = Dropout({{uniform(0, 1)}})(onehot_x2) onehot_x2 = keras.layers.advanced_activations.PReLU(alpha_initializer='zeros', alpha_regularizer=None,alpha_constraint=None, shared_axes=None)(onehot_x2) onehot_x2 = core.Flatten()(onehot_x2) output_onehot_seq = concatenate([onehot_x, onehot_x2], axis=-1) final_output = concatenate([output_onehot_sec, output_onehot_seq]) dense_out = Dense({{choice([20,30,50,60,64,70,80,90,100, 128, 256, 512, 1024])}}, kernel_initializer='glorot_normal', activation='softplus', name='dense_concat')(final_output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal', name='6')(dense_out) ########## Set Net ########## cnn = Model(inputs=[input_1,input_2], outputs=out) cnn.summary() adam = Adam(lr={{uniform(0.0001, 0.1)}}) nadam = Nadam(lr={{uniform(0.0001, 0.1)}}) rmsprop = RMSprop(lr={{uniform(0.0001, 0.1)}}) sgd = SGD(lr={{uniform(0.0001, 0.1)}}) choiceval = {{choice(['adam', 'sgd', 'rmsprop','nadam'])}} if choiceval == 'adam': optim = adam elif choiceval == 'rmsprop': optim = rmsprop elif choiceval=='nadam': optim = nadam else: optim = sgd globalvars.globalVar += 1 #early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=20, verbose=1, mode='auto') cnn.compile(loss='binary_crossentropy', optimizer=optim, metrics=[keras.metrics.binary_accuracy]) # Nadam early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint(filepath='%d-secstr_seq_denseconcat.h5' % globalvars.globalVar, verbose=1,save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1,trainX2], trainY1, batch_size={{choice([32,64,128,256,512])}}, nb_epoch=500,validation_data=([valX1,valX2], valY1),callbacks=[checkpointer,early_stopping,tensorboard],class_weight=cwt.class_weights) myjson_file = "myhist_" +"_dict" + "_hyperas_model_trial_" +str(globalvars.globalVar) json.dump(fitHistory.history, open(myjson_file, 'w')) score, acc = cnn.evaluate([valX1, valX2], valY1, batch_size=32) pred_proba = cnn.predict([valX1,valX2], batch_size=32) pred_score = pred_proba[:, 1] true_class = valY1[:, 1] f1_sc = f1_score(true_class,pred_score) print('F1 score:', f1_sc) print('Test score:', score) print('accuracy:', acc) return {'loss': -f1_sc, 'status': STATUS_OK, 'model': cnn}
def architecture(): # put the normalization fucntion inside the model ensure preprocess using Lambda layer # There were many issue depending on using the lambda layer and in this waa it works def resize_normalize(image): import cv2 from keras.backend import tf as ktf """ Applies preprocessing pipeline to an image: crops `top` and `bottom` portions of image, resizes to 66*200 px and scales pixel values to [0, 1]. """ # resize to width 200 and high 66 liek recommended # in the nvidia paper for the used CNN # image = cv2.resize(image, (66, 200)) #first try resized = ktf.image.resize_images(image, (32, 128)) #normalize 0-1 resized = resized / 255.0 - 0.5 return resized print('I am inside call of architecture') #initialize model model = Sequential() #dropout = 0.5 nonlinear = 'tanh' print('I am before call of cropping layer') ### Convolution layers and parameters were taken from the "nvidia paper" on end-to-end autonomous steering. model.add( Cropping2D(cropping=((60, 20), (1, 1)), input_shape=(160, 320, 3))) print('I am before call of Lambda') model.add( Lambda(resize_normalize, input_shape=(160, 320, 3), output_shape=(32, 128, 3))) # Model architecture model.add( Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(32, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(500, activation='relu')) model.add(core.Dropout(.5)) model.add(core.Dense(100, activation='relu')) model.add(core.Dropout(.25)) model.add(core.Dense(20, activation='relu')) model.add(core.Dense(1)) model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') #model.add(Lambda(lambda x: resize_normalize(x), input_shape=(80,318,3), output_shape=(66, 200, 3))) # model.add(Convolution2D(24, 5, 5, name='conv1', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(36, 5, 5, name='conv2', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(48, 5, 5, name='conv3', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(64, 3, 3, name='conv4', activation=nonlinear)) # model.add(Convolution2D(64, 3, 3, name='conv5', activation=nonlinear)) # ### Regression # model.add(Flatten()) # model.add(Dropout(dropout)) # model.add(Dense(1164, name='hidden1', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(100, name='hidden2', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(50, name='hidden3', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(10, name='hidden4', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(1, name='output', activation=nonlinear)) # #model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') # model.compile(optimizer='adam', loss='mse') print('I am finished build the model') print(model.summary()) return model
activation='relu', padding='same', data_format='channels_first')(up3) conv6 = Dropout(0.2)(conv6) conv6 = Conv2D(16, (3, 3), activation='relu', padding='same', data_format='channels_first')(conv6) conv7 = Conv2D(3, (1, 1), activation='relu', padding='same', data_format='channels_first')(conv6) conv7 = core.Reshape((3, 128 * 128))(conv7) #conv6 = core.Permute((3,1))(conv6) conv7 = core.Flatten()(conv7) #conv7 = core.Dense(64)(conv7) #conv7 = core.Activation('relu')(conv7) #conv7 = Dropout(0.2)(conv7) conv7 = core.Dense(2)(conv7) ############ conv8 = core.Activation('softmax')(conv7) model = Model(input=inputs, output=conv8) # sgd = SGD(lr=0.01, decay=1e-6, momentum=0.3, nesterov=False) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
def test_flatten(): layer = core.Flatten() _runner(layer)
def MCNN_26(trainX1, trainX2, trainY1, valX1, valX2, valY1, input_1, input_2, i, class_weights, t): if (t == 0): print("####################################bootstrap iteration ", t, "#############fold iteration ", i, "\n") onehot_secstr = conv.Conv1D( 5, 10, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.011206678796947282), padding='valid', name='0_secstr')(input_1) onehot_secstr = Dropout(0.9942741825824339)(onehot_secstr) onehot_secstr = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_secstr) onehot_secstr = core.Flatten()(onehot_secstr) onehot_secstr2 = conv.Conv1D( 9, 4, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.04663753230167181), padding='valid', name='1_secstr')(input_1) onehot_secstr2 = Dropout(0.4084429796653032)(onehot_secstr2) onehot_secstr2 = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_secstr2) onehot_secstr2 = core.Flatten()(onehot_secstr2) output_onehot_sec = concatenate([onehot_secstr, onehot_secstr2], axis=-1) onehot_x = conv.Conv1D(5, 10, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.032491576988669696), padding='valid', name='0')(input_2) onehot_x = Dropout(0.5471399358933519)(onehot_x) onehot_x = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_x) onehot_x = core.Flatten()(onehot_x) onehot_x2 = conv.Conv1D(9, 4, kernel_initializer='glorot_normal', kernel_regularizer=l2(0.021775346680719416), padding='valid', name='1')(input_2) onehot_x2 = Dropout(0.10926522237224338)(onehot_x2) onehot_x2 = keras.layers.advanced_activations.PReLU( alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)(onehot_x2) onehot_x2 = core.Flatten()(onehot_x2) output_onehot_seq = concatenate([onehot_x, onehot_x2], axis=-1) final_output = concatenate([output_onehot_sec, output_onehot_seq]) dense_out = Dense(30, kernel_initializer='glorot_normal', activation='softplus', name='dense_concat')(final_output) out = Dense(2, activation="softmax", kernel_initializer='glorot_normal', name='6')(dense_out) ########## Set Net ########## cnn = Model(inputs=[input_1, input_2], outputs=out) cnn.load_weights('weightsfile_hyperas26.h5') cnn.summary() adam = Adam(lr=0.06971582946481189) nadam = Nadam(lr=0.010482932560304255) rmsprop = RMSprop(lr=0.031598749327261345) sgd = SGD(lr=0.008615890670714792) optim = sgd ## choiceval = {{choice(['adam', 'sgd', 'rmsprop','nadam'])}} ## if choiceval == 'adam': ## optim = adam ## elif choiceval == 'rmsprop': ## optim = rmsprop ## elif choiceval=='nadam': ## optim = nadam ## else: ## optim = sgd cnn.compile(loss='binary_crossentropy', optimizer=optim, metrics=[keras.metrics.binary_accuracy]) # Nadam early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint( filepath='%d-secstr_seq_denseconcat_60perc_trial26.h5' % i, verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1, trainX2], trainY1, batch_size=32, nb_epoch=500, validation_data=([valX1, valX2], valY1), callbacks=[checkpointer, early_stopping], class_weight=class_weights) myjson_file = "myhist_" + "dict_" + "secstr_seq_denseconcat_60perc_trial26_" + str( i) json.dump(fitHistory.history, open(myjson_file, 'w')) return cnn, fitHistory else: print("####################################bootstrap iteration ", t, "#############fold iteration ", i, "\n") cnn = models.load_model('%d-secstr_seq_denseconcat_60perc_trial26.h5' % i) early_stopping = EarlyStopping(monitor='val_loss', patience=20) checkpointer = ModelCheckpoint( filepath='%d-secstr_seq_denseconcat_60perc_trial26.h5' % i, verbose=1, save_best_only=True, monitor='val_loss', mode='min') fitHistory = cnn.fit([trainX1, trainX2], trainY1, batch_size=32, nb_epoch=500, validation_data=([valX1, valX2], valY1), class_weight=class_weights, callbacks=[checkpointer, early_stopping]) myjson_file = "myhist_" + "dict_" + "secstr_seq_denseconcat_60perc_trial26_" + str( i) json.dump(fitHistory.history, open(myjson_file, 'a')) return cnn, fitHistory