def Keras_B01(Xtr,Ytr,Xte,num_list, cat_list, Din, Dout,cv_i): X_list = [] for col in cat_list: X_list.append(Xte[col].values) X_list.append(Xte[num_list].values) X_test = X_list l2_emb = 0.00001 #emb_layers=[] cat_out = [] cat_in = [] #cat var for idx, var_name in enumerate(cat_list): x_in = Input(shape=(1,), dtype='int64', name=var_name+'_in') input_dim = Din[var_name] output_dim = Dout[var_name] x_out = Embedding(input_dim, output_dim, input_length=1, name = var_name, embeddings_regularizer=l1(l2_emb))(x_in) flatten_c = Flatten()(x_out) #emb_layers.append(x_out) cat_in.append(x_in) cat_out.append(flatten_c) x_emb = layers.concatenate(cat_out,name = 'emb') #continuous variables cont_in = Input(shape=(len(num_list),), name='continuous_input') cont_out = Lambda(expand_dims, expand_dims_output_shape)(cont_in) x_num = Flatten(name = 'num')(cont_out) cat_in.append(cont_in) #merge x = layers.concatenate([x_emb,x_num],name = 'emb_num') x = Dense(512 ,activation='relu')(x) x = PReLU()(x) x = Dropout(0.6)(x) x = Dense(64)(x) x = PReLU()(x) x = Dropout(0.3)(x) x = Dense(32)(x) x = PReLU()(x) x = Dropout(0.2)(x) x = Dense(1, activation='sigmoid')(x) model = Model(inputs = cat_in, outputs = x) model.compile(optimizers.Adam(), loss='binary_crossentropy', metrics=['accuracy']) batch_size = 256 model.fit_generator(generator=batch_generator(Xtr, Ytr,cat_list,num_list, batch_size), epochs=10, verbose=0, steps_per_epoch= np.floor(Xtr.shape[0]/batch_size)) Yt = model.predict(X_test).flatten() K.clear_session() return Yt
y, encoder = preprocess_labels(labels) X_test, ids = load_data('test.csv', train=False) X_test, _ = preprocess_data(X_test, scaler) nb_classes = y.shape[1] print(nb_classes, 'classes') dims = X.shape[1] print(dims, 'dims') print('Building model...') model = Sequential() model.add(Dense(512, input_shape=(dims, ))) model.add(PReLU()) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(512)) model.add(PReLU()) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(512)) model.add(PReLU()) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax'))
xg_test = xgb.DMatrix(T) p_w = np.zeros((T.shape[0], 9), np.float32) #generate an ensemble of 10 DNN models and 10 GBM models for r in range(10): nb_classes = yt.shape[1] print(nb_classes, 'classes') dims = X.shape[1] print(dims, 'dims') np.random.seed((r + 1) * 10) model = Sequential() model.add(Dense(dims, 1000, init='glorot_uniform')) model.add(PReLU((1000, ))) model.add(BatchNormalization((1000, ))) model.add(Dropout(0.5)) model.add(Dense(1000, 500, init='glorot_uniform')) model.add(PReLU((500, ))) model.add(BatchNormalization((500, ))) model.add(Dropout(0.5)) model.add(Dense(500, nb_classes, init='glorot_uniform')) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer="adagrad") print("Training DNN model...")
padding='post', truncating='post', value=0.) array_train_x = np.reshape( array_train_x, [array_train_x.shape[0], array_train_x.shape[1], 1]) rnn_model = Sequential() init_one = keras.initializers.Ones() ##rnn_model.add(GRU(rnn_hidden_node, input_shape=(None, 1), kernel_initializer=init_one)) rnn_model.add( LSTM(rnn_hidden_node, input_shape=(None, 1), kernel_initializer=init_one)) #rnn_model.add(SimpleRNN(rnn_hidden_node, input_shape=(None, 1), kernel_initializer=init_one)) rnn_model.add(BatchNormalization()) rnn_model.add(PReLU()) rnn_model.add(Dropout(rnn_dropout)) rnn_model.add(Dense(1)) rnn_model.add(Activation('sigmoid')) rnn_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy']) array_train_x = sequence.pad_sequences(rnn_train_x, maxlen=None, dtype='float64', padding='post', truncating='post', value=0.) array_train_x = np.reshape( array_train_x, [array_train_x.shape[0], array_train_x.shape[1], 1])
def bottleneck(inp, output, internal_scale=4, asymmetric=0, dilated=0, downsample=False, dropout_rate=0.1): # main branch internal = output // internal_scale encoder = inp # 1x1 input_stride = 2 if downsample else 1 # the 1st 1x1 projection is replaced with a 2x2 convolution when downsampling encoder = Conv2D( internal, (input_stride, input_stride), # padding='same', strides=(input_stride, input_stride), use_bias=False)(encoder) # Batch normalization + PReLU encoder = BatchNormalization(momentum=0.1)( encoder) # enet uses momentum of 0.1, keras default is 0.99 encoder = PReLU(shared_axes=[1, 2])(encoder) # conv if not asymmetric and not dilated: encoder = Conv2D(internal, (3, 3), padding='same')(encoder) elif asymmetric: encoder = Conv2D(internal, (1, asymmetric), padding='same', use_bias=False)(encoder) encoder = Conv2D(internal, (asymmetric, 1), padding='same')(encoder) elif dilated: encoder = Conv2D(internal, (3, 3), dilation_rate=(dilated, dilated), padding='same')(encoder) else: raise (Exception('You shouldn\'t be here')) encoder = BatchNormalization(momentum=0.1)( encoder) # enet uses momentum of 0.1, keras default is 0.99 encoder = PReLU(shared_axes=[1, 2])(encoder) # 1x1 encoder = Conv2D(output, (1, 1), use_bias=False)(encoder) encoder = BatchNormalization(momentum=0.1)( encoder) # enet uses momentum of 0.1, keras default is 0.99 encoder = SpatialDropout2D(dropout_rate)(encoder) other = inp # other branch if downsample: other = MaxPooling2D()(other) other = Permute((1, 3, 2))(other) pad_feature_maps = output - inp.get_shape().as_list()[3] tb_pad = (0, 0) lr_pad = (0, pad_feature_maps) other = ZeroPadding2D(padding=(tb_pad, lr_pad))(other) other = Permute((1, 3, 2))(other) encoder = add([encoder, other]) encoder = PReLU(shared_axes=[1, 2])(encoder) return encoder
json.dump(word2idx, j) autoencoder = Sequential() autoencoder.add( Embedding(input_dim=vocab_size, output_dim=emb_dim, input_length=maxlen, weights=[embedding_weights])) autoencoder.add(Dropout(0.5)) autoencoder.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(300, ))) autoencoder.add(Dropout(0.5)) autoencoder.add(Dense(512)) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(RepeatVector(maxlen)) autoencoder.add(TimeDistributed(Dense(300))) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print_summary(autoencoder.layers)
def build_cnn(dim, n_classes): model = Sequential() model.add( Convolution2D(96, 11, 11, subsample=(4, 4), input_shape=(dim, dim, 3), init='glorot_uniform', border_mode='same')) model.add(PReLU()) model.add(MaxPooling2D((3, 3), strides=(2, 2))) model.add( Convolution2D(256, 5, 5, subsample=(1, 1), init='glorot_uniform', border_mode='same')) model.add(PReLU()) model.add(MaxPooling2D((3, 3), strides=(2, 2))) model.add( Convolution2D(384, 3, 3, subsample=(1, 1), init='glorot_uniform', border_mode='same')) model.add(PReLU()) model.add( Convolution2D(384, 3, 3, subsample=(1, 1), init='glorot_uniform', border_mode='same')) model.add(PReLU()) model.add( Convolution2D(256, 3, 3, subsample=(1, 1), init='glorot_uniform', border_mode='same')) model.add(PReLU()) model.add(MaxPooling2D((3, 3), strides=(2, 2))) model.add(Flatten()) model.add(Dropout(0.5)) model.add(Dense(2048, init='glorot_uniform')) model.add(PReLU()) model.add(Dropout(0.5)) model.add(Dense(256, init='glorot_uniform')) model.add(PReLU()) model.add(Dense(n_classes, init='glorot_uniform', activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def Regularize(layer, params, shared_layers=False, name='', apply_noise=True, apply_batch_normalization=True, apply_prelu=True, apply_dropout=True, apply_l2=True, trainable=True): """ Apply the regularization specified in parameters to the layer :param layer: Layer to regularize :param params: Params specifying the regularizations to apply :param shared_layers: Boolean indicating if we want to get the used layers for applying to a shared-layers model. :param name: Name prepended to regularizer layer :param apply_noise: If False, noise won't be applied, independently of params :param apply_dropout: If False, dropout won't be applied, independently of params :param apply_prelu: If False, prelu won't be applied, independently of params :param apply_batch_normalization: If False, batch normalization won't be applied, independently of params :param apply_l2: If False, l2 normalization won't be applied, independently of params :return: Regularized layer """ shared_layers_list = [] if apply_noise and params.get('USE_NOISE', False): shared_layers_list.append( GaussianNoise(params.get('NOISE_AMOUNT', 0.01), name=name + '_gaussian_noise', trainable=trainable)) if apply_batch_normalization and params.get('USE_BATCH_NORMALIZATION', False): if params.get('WEIGHT_DECAY'): l2_gamma_reg = l2(params['WEIGHT_DECAY']) l2_beta_reg = l2(params['WEIGHT_DECAY']) else: l2_gamma_reg = None l2_beta_reg = None bn_mode = params.get('BATCH_NORMALIZATION_MODE', 0) shared_layers_list.append( BatchNormalization(mode=bn_mode, gamma_regularizer=l2_gamma_reg, beta_regularizer=l2_beta_reg, name=name + '_batch_normalization', trainable=trainable)) if apply_prelu and params.get('USE_PRELU', False): shared_layers_list.append( PReLU(name=name + '_PReLU', trainable=trainable)) if apply_dropout and params.get('DROPOUT_P', 0) > 0: shared_layers_list.append( Dropout(params.get('DROPOUT_P', 0.5), name=name + '_dropout', trainable=trainable)) if apply_l2 and params.get('USE_L2', False): shared_layers_list.append( Lambda(L2_norm, name=name + '_L2_norm', trainable=trainable)) # Apply all the previously built shared layers for l in shared_layers_list: layer = l(layer) result = layer # Return result or shared layers too if shared_layers: return result, shared_layers_list return result
def run_network_keras(hidden_connections, hidden_layers, loss, single_output=None, single_input=None, normalise_input=None, normalise_output=None, input_filter_types=None, use_graph=False, unknown_input_handler=None): config['input_filter_types'] = input_filter_types train_data = get_training_data(config, tmp_file, single_output, single_input, normalise_input, normalise_output, unknown_input_handler, percentile_bin=config['percentile_bin'], erase_above=config['erase_above']) test_data = config['test_data'] # number of test sets train_in = train_data['train_in'] train_out = train_data['train_out'] test_in = train_data['test_in'] test_out = train_data['test_out'] redshifts_train = train_data['redshifts_train'] galaxy_ids_test = train_data['galaxy_ids_test'] out_normaliser = train_data['out_normaliser'] mean_in = train_data['mean_in'] mean_out = train_data['mean_out'] stddev_in = train_data['stddev_in'] stddev_out = train_data['stddev_out'] print np.shape(train_in) print np.shape(train_out) print np.shape(test_in) print np.shape(test_out) LOG.info('Compiling neural network model') optimiser = SGD(lr=learning_params['learning_rate'], momentum=learning_params['momentum'], decay=learning_params['decay'], nesterov=True) input_dim = len(train_in[0]) try: output_dim = len(train_out[0]) except: output_dim = 1 model = Sequential() model.add(Dense(output_dim=hidden_connections, input_dim=input_dim)) model.add(PReLU()) model.add(Dense(output_dim=input_dim, input_dim=hidden_connections)) model.compile(loss='mse', optimizer=RMSprop(lr=0.001), class_mode='binary') model.fit(train_in, train_in, 5000, 10000, validation_split=0.3, verbose=True, show_accuracy=True) for i in range(0, 30): ans = model.predict(np.array([test_in[i]])) print 'Test', test_in[i] print 'Ans', ans[0] print print exit()
def _cnn(imgs_dim, compile_=True): model = Sequential() model.add(_convolutional_layer(nb_filter=16, input_shape=imgs_dim)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=16)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(_convolutional_layer(nb_filter=32)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=32)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=32)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(_convolutional_layer(nb_filter=64)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=64)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=64)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(_convolutional_layer(nb_filter=128)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=128)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=128)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(_convolutional_layer(nb_filter=256)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=256)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(_convolutional_layer(nb_filter=256)) model.add(BatchNormalization(axis=1)) model.add(PReLU(alpha_initializer=W_INIT)) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(Dropout(rate=dropout)) model.add(Flatten()) model.add(_dense_layer(output_dim=PENULTIMATE_SIZE)) model.add(BatchNormalization()) model.add(PReLU(alpha_initializer=W_INIT)) if compile_: model.add(Dropout(rate=dropout)) # Output: a vector of size (# of classes). model.add(_dense_layer(output_dim=SOFTMAX_SIZE)) model.add(BatchNormalization()) model.add(Activation(activation='softmax')) return compile_model(model) # return model return model
model.add(BatchNormalization((64,))) model.add(Dropout(0.5)) model.add(Dense(64, nb_classes, init='glorot_uniform')) model.add(Activation('softmax')) """ NN = 1024 dropout = 0.7 model = Sequential() #model.add(Dense(dims, 30, W_regularizer = l1(.01))) model.add(Dense(dims, 1024, init='glorot_uniform')) model.add(Activation('tanh')) model.add(PReLU((1024, ))) model.add(BatchNormalization((1024, ))) model.add(Dropout(dropout)) model.add(Dense(1024, NN, init='glorot_uniform')) model.add(Activation('tanh')) model.add(PReLU((NN, ))) model.add(BatchNormalization((NN, ))) model.add(Dropout(dropout)) model.add(Dense(NN, 20, init='glorot_uniform')) model.add(Activation('tanh')) model.add(PReLU((20, ))) model.add(BatchNormalization((20, ))) model.add(Dropout(dropout))
def Blender_CNN(num_of_res, num_of_inp, params): from keras.models import Sequential from keras.layers.core import Dense, Dropout, Flatten, Activation from keras.layers.normalization import BatchNormalization from keras.layers.advanced_activations import PReLU level1_size = 300 if 'level1_size' in params: level1_size = params['level1_size'] level2_size = 300 if 'level2_size' in params: level2_size = params['level2_size'] level3_size = 250 if 'level3_size' in params: level3_size = params['level3_size'] dropout_val_1 = 0.1 if 'dropout_val_1' in params: dropout_val_1 = params['dropout_val_1'] dropout_val_2 = 0.1 if 'dropout_val_2' in params: dropout_val_2 = params['dropout_val_2'] dropout_val_3 = 0.1 if 'dropout_val_3' in params: dropout_val_3 = params['dropout_val_3'] activation_1 = 'prelu' if 'activation_1' in params: activation_1 = params['activation_1'] activation_2 = 'prelu' if 'activation_2' in params: activation_2 = params['activation_2'] activation_3 = 'prelu' if 'activation_3' in params: activation_3 = params['activation_3'] use_3rd_level = 1 if 'use_3rd_level' in params: use_3rd_level = params['use_3rd_level'] model = Sequential() model.add(Dense(level1_size, input_shape=(num_of_inp,))) if activation_1 == 'prelu': model.add(PReLU()) elif activation_1 == 'relu': model.add(Activation('relu')) else: model.add(Activation('elu')) model.add(Dropout(dropout_val_1)) model.add(Dense(level2_size)) if activation_2 == 'prelu': model.add(PReLU()) elif activation_2 == 'relu': model.add(Activation('relu')) else: model.add(Activation('elu')) model.add(Dropout(dropout_val_2)) if use_3rd_level == 1: model.add(Dense(level3_size)) if activation_3 == 'prelu': model.add(PReLU()) elif activation_3 == 'relu': model.add(Activation('relu')) else: model.add(Activation('elu')) model.add(Dropout(dropout_val_3)) model.add(Dense(num_of_res, activation='sigmoid')) return model
@author: Emrick """ from keras.models import Sequential from keras.models import Model from keras.layers import Convolution2D from keras.layers import MaxPooling2D from keras.layers import Flatten from keras.layers import Dense from keras.layers import Input from keras.layers.core import Dropout from keras.layers.advanced_activations import PReLU identifier = Sequential() x = Input((64, 64, 1)) y = (Convolution2D(32, 3, 3, border_mode="valid"))(x) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Convolution2D(32, 3, 3, border_mode="valid"))(y) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Convolution2D(32, 3, 3, border_mode="valid"))(y) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Flatten())(y) y = (Dense(output_dim=512, activation="relu"))(y) #y = (Dense(output_dim=256,activation = "relu"))(y) y = (Dense(output_dim=256, activation="relu"))(y)
def keras_mlp(train_feat, valid_feat, train_target, valid_target, arch, slr, keepprob, problem, pred_file, batch_size, epochs): # Params batch_size = batch_size epochs = epochs # Standardize initial inputs to 0-1 range train_feat, valid_feat = scale_0_1_multiple(train_feat, valid_feat) print("SHAPES!!!", train_feat.shape, valid_feat.shape) if problem == "classification": print("Classification problem") # Get class weights ones = np.count_nonzero(train_target) zeros = train_target.shape[0] - ones zero_weight = float(zeros) / ones # class_weight = {0:1.0, 1:zero_weight} # class_weight = {0:zero_weight, 1:1.0} class_weight = {0: 1.0, 1: 2.5} print(class_weight) # One-hot encode labels (2-Classes) y_train = to_categorical(train_target, num_classes=2) y_valid = to_categorical(valid_target, num_classes=2) elif problem == "regression": y_train = train_target y_valid = valid_target class_weight = None # Architecture init_features = train_feat.shape[1] print("init arch: ", init_features, arch) layers = arch_layers(init_features, arch) print(layers) # Build KERAS Fully connected network model = Sequential() model.add(Dense(layers[1], activation=PReLU(), input_dim=layers[0])) model.add(BatchNormalization()) model.add(Dropout(keepprob)) for l in layers[2:]: model.add(Dense(l, activation=PReLU())) model.add(BatchNormalization()) model.add(Dropout(keepprob)) # Add the output layer and compile if problem == "classification": model.add(Dense(2, activation='softmax')) # Optimizer optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999) # Compile model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=[metrics.binary_crossentropy]) elif problem == "regression": model.add(Dense(1)) # Optimizer optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999) # Compile model.compile(optimizer=optimizer, loss="mse", metrics=["mse"]) print(model.summary()) # Run model # reduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x) early_stopping = EarlyStopping(monitor='val_loss', patience=20) # results = model.fit(train_feat, y_train, batch_size=batch_size, epochs=epochs, # validation_data=(valid_feat, y_valid), verbose=2, # callbacks=[reduce_lr], class_weight=class_weight) # model.fit(x=np.concatenate((train_feat, valid_feat), axis=0), # y=np.concatenate((y_train, y_valid), axis=0), # batch_size=batch_size, epochs=5, validation_split=0.1, verbose=2, # callbacks=[reduce_lr], class_weight=class_weight) results = model.fit(x=train_feat, y=y_train, batch_size=batch_size, epochs=200, validation_split=0.1, verbose=2, callbacks=[early_stopping], class_weight=class_weight) # Clean up # K.clear_session() return results, model, early_stopping
X_test, _ = preprocess_data(X_test_list, scaler) nb_classes = Y.shape[1] print(nb_classes, 'classes') dims = X.shape[1] print(dims, 'dims') print("Building model...") neuro_num = 16 # setup deep NN model = Sequential() model.add(Dense(dims, neuro_num, init='glorot_uniform')) model.add(PReLU((neuro_num,))) model.add(BatchNormalization((neuro_num,))) model.add(Dropout(0.5)) model.add(Dense(neuro_num, neuro_num, init='glorot_uniform')) model.add(PReLU((neuro_num,))) model.add(BatchNormalization((neuro_num,))) model.add(Dropout(0.5)) model.add(Dense(neuro_num, neuro_num, init='glorot_uniform')) model.add(PReLU((neuro_num,))) model.add(BatchNormalization((neuro_num,))) model.add(Dropout(0.5)) model.add(Dense(neuro_num, nb_classes, init='glorot_uniform')) model.add(Activation('sigmoid'))
def create_dueling_net(img_channels, img_rows, img_cols, n_conv1=32, n_conv2=64, n_conv3=64, n_out1=512, n_out2=-1, lr=.001, n_actions=4, loss='mse', use_perm_drop=False, drop_o=.25): def make_output(x): x = -K.mean(x, axis=1, keepdims=True) x = K.tile(x, 4) return x def make_output_shape(input_shape): shape = list(input_shape) return tuple(shape) def perm_drop(x): return K.dropout(x, .25) # input for the netwrok input = Input(shape=(img_channels, img_rows, img_cols)) # conv layers - shared by both netwroks conv1 = Convolution2D(n_conv1, 5, 5, border_mode='same', subsample=(2, 2))(input) prelu1 = PReLU()(conv1) conv2 = Convolution2D(n_conv2, 3, 3, border_mode='same', subsample=(2, 2))(prelu1) prelu2 = PReLU()(conv2) conv3 = Convolution2D(n_conv2, 3, 3, border_mode='same')(prelu2) prelu3 = PReLU()(conv3) flatten = Flatten()(prelu3) # A(s,a) dense11 = Dense(n_out1)(flatten) if not use_perm_drop: prelu31 = PReLU()(dense11) else: prelu310 = PReLU()(dense11) prelu31 = Lambda(perm_drop, output_shape=make_output_shape)(prelu310) dense21 = Dense(n_actions)(prelu31) out1 = Activation('linear')(dense21) # V(s) dense12 = Dense(n_out1)(flatten) if not use_perm_drop: prelu32 = PReLU()(dense12) else: prelu320 = PReLU()(dense12) prelu32 = Lambda(perm_drop, output_shape=make_output_shape)(prelu320) dense22 = Dense(1)(prelu32) out2 = Activation('linear')(dense22) out2 = RepeatVector(n_actions)(out2) out2 = Reshape((n_actions, ))(out2) # - E[ A(s,a) ] out3 = Lambda(make_output, output_shape=make_output_shape)(out1) output = merge([out1, out2, out3], mode='sum', concat_axis=1) model = Model(input=input, output=output) model.compile(loss=loss, optimizer=adam(lr=lr)) return model
def buildNN(architecture, training_params, input_dim): """Lay out a Neural Network as described in the YAML file.""" current_units = input_dim model = Sequential() is_first_layer = True for layer in architecture: layer_name = layer.keys()[0] layer_params = layer[layer_name] if layer_name == 'Activation': model.add(Activation(layer_params['type'])) if layer_name == 'PReLU': model.add(PReLU()) if layer_name == 'Dropout': if is_first_layer: model.add( Dropout(layer_params['p'], input_shape=(None, current_units))) else: model.add(Dropout(layer_params['p'])) if layer_name == 'Dense': model.add(Dense(layer_params['num_units'])) current_units = layer_params['num_units'] if layer_name == 'Conv': # a filter covers all predictions for a single time sample # strides are made between time samples model.add(Conv2D(layer_params['nb_filters'], 1)) current_units = layer_params['nb_filters'] * training_params[ 'num_strides'] if layer_name == 'GRU': model.add( GRU(layer_params['num_units'], input_shape=((None, input_dim) if is_first_layer else None), dropout=(layer_params['dropout'] if 'dropout' in layer_params else 0.0), return_sequences=layer_params['next_GRU'])) current_units = layer_params['num_units'] if layer_name == 'LSTM': model.add( LSTM(layer_params['num_units'], return_sequences=layer_params['next_GRU'])) current_units = layer_params['num_units'] if layer_name == 'BatchNormalization': model.add(BatchNormalization()) if layer_name == 'Flatten': model.add(Flatten()) if layer_name == 'Output': model.add(Dense(N_EVENTS)) model.add(Activation('sigmoid')) is_first_layer = False if not training_params.has_key( 'optim') or training_params['optim'] == 'sgd': optim = SGD(lr=training_params['lr'], decay=float(training_params['decay']), momentum=training_params['momentum'], nesterov=True) elif training_params['optim'] == 'adam': optim = Adam(lr=training_params['lr']) model.compile(loss='binary_crossentropy', optimizer=optim) return model
def getModelGivenModelOptions(self, options): np.random.seed(1234) import keras from keras.models import Sequential from keras.layers.core import Dropout, Reshape, Dense, Activation, Flatten from keras.layers.convolutional import Convolution2D, MaxPooling2D from keras.optimizers import Adadelta, SGD, RMSprop from keras.constraints import maxnorm from keras.layers.advanced_activations import PReLU from keras.layers.normalization import BatchNormalization from keras.regularizers import l1, l2, activity_l1, activity_l2 model = Sequential() model.add( Convolution2D(300, 4, 19, input_shape=(1, 4, int(options.seq_length)), W_learning_rate_multiplier=10.0)) model.add(BatchNormalization(mode=0, axis=1)) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(1, 3))) model.add( Convolution2D(200, 1, 11, W_learning_rate_multiplier=5.0, W_constraint=maxnorm(m=7))) model.add(BatchNormalization(mode=0, axis=1)) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(1, 4))) model.add(Convolution2D(200, 1, 7, W_constraint=maxnorm(m=7))) model.add(BatchNormalization(mode=0, axis=1)) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(1, 4))) model.add(Flatten()) model.add( Dense(1000, activity_regularizer=activity_l1(0.00001), W_constraint=maxnorm(m=7))) model.add(PReLU()) model.add(Dropout(0.3)) model.add( Dense(1000, activity_regularizer=activity_l1(0.00001), W_constraint=maxnorm(m=7))) model.add(PReLU()) model.add(Dropout(0.3)) model.add(Dense(options.num_labels)) model.add(Activation("sigmoid")) adam = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(loss="binary_crossentropy", optimizer=adam) # model.save_weights('/mnt/lab_data/kundaje/users/pgreens/projects/hematopoiesis/models/saved_weights/dec_18_yaml_150K_neg_no_hema_diff_peaks_Leuk_75M_BassetModel_10xlearnrate_noweightnorml1.h5') if options.init_weights_file != None: model.save_weights(options.init_weights_file) return model
def build_model(vectors, shape, settings): '''Compile the model.''' max_length, nr_hidden, nr_class, ncols = shape # Declare inputs. ids1 = Input(shape=(max_length, ), dtype='int32', name='words1') ids2 = Input(shape=(max_length, ), dtype='int32', name='words2') # Construct operations, which we'll chain together. embed = _StaticEmbedding(vectors, max_length, nr_hidden, dropout=0.2, nr_tune=5000) if settings['gru_encode']: encode = _BiRNNEncoding(max_length, nr_hidden, dropout=settings['dropout']) attend = _Attention(max_length, nr_hidden, dropout=settings['dropout']) align = _SoftAlignment(max_length, nr_hidden) compare = _Comparison(max_length, nr_hidden, dropout=settings['dropout']) entail = _Entailment(nr_hidden, nr_class, dropout=settings['dropout']) # Declare the model as a computational graph. sent1 = embed(ids1) # Shape: (i, n) sent2 = embed(ids2) # Shape: (j, n) if settings['gru_encode']: sent1 = encode(sent1) sent2 = encode(sent2) attention = attend(sent1, sent2) # Shape: (i, j) align1 = align(sent2, attention) align2 = align(sent1, attention, transpose=True) feats1 = compare(sent1, align1) feats2 = compare(sent2, align2) scores = entail(feats1, feats2) dense_input = Input(shape=(ncols, )) d = Dense(256, kernel_initializer='he_normal')(dense_input) d = PReLU()(d) d = BatchNormalization()(d) d = Dropout(0.4)(d) d2 = Dense(512, kernel_initializer='he_normal')(d) d2 = PReLU()(d2) d2 = BatchNormalization()(d2) d2 = Dropout(0.2)(d2) d3 = Dense(512, kernel_initializer='he_normal')(d2) d3 = PReLU()(d3) d3 = BatchNormalization()(d3) d3 = Dropout(0.2)(d3) merged = concatenate([feats1, feats2, d3]) merged = Dropout(0.2)(merged) merged = BatchNormalization()(merged) merged = Dense(512)(merged) merged = PReLU()(merged) merged = Dropout(0.2)(merged) merged = BatchNormalization()(merged) preds = Dense(1, activation='sigmoid')(merged) model = Model(input=[ids1, ids2, dense_input], output=[preds]) model.compile(optimizer=Adam(lr=settings['lr']), loss='binary_crossentropy', metrics=['accuracy']) return model
def get_model(args): # Dataset config config = data_constants[args.dataset.lower()] inputCNNshape = config['lstm_inputCNNshape'] inputMLPshape = config['lstm_inputMLPshape'] nb_classes = config['nb_classes'] # Build the CNN - pre-cross-connections inputCNN = Input(shape=inputCNNshape) inputNorm = TimeDistributed(Flatten())(inputCNN) inputNorm = Masking(mask_value=0.)(inputNorm) inputNorm = TimeDistributed(Reshape((90, 120, 1)))(inputNorm) inputNorm = BatchNormalization(axis=1)(inputNorm) conv = TimeDistributed( Convolution2D(8, 3, 3, border_mode='same', activation='relu'), name='conv11')(inputNorm) pool = TimeDistributed( MaxPooling2D((2,2), strides=(2, 2)), name='maxpool1')(conv) # Build the MLP - pre-cross-connections inputMLP = Input(shape=inputMLPshape) inputMasked = Masking(mask_value=0., input_shape=inputMLPshape)(inputMLP) fcMLP = TimeDistributed( Dense(32, activation='relu'), name='fc1')(inputMasked) # Add the 1st round of cross-connections - CNN to MLP x21 = TimeDistributed(Convolution2D(8, 1, 1, border_mode='same'))(pool) x21 = TimeDistributed(PReLU())(x21) x21 = TimeDistributed(Flatten())(x21) x21 = TimeDistributed(Dense(32))(x21) x21 = TimeDistributed(PReLU())(x21) # Add 1st shortcut (residual connection) from CNN input to MLP short1_2dto1d = TimeDistributed(MaxPooling2D((4,4), strides=(4,4)))(inputNorm) short1_2dto1d = TimeDistributed(Flatten())(short1_2dto1d) short1_2dto1d = TimeDistributed(Dense(32))(short1_2dto1d) short1_2dto1d = TimeDistributed(PReLU())(short1_2dto1d) # Cross-connections - MLP to CNN x12 = TimeDistributed(Dense(14*29))(fcMLP) x12 = TimeDistributed(PReLU())(x12) x12 = TimeDistributed(Reshape((14,29,1)))(x12) x12 = TimeDistributed(Conv2DTranspose(8, (32, 32), padding='valid'))(x12) x12 = TimeDistributed(PReLU(), name='x1_to_cnn')(x12) # 1st shortcut (residual connection) from MLP input to CNN short1_1dto2d = TimeDistributed(Dense(14*29))(inputMasked) short1_1dto2d = TimeDistributed(PReLU())(short1_1dto2d) short1_1dto2d = TimeDistributed(Reshape((14,29,1)))(short1_1dto2d) short1_1dto2d = TimeDistributed(Conv2DTranspose(8, (32, 32), padding='valid'))(short1_1dto2d) short1_1dto2d = TimeDistributed(PReLU(), name='res1_to_cnn')(short1_1dto2d) # CNN - post-cross-connections 1 pool = add([pool, short1_1dto2d]) merged = concatenate([pool, x12]) conv = TimeDistributed( Convolution2D(16, 3, 3, border_mode='same', activation='relu'), name='conv21')(merged) pool = TimeDistributed( MaxPooling2D((2,2), strides=(2, 2)), name='maxpool2')(conv) # MLP - post-cross-connections 1 fcMLP = add([fcMLP, short1_2dto1d]) fcMLP = concatenate([fcMLP, x21]) fcMLP = TimeDistributed( Dense(32, activation='relu'), name='fc2')(fcMLP) # Add the 2nd round of cross-connections - CNN to MLP x21 = TimeDistributed(Convolution2D(16, 1, 1, border_mode='same'))(pool) x21 = TimeDistributed(PReLU())(x21) x21 = TimeDistributed(Flatten())(x21) x21 = TimeDistributed(Dense(64))(x21) x21 = TimeDistributed(PReLU())(x21) # Add 2nd shortcut (residual connection) from CNN input to MLP short2_2dto1d = TimeDistributed(MaxPooling2D((8,8), strides=(8,4)))(inputNorm) short2_2dto1d = TimeDistributed(Flatten())(short2_2dto1d) short2_2dto1d = TimeDistributed(Dense(32))(short2_2dto1d) short2_2dto1d = TimeDistributed(PReLU())(short2_2dto1d) # Cross-connections - MLP to CNN x12 = TimeDistributed(Dense(7*15))(fcMLP) x12 = TimeDistributed(PReLU())(x12) x12 = TimeDistributed(Reshape((7,15,1)))(x12) x12 = TimeDistributed(Conv2DTranspose(16, (16,16), padding='valid'))(x12) x12 = TimeDistributed(PReLU(), name='x2_to_cnn')(x12) # 2nd shortcut (residual connection) from MLP input to CNN short2_1dto2d = TimeDistributed(Dense(7*15))(inputMasked) short2_1dto2d = TimeDistributed(PReLU())(short2_1dto2d) short2_1dto2d = TimeDistributed(Reshape((7,15,1)))(short2_1dto2d) short2_1dto2d = TimeDistributed(Conv2DTranspose(16, (16, 16), padding='valid'))(short2_1dto2d) short2_1dto2d = TimeDistributed(PReLU(), name='res2_to_cnn')(short2_1dto2d) # CNN - post-cross-connections 2 pool = add([pool, short2_1dto2d]) merged = concatenate([pool, x12]) reshape = TimeDistributed( Flatten(), name='flatten1')(merged) fcCNN = TimeDistributed( Dense(64, activation='relu'), name='fcCNN')(reshape) # Merge the models fcMLP = add([fcMLP, short2_2dto1d]) merged = concatenate([fcCNN, fcMLP, x21]) merged = BatchNormalization(axis=1, name='mergebn')(merged) merged = Dropout(0.5, name='mergedrop')(merged) lstm = LSTM(64)(merged) out = Dense(nb_classes, activation='softmax')(lstm) # Return the model object model = Model(input=[inputCNN, inputMLP], output=out) return model
model4.add(BatchNormalization()) model5 = Sequential() model5.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2)) model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) model6 = Sequential() model6.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2)) model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) merged_model = Sequential() merged_model.add( Merge([model1, model2, model3, model4, model5, model6], mode='concat')) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU())
def GetNetArchitecture(input_shape): model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=input_shape)) model.add(Convolution2D(64, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add( PReLU(alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None)) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, (3, 3), activation='linear')) model.add(BatchNormalization()) model.add(Activation('relu')) # model.add(MaxPooling2D((2,2), strides=(2,2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) # Vgg 16 uses 232 model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) #vgg 16 uses 232 model.add(Dropout(0.5)) model.add(Dense(2, activation='softmax')) return model
def _bn_prelu(input_layer): """Helper to build a BN -> prelu block the activation function could either be relu or prelu """ norm = BatchNormalization()(input_layer) return PReLU()(norm)
def model(): input_1 = Input(shape=(7, 7, 200)) input_2 = Input(shape=(27, 27, 30)) # BAM CAB_conv1 = Conv2D( 16, (3, 3), padding='same', strides=(1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4), # use_bias=False )(input_1) CAB_bn1 = BatchNormalization()(CAB_conv1) CAB_relu1 = PReLU()(CAB_bn1) CAB_avg_pool1 = AveragePooling2D()(CAB_relu1) CAB_conv4 = Conv2D( 32, (3, 3), padding='same', strides=(1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4), # use_bias=False )(CAB_avg_pool1) CAB_bn4 = BatchNormalization()(CAB_conv4) CAB_relu4 = PReLU()(CAB_bn4) CAB_conv5 = Conv2D( 32, (3, 3), padding='same', strides=(1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4), # use_bias=False )(CAB_relu4) CAB_bn5 = BatchNormalization()(CAB_conv5) CAB_relu5 = PReLU()(CAB_bn5) CAB_global_pool = GlobalAveragePooling2D()(CAB_relu5) # =================================================================================================================== CAB_reshape = Reshape( (1, CAB_global_pool._keras_shape[1]))(CAB_global_pool) CAB_conv6 = Conv1D(50, (32), padding='same', strides=(1), kernel_initializer='glorot_uniform', use_bias=False)(CAB_reshape) CAB_bn6 = BatchNormalization()(CAB_conv6) CAB_relu6 = PReLU()(CAB_bn6) CAB_conv7 = Conv1D(200, (50), padding='same', strides=(1), kernel_initializer='glorot_uniform', use_bias=False)(CAB_relu6) CAB_bn7 = BatchNormalization()(CAB_conv7) CAB_sigmoid = Activation('sigmoid')(CAB_bn7) # ================================================================================================================= CAB_mul = multiply([input_1, CAB_sigmoid]) # Spectral feature extraction input_spe = Reshape((CAB_mul._keras_shape[1], CAB_mul._keras_shape[2], CAB_mul._keras_shape[3], 1))(CAB_mul) # input_spe = Reshape((input_1._keras_shape[1], input_1._keras_shape[2], input_1._keras_shape[3], 1))(input_1) conv_spe1 = Conv3D(32, (1, 1, 7), padding='valid', strides=(1, 1, 2))(input_spe) bn_spe1 = BatchNormalization()(conv_spe1) relu_spe1 = PReLU()(bn_spe1) conv_spe11 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe1) bn_spe11 = BatchNormalization()(conv_spe11) relu_spe11 = PReLU()(bn_spe11) # ==============================resnet block=========================================== blockconv_spe1 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe11) blockbn_spe1 = BatchNormalization()(blockconv_spe1) blockrelu_spe1 = PReLU()(blockbn_spe1) conv_spe2 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe1) add_spe1 = add([relu_spe11, conv_spe2]) bn_spe2 = BatchNormalization()(add_spe1) relu_spe2 = PReLU()(bn_spe2) blockconv_spe2 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe2) blockbn_spe2 = BatchNormalization()(blockconv_spe2) blockrelu_spe2 = PReLU()(blockbn_spe2) conv_spe4 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe2) add_spe2 = add([relu_spe2, conv_spe4]) bn_spe4 = BatchNormalization()(add_spe2) relu_spe4 = PReLU()(bn_spe4) blockconv_spe3 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe4) blockbn_spe3 = BatchNormalization()(blockconv_spe3) blockrelu_spe3 = PReLU()(blockbn_spe3) conv_spe41 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe3) add_spe3 = add([relu_spe4, conv_spe41]) # =================================================================================================== bn_spe41 = BatchNormalization()(add_spe3) relu_spe41 = PReLU()(bn_spe41) add_all_spe = add([relu_spe2, relu_spe4, relu_spe41]) conv_spe6 = Conv3D(8, (1, 1, 97), padding='valid', strides=(1, 1, 1))(add_all_spe) bn_spe6 = BatchNormalization()(conv_spe6) relu_spe6 = PReLU()(bn_spe6) # =========================== Spatial feature extraction===================================================== input_spa = Reshape((input_2._keras_shape[1], input_2._keras_shape[2], input_2._keras_shape[3], 1))(input_2) conv_spa1 = Conv3D(16, (5, 5, 30), padding='valid', strides=(1, 1, 1))(input_spa) print('conv_spa1 shape:', conv_spa1.shape) bn_spa1 = BatchNormalization()(conv_spa1) relu_spa1 = PReLU()(bn_spa1) reshape_spa1 = Reshape( (relu_spa1._keras_shape[1], relu_spa1._keras_shape[2], relu_spa1._keras_shape[4], relu_spa1._keras_shape[3]))(relu_spa1) conv_spa11 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1, 1, 1))(reshape_spa1) bn_spa11 = BatchNormalization()(conv_spa11) relu_spa11 = PReLU()(bn_spa11) # SAM VIS_conv1 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_spa11) VIS_BN1 = BatchNormalization()(VIS_conv1) VIS_relu1 = Activation('relu')(VIS_BN1) VIS_SHAPE1 = Reshape( (VIS_relu1._keras_shape[1] * VIS_relu1._keras_shape[2], VIS_relu1._keras_shape[4]))(VIS_relu1) VIS_conv2 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_spa11) VIS_BN2 = BatchNormalization()(VIS_conv2) VIS_relu2 = Activation('relu')(VIS_BN2) VIS_SHAPE2 = Reshape( (VIS_relu2._keras_shape[1] * VIS_relu2._keras_shape[2], VIS_relu2._keras_shape[4]))(VIS_relu2) trans_VIS_SHAPE2 = Permute((2, 1))(VIS_SHAPE2) VIS_conv3 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_spa11) VIS_BN3 = BatchNormalization()(VIS_conv3) VIS_relu3 = Activation('relu')(VIS_BN3) VIS_SHAPE3 = Reshape( (VIS_relu3._keras_shape[1] * VIS_relu3._keras_shape[2], VIS_relu3._keras_shape[4]))(VIS_relu3) VIS_mul1 = dot([VIS_SHAPE1, trans_VIS_SHAPE2], axes=(2, 1)) VIS_sigmoid = Activation('sigmoid')(VIS_mul1) VIS_mul2 = dot([VIS_sigmoid, VIS_SHAPE3], axes=(2, 1)) VIS_SHAPEall = Reshape((23, 23, 16, 1))(VIS_mul2) VIS_conv4 = Conv3D(16, (16, 1, 1), padding='same', strides=(1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(VIS_SHAPEall) VIS_BN4 = BatchNormalization()(VIS_conv4) VIS_ADD = add([relu_spa11, VIS_BN4]) # ===================================resnet block================================================ blockconv_spa1 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1, 1, 1))(VIS_ADD) blockbn_spa1 = BatchNormalization()(blockconv_spa1) blockrelu_spa1 = PReLU()(blockbn_spa1) conv_spa2 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa1) add_spa1 = add([VIS_ADD, conv_spa2]) bn_spa2 = BatchNormalization()(add_spa1) relu_spa2 = PReLU()(bn_spa2) blockconv_spa2 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(relu_spa2) blockbn_spa2 = BatchNormalization()(blockconv_spa2) blockrelu_spa2 = PReLU()(blockbn_spa2) conv_spa4 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa2) add_spa2 = add([relu_spa2, conv_spa4]) bn_spa4 = BatchNormalization()(add_spa2) relu_spa4 = PReLU()(bn_spa4) blockconv_spa3 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(relu_spa4) blockbn_spa3 = BatchNormalization()(blockconv_spa3) blockrelu_spa3 = PReLU()(blockbn_spa3) conv_spa41 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa3) add_spa3 = add([relu_spa4, conv_spa41]) bn_spa41 = BatchNormalization()(add_spa3) relu_spa41 = PReLU()(bn_spa41) # spa-multi-feature-fusion add_all_spa = add([relu_spa2, relu_spa4, relu_spa41]) # feature alignment conv_spa6 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(add_all_spa) bn_spa6 = BatchNormalization()(conv_spa6) relu_spa6 = PReLU()(bn_spa6) conv_spa7 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa6) bn_spa7 = BatchNormalization()(conv_spa7) relu_spa7 = PReLU()(bn_spa7) conv_spa8 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa7) bn_spa8 = BatchNormalization()(conv_spa8) relu_spa8 = PReLU()(bn_spa8) conv_spa81 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa8) bn_spa81 = BatchNormalization()(conv_spa81) relu_spa81 = PReLU()(bn_spa81) conv_spa9 = Conv3D(8, (1, 1, 16), padding='valid', strides=(1, 1, 1))(relu_spa81) bn_spa9 = BatchNormalization()(conv_spa9) relu_spa9 = PReLU()(bn_spa9) # Spectral features and spatial features are connected feature_fusion = concatenate([relu_spe6, relu_spa9]) reshape_all = Reshape( (feature_fusion._keras_shape[1], feature_fusion._keras_shape[2], feature_fusion._keras_shape[4], feature_fusion._keras_shape[3]))(feature_fusion) # joint spectral-spatial feature extraction conv_all1 = Conv3D(16, (3, 3, 3), padding='same', strides=(1, 1, 1))(reshape_all) print('convall1 shape:', conv_all1.shape) bn_all1 = BatchNormalization()(conv_all1) relu_all1 = PReLU()(bn_all1) # SAM2 VIS_conv11 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_all1) VIS_BN11 = BatchNormalization()(VIS_conv11) VIS_relu11 = Activation('relu')(VIS_BN11) VIS_SHAPE11 = Reshape( (VIS_relu11._keras_shape[1] * VIS_relu11._keras_shape[2], VIS_relu11._keras_shape[4]))(VIS_relu11) VIS_conv21 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_all1) VIS_BN21 = BatchNormalization()(VIS_conv21) VIS_relu21 = Activation('relu')(VIS_BN21) VIS_SHAPE21 = Reshape( (VIS_relu21._keras_shape[1] * VIS_relu21._keras_shape[2], VIS_relu21._keras_shape[4]))(VIS_relu21) trans_VIS_SHAPE21 = Permute((2, 1))(VIS_SHAPE21) VIS_conv31 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(relu_all1) VIS_BN31 = BatchNormalization()(VIS_conv31) VIS_relu31 = Activation('relu')(VIS_BN31) VIS_SHAPE31 = Reshape( (VIS_relu31._keras_shape[1] * VIS_relu31._keras_shape[2], VIS_relu31._keras_shape[4]))(VIS_relu31) VIS_mul11 = dot([VIS_SHAPE11, trans_VIS_SHAPE21], axes=(2, 1)) VIS_sigmoid1 = Activation('sigmoid')(VIS_mul11) VIS_mul21 = dot([VIS_sigmoid1, VIS_SHAPE31], axes=(2, 1)) VIS_SHAPEall1 = Reshape((7, 7, 16, 1))(VIS_mul21) VIS_conv41 = Conv3D(16, (16, 1, 1), padding='same', strides=(1), kernel_initializer='glorot_uniform', kernel_regularizer=l2(1e-4))(VIS_SHAPEall1) VIS_BN41 = BatchNormalization()(VIS_conv41) VIS_ADD1 = add([relu_all1, VIS_BN41]) conv_all2 = Conv3D(16, (3), padding='valid', strides=(1, 1, 1))(VIS_ADD1) bn_all2 = BatchNormalization()(conv_all2) relu_all2 = PReLU()(bn_all2) flatten = Flatten()(relu_all2) dense = Dense(units=512, activation="relu", kernel_initializer="he_normal")(flatten) drop = Dropout(0.5)(dense) dense_2 = Dense(units=256, activation="relu", kernel_initializer="he_normal")(drop) drop1 = Dropout(0.5)(dense_2) dense_3 = Dense(units=nb_classes, activation="softmax", kernel_initializer="he_normal")(drop1) model = Model(inputs=[input_1, input_2], outputs=dense_3) sgd = keras.optimizers.sgd(lr=0.001, momentum=0.9) # adam = Adam(lr=0.001) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) model.summary() return model
#============================================================================================================================================================ from keras.layers.core import Dense, Activation, Dropout from keras.layers.recurrent import LSTM from keras.models import Sequential, load_model from keras.optimizers import Adam from keras.layers.advanced_activations import PReLU from keras.callbacks import EarlyStopping from keras.utils import plot_model model = Sequential() model.add(LSTM(input_dim=1, output_dim=50, return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(input_dim=50, output_dim=100, return_sequences=False)) model.add(Dropout(0.2)) model.add(Dense(output_dim=1)) model.add(PReLU( )) ############sigmoid,tanh,relu,Leakyrelu,LeakyRelu(alpha = 0.001),PReLU() optimizer = Adam(lr=0.0001, decay=1e-6) model.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) if os.path.exists('log/stock_price_pred_model.h5'): model = load_model('log/stock_price_pred_model.h5') #============================================================================================================================================================ #fit에 사용되는 valid값 저장 #============================================================================================================================================================ def valid_increasing(model, X_train, Y_train, epochs, val_acc): epochs = epochs for i in range(epochs): val_split = i / 100 + 0.001 history = model.fit(X_train, Y_train,
print('Shape of x_test:', x_test.shape) print("Preparing x_test...") for c in x_test.dtypes[x_test.dtypes == object].index.values: x_test[c] = (x_test[c] == True) sc = StandardScaler() x_train = sc.fit_transform(x_train) len_x = int(x_train.shape[1]) print("len_x is:", len_x) # Neural Network print("\nSetting up neural network model...") nn = Sequential() nn.add(Dense(units=400, kernel_initializer='normal', input_dim=len_x)) nn.add(PReLU()) nn.add(Dropout(.4)) nn.add(Dense(units=160, kernel_initializer='normal')) nn.add(PReLU()) nn.add(BatchNormalization()) nn.add(Dropout(.63)) nn.add(Dense(units=64, kernel_initializer='normal')) nn.add(PReLU()) nn.add(BatchNormalization()) nn.add(Dropout(.45)) nn.add(Dense(units=28, kernel_initializer='normal')) nn.add(PReLU()) nn.add(BatchNormalization()) nn.add(Dropout(.5)) nn.add(Dense(1, kernel_initializer='normal')) nn.compile(loss='mae', optimizer=Adam(lr=4e-3, decay=1e-4))
def create_model(nb_classes, input_shape, config=None): """Create a VGG-16 like model.""" if len(input_shape) != 3: raise Exception("Input shape should be a tuple (nb_channels, nb_rows, " "nb_cols) or (nb_rows, nb_cols, nb_channels), " "depending on your backend.") if config is None: config = {'model': {}} min_feature_map_dimension = min(input_shape[:2]) if min_feature_map_dimension < 32: print("ERROR: Please upsample the feature maps to have at least " "a size of 32 x 32. Currently, it has {}".format(input_shape)) nb_filter = 32 if 'l2' not in config['model']: config['model']['l2'] = 0.0001 # Network definition # input_shape = (None, None, 3) # for fcn input_ = Input(shape=input_shape) x = input_ # Scale feature maps down to [63, 32] x [63, 32] tmp = min_feature_map_dimension / 32. if tmp >= 2: while tmp >= 2.: for _ in range(2): x = Convolution2D(nb_filter, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2( config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x) nb_filter *= 2 tmp /= 2 # 32x32 x = Convolution2D(nb_filter + 37, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) x = Convolution2D(nb_filter + 37, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) # 16x16 x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x) x = Convolution2D(2 * nb_filter, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) x = Convolution2D(2 * nb_filter, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) # 8x8 x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x) x = Convolution2D(2 * nb_filter, (3, 3), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']))(x) x = BatchNormalization()(x) x = PReLU()(x) # 4x4 x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x) x = Convolution2D(512, (4, 4), padding='valid', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']), use_bias=False)(x) x = BatchNormalization()(x) x = PReLU()(x) x = Dropout(0.5)(x) # 1x1 x = Convolution2D(512, (1, 1), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']), use_bias=False)(x) x = BatchNormalization()(x) x = PReLU()(x) x = Dropout(0.5)(x) x = Convolution2D(nb_classes, (1, 1), padding='same', kernel_initializer='he_uniform', kernel_regularizer=l2(config['model']['l2']), use_bias=False)(x) x = GlobalAveragePooling2D()(x) # Adjust for FCN x = BatchNormalization()(x) x = Activation('softmax')(x) model = Model(inputs=input_, outputs=x) return model
y, encoder = preprocess_labels(labels) X_test, ids = load_data('test.csv', train=False) X_test, _ = preprocess_data(X_test, scaler) nb_classes = y.shape[1] print(nb_classes, 'classes') dims = X.shape[1] print(dims, 'dims') print("Building model...") model = Sequential() model.add(Dense(dims, 512, init='glorot_uniform')) model.add(PReLU((512, ))) model.add(BatchNormalization((512, ))) model.add(Dropout(0.5)) model.add(Dense(512, 512, init='glorot_uniform')) model.add(PReLU((512, ))) model.add(BatchNormalization((512, ))) model.add(Dropout(0.5)) model.add(Dense(512, 512, init='glorot_uniform')) model.add(PReLU((512, ))) model.add(BatchNormalization((512, ))) model.add(Dropout(0.5)) model.add(Dense(512, nb_classes, init='glorot_uniform')) model.add(Activation('softmax'))
def Conv1DLSTMbranchedV2(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) conv1 = Conv1D(filters=128, kernel_size=1, padding='same', activation='relu') conv2 = Conv1D(filters=128, kernel_size=2, padding='same', activation='relu') conv3 = Conv1D(filters=128, kernel_size=3, padding='same', activation='relu') conv4 = Conv1D(filters=128, kernel_size=4, padding='same', activation='relu') conv5 = Conv1D(filters=32, kernel_size=5, padding='same', activation='relu') conv6 = Conv1D(filters=32, kernel_size=6, padding='same', activation='relu') conv1a = conv1(embed_input_) glob1a = GlobalAveragePooling1D()(conv1a) conv2a = conv2(embed_input_) glob2a = GlobalAveragePooling1D()(conv2a) conv3a = conv3(embed_input_) glob3a = GlobalAveragePooling1D()(conv3a) conv4a = conv4(embed_input_) glob4a = GlobalAveragePooling1D()(conv4a) conv5a = conv5(embed_input_) glob5a = GlobalAveragePooling1D()(conv5a) conv6a = conv6(embed_input_) glob6a = GlobalAveragePooling1D()(conv6a) if params['bidirectional']: rnn_branch = Bidirectional( GRU(params['lstm_units'], return_sequences=True))(embed_input_) else: rnn_branch = GRU(params['lstm_units'], return_sequences=True)(embed_input_) # rnn_branch = AttentionWithContext()(rnn_branch) rnn_branch = GlobalAveragePooling1D()(rnn_branch) mlp_input = Input(shape=(params['num_columns'],)) mlp = BatchNormalization()(mlp_input) mlp = Dense(256)(mlp) mlp = PReLU()(mlp) mlp = BatchNormalization()(mlp) mlp = Dense(256)(mlp) mlp = PReLU()(mlp) merge_layer = concatenate([glob1a, glob2a, glob3a, glob4a, glob5a, glob6a, rnn_branch, mlp]) x = Dropout(0.2)(merge_layer) x = BatchNormalization()(x) x = Dense(256)(x) x = PReLU()(x) x = Dropout(0.2)(x) x = BatchNormalization()(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=[input_, mlp_input], outputs=x) model.compile(loss='binary_crossentropy', optimizer=params['optimizer'], metrics=['accuracy']) return model
def test_prelu(): from keras.layers.advanced_activations import PReLU np.random.seed(1337) inp = get_standard_values() for train in [True, False]: # test with custom weights alphas = np.random.random(inp.shape) layer = PReLU(weights=alphas, input_shape=inp.flatten().shape) # calling build here causes an error, unclear if this is a bug # layer.build() layer.input = K.variable(inp) outp = K.eval(layer.get_output(train)) assert_allclose(inp, outp) layer.input = K.variable(-inp) outp = K.eval(layer.get_output(train)) assert_allclose(-alphas*inp, outp) # test with default weights layer = PReLU(input_shape=inp.flatten().shape) # layer.build() layer.input = K.variable(inp) outp = K.eval(layer.get_output(train)) assert_allclose(inp, outp) layer.input = K.variable(-inp) outp = K.eval(layer.get_output(train)) assert_allclose(0., alphas*outp) layer.get_config()
def model_generator(self, units=512, dropout=0.5, reg=lambda: regularizers.l1_l2(l1=1e-7, l2=1e-7)): decoder = Sequential(name="decoder") h = 5 decoder.add( Dense(units * 4 * 4, input_dim=self.latent_dim, kernel_regularizer=reg())) # check channel order on below decoder.add(Reshape((4, 4, units))) # decoder.add(SpatialDropout2D(dropout)) #decoder.add(LeakyReLU(0.2)) decoder.add(PReLU()) decoder.add( Conv2D(units // 2, (h, h), padding='same', kernel_regularizer=reg())) # decoder.add(SpatialDropout2D(dropout)) #decoder.add(LeakyReLU(0.2)) decoder.add(PReLU()) decoder.add(UpSampling2D(size=(2, 2))) decoder.add( Conv2D(units // 2, (h, h), padding='same', kernel_regularizer=reg())) # decoder.add(SpatialDropout2D(dropout)) #decoder.add(LeakyReLU(0.2)) decoder.add(PReLU()) decoder.add(UpSampling2D(size=(2, 2))) decoder.add( Conv2D(units // 4, (h, h), padding='same', kernel_regularizer=reg())) # decoder.add(SpatialDropout2D(dropout)) #decoder.add(LeakyReLU(0.2)) decoder.add(PReLU()) decoder.add(UpSampling2D(size=(2, 2))) decoder.add(Conv2D(3, (h, h), padding='same', kernel_regularizer=reg())) # add one more PReLU for fine scale detail? # added another upsampling step to get to 64 x 64 #decoder.add(LeakyReLU(0.2)) decoder.add(PReLU()) decoder.add(UpSampling2D(size=(2, 2))) decoder.add(Conv2D(3, (h, h), padding='same', kernel_regularizer=reg())) decoder.add(Activation('sigmoid')) #decoder.summary() # above assumes a particular output dimension, instead try below #decoder.add(Dense(np.prod(self.img_shape), activation='sigmoid')) #decoder.add(Reshape(self.img_shape)) decoder.summary() z = Input(shape=(self.latent_dim, )) img = decoder(z) return Model(z, img)