def one_block_model(self, input_tensor): """ Method to model one cnn. It doesn't compile the model. :param input_tensor: tensor, to feed the two path :return: output: tensor, the output of the cnn """ # localPath loc_path = Conv2D(64, (7, 7), data_format='channels_first', padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.), kernel_initializer='lecun_uniform', bias_initializer='zeros')(input_tensor) loc_path = MaxPooling2D(pool_size=(4, 4), data_format='channels_first', strides=1, padding='valid')(loc_path) loc_path = Dropout(self.dropout_rate)(loc_path) loc_path = Conv2D(64, (3, 3), data_format='channels_first', padding='valid', activation='relu', use_bias=True, kernel_initializer='lecun_uniform', bias_initializer='zeros', kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(loc_path) loc_path = MaxPooling2D(pool_size=(2, 2), data_format='channels_first', strides=1, padding='valid')(loc_path) loc_path = Dropout(self.dropout_rate)(loc_path) # globalPath glob_path = Conv2D(160, (13, 13), data_format='channels_first', strides=1, padding='valid', activation='relu', use_bias=True, kernel_initializer='lecun_uniform', bias_initializer='zeros', kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input_tensor) glob_path = Dropout(self.dropout_rate)(glob_path) # concatenation of the two path path = Concatenate(axis=1)([loc_path, glob_path]) # output layer output = Conv2D(5, (21, 21), data_format='channels_first', strides=1, padding='valid', activation='softmax', use_bias=True, kernel_initializer='lecun_uniform', bias_initializer='zeros')(path) return output
def one_block_model(self, input_tensor): """ Model for the twoPathways CNN. It doesn't compile the model. The consist of two streams, namely: local_path anc global_path joined in a final stream named path local_path is articulated through: 1st convolution 64x7x7 + relu 1st maxpooling 4x4 1st Dropout with rate: 0.5 2nd convolution 64x3x3 + relu 2nd maxpooling 2x2 2nd droput with rate: 0.5 global_path is articulated through: convolution 160x13x13 + relu dropout with rate: 0.5 path is articulated through: convolution 5x21x21 :param input_tensor: tensor, to feed the two path :return: output: tensor, the output of the cnn """ # localPath loc_path = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input_tensor) loc_path = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(loc_path) loc_path = Dropout(self.dropout_rate)(loc_path) loc_path = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(loc_path) loc_path = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(loc_path) loc_path = Dropout(self.dropout_rate)(loc_path) # globalPath glob_path = Conv2D(160, (13, 13), strides=1, padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input_tensor) glob_path = Dropout(self.dropout_rate)(glob_path) # concatenation of the two path path = Concatenate(axis=-1)([loc_path, glob_path]) # output layer output = Conv2D(5, (21, 21), strides=1, padding='valid', activation='softmax', use_bias=True)(path) return output
def test_kernel_regularization(): x_train, y_train = get_data() for reg in [regularizers.l1(), regularizers.l2(), regularizers.l1_l2()]: model = create_model(kernel_regularizer=reg) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 1 model.train_on_batch(x_train, y_train)
def __init__(self, learning_rate=None, vocab_size=None, embedding_size=None, rnn_output_size=None, dropout_rate=None, bidirectional_rnn=None, rnn_type=None, rnn_layers=None, l1_reg=None, l2_reg=None, initializer=None, word_vector_init=None): """ If an arg is None, it will get its value from config.active_config. """ self._learning_rate = learning_rate or active_config().learning_rate self._vocab_size = vocab_size or active_config().vocab_size self._embedding_size = embedding_size or active_config().embedding_size self._rnn_output_size = (rnn_output_size or active_config().rnn_output_size) self._dropout_rate = dropout_rate or active_config().dropout_rate self._rnn_type = rnn_type or active_config().rnn_type self._rnn_layers = rnn_layers or active_config().rnn_layers self._word_vector_init = (word_vector_init or active_config().word_vector_init) self._initializer = initializer or active_config().initializer if self._initializer == 'vinyals_uniform': self._initializer = RandomUniform(-0.08, 0.08) if bidirectional_rnn is None: self._bidirectional_rnn = active_config().bidirectional_rnn else: self._bidirectional_rnn = bidirectional_rnn l1_reg = l1_reg or active_config().l1_reg l2_reg = l2_reg or active_config().l2_reg self._regularizer = l1_l2(l1_reg, l2_reg) self._keras_model = None if self._vocab_size is None: raise ValueError('config.active_config().vocab_size cannot be ' 'None! You should check your config or you can ' 'explicitly pass the vocab_size argument.') if self._rnn_type not in ('lstm', 'gru'): raise ValueError('rnn_type must be either "lstm" or "gru"!') if self._rnn_layers < 1: raise ValueError('rnn_layers must be >= 1!') if self._word_vector_init is not None and self._embedding_size != 300: raise ValueError('If word_vector_init is not None, embedding_size ' 'must be 300')
def test_kernel_regularization(): (x_train, y_train), (x_test, y_test) = get_data() for reg in [regularizers.l1(), regularizers.l2(), regularizers.l1_l2()]: model = create_model(kernel_regularizer=reg) model.compile(loss='categorical_crossentropy', optimizer='sgd') assert len(model.losses) == 1 model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
def _create_layers(self, input_layer): """ Create the encoding and the decoding layers of the deep autoencoder. :param input_layer: Input size. :return: self """ encode_layer = input_layer for i, l in enumerate(self.n_hidden): encode_layer = Dense(units=l, name='encoder_%d' % i, activation=self.enc_activation[i], kernel_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]), bias_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]))(encode_layer) self._decode_layer = encode_layer for i, l in enumerate(self.n_hidden[-2:-(len(self.n_hidden)+1):-1] + [K.int_shape(input_layer)[1]]): self._decode_layer = Dense(units=l, name='decoder_%d' % i, activation=self.dec_activation[i])(self._decode_layer)
def _create_layers(self, input_shape, n_output): """ Create the finetuning model :param input_shape: :param n_output: :return: self """ # Hidden layers for i, l in enumerate(self.layers): self._model.add(Dense(input_shape=[input_shape[1] if i == 0 else None], units=l.n_hidden, weights=l.get_model_parameters()['enc'], activation=l.enc_activation, kernel_regularizer=l1_l2(l.l1_reg, l.l2_reg), bias_regularizer=l1_l2(l.l1_reg, l.l2_reg))) if self.dropout[i] > 0: self._model.add(Dropout(rate=self.dropout[i])) # Output layer self._model.add(Dense(units=n_output, activation=self.out_activation))
def l1l2_penalty_reg(alpha=1.0, l1_ratio=0.5): '''Calculate L1 and L2 penalties for a Keras layer This follows the same formulation as in the R package glmnet and Sklearn Args: alpha ([float]): amount of regularization. l1_ratio ([float]): portion of L1 penalty. Setting to 1.0 equals Lasso. ''' if l1_ratio == .0: return l2(alpha) elif l1_ratio == 1.: return l1(alpha) else: return l1_l2(l1_ratio*alpha, 1./2*(1 - l1_ratio)*alpha)
def wide_deep(df_train, df_test, wide_cols, x_cols, embedding_cols, cont_cols, method): """Run the wide and deep model. Parameters are the same as those for the wide and deep functions """ # Default model_type is "wide_deep" X_train_wide, y_train_wide, X_test_wide, y_test_wide = \ wide(df_train, df_test, wide_cols, x_cols, target, model_type, method) X_train_deep, y_train_deep, X_test_deep, y_test_deep, deep_inp_embed, deep_inp_layer = \ deep(df_train, df_test, embedding_cols, cont_cols, target, model_type, method) X_tr_wd = [X_train_wide] + X_train_deep Y_tr_wd = y_train_deep # wide or deep is the same here X_te_wd = [X_test_wide] + X_test_deep Y_te_wd = y_test_deep # wide or deep is the same here activation, loss, metrics = fit_param[method] if metrics: metrics = [metrics] # WIDE w = Input(shape=(X_train_wide.shape[1],), dtype='float32', name='wide') # DEEP: the output of the 50 neurons layer will be the deep-side input d = merge(deep_inp_embed, mode='concat') d = Flatten()(d) d = BatchNormalization()(d) d = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(d) d = Dense(50, activation='relu', name='deep')(d) # WIDE + DEEP wd_inp = concatenate([w, d]) wd_out = Dense(Y_tr_wd.shape[1], activation=activation, name='wide_deep')(wd_inp) wide_deep = Model(inputs=[w] + deep_inp_layer, outputs=wd_out) wide_deep.compile(optimizer=Adam(lr=0.01), loss=loss, metrics=metrics) wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=10, batch_size=128) # Maybe you want to schedule a second search with lower learning rate # wide_deep.optimizer.lr = 0.0001 # wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=10, batch_size=128) results = wide_deep.evaluate(X_te_wd, Y_te_wd) print("\n", results)
model.add(Conv1D(32, kernel_size=3, activation='relu')) model.add(Conv1D(32, kernel_size=3, activation='relu')) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.2)) #model.add(GlobalAveragePooling1D()) #model.add(Conv1D(8,strides =1, kernel_size = 3)) #16 #model.add(LeakyReLU(alpha=0.1)) #model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) #this layer ok? model.add(Dense(500, activation='relu')) #model.add(LeakyReLU()) model.add(Dense(300, activation='relu')) #model.add(LeakyReLU()) #model.add(Dense(20, activation = 'relu')) #model.add(LeakyReLU()) model.add(Dense(8, activation='softmax', activity_regularizer=l1_l2())) #model.add(Dense(8, activation='softmax')) ''' #model MLP model = Sequential() model.add(Dense(20, input_dim=marco, activation='relu')) #model.add(Dense(200, activation='relu')) model.add(Dense(catenumber , activation='softmax')) ''' model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizers.Adam(lr=0.0005), metrics=['accuracy']) print(model.summary()) model.fit(X_train, y_train, batch_size=batch_size,
def deep(df_train, df_test, embedding_cols, cont_cols, target, model_type, method): """Run the deep model. Two layers of 100 and 50 neurons. In a decent, finished code these would be tunable. Params: ------- df_train, df_test: train and test datasets embedding_cols: columns to be passed as embeddings cont_cols : numerical columns to be combined with the embeddings target : the target feature model_type : accepts "deep" and "wide_deep" (or anything that is not "wide"). If "wide_deep" the function will build and returns the inputs but NOT run any model method : the fitting method. accepts regression, logistic and multiclass Returns: -------- if "deep": print the results obtained on the test set in the terminal. if "wide_deep": X_train, y_train, X_test, y_test: the inputs required to build wide and deep inp_embed, inp_layer: the embedding layers and the input tensors for Model() """ df_train['IS_TRAIN'] = 1 df_test['IS_TRAIN'] = 0 df_deep = pd.concat([df_train, df_test]) deep_cols = embedding_cols + cont_cols # I 'd say that adding numerical columns to embeddings can be done in two ways: # 1_. normalise the values in the dataframe and pass them to the network # 2_. add BatchNormalization() layer. (I am not entirely sure this is right) # I'd say option 1 is the correct one. 2 performs better, which does not say much, but... # 1_. Scaling in the dataframe # scaler = MinMaxScaler() # cont_df = df_deep[cont_cols] # cont_norm_df = pd.DataFrame(scaler.fit_transform(df_train[cont_cols])) # cont_norm_df.columns = cont_cols # for c in cont_cols: df_deep[c] = cont_norm_df[c] df_deep, unique_vals = val2idx(df_deep, embedding_cols) train = df_deep[df_deep.IS_TRAIN == 1].drop('IS_TRAIN', axis=1) test = df_deep[df_deep.IS_TRAIN == 0].drop('IS_TRAIN', axis=1) embeddings_tensors = [] n_factors = 8 reg = 1e-3 for ec in embedding_cols: layer_name = ec + '_inp' t_inp, t_build = embedding_input( layer_name, unique_vals[ec], n_factors, reg) embeddings_tensors.append((t_inp, t_build)) del (t_inp, t_build) continuous_tensors = [] for cc in cont_cols: layer_name = cc + '_in' t_inp, t_build = continous_input(layer_name) continuous_tensors.append((t_inp, t_build)) del (t_inp, t_build) X_train = [train[c] for c in deep_cols] y_train = np.array(train[target].values).reshape(-1, 1) X_test = [test[c] for c in deep_cols] y_test = np.array(test[target].values).reshape(-1, 1) if method == 'multiclass': y_train = onehot(y_train) y_test = onehot(y_test) inp_layer = [et[0] for et in embeddings_tensors] inp_layer += [ct[0] for ct in continuous_tensors] inp_embed = [et[1] for et in embeddings_tensors] inp_embed += [ct[1] for ct in continuous_tensors] if model_type == 'deep': activation, loss, metrics = fit_param[method] if metrics: metrics = [metrics] d = merge(inp_embed, mode='concat') d = Flatten()(d) # 2_. layer to normalise continous columns with the embeddings d = BatchNormalization()(d) d = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(d) # d = Dropout(0.5)(d) # Dropout don't seem to help in this model d = Dense(50, activation='relu')(d) # d = Dropout(0.5)(d) # Dropout don't seem to help in this model d = Dense(y_train.shape[1], activation=activation)(d) deep = Model(inp_layer, d) deep.compile(Adam(0.01), loss=loss, metrics=metrics) deep.fit(X_train, y_train, batch_size=64, nb_epoch=10) results = deep.evaluate(X_test, y_test) print("\n", results) else: return X_train, y_train, X_test, y_test, inp_embed, inp_layer
def keras_build_fn(num_feature, num_output, is_sparse, embedding_dim=-1, num_hidden_layer=2, hidden_layer_dim=512, activation='elu', learning_rate=1e-3, dropout=0.5, l1=0.0, l2=0.0, loss='categorical_crossentropy'): """Initializes and compiles a Keras DNN model using the Adam optimizer. Args: num_feature: number of features num_output: number of outputs (targets, e.g., classes)) is_sparse: boolean whether input data is in sparse format embedding_dim: int number of nodes in embedding layer; if value is <= 0 then no embedding layer will be present in the model num_hidden_layer: number of hidden layers hidden_layer_dim: int number of nodes in the hidden layer(s) activation: string activation function for hidden layers; see https://keras.io/activations/ learning_rate: float learning rate for Adam dropout: float proportion of nodes to dropout; values in [0, 1] l1: float strength of L1 regularization on weights l2: float strength of L2 regularization on weights loss: string loss function; see https://keras.io/losses/ Returns: model: Keras.models.Model compiled Keras model """ assert num_hidden_layer >= 1 inputs = Input(shape=(num_feature, ), sparse=is_sparse) activation_func_args = () if activation.lower() == 'prelu': activation_func = PReLU elif activation.lower() == 'leakyrelu': activation_func = LeakyReLU elif activation.lower() == 'elu': activation_func = ELU elif activation.lower() == 'thresholdedrelu': activation_func = ThresholdedReLU else: activation_func = Activation activation_func_args = (activation) if l1 > 0 and l2 > 0: reg_init = lambda: regularizers.l1_l2(l1, l2) elif l1 > 0: reg_init = lambda: regularizers.l1(l1) elif l2 > 0: reg_init = lambda: regularizers.l2(l2) else: reg_init = lambda: None if embedding_dim > 0: # embedding layer e = Dense(embedding_dim)(inputs) x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(e) x = activation_func(*activation_func_args)(x) x = Dropout(dropout)(x) else: x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(inputs) x = activation_func(*activation_func_args)(x) x = Dropout(dropout)(x) # add additional hidden layers for _ in range(num_hidden_layer - 1): x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(x) x = activation_func(*activation_func_args)(x) x = Dropout(dropout)(x) x = Dense(num_output)(x) preds = Activation('softmax')(x) model = Model(inputs=inputs, outputs=preds) model.compile(optimizer=Adam(lr=learning_rate), loss=loss) return model
# Initial variables reg1_ = 0 reg2_ = 0 lrs_ = 0.1 ac_in_ = 'selu' ac_hid_ = 'elu' ac_out_ = 'relu' # Define the layer which will be used and give them a name. This is useful for # accessing the weights and biases. input_layer = Dense(5, input_dim=5, activation=ac_in_, kernel_regularizer=regularizers.l1_l2(0, 0)) #hidden_layer1 = Dense(3, activation = ac_hid_, # kernel_regularizer = regularizers.l1_l2(0,0)) hidden_layer2 = Dense(10, activation=ac_hid_, kernel_regularizer=regularizers.l1_l2(0, 0)) # hidden_layer3 = Dense(5, activation = ac_hid_, # kernel_regularizer = regularizers.l1_l2(0,0)) hidden_layer4 = Dense(100, activation=ac_hid_, kernel_regularizer=regularizers.l1_l2(0, 0)) hidden_layer5 = Dense(100, activation=ac_hid_, kernel_regularizer=regularizers.l1_l2(0, 0)) hidden_layer6 = Dense(100, activation=ac_hid_,
def twoBlocksDCNN(self): """ :param in_channels: int, number of input channel :param in_shape: int, dim of the input image :return: Model, TwoPathCNN compiled """ input = Input(shape=(65, 65, 4)) # localPath locPath = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input) locPath = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(locPath) locPath = Dropout(self.dropout_rate)(locPath) locPath = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(locPath) locPath = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(locPath) locPath = Dropout(self.dropout_rate)(locPath) # globalPath globPath = Conv2D(160, (13, 13), strides=1, padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input) globPath = Dropout(self.dropout_rate)(globPath) # concatenation of the two path path = Concatenate(axis=-1)([locPath, globPath]) # output layer cnn1 = Conv2D(5, (21, 21), padding='valid', activation='softmax', use_bias=True)(path) #second CNN input_cnn2 = Input(shape=(33, 33, 4)) conc_input = Concatenate(axis=-1)([input_cnn2, cnn1]) locPath2 = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(conc_input) locPath2 = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(locPath2) locPath2 = Dropout(self.dropout_rate)(locPath2) locPath2 = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(locPath2) locPath2 = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(locPath2) locPath2 = Dropout(self.dropout_rate)(locPath2) # globalPath globPath2 = Conv2D(160, (13, 13), padding='valid', activation='relu', use_bias=True, kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate), kernel_constraint=max_norm(2.), bias_constraint=max_norm(2.))(input_cnn2) globPath2 = Dropout(self.dropout_rate)(globPath2) # concatenation of the two path path2 = Concatenate(axis=-1)([locPath2, globPath2]) # output layer output = Conv2D(5, (21, 21), strides=1, padding='valid', activation='softmax', use_bias=True)(path2) #compiling model model = Model(inputs=[input, input_cnn2], outputs=output) sgd = SGD(lr=self.learning_rate, momentum=self.momentum_rate, decay=self.decay_rate, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) print 'DCNN done!' return model
def validate(X,Y,epoch): dimension=X.shape[1] print('Starting Execution of CV') kfold = KFold(n_splits=5, shuffle=True) cvscores = [] trainingscores =[] split=0 best_lr = 0.01 best_bs = 256 dropout=0.1 initializer='lecun_uniform' for train, test in kfold.split(X,Y): model = Sequential() model.add(Dense(units=96, activation='softsign', input_dim=dimension, kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001))) model.add(Dropout(dropout)) model.add(Dense(units=96, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001))) model.add(Dense(units=48, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001))) model.add(Dense(units=48, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001))) sgd = SGD(lr=best_lr) model.add(Dense(units=1, activation='linear')) model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy']) # Fit the model model.fit(X[train], Y[train], epochs=epoch, batch_size=best_bs, verbose=False) y_pred = model.predict(X[test]) y_train = model.predict(X[train]) y_train = y_train.flatten() y_pred = y_pred.flatten() try: training_error = metrics.mean_absolute_error(Y[train], y_train) error = metrics.mean_absolute_error(Y[test], y_pred) trainingscores.append(training_error) except: print("Input contains null values. Skipping Config.") continue cvscores.append(error) split=split+1 print("Validation Score: %.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores))) print("Training Score: %.2f%% (+/- %.2f%%)" % (np.mean(trainingscores), np.std(trainingscores))) return
model = Sequential() model.add( Conv2D(32, kernel_size=3, padding='same', activation='relu', input_shape=(32, 32, 3))) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2), activation='relu')) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(Flatten()) model.add(Dense(10, activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(Dense(10, activation='softmax')) #3. 컴파일, 훈련 model.compile(loss='categorical_crossentropy', optimizer=Adam(1e-4), metrics=['acc']) modelpath = './model/cifar100/{epoch:02d}-{val_loss:.4f}.hdf5' checkpoint = ModelCheckpoint(filepath=modelpath, monitor='val_loss', save_best_only=True, mode='auto')
width_shift_range=0.15, height_shift_range=0.15, rescale=0.3, horizontal_flip=False, vertical_flip=False) datagen.fit(train_x) inp = Input(shape=(96, 100, 1)) x = inp x = BatchNormalization()(x) x = Conv2D(filters=4, strides=1, kernel_size=(1, 1), padding="same", kernel_regularizer=regularizers.l1_l2())(x) x = Conv2D(filters=4, strides=1, kernel_size=(1, 1), padding="same", kernel_regularizer=regularizers.l1_l2())(x) x = MaxPool2D(pool_size=(1, 1))(x) x = BatchNormalization()(x) x = Conv2D(filters=32, strides=1, kernel_size=(5, 5), padding="same", kernel_regularizer=regularizers.l1_l2())(x) x = Conv2D(filters=32, strides=1,
X_test_deep = [test[c] for c in deep_cols] Y_test_deep = test[target] X_tr_wd = [X_train_wide] + X_train_deep Y_tr_wd = Y_train_deep # wide or deep is the same here X_te_wd = [X_test_wide] + X_test_deep Y_te_wd = Y_test_deep # wide or deep is the same here #WIDE wide_inp = Input(shape=(X_train_wide.shape[1],), dtype='float32', name='wide_inp') #DEEP deep_inp = merge([ge, xyz, ag, fb, it, cp, cpa], mode='concat') deep_inp = Flatten()(deep_inp) # layer to normalise continous columns with the embeddings deep_inp = BatchNormalization()(deep_inp) deep_inp = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(deep_inp) deep_inp = Dense(50, activation='relu',name='deep_inp')(deep_inp) #WIDE + DEEP wide_deep_inp = concatenate([wide_inp, deep_inp]) wide_deep_out = Dense(1, activation='sigmoid', name='wide_deep_out')(wide_deep_inp) wide_deep = Model(inputs=[wide_inp, gender, age, xyz_campaign, fb_campaign_id,cpco, cpcoa], outputs=wide_deep_out) wide_deep.compile(optimizer=Adam(lr=0.001),loss='mse', metrics=['accuracy']) wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=50, batch_size=80) # wide_deep.optimizer.lr = 0.001 # wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=5, batch_size=64) results = wide_deep.evaluate(X_te_wd, Y_te_wd) print "\n Results with wide and deep model: %.3f" % results[1]
def model_Pnem4(img_dims): inputs = Input(shape=(img_dims, img_dims, 3)) # Ist conv block x = Conv2D(filters=64, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(inputs) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=64, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) #x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) x = MaxPooling2D(pool_size=(4, 4))(x) # 2nd conv block x = Conv2D(filters=128, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=128, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x) # 3rd conv block x = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x) # Fourth conv block x = Conv2D(filters=384, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=384, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x) # fifth conv block x = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x) # sixth conv block x = Conv2D(filters=1024, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(filters=1024, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x) # FC layer x = Flatten()(x) x = Dense(units=2048, activation='relu')(x) x = Dropout(rate=0.4)(x) x = Dense(units=1024, activation='relu')(x) x = Dropout(rate=0.3)(x) x = Dense(units=512, activation='relu')(x) x = Dropout(rate=0.2)(x) x = Dense(units=256, activation='relu')(x) x = Dropout(rate=0.2)(x) x = Dense(units=128, activation='relu')(x) x = Dropout(rate=0.1)(x) # Output layer output = Dense(units=3, activation='softmax', name='dense_out')(x) # Creating model and compiling model = Model(inputs=inputs, outputs=output) model.summary() return model
print("y_val ", y_val.shape) print("partial_y_train ", partial_y_train.shape) # NN MODEL # Use of DROPOUT #model = models.Sequential() #model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001), activation='relu', input_shape=(10000,))) #model.add(layers.Dropout(0.5)) #model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001),activation='relu')) #model.add(layers.Dropout(0.5)) #model.add(layers.Dense(1, activation='sigmoid')) # Use of REGULARIZATION model = models.Sequential() model.add(layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),activation='relu', input_shape=(10000,))) model.add(layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) # REGULARIZERS L1 L2 #regularizers.l1(0.001) #regularizers.l2(0.001) regularizers.l1_l2(l1=0.001, l2=0.001) # OPTIMIZERS #model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss=losses.binary_crossentropy, metrics=[metrics.binary_accuracy]) model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['accuracy']) # FIT / TRAIN model NumEpochs = 28
model.add(Dropout(0.2)) model.add(Conv1D(filters=100, kernel_size=4, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=4)) model.add(CuDNNGRU(100, return_sequences=True)) model.add(GlobalMaxPooling1D(input_shape=(25, 100))) model.add( Dense(3, activation='softmax', activity_regularizer=regularizers.l1_l2(0.01))) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['categorical_accuracy']) print(model.summary()) history = model.fit(X_train, y_train, batch_size=BATCH, epochs=10, validation_data=(X_val, y_val)) model.save_weights('./hate_detector_trained.h5')
# embed_xvals = np.array(embed_xvals) ut = get_unique_tokens(prep_xvals) num_words = len(ut) word_index = buildWordToInt(ut, []) embed_xvals = prep_feed_model(prep_xvals, word_index) rawtest_x, rawtest_y = get_test_set() rawtest_x = myTokenize(rawtest_x) test_x = prep_feed_model(rawtest_x, word_index) test_y = intents_to_categorical(rawtest_y) save_to_json("xval_man_tokens.json", str(word_index)) reg = l2(0.01) reg_2 = l1_l2() # embed_init = RandomUniform(seed=313) embed_init = RandomUniform(seed=15) # Activity reg deals with outputs # Embeddings regs deals with hidden values my_embedding = Embedding(num_words, embedding_vector_length, embeddings_initializer=embed_init, activity_regularizer=reg, embeddings_constraint=MaxNorm(max_value=2, axis=0), embeddings_regularizer=None, input_length=max_review_length, trainable=True)
folder = "D:\PickledData\\" # create the MLP and CNN models mlp = mg.create_mlp(6, regress=False) cnn = mg.create_cnn(512, 512, 1, regress=False) # create the input to our final set of layers as the *output* of both # the MLP and CNN combinedInput = concatenate([mlp.output, cnn.output]) # our final FC layer head will have two dense layers, the final one # being our regression head x = Dense(4, activation="relu")(combinedInput) comb_output = (Dense(1,activation='linear', kernel_regularizer=regularizers.l1_l2(l1 = 0.001,l2 = 0.001)))(x) # our final model will accept categorical/numerical data on the MLP # input and images on the CNN input, outputting a single value model = Model(inputs=[mlp.input, cnn.input], outputs=comb_output) adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) model.compile(loss="mean_absolute_percentage_error", optimizer=adam,metrics=['mae']) model.summary() tensorboard = TensorBoard(log_dir = "/logs/mixed_data_3/") early_stop= EarlyStopping(monitor = 'val_loss', patience = 3,restore_best_weights=True) model.fit_generator(generator = pd.data_generator(folder,'train',25), validation_data = pd.data_generator(folder,'test',25), steps_per_epoch=(240),
print("\n\n----" + log_key + "----", file=log_file) print("------------------------------------------------") print("%d) Implementation Model: %s" % (r + 1, dl_model)) print("------------------------------------------------") start_time = time.time() # START: Training Time Tracker K.clear_session() # Kills current TF comp-graph & creates a new one model = Sequential() if (dl_model == "RLVECN_Link_Pred"): model.add( Conv1D(filters=(embed_dim * 4) + 1, kernel_size=3, strides=1, input_shape=(train_X.shape[1], train_X.shape[2]), kernel_initializer='glorot_uniform', kernel_regularizer=l1_l2(l1=0.00, l2=0.04), use_bias=True, padding='same', data_format='channels_last')) model.add( Conv1D(filters=(embed_dim * 4) + 1, kernel_size=3, strides=1, kernel_initializer='glorot_uniform', kernel_regularizer=l1_l2(l1=0.00, l2=0.04), use_bias=True, padding='same', data_format='channels_last')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(
# NN MODEL # Use of DROPOUT #model = models.Sequential() #model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001), activation='relu', input_shape=(10000,))) #model.add(layers.Dropout(0.5)) #model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001),activation='relu')) #model.add(layers.Dropout(0.5)) #model.add(layers.Dense(1, activation='sigmoid')) # Use of REGULARIZATION model = models.Sequential() model.add( layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001), activation='relu', input_shape=(10000, ))) model.add( layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001), activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) # REGULARIZERS L1 L2 #regularizers.l1(0.001) #regularizers.l2(0.001) regularizers.l1_l2(l1=0.001, l2=0.001) # OPTIMIZERS #model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss=losses.binary_crossentropy, metrics=[metrics.binary_accuracy])
def train_model(training_set, valid_set, hidden_units, n_epochs=250, batch_size=10, n_prints=25, actv='relu', optimizer='adam', reg=0.0001, regtype = None): if regtype == 'l1': regularizer = regularizers.l1(reg) elif regtype == 'l2': regularizer=regularizers.l2(reg) elif regtype == 'l1l2': regularizer=regularizers.l1_l2(reg) else: regularizer=None X_train = training_set[:,:-1] Y_train = convert_to_onehot(training_set[:,-1].astype('int'), N_OUT) X_valid = valid_set[:,:-1] Y_valid = convert_to_onehot(valid_set[:,-1].astype('int'), N_OUT) model = Sequential() # adding layers to neural network for i in range(len(hidden_units) + 1): if i == 0 : model.add(Dense(hidden_units[i], input_dim=N_IN, activation=actv, kernel_regularizer=regularizer)) elif i == len(hidden_units): model.add(Dense(N_OUT, activation='softmax')) else: model.add(Dense(hidden_units[i], activation=actv, kernel_regularizer=regularizer)) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) print("\nArchitecture of hidden layers : %s" % str(hidden_units)) train_error = [] test_error = [] for i in range(n_prints): scores_train = model.evaluate(X_train, Y_train, verbose=0) scores_test = model.evaluate(X_valid, Y_valid, verbose=0) print("\nProgress : %d / %d epochs" %(i * n_epochs //n_prints, n_epochs)) print("Train %s: %.2f%%" % (model.metrics_names[1], scores_train[1]*100)) print("Valid %s: %.2f%%" % (model.metrics_names[1], scores_test[1]*100)) train_error.append(1 - scores_train[1]) test_error.append(1 - scores_test[1]) # Fit the model model.fit(X_train, Y_train, epochs=1, batch_size=batch_size, verbose=0) print("\nResults") print("Train %s: %.2f%%" % (model.metrics_names[1], scores_train[1]*100)) print("Valid %s: %.2f%%" % (model.metrics_names[1], scores_test[1]*100)) train_error.append(1 - scores_train[1]) test_error.append(1 - scores_test[1]) train_error = np.array(train_error) test_error = np.array(test_error) plt.plot(np.arange(n_epochs+1), train_error, 'k', label='Train error') plt.plot(np.arange(n_epochs+1), test_error, 'k--', label='Test error') plt.xlabel("Number of training epochs") plt.ylabel("Classification error") plt.legend(fancybox=True, shadow=True) plt.title("Architecture of hidden layers : %s" % str(hidden_units)) archstr = str(hidden_units).strip('[]').replace(' ','_').replace(',','') #plt.savefig("figures/ecg_%s.png"%archstr) plt.savefig("figures/sdss_%s_%f.png"%(archstr, reg)) plt.clf() np.savetxt("log/sdss_train_%s_%f.txt"%(archstr, reg), train_error) np.savetxt("log/sdss_test_%s_%f.txt"%(archstr, reg), test_error) case_str_train = "sdss_train_%s_%s_%f"%(archstr, regtype, reg) case_str_test = "sdss_test_%s_%s_%f"%(archstr, regtype, reg) log_dict[case_str_train] = (1-scores_train[1]) log_dict[case_str_test] = (1-scores_test[1])
print('y_train :',y_train.shape) print('y_test :',y_test.shape) #2. 데이터 정규화 x_train = x_train/255 x_test = x_test/255 print('x_train :',x_train.shape) print('x_test :',x_test.shape) #.3 모델구성 model = Sequential() model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu', input_shape=(32,32,3))) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')) model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')) model.add(Conv2D(128, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(Conv2D(128, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')) model.add(Flatten()) model.add(Dense(256, activation='relu', kernel_regularizer=l1_l2(0.001))) model.add(Dense(10, activation='softmax'))
def build_model(input_layer, start_neurons): # 128 -> 64 conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(input_layer) conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(conv1) batch1 = BatchNormalization()(conv1) reg1 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv1) pool1 = MaxPooling2D((2, 2))(conv1) pool1 = Dropout(0.25)(pool1) # 64 -> 32 conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(pool1) conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(conv2) batch2 = BatchNormalization()(conv2) reg2 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv2) pool2 = MaxPooling2D((2, 2))(conv2) pool2 = Dropout(0.5)(pool2) # 32 -> 16 conv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(pool2) conv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(conv3) batch3 = BatchNormalization()(conv3) reg3 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv3) pool3 = MaxPooling2D((2, 2))(conv3) pool3 = Dropout(0.5)(pool3) # 16 -> 8 conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(pool3) conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(conv4) batch4 = BatchNormalization()(conv4) reg4 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv4) pool4 = MaxPooling2D((2, 2))(conv4) pool4 = Dropout(0.5)(pool4) # Middle convm = Conv2D(start_neurons * 16, (3, 3), activation="relu", padding="same")(pool4) convm = Conv2D(start_neurons * 16, (3, 3), activation="relu", padding="same")(convm) # 8 -> 16 deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding="same")(convm) uconv4 = concatenate([deconv4, conv4]) uconv4 = Dropout(0.5)(uconv4) uconv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(uconv4) uconv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(uconv4) # 16 -> 32 deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4) uconv3 = concatenate([deconv3, conv3]) uconv3 = Dropout(0.5)(uconv3) uconv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(uconv3) uconv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(uconv3) # 32 -> 64 deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding="same")(uconv3) uconv2 = concatenate([deconv2, conv2]) uconv2 = Dropout(0.5)(uconv2) uconv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(uconv2) uconv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(uconv2) # 64 -> 128 deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2) uconv1 = concatenate([deconv1, conv1]) uconv1 = Dropout(0.25)(uconv1) uconv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(uconv1) uconv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(uconv1) output_layer = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(uconv1) return output_layer
# Get hdf5 file # Please read the README_Info in GAN-TransferLearning for information about how to get Blondes64_Transfer.h5 hdf5_file = os.path.join("PATH TO DATASET", "Blondes64_Transfer.h5") with h5py.File(hdf5_file, "r") as hf: X_train = hf["data"][()] #[()] makes it read the whole thing X_train = X_train.astype(np.float32) / 255 #---------------- # HYPERPARAMETERS #---------------- randomDim = 100 adam = Adam(lr=0.0002, beta_1=0.5) reg = lambda: l1_l2(l1=1e-7, l2=1e-7) dropout = 0 # Load the old models # Please read the README_Info in GAN-TransferLearning for information about how to get the weight-files below old_discriminator = 'dcgan64_discriminator_transfer.h5' old_generator = 'dcgan64_generator_transfer.h5' # There is a strange bug if the optimizer is loaded from last network therefore just delete them with h5py.File(old_generator, 'a') as f: if 'optimizer_weights' in f.keys(): del f['optimizer_weights'] with h5py.File(old_discriminator, 'a') as f: if 'optimizer_weights' in f.keys():
def double_tanh(x): return (K.tanh(x) * 2) get_custom_objects().update({'double_tanh': Double_Tanh(double_tanh)}) # Model Generation model = Sequential() #check https://machinelearningmastery.com/use-weight-regularization-lstm-networks-time-series-forecasting/ model.add( LSTM(25, input_shape=(20, 1), dropout=0.0, kernel_regularizer=l1_l2(0.00, 0.00), bias_regularizer=l1_l2(0.00, 0.00))) model.add(Dense(1)) model.add(Activation(double_tanh)) model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mse', 'mae']) #, kernel_regularizer=l1_l2(0,0.1), bias_regularizer=l1_l2(0,0.1), print(model.metrics_names) # Fitting the Model model_scores = {} Reg = False d = 'hybrid_LSTM' if Reg:
def test_arg_l1_reg_and_l2_reg(self, model): model._regularizer = l1_l2(0.01, 0.01) self._build_and_assert(model)
def Conv2DClassifierIn1(x_train, y_train, x_test, y_test): summary = True verbose = 1 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = {{choice([32, 64, 128, 256, 512])}} epoch = {{choice([25, 50, 75, 100, 125, 150, 175, 200])}} conv_block = {{choice(['two', 'three', 'four'])}} conv1_num = {{choice([8, 16, 32, 64])}} conv2_num = {{choice([16, 32, 64, 128])}} conv3_num = {{choice([32, 64, 128])}} conv4_num = {{choice([32, 64, 128, 256])}} dense1_num = {{choice([128, 256, 512])}} dense2_num = {{choice([64, 128, 256])}} l1_regular_rate = {{uniform(0.00001, 1)}} l2_regular_rate = {{uniform(0.000001, 1)}} drop1_num = {{uniform(0.1, 1)}} drop2_num = {{uniform(0.0001, 1)}} activator = {{choice(['elu', 'relu', 'tanh'])}} optimizer = {{choice(['adam', 'rmsprop', 'SGD'])}} #--------------------------------------------------------------------------------------------------------------- kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' loss_type = 'binary_crossentropy' metrics = ['accuracy'] my_callback = None # early_stopping = EarlyStopping(monitor='val_loss', patience=4) # checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5', # verbose=1, # save_best_only=True) # my_callback = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, # patience=5, min_lr=0.0001) # build -------------------------------------------------------------------------------------------------------- input_layer = Input(shape=x_train.shape[1:]) conv = layers.Conv2D(conv1_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(input_layer) conv = layers.Conv2D(conv1_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(conv) if conv_block == 'two': conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) elif conv_block == 'three': conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv3_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv3_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) elif conv_block == 'four': conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv2_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv3_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv3_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv4_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(pool) conv = layers.Conv2D(conv4_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm) flat = layers.Flatten()(pool) drop = layers.Dropout(drop1_num)(flat) dense = layers.Dense(dense1_num, activation=activator, kernel_regularizer=regularizers.l1_l2( l1=l1_regular_rate, l2=l2_regular_rate))(drop) BatchNorm = layers.BatchNormalization(axis=-1)(dense) drop = layers.Dropout(drop2_num)(BatchNorm) dense = layers.Dense(dense2_num, activation=activator, kernel_regularizer=regularizers.l1_l2( l1=l1_regular_rate, l2=l2_regular_rate))(drop) output_layer = layers.Dense(len(np.unique(y_train)), activation='softmax')(dense) model = models.Model(inputs=input_layer, outputs=output_layer) if summary: model.summary() # train(self): class_weights = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train.reshape(-1)) class_weights_dict = dict(enumerate(class_weights)) model.compile( optimizer=optimizer, loss=loss_type, metrics=metrics # accuracy ) result = model.fit(x=x_train, y=y_train, batch_size=batch_size, epochs=epoch, verbose=verbose, callbacks=my_callback, validation_data=(x_test, y_test), shuffle=True, class_weight=class_weights_dict) validation_acc = np.amax(result.history['val_acc']) print('Best validation acc of epoch:', validation_acc) return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def get_regularizer(l1=0.01, l2=0.01): return l1_l2(l1=l1, l2=l2)
def build_model(self, n_features): """ The method builds a new member of the ensemble and returns it. """ # derived parameters self.hyperparameters['n_members'] = self.hyperparameters[ 'n_segments'] * self.hyperparameters['n_members_segment'] # initialize optimizer and early stopping self.optimizer = Adam(lr=self.hyperparameters['lr'], beta_1=0.9, beta_2=0.999, epsilon=None, decay=0., amsgrad=False) self.es = EarlyStopping(monitor=f'val_{self.loss_name}', min_delta=0.0, patience=self.hyperparameters['patience'], verbose=1, mode='min', restore_best_weights=True) inputs = Input(shape=(n_features, )) h = GaussianNoise(self.hyperparameters['noise_in'], name='noise_input')(inputs) for i in range(self.hyperparameters['layers']): h = Dense(self.hyperparameters['neurons'], activation=self.hyperparameters['activation'], kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_hidden'], self.hyperparameters['l2_hidden']), kernel_initializer='random_uniform', bias_initializer='zeros', name=f'hidden_{i}')(h) h = Dropout(self.hyperparameters['dropout'], name=f'hidden_dropout_{i}')(h) mu = Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_mu'], self.hyperparameters['l2_mu']), kernel_initializer='random_uniform', bias_initializer='zeros', name='mu_output')(h) mu = GaussianNoise(self.hyperparameters['noise_mu'], name='noise_mu')(mu) if self.hyperparameters['pdf'] == 'normal' or self.hyperparameters[ 'pdf'] == 'skewed': sigma = Dense(1, activation='softplus', kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_sigma'], self.hyperparameters['l2_sigma']), kernel_initializer='random_uniform', bias_initializer='zeros', name='sigma_output')(h) sigma = GaussianNoise(self.hyperparameters['noise_sigma'], name='noise_sigma')(sigma) if self.hyperparameters['pdf'] == 'skewed': alpha = Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2( self.hyperparameters['l1_alpha'], self.hyperparameters['l2_alpha']), kernel_initializer='random_uniform', bias_initializer='zeros', name='alpha_output')(h) alpha = GaussianNoise(self.hyperparameters['noise_alpha'], name='noise_alpha')(alpha) if self.hyperparameters['pdf'] is None: outputs = mu elif self.hyperparameters['pdf'] == 'normal': outputs = concatenate([mu, sigma]) elif self.hyperparameters['pdf'] == 'skewed': outputs = concatenate([mu, sigma, alpha]) model = Model(inputs=inputs, outputs=outputs) return model
from keras.layers import TimeDistributed, Bidirectional from keras import regularizers from ..custom_layers.lrn import LRN fillerMap = { 'constant': 'Constant', 'uniform': 'RandomUniform', 'gaussian': 'RandomNormal', 'xavier': 'glorot_normal', 'msra': 'he_normal' } regularizerMap = { 'l1': regularizers.l1(), 'l2': regularizers.l2(), 'l1_l2': regularizers.l1_l2(), 'L1L2': regularizers.l1_l2(), 'None': None } constraintMap = { 'max_norm': 'max_norm', 'non_neg': 'non_neg', 'unit_norm': 'unit_norm', 'MaxNorm': 'max_norm', 'NonNeg': 'non_neg', 'UnitNorm': 'unit_norm', 'None': None }
def PGNN_train_test(optimizer_name, optimizer_val, drop_rate, iteration, n_layers, n_nodes, tr_size, pre_train, use_YPhy, lake_name): # Hyper-parameters of the training process # batch_size = int(tr_size/2) batch_size = 1000 num_epochs = 1000 val_frac = 0.2 patience_val = 100 # Initializing results filename exp_name = "Pre-train" + optimizer_name + '_drop' + str(drop_rate) + '_nL' + str(n_layers) + '_nN' + str(n_nodes) + '_trsize' + str(tr_size) + '_iter' + str(iteration) exp_name = exp_name.replace('.','pt') results_dir = '../../../../results/Lake/' model_name = results_dir + exp_name + '.h5' # storing the trained model results_name = results_dir + exp_name + '_results.dat' # storing the results of the model # Load features (Xc) and target values (Y) data_dir = '../../../../data/' filename = lake_name + '.mat' mat = spio.loadmat(data_dir + filename, squeeze_me=True, variable_names=['Y','Xc_doy','Modeled_temp']) Xc = mat['Xc_doy'] Y = mat['Y'] Xc = Xc[:,:-1] # remove Y_phy, physics model outputs # train and test data trainX, testX, trainY, testY = train_test_split(Xc, Y, train_size=tr_size/Xc.shape[0], test_size=tr_size/Xc.shape[0], random_state=42, shuffle=True) ## train and test data #trainX, trainY = Xc[:tr_size,:], Y[:tr_size] #testX, testY = Xc[-50:,:], Y[-50:] dependencies = {'root_mean_squared_error': root_mean_squared_error} # load the pre-trained model using non-calibrated physics-based model predictions (./data/unlabeled.dat) loaded_model = load_model(results_dir + pre_train, custom_objects=dependencies) # Creating the model model = Sequential() for layer in np.arange(n_layers): if layer == 0: model.add(Dense(n_nodes, activation='relu', input_shape=(np.shape(trainX)[1],))) else: model.add(Dense(n_nodes, activation='relu', kernel_regularizer=l1_l2(l1=.00, l2=.00))) # model.add(Dropout(rate=drop_rate)) model.add(MCDropout(rate=drop_rate)) model.add(Dense(1, activation='linear')) # pass the weights to all layers but 1st input layer, whose dimensions are updated for new_layer, layer in zip(model.layers[1:], loaded_model.layers[1:]): new_layer.set_weights(layer.get_weights()) model.compile(loss='mean_squared_error', optimizer=optimizer_val, metrics=[root_mean_squared_error]) early_stopping = EarlyStopping(monitor='val_loss', patience=patience_val,verbose=1) print('Running...' + optimizer_name) history = model.fit(trainX, trainY, batch_size=batch_size, epochs=num_epochs, verbose=0, validation_split=val_frac, callbacks=[early_stopping, TerminateOnNaN()]) test_score = model.evaluate(testX, testY, verbose=1) print(test_score) # scale the uniform numbers to original space # max and min value in each column max_in_column_Xc = np.max(trainX,axis=0) min_in_column_Xc = np.min(trainX,axis=0) # Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc) Xc_org = Xx*(max_in_column_Xc-min_in_column_Xc) + min_in_column_Xc samples = [] for i in range(int(nsim)): #print("simulation num:",i) predictions = model.predict(Xc_org) samples.append(predictions) return np.array(samples)
def unet(input_shape=(256, 256, 1), l1=0.00001, l2=0.005): # Hyper-parameters values initializer = 'he_normal' pool_size = (2, 2) # Compute input shape, receptive field and output shape after softmax activation input_shape = input_shape # Activations, regularizers and optimizers regularizer = l1_l2(l1=l1, l2=l2) # Architecture definition # INPUT x = Input(shape=input_shape, name='V-net_input') # First block (down) first_conv = Conv2D(8, 3, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_initial', padding='same')(x) tmp = BatchNormalization(axis=3, name='batch_norm_1.1')(first_conv) tmp = Activation('relu')(tmp) z1 = Conv2D(8, 3, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_1.1', padding='same')(tmp) c11 = Conv2D(8, 1, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_1.1', padding='same')(x) end_11 = Add()([z1, c11]) # Second block (down) tmp = BatchNormalization(axis=3, name='batch_norm_2.1')(end_11) tmp = Activation('relu')(tmp) tmp = Conv2D(16, 2, strides=(2, 2), kernel_initializer=initializer, kernel_regularizer=regularizer, name='downpool_1', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_2.2')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(16, 3, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_2.1', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_2.3')(tmp) tmp = Activation('relu')(tmp) z2 = Conv2D(16, 3, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_2.2', padding='same')(tmp) c21 = MaxPooling2D(pool_size=pool_size, name='pool_1')(end_11) c21 = Conv2D(16, 1, kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_2.1', padding='same')(c21) end_21 = Add()([z2, c21]) # Third block (down) tmp = BatchNormalization(axis=3, name='batch_norm_3.1')(end_21) tmp = Activation('relu')(tmp) tmp = Conv2D(32, 2, strides=(2, 2), kernel_initializer=initializer, kernel_regularizer=regularizer, name='downpool_2', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_3.2')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(32, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_3.1', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_3.3')(tmp) tmp = Activation('relu')(tmp) z3 = Conv2D(32, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_3.2', padding='same')(tmp) c31 = MaxPooling2D(pool_size=pool_size, name='pool_2')(end_21) c31 = Conv2D(32, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_3.1', padding='same')(c31) end_31 = Add()([z3, c31]) # Fourth block (down) tmp = BatchNormalization(axis=3, name='batch_norm_4.1')(end_31) tmp = Activation('relu')(tmp) tmp = Conv2D(64, (2, 2), strides=(2, 2), kernel_initializer=initializer, kernel_regularizer=regularizer, name='downpool_3', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_4.2')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(64, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_4.1', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_4.3')(tmp) tmp = Activation('relu')(tmp) z4 = Conv2D(64, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_4.2', padding='same')(tmp) c41 = MaxPooling2D(pool_size=pool_size, name='pool_3')(end_31) c41 = Conv2D(64, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_4.1', padding='same')(c41) end_41 = Add()([z4, c41]) # Fifth block tmp = BatchNormalization(axis=3, name='batch_norm_5.1')(end_41) tmp = Activation('relu')(tmp) tmp = Conv2D(128, (2, 2), strides=(2, 2), kernel_initializer=initializer, kernel_regularizer=regularizer, name='downpool_4', padding='same')(tmp) tmp = BatchNormalization(axis=4, name='batch_norm_5.2')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(128, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_5.1', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_5.3')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(128, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_5.2', padding='same')(tmp) # inflection point c5 = MaxPooling2D(pool_size=pool_size, name='pool_4')(end_41) c5 = Conv2D(128, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_5', padding='same')(c5) end_5 = Add()([tmp, c5]) # Fourth block (up) tmp = BatchNormalization(axis=3, name='batch_norm_4.4')(end_5) tmp = Activation('relu')(tmp) tmp = UpSampling2D(size=pool_size, name='up_4')(tmp) tmp = Conv2D(64, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_up_4', padding='same')(tmp) tmp = Concatenate(axis=3)([tmp, z4]) tmp = BatchNormalization(axis=3, name='batch_norm_4.5')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(64, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_4.3', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_4.6')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(64, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_4.4', padding='same')(tmp) c42 = UpSampling2D(size=pool_size, name='up_4conn')(end_5) c42 = Conv2D(64, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_4.2', padding='same')(c42) end_42 = Add()([tmp, c42]) # Third block (up) tmp = BatchNormalization(axis=3, name='batch_norm_3.4')(end_42) tmp = Activation('relu')(tmp) tmp = UpSampling2D(size=pool_size, name='up_3')(tmp) tmp = Conv2D(32, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_up_3', padding='same')(tmp) tmp = Concatenate(axis=3)([tmp, z3]) tmp = BatchNormalization(axis=3, name='batch_norm_3.5')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(32, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_3.3', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_3.6')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(32, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_3.4', padding='same')(tmp) c32 = UpSampling2D(size=pool_size, name='up_3conn')(end_42) c32 = Conv2D(32, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_3.2', padding='same')(c32) end_32 = Add()([tmp, c32]) # Second block (up) tmp = BatchNormalization(axis=3, name='batch_norm_2.4')(end_32) tmp = Activation('relu')(tmp) tmp = UpSampling2D(size=pool_size, name='up_2')(tmp) tmp = Conv2D(16, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_up_2', padding='same')(tmp) tmp = Concatenate(axis=3)([tmp, z2]) tmp = BatchNormalization(axis=3, name='batch_norm_2.5')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(16, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_2.3', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_2.6')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(16, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_2.4', padding='same')(tmp) c22 = UpSampling2D(size=pool_size, name='up_2conn')(end_32) c22 = Conv2D(16, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_2.2', padding='same')(c22) end_22 = Add()([tmp, c22]) # First block (up) tmp = BatchNormalization(axis=3, name='batch_norm_1.4')(end_22) tmp = Activation('relu')(tmp) tmp = UpSampling2D(size=pool_size, name='up_1')(tmp) tmp = Conv2D(8, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_up_1', padding='same')(tmp) tmp = Concatenate(axis=3)([tmp, z1]) tmp = BatchNormalization(axis=3, name='batch_norm_1.5')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(8, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_1.3', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_1.6')(tmp) tmp = Activation('relu')(tmp) tmp = Conv2D(8, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_1.4', padding='same')(tmp) c12 = UpSampling2D(size=pool_size, name='up_1_conn')(end_22) c12 = Conv2D(8, (1, 1), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_conn_1.2', padding='same')(c12) end_12 = Add()([tmp, c12]) # Final convolution tmp = BatchNormalization(axis=3, name='batch_norm_1.7')(end_12) tmp = Activation('relu')(tmp) tmp = Conv2D(8, (3, 3), kernel_initializer=initializer, kernel_regularizer=regularizer, name='conv_pre_softmax', padding='same')(tmp) tmp = BatchNormalization(axis=3, name='batch_norm_pre_softmax')(tmp) in_softmax = Activation('relu')(tmp) classification = Conv2D(313, (1, 1, 1), kernel_initializer=initializer, name='final_convolution_1x1x1')(in_softmax) y = softmax(classification) model = Model(inputs=x, outputs=y) return model
def build_keras_model(args, modelEpsilon, input_shape, hiddenShrinkage, M , y , M_validation = None, y_validation = None) : hLayerCount = args.hidCount k.set_image_dim_ordering('tf') # otherwise it would give 'negative dimension size' for maxpool operations #k.image_dim_ordering() BNEnabled = int(args.bnorm) == 1 decay_Enabled = int(args.lr_decay) == 1 shrinkage = hiddenShrinkage # k.set_image_dim_ordering('tf') tf.set_random_seed(args.randomSeed) ## tf runs off a different random generator, let it be random for now, to be able to see if it really is the random init that is causing the 'zero' results? if args.optimizer == 1 : optimizer=Adam(lr=args.learnRate, epsilon=modelEpsilon, beta_1=args.momentum, beta_2=0.999, decay=args.lr_decay) # for float16, otherwise we get NaNs else : optimizer = SGD(lr=args.learnRate, momentum=args.momentum, decay=args.lr_decay) # , beta_1=0.99, since the new Batchnorm logic this isn't needed loss = 'mean_squared_error' accMetrc = 'mae' # Set up the base model. model = Sequential() lastLayerSize = args.firstLayerSize #lastLayerSize_MAX # Input = knet_main.knnLayer( myNet,np.array([-1]), knet_main.LAYER_SUBTYPE_INPUT) w_reg = l1_l2(l1=0.0, l2=hiddenShrinkage) # if conv was enabled we then do NOT regularize stuff at the first FC layer as we only want to regularize by h2 once if args.convLayers > 0 : lastOutput = input_shape[0] # for conv nets it is channels last format for Tensorflow, so the first element of the input shape is the actual number of SNPs print("Adding "+str(args.convLayers)+" conv layers, with initial input dimension: " + str(lastOutput), flush=True) currentNumFilters= args.convFilters currentStride = 3 filter_size = 5 # as it turns out it is not actually a problem if the conv outputs something that isn't an integer, so we just need to downsample it model.add(Conv1D(currentNumFilters, filter_size, input_shape=(input_shape),kernel_regularizer=w_reg, kernel_initializer='he_normal' , padding="same", strides=currentStride )) if BNEnabled : model.add(BatchNormalization()) addActivation(model,args.hidAct) if args.dropout != -1 : model.add(Dropout(args.dropout)) # k.floatx() # 'float32' # args.convLayers = 2 lastOutput = (lastOutput - filter_size +2) / currentStride + 1 lastOutput = int(lastOutput) # as these can only be integers print("filter size : " + str(filter_size), flush=True) #i=1 shrinkage = 0.0 currentStride = 1 pool_size = 2 for i in range(1, args.convLayers +1) : # decide on filter size, depending on input, Conv layers must always produce even outputs so that maxpool can half them filter_size = 3 if lastOutput % 2 != 0 : filter_size = 4 # if the current output is not even, then we have to use a filter size of 4, otherwise we get fractions after the maxpool operation ## currentNumFilters = (i+1) * args.convFilters currentNumFilters = currentNumFilters // 2 model.add(Conv1D(currentNumFilters, filter_size,kernel_regularizer=None, kernel_initializer='he_normal' , padding="same", strides=currentStride)) if BNEnabled : model.add(BatchNormalization()) addActivation(model,args.hidAct) if args.dropout != -1 : model.add(Dropout(args.dropout)) lastOutput = (lastOutput - filter_size +2) / currentStride + 1 lastOutput = int(lastOutput) # as these can only be integers print("filter size affter Conv ("+str(i)+") : " + str(filter_size) + " / output: " + str(lastOutput), flush=True) lastOutput = (lastOutput - pool_size) / pool_size + 1 # compute what dimensions the conv+maxpool operations are going to leave for the next layer print("filter size affter Maxpool ("+str(i)+") : " + str(filter_size) + " / output: " + str(lastOutput), flush=True) model.add(MaxPooling1D()) # the default is 2 model.add(Flatten()) # Flatten the data for input into the plain hidden layer for i in range(1,hLayerCount+1) : # iterate 1 based, otherwise we will get a reduction after the first layer, no matter the widthReductionRate, as 0 is divisible by anything if i > 1 or args.convLayers > 0 : shrinkageParam = w_reg # only add regularizer for first layer, subsequent layers will always have none else : shrinkageParam = None #if i == (hLayerCount-1) : lastWidth = 2 # enforce so that the last widht is always 2, ie 1 neuron makes it MORE like the other LESS likely if i == 1: model.add(Dense(lastLayerSize,kernel_regularizer=shrinkageParam, kernel_initializer='he_normal', input_shape = input_shape)) else : model.add(Dense(lastLayerSize,kernel_regularizer=shrinkageParam, kernel_initializer='he_normal')) if BNEnabled : model.add(BatchNormalization()) addActivation(model,args.hidAct) if args.dropout != -1 : model.add(Dropout(args.dropout)) print("added layer at depth: " + str(i) + " with width: " + str(lastLayerSize) + " / shrinkage: " + str(shrinkage)) # control the 'fatness' of the network: we reduce the width at a given rate: if this is 1, then at every subsequent layer, if its 2, then every 2nd layer etc if i % args.widthReductionRate == 0 : lastLayerSize = lastLayerSize // 2 if lastLayerSize < 2 : break # if # if hLayerCount == 0 : model.add(Dense(1, kernel_initializer='he_normal',kernel_regularizer=w_reg, input_shape = input_shape)) else : model.add(Dense(1, kernel_initializer='he_normal',kernel_regularizer=shrinkageParam)) if len( y.shape) > 1 : model.add(Activation('softmax')) # Compile the model. model.compile(loss=loss, optimizer=optimizer, metrics=[accMetrc]) return(model)
nn_y_train = nn_y_train.reshape(len(nn_y_train), 1) nn_y_train = onehot_encoder.fit_transform(nn_y_train) nn_y_validation = nn_y_validation.reshape(len(nn_y_validation), 1) nn_y_validation = onehot_encoder.fit_transform(nn_y_validation) test_y_01 = test_y_true.reshape(len(test_y_true), 1) test_y_01 = onehot_encoder.fit_transform(test_y_01) # Training neural network model = Sequential() # first hidden layer model.add( Dense(units=LAYER1_NODES, input_dim=INPUT_NODES, activation="relu", kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2))) # second hidden layer model.add( Dense(units=LAYER2_NODES, activation='relu', kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2))) # output layer model.add(Dense(units=OUTPUT_NODES, activation='softmax')) # compile models with the learning rates set opt = optimizers.adam(learning_rate=LEARNING_RATE) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) history = model.fit(
for l1 in range(1, 5): for l2 in range(1, 5): for l3 in range(1, 5): Hlayer1 = 128 * l1 Hlayer2 = 128 * l2 Hlayer3 = 128 * l3 model = Sequential([ Dense(256, input_dim=202, kernel_regularizer=regularizers.l1(lamda1), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer1, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer2, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(Hlayer3, kernel_regularizer=regularizers.l1_l2(0), kernel_initializer='random_uniform'), Activation('relu'), Dense(51, kernel_regularizer=regularizers.l1(0), kernel_initializer='random_uniform'), Activation('softmax') ])
def PGNN_train_test(optimizer_name, optimizer_val, drop_frac, use_YPhy, iteration, n_layers, n_nodes, tr_size, lamda, reg, samp): # fix_seeds(ss) # Hyper-parameters of the training process # batch_size = tr_size batch_size = 1 num_epochs = 300 val_frac = 0.25 patience_val = 80 # Initializing results filename exp_name = "DNN_loss" + optimizer_name + '_drop' + str( drop_frac) + '_usePhy' + str(use_YPhy) + '_nL' + str( n_layers) + '_nN' + str(n_nodes) + '_trsize' + str( tr_size) + '_lamda' + str(lamda) + '_iter' + str(iteration) exp_name = exp_name.replace('.', 'pt') results_dir = '../results/' model_name = results_dir + exp_name + '_model.h5' # storing the trained model if reg == True and samp == 25: results_name = results_dir + exp_name + '_results_25_regularizer.dat' # storing the results of the model elif reg == False and samp == 25: results_name = results_dir + exp_name + '_results_25.dat' # storing the results of the model elif reg == True and samp == 1519: results_name = results_dir + exp_name + '_results_1519_regularizer.dat' # storing the results of the model elif reg == False and samp == 1519: results_name = results_dir + exp_name + '_results_1519.dat' # storing the results of the model # Load labeled data data = np.loadtxt('../data/labeled_data.dat') # data = np.loadtxt('../data/labeled_data_BK_constw_unique.dat') # data = np.loadtxt('../data/labeled_data_BK_constw_v2.dat') x_labeled = data[:, : 2] # -2 because we do not need porosity predictions y_labeled = data[:, -3: -1] # dimensionless bond length and porosity measurements if samp == 25: data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_25.dat') x_unlabeled = data[:, :] elif samp == 1519: data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_1525.dat') x_unlabeled = data[:, :] x_unlabeled1 = x_unlabeled[:1303, :] x_unlabeled2 = x_unlabeled[-6:, :] x_unlabeled = np.vstack((x_unlabeled1, x_unlabeled2)) # initial porosity init_poro = x_unlabeled[:, -1] x_unlabeled = x_unlabeled[:, :2] # data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_1519.dat') # x_unlabeled = data[:1303, :] # 1303 last regular sample: 260, 46 # x_unlabeled_non = x_unlabeled # normalize dataset with MinMaxScaler scaler = preprocessing.MinMaxScaler(feature_range=(0.0, 1.0)) # scaler = preprocessing.StandardScaler() x_labeled = scaler.fit_transform(x_labeled) # y_labeled = scaler.fit_transform(y_labeled) x_unlabeled = scaler.fit_transform(x_unlabeled) # # initial porosity & physics outputs are removed # x_unlabeled = x_unlabeled[:, :-3] # train and test data trainX, trainY = x_labeled[:tr_size, :], y_labeled[:tr_size] # testX, testY = x_labeled[tr_size:,:], y_labeled[tr_size:] testX, testY = x_labeled[tr_size:, :], y_labeled[tr_size:] if use_YPhy == 0: # Removing the last column from x_unlabeled (corresponding to Y_PHY) x_unlabeled = x_unlabeled[:, :-1] # Creating the model model = Sequential() for layer in np.arange(n_layers): if layer == 0: model.add( Dense(n_nodes, activation='relu', input_shape=(np.shape(trainX)[1], ))) else: if reg: model.add( Dense(n_nodes, activation='relu', kernel_regularizer=l1_l2(l1=.001, l2=.001))) else: model.add(Dense(n_nodes, activation='relu')) # model.add(Dropout(rate=drop_frac)) model.add(MCDropout(rate=drop_frac)) model.add(Dense(2, activation='linear')) # physics-based regularization uinp_sc = K.constant(value=x_unlabeled) # unlabeled input data lam1 = K.constant(value=lamda[0]) # regularization hyper-parameter lam2 = K.constant(value=lamda[1]) # regularization hyper-parameter lam3 = K.constant(value=lamda[2]) # regularization hyper-parameter lam4 = K.constant(value=lamda[3]) # regularization hyper-parameter predictions = model(uinp_sc) # model output at depth i # porosity = K.relu(predictions[:,1]) phyloss1 = bond(predictions[:, 0]) # physics loss 1 # uinp = K.constant(value=x_unlabeled_non) # unlabeled input data phyloss2 = poros(init_poro, predictions[:, 1]) # physics loss 1 phyloss3 = strength1(predictions[:, 0], predictions[:, 1]) phyloss4 = strength2(predictions[:, 0], predictions[:, 1]) totloss = combined_loss( [phyloss1, phyloss2, phyloss3, phyloss4, lam1, lam2, lam3, lam4]) phyloss = phy_loss_mean( [phyloss1, phyloss2, phyloss3, phyloss4, lam1, lam2, lam3, lam4]) model.compile(loss=totloss, optimizer=optimizer_val, metrics=[phyloss, root_mean_squared_error]) early_stopping = EarlyStopping(monitor='val_loss', patience=patience_val, verbose=1) # print('Running...' + optimizer_name) history = model.fit(trainX, trainY, batch_size=batch_size, epochs=num_epochs, verbose=0, validation_split=val_frac, callbacks=[early_stopping, TerminateOnNaN()]) # early_stopping = EarlyStopping(monitor='loss', patience=patience_val, verbose=1) # history = model.fit(trainX, trainY, # batch_size=batch_size, # epochs=num_epochs, # verbose=1, # callbacks=[early_stopping, TerminateOnNaN()]) # test_score = model.evaluate(testX, testY, verbose=0) # predictions = model.predict(x_labeled) # model output at depth i # print(np.sort(predictions[:,0], axis=0)) # predictions = model.predict(x_unlabeled) # model output at depth i # print(np.sort(predictions[:,0], axis=0)) # print('iter: ' + str(iteration) + ' useYPhy: ' + str(use_YPhy) + # ' nL: ' + str(n_layers) + ' nN: ' + str(n_nodes) + # ' lamda1: ' + str(lamda[0]) + ' lamda2: ' + str(lamda[1]) + ' trsize: ' + str(tr_size) + # ' TestRMSE: ' + str(test_score[2]) + ' PhyLoss: ' + str(test_score[1]), ' TestLoss: ' + str(test_score[0]), "\n") # # print('iter: ' + str(iteration) + ' TestRMSE: ' + str(test_score[2]) + ' PhyLoss: ' + str(test_score[1]), "\n") # # model.save(model_name) # # save results # results = {'train_loss_1':history.history['loss_1'], # 'val_loss_1':history.history['val_loss_1'], # 'train_rmse':history.history['root_mean_squared_error'], # 'val_rmse':history.history['val_root_mean_squared_error'], # 'test_rmse':test_score[2], # 'PhyLoss':test_score[1]} # results = {'train_loss_1':history.history['loss_1'], # 'train_rmse':history.history['root_mean_squared_error'], # 'test_rmse':test_score[2], # 'PhyLoss':test_score[1]} # save_obj(results, results_name) # predictions = model.predict(testX) # return results, results_name, predictions, testY, test_score[2], trainY test_score = model.evaluate(testX, testY, verbose=1) print(test_score) samples = [] for i in range(int(nsim)): print("simulation num:", i) predictions = model.predict(Xx) predictions = predictions[:, 1] samples.append(predictions) return np.array(samples)
# create the MLP and CNN models mlp = mg.create_mlp(4, regress=False) cnn = mg.create_cnn(508, 508, 1, regress=False) # create the input to our final set of layers as the *output* of both # the MLP and CNN combinedInput = concatenate([mlp.output, cnn.output]) # our final FC layer head will have two dense layers, the final one # being our regression head x = Dense(4, activation="relu")(combinedInput) #for vector output we have to have many output nodes out_0 = (Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x) out_100 = (Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x) out_200 = (Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x) out_300 = (Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x) out_400 = (Dense(1, activation='linear', kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x) # our final model will accept categorical/numerical data on the MLP # input and images on the CNN input, outputting a single value
class KerasRegularizersTest(keras_parameterized.TestCase, parameterized.TestCase): def create_model(self, kernel_regularizer=None, activity_regularizer=None): model = keras.models.Sequential() model.add( keras.layers.Dense(NUM_CLASSES, kernel_regularizer=kernel_regularizer, activity_regularizer=activity_regularizer, input_shape=(DATA_DIM, ))) return model def get_data(self): (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=10, test_samples=10, input_shape=(DATA_DIM, ), num_classes=NUM_CLASSES) y_train = np_utils.to_categorical(y_train, NUM_CLASSES) y_test = np_utils.to_categorical(y_test, NUM_CLASSES) return (x_train, y_train), (x_test, y_test) def create_multi_input_model_from(self, layer1, layer2): input_1 = keras.layers.Input(shape=(DATA_DIM, )) input_2 = keras.layers.Input(shape=(DATA_DIM, )) out1 = layer1(input_1) out2 = layer2(input_2) out = keras.layers.Average()([out1, out2]) model = keras.models.Model([input_1, input_2], out) model.add_loss(keras.backend.mean(out2)) model.add_loss(tf.reduce_sum(input_1)) return model @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_kernel_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(kernel_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.assertEqual(len(model.losses), 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ('l2_zero', keras.regularizers.l2(0.)), ]) def test_activity_regularization(self, regularizer): (x_train, y_train), _ = self.get_data() model = self.create_model(activity_regularizer=regularizer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.assertEqual(len(model.losses), 1 if tf.executing_eagerly() else 1) model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0) @keras_parameterized.run_all_keras_modes @keras_parameterized.run_with_all_model_types def test_zero_regularization(self): # Verifies that training with zero regularization works. x, y = np.ones((10, 10)), np.ones((10, 3)) model = testing_utils.get_model_from_layers([ keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0)) ], input_shape=(10, )) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) model.fit(x, y, batch_size=5, epochs=1) def test_custom_regularizer_saving(self): def my_regularizer(weights): return tf.reduce_sum(tf.abs(weights)) inputs = keras.Input((10, )) outputs = keras.layers.Dense(1, kernel_regularizer=my_regularizer)(inputs) model = keras.Model(inputs, outputs) model2 = model.from_config( model.get_config(), custom_objects={'my_regularizer': my_regularizer}) self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) model = self.create_multi_input_model_from(dense_layer, dense_layer) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.assertLen(model.losses, 5) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_model(self, regularizer): dense_layer = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) input_tensor = keras.layers.Input(shape=(DATA_DIM, )) dummy_model = keras.models.Model(input_tensor, dense_layer(input_tensor)) model = self.create_multi_input_model_from(dummy_model, dummy_model) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.assertLen(model.losses, 6) @keras_parameterized.run_all_keras_modes @parameterized.named_parameters([ ('l1', regularizers.l1()), ('l2', regularizers.l2()), ('l1_l2', regularizers.l1_l2()), ]) def test_regularization_shared_layer_in_different_models( self, regularizer): shared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer, activity_regularizer=regularizer) models = [] for _ in range(2): input_tensor = keras.layers.Input(shape=(DATA_DIM, )) unshared_dense = keras.layers.Dense(NUM_CLASSES, kernel_regularizer=regularizer) out = unshared_dense(shared_dense(input_tensor)) models.append(keras.models.Model(input_tensor, out)) model = self.create_multi_input_model_from(layer1=models[0], layer2=models[1]) model.compile(loss='categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) # We expect to see 9 losses on the model: # - 2 from the 2 add_loss calls on the outer model. # - 3 from the weight regularizers on the shared_dense layer, unshared_dense # in inner model 1, unshared_dense in inner model 2. # - 4 from activity regularizers on the shared_dense layer. self.assertLen(model.losses, 9) def test_deserialization_error(self): with self.assertRaisesRegex(ValueError, 'Could not interpret regularizer'): keras.regularizers.get(0) @parameterized.named_parameters([ ('l1', regularizers.l1(l1=None), 0.01), ('l2', regularizers.l2(l2=None), 0.01), ('l1_l2', regularizers.l1_l2(l1=None, l2=None), 0.), ]) def test_default_value_when_init_with_none(self, regularizer, expected_value): expected_value = np.asarray(expected_value) if hasattr(regularizer, 'l1'): self.assertAllClose(regularizer.l1, expected_value) if hasattr(regularizer, 'l2'): self.assertAllClose(regularizer.l2, expected_value)
论文中的损失函数 :param y_true: 真实值 :param y_pred: 预测值 :return: 修订版MAPE """ diff = K.square(y_true - y_pred) / K.clip(K.square(y_true), 1, None) weight = 10 return K.mean(K.square(y_pred - y_true) + weight * diff) LOSS_NAME_FUNC = { 'mix': metric_loss, 'rmse': metric_rmse, 'mape': metric_mape, 'mae': metric_mae, 'mse': metric_mse } def get_loss_func(name): assert name in LOSS_NAME_FUNC return LOSS_NAME_FUNC[name] REGULAR_NAME_FUNC = {'l1': l1(), 'l2': l2(), 'l1_l2': l1_l2(), 'none': None} def get_regularizer(name): assert name in REGULAR_NAME_FUNC return REGULAR_NAME_FUNC[name]
def Conv2DClassifierIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test, class_weights_dict, obj): K.clear_session() CUDA = '2' summary = False verbose = 0 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = {{choice([32, 64])}} # batch_size = 64 # epoch = 2 epoch = {{choice([1, 2])}} kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' activator = 'relu' regular_rate = (0.001, 0.001) dropout_rate = 0.1 optimizer = 'adam' loss_type = 'mse' # metrics=('accuracy',acc, mcc, mcc_concise, recall_p, recall_n, precision_p, precision_n, tp_Concise,tn_Concise,fp_Concise,fn_Concise, recall_p_Concise,recall_n_Concise,precision_p_Concise,precision_n_Concise) metrics = ('mae', ) callbacks = None # config TF----------------------------------------------------------------------------------------------------- os.environ['CUDA_VISIBLE_DEVICES'] = CUDA os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' config = tf.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.Session(config=config)) # build -------------------------------------------------------------------------------------------------------- # mCNN feature inputs as a whole 2D array input_layer = Input(shape=x_train.shape[1:]) conv1 = layers.Conv2D(16, kernel_size, kernel_initializer=initializer, activation=activator)(input_layer) conv2 = layers.Conv2D(32, kernel_size, kernel_initializer=initializer, activation=activator)(conv1) pool1 = layers.MaxPooling2D(pool_size, padding=padding_style)(conv2) conv3 = layers.Conv2D(64, kernel_size, kernel_initializer=initializer, activation=activator, kernel_regularizer=regularizers.l1_l2( l1=regular_rate[0], l2=regular_rate[1]))(pool1) conv3_BatchNorm = layers.BatchNormalization(axis=-1)(conv3) pool2 = layers.MaxPooling2D(pool_size, padding=padding_style)(conv3_BatchNorm) conv4 = layers.Conv2D(128, kernel_size, kernel_initializer=initializer, activation=activator, kernel_regularizer=regularizers.l1_l2( l1=regular_rate[0], l2=regular_rate[1]))(pool2) pool3 = layers.MaxPooling2D(pool_size, padding=padding_style)(conv4) flat = layers.Flatten()(pool3) dense = layers.Dense(1024, activation=activator)(flat) dense_BatchNorm = layers.BatchNormalization(axis=-1)(dense) drop = layers.Dropout(dropout_rate)(dense_BatchNorm) # output_layer = layers.Dense(len(np.unique(y_train)),activation='softmax')(drop) output_layer = layers.Dense(1)(drop) model = models.Model(inputs=input_layer, outputs=output_layer) if summary: model.summary() # train(self): # class_weights_dict = dict(enumerate(class_weights)) # print(class_weights_dict) model.compile( optimizer=optimizer, loss=loss_type, metrics=list(metrics) # accuracy ) K.set_session(tf.Session(graph=model.output.graph)) init = K.tf.global_variables_initializer() K.get_session().run(init) result = model.fit( x=x_train, y=ddg_train, batch_size=batch_size, epochs=epoch, verbose=verbose, callbacks=callbacks, validation_data=(x_test, ddg_test), shuffle=True, ) # print('\n----------History:' # '\n%s'%result.history) if obj == 'test_report_reg': pearson_coeff, std = test_report_reg(model, x_test, ddg_test) print('\n----------Predict:\npearson_coeff: %s, std: %s' % (pearson_coeff, std)) objective = pearson_coeff * 2 + std return {'loss': -objective, 'status': STATUS_OK} elif obj == 'val_mae': print(result.history) validation_mae = np.amax(result.history['val_mean_absolute_error']) print('Best validation mae of epoch:', validation_mae) return {'loss': validation_mae, 'status': STATUS_OK}