def one_block_model(self, input_tensor):
"""
Method to model one cnn. It doesn't compile the model.
:param input_tensor: tensor, to feed the two path
:return: output: tensor, the output of the cnn
"""
# localPath
loc_path = Conv2D(64, (7, 7), data_format='channels_first', padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.), kernel_initializer='lecun_uniform', bias_initializer='zeros')(input_tensor)
loc_path = MaxPooling2D(pool_size=(4, 4), data_format='channels_first', strides=1, padding='valid')(loc_path)
loc_path = Dropout(self.dropout_rate)(loc_path)
loc_path = Conv2D(64, (3, 3), data_format='channels_first', padding='valid', activation='relu', use_bias=True,
kernel_initializer='lecun_uniform', bias_initializer='zeros',
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(loc_path)
loc_path = MaxPooling2D(pool_size=(2, 2), data_format='channels_first', strides=1, padding='valid')(loc_path)
loc_path = Dropout(self.dropout_rate)(loc_path)
# globalPath
glob_path = Conv2D(160, (13, 13), data_format='channels_first', strides=1, padding='valid', activation='relu', use_bias=True,
kernel_initializer='lecun_uniform', bias_initializer='zeros',
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input_tensor)
glob_path = Dropout(self.dropout_rate)(glob_path)
# concatenation of the two path
path = Concatenate(axis=1)([loc_path, glob_path])
# output layer
output = Conv2D(5, (21, 21), data_format='channels_first', strides=1, padding='valid', activation='softmax', use_bias=True,
kernel_initializer='lecun_uniform', bias_initializer='zeros')(path)
return output
def one_block_model(self, input_tensor):
"""
Model for the twoPathways CNN.
It doesn't compile the model.
The consist of two streams, namely:
local_path anc global_path joined
in a final stream named path
local_path is articulated through:
1st convolution 64x7x7 + relu
1st maxpooling 4x4
1st Dropout with rate: 0.5
2nd convolution 64x3x3 + relu
2nd maxpooling 2x2
2nd droput with rate: 0.5
global_path is articulated through:
convolution 160x13x13 + relu
dropout with rate: 0.5
path is articulated through:
convolution 5x21x21
:param input_tensor: tensor, to feed the two path
:return: output: tensor, the output of the cnn
"""
# localPath
loc_path = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input_tensor)
loc_path = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(loc_path)
loc_path = Dropout(self.dropout_rate)(loc_path)
loc_path = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(loc_path)
loc_path = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(loc_path)
loc_path = Dropout(self.dropout_rate)(loc_path)
# globalPath
glob_path = Conv2D(160, (13, 13), strides=1, padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input_tensor)
glob_path = Dropout(self.dropout_rate)(glob_path)
# concatenation of the two path
path = Concatenate(axis=-1)([loc_path, glob_path])
# output layer
output = Conv2D(5, (21, 21), strides=1, padding='valid', activation='softmax', use_bias=True)(path)
return output
def test_kernel_regularization():
x_train, y_train = get_data()
for reg in [regularizers.l1(),
regularizers.l2(),
regularizers.l1_l2()]:
model = create_model(kernel_regularizer=reg)
model.compile(loss='categorical_crossentropy', optimizer='sgd')
assert len(model.losses) == 1
model.train_on_batch(x_train, y_train)
def __init__(self,
learning_rate=None,
vocab_size=None,
embedding_size=None,
rnn_output_size=None,
dropout_rate=None,
bidirectional_rnn=None,
rnn_type=None,
rnn_layers=None,
l1_reg=None,
l2_reg=None,
initializer=None,
word_vector_init=None):
"""
If an arg is None, it will get its value from config.active_config.
"""
self._learning_rate = learning_rate or active_config().learning_rate
self._vocab_size = vocab_size or active_config().vocab_size
self._embedding_size = embedding_size or active_config().embedding_size
self._rnn_output_size = (rnn_output_size or
active_config().rnn_output_size)
self._dropout_rate = dropout_rate or active_config().dropout_rate
self._rnn_type = rnn_type or active_config().rnn_type
self._rnn_layers = rnn_layers or active_config().rnn_layers
self._word_vector_init = (word_vector_init or
active_config().word_vector_init)
self._initializer = initializer or active_config().initializer
if self._initializer == 'vinyals_uniform':
self._initializer = RandomUniform(-0.08, 0.08)
if bidirectional_rnn is None:
self._bidirectional_rnn = active_config().bidirectional_rnn
else:
self._bidirectional_rnn = bidirectional_rnn
l1_reg = l1_reg or active_config().l1_reg
l2_reg = l2_reg or active_config().l2_reg
self._regularizer = l1_l2(l1_reg, l2_reg)
self._keras_model = None
if self._vocab_size is None:
raise ValueError('config.active_config().vocab_size cannot be '
'None! You should check your config or you can '
'explicitly pass the vocab_size argument.')
if self._rnn_type not in ('lstm', 'gru'):
raise ValueError('rnn_type must be either "lstm" or "gru"!')
if self._rnn_layers < 1:
raise ValueError('rnn_layers must be >= 1!')
if self._word_vector_init is not None and self._embedding_size != 300:
raise ValueError('If word_vector_init is not None, embedding_size '
'must be 300')
def test_kernel_regularization():
(x_train, y_train), (x_test, y_test) = get_data()
for reg in [regularizers.l1(),
regularizers.l2(),
regularizers.l1_l2()]:
model = create_model(kernel_regularizer=reg)
model.compile(loss='categorical_crossentropy', optimizer='sgd')
assert len(model.losses) == 1
model.fit(x_train, y_train, batch_size=batch_size,
epochs=epochs, verbose=0)
def _create_layers(self, input_layer):
""" Create the encoding and the decoding layers of the deep autoencoder.
:param input_layer: Input size.
:return: self
"""
encode_layer = input_layer
for i, l in enumerate(self.n_hidden):
encode_layer = Dense(units=l,
name='encoder_%d' % i,
activation=self.enc_activation[i],
kernel_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]),
bias_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]))(encode_layer)
self._decode_layer = encode_layer
for i, l in enumerate(self.n_hidden[-2:-(len(self.n_hidden)+1):-1] + [K.int_shape(input_layer)[1]]):
self._decode_layer = Dense(units=l,
name='decoder_%d' % i,
activation=self.dec_activation[i])(self._decode_layer)
def _create_layers(self, input_shape, n_output):
""" Create the finetuning model
:param input_shape:
:param n_output:
:return: self
"""
# Hidden layers
for i, l in enumerate(self.layers):
self._model.add(Dense(input_shape=[input_shape[1] if i == 0 else None],
units=l.n_hidden,
weights=l.get_model_parameters()['enc'],
activation=l.enc_activation,
kernel_regularizer=l1_l2(l.l1_reg, l.l2_reg),
bias_regularizer=l1_l2(l.l1_reg, l.l2_reg)))
if self.dropout[i] > 0:
self._model.add(Dropout(rate=self.dropout[i]))
# Output layer
self._model.add(Dense(units=n_output, activation=self.out_activation))
def l1l2_penalty_reg(alpha=1.0, l1_ratio=0.5):
'''Calculate L1 and L2 penalties for a Keras layer
This follows the same formulation as in the R package glmnet and Sklearn
Args:
alpha ([float]): amount of regularization.
l1_ratio ([float]): portion of L1 penalty. Setting to 1.0 equals
Lasso.
'''
if l1_ratio == .0:
return l2(alpha)
elif l1_ratio == 1.:
return l1(alpha)
else:
return l1_l2(l1_ratio*alpha, 1./2*(1 - l1_ratio)*alpha)
def wide_deep(df_train, df_test, wide_cols, x_cols, embedding_cols, cont_cols, method):
"""Run the wide and deep model. Parameters are the same as those for the
wide and deep functions
"""
# Default model_type is "wide_deep"
X_train_wide, y_train_wide, X_test_wide, y_test_wide = \
wide(df_train, df_test, wide_cols, x_cols, target, model_type, method)
X_train_deep, y_train_deep, X_test_deep, y_test_deep, deep_inp_embed, deep_inp_layer = \
deep(df_train, df_test, embedding_cols, cont_cols, target, model_type, method)
X_tr_wd = [X_train_wide] + X_train_deep
Y_tr_wd = y_train_deep # wide or deep is the same here
X_te_wd = [X_test_wide] + X_test_deep
Y_te_wd = y_test_deep # wide or deep is the same here
activation, loss, metrics = fit_param[method]
if metrics: metrics = [metrics]
# WIDE
w = Input(shape=(X_train_wide.shape[1],), dtype='float32', name='wide')
# DEEP: the output of the 50 neurons layer will be the deep-side input
d = merge(deep_inp_embed, mode='concat')
d = Flatten()(d)
d = BatchNormalization()(d)
d = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(d)
d = Dense(50, activation='relu', name='deep')(d)
# WIDE + DEEP
wd_inp = concatenate([w, d])
wd_out = Dense(Y_tr_wd.shape[1], activation=activation, name='wide_deep')(wd_inp)
wide_deep = Model(inputs=[w] + deep_inp_layer, outputs=wd_out)
wide_deep.compile(optimizer=Adam(lr=0.01), loss=loss, metrics=metrics)
wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=10, batch_size=128)
# Maybe you want to schedule a second search with lower learning rate
# wide_deep.optimizer.lr = 0.0001
# wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=10, batch_size=128)
results = wide_deep.evaluate(X_te_wd, Y_te_wd)
print("\n", results)
model.add(Conv1D(32, kernel_size=3, activation='relu'))
model.add(Conv1D(32, kernel_size=3, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.2))
#model.add(GlobalAveragePooling1D())
#model.add(Conv1D(8,strides =1, kernel_size = 3)) #16
#model.add(LeakyReLU(alpha=0.1))
#model.add(MaxPooling1D(pool_size=2))
model.add(Flatten()) #this layer ok?
model.add(Dense(500, activation='relu'))
#model.add(LeakyReLU())
model.add(Dense(300, activation='relu'))
#model.add(LeakyReLU())
#model.add(Dense(20, activation = 'relu'))
#model.add(LeakyReLU())
model.add(Dense(8, activation='softmax', activity_regularizer=l1_l2()))
#model.add(Dense(8, activation='softmax'))
'''
#model MLP
model = Sequential()
model.add(Dense(20, input_dim=marco, activation='relu'))
#model.add(Dense(200, activation='relu'))
model.add(Dense(catenumber , activation='softmax'))
'''
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=optimizers.Adam(lr=0.0005),
metrics=['accuracy'])
print(model.summary())
model.fit(X_train,
y_train,
batch_size=batch_size,
def deep(df_train, df_test, embedding_cols, cont_cols, target, model_type, method):
"""Run the deep model. Two layers of 100 and 50 neurons. In a decent,
finished code these would be tunable.
Params:
-------
df_train, df_test: train and test datasets
embedding_cols: columns to be passed as embeddings
cont_cols : numerical columns to be combined with the embeddings
target : the target feature
model_type : accepts "deep" and "wide_deep" (or anything that is not
"wide"). If "wide_deep" the function will build and returns the inputs
but NOT run any model
method : the fitting method. accepts regression, logistic and multiclass
Returns:
--------
if "deep":
print the results obtained on the test set in the terminal.
if "wide_deep":
X_train, y_train, X_test, y_test: the inputs required to build wide and deep
inp_embed, inp_layer: the embedding layers and the input tensors for Model()
"""
df_train['IS_TRAIN'] = 1
df_test['IS_TRAIN'] = 0
df_deep = pd.concat([df_train, df_test])
deep_cols = embedding_cols + cont_cols
# I 'd say that adding numerical columns to embeddings can be done in two ways:
# 1_. normalise the values in the dataframe and pass them to the network
# 2_. add BatchNormalization() layer. (I am not entirely sure this is right)
# I'd say option 1 is the correct one. 2 performs better, which does not say much, but...
# 1_. Scaling in the dataframe
# scaler = MinMaxScaler()
# cont_df = df_deep[cont_cols]
# cont_norm_df = pd.DataFrame(scaler.fit_transform(df_train[cont_cols]))
# cont_norm_df.columns = cont_cols
# for c in cont_cols: df_deep[c] = cont_norm_df[c]
df_deep, unique_vals = val2idx(df_deep, embedding_cols)
train = df_deep[df_deep.IS_TRAIN == 1].drop('IS_TRAIN', axis=1)
test = df_deep[df_deep.IS_TRAIN == 0].drop('IS_TRAIN', axis=1)
embeddings_tensors = []
n_factors = 8
reg = 1e-3
for ec in embedding_cols:
layer_name = ec + '_inp'
t_inp, t_build = embedding_input(
layer_name, unique_vals[ec], n_factors, reg)
embeddings_tensors.append((t_inp, t_build))
del (t_inp, t_build)
continuous_tensors = []
for cc in cont_cols:
layer_name = cc + '_in'
t_inp, t_build = continous_input(layer_name)
continuous_tensors.append((t_inp, t_build))
del (t_inp, t_build)
X_train = [train[c] for c in deep_cols]
y_train = np.array(train[target].values).reshape(-1, 1)
X_test = [test[c] for c in deep_cols]
y_test = np.array(test[target].values).reshape(-1, 1)
if method == 'multiclass':
y_train = onehot(y_train)
y_test = onehot(y_test)
inp_layer = [et[0] for et in embeddings_tensors]
inp_layer += [ct[0] for ct in continuous_tensors]
inp_embed = [et[1] for et in embeddings_tensors]
inp_embed += [ct[1] for ct in continuous_tensors]
if model_type == 'deep':
activation, loss, metrics = fit_param[method]
if metrics:
metrics = [metrics]
d = merge(inp_embed, mode='concat')
d = Flatten()(d)
# 2_. layer to normalise continous columns with the embeddings
d = BatchNormalization()(d)
d = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(d)
# d = Dropout(0.5)(d) # Dropout don't seem to help in this model
d = Dense(50, activation='relu')(d)
# d = Dropout(0.5)(d) # Dropout don't seem to help in this model
d = Dense(y_train.shape[1], activation=activation)(d)
deep = Model(inp_layer, d)
deep.compile(Adam(0.01), loss=loss, metrics=metrics)
deep.fit(X_train, y_train, batch_size=64, nb_epoch=10)
results = deep.evaluate(X_test, y_test)
print("\n", results)
else:
return X_train, y_train, X_test, y_test, inp_embed, inp_layer
def keras_build_fn(num_feature,
num_output,
is_sparse,
embedding_dim=-1,
num_hidden_layer=2,
hidden_layer_dim=512,
activation='elu',
learning_rate=1e-3,
dropout=0.5,
l1=0.0,
l2=0.0,
loss='categorical_crossentropy'):
"""Initializes and compiles a Keras DNN model using the Adam optimizer.
Args:
num_feature: number of features
num_output: number of outputs (targets, e.g., classes))
is_sparse: boolean whether input data is in sparse format
embedding_dim: int number of nodes in embedding layer; if value is <= 0 then
no embedding layer will be present in the model
num_hidden_layer: number of hidden layers
hidden_layer_dim: int number of nodes in the hidden layer(s)
activation: string
activation function for hidden layers; see https://keras.io/activations/
learning_rate: float learning rate for Adam
dropout: float proportion of nodes to dropout; values in [0, 1]
l1: float strength of L1 regularization on weights
l2: float strength of L2 regularization on weights
loss: string
loss function; see https://keras.io/losses/
Returns:
model: Keras.models.Model
compiled Keras model
"""
assert num_hidden_layer >= 1
inputs = Input(shape=(num_feature, ), sparse=is_sparse)
activation_func_args = ()
if activation.lower() == 'prelu':
activation_func = PReLU
elif activation.lower() == 'leakyrelu':
activation_func = LeakyReLU
elif activation.lower() == 'elu':
activation_func = ELU
elif activation.lower() == 'thresholdedrelu':
activation_func = ThresholdedReLU
else:
activation_func = Activation
activation_func_args = (activation)
if l1 > 0 and l2 > 0:
reg_init = lambda: regularizers.l1_l2(l1, l2)
elif l1 > 0:
reg_init = lambda: regularizers.l1(l1)
elif l2 > 0:
reg_init = lambda: regularizers.l2(l2)
else:
reg_init = lambda: None
if embedding_dim > 0:
# embedding layer
e = Dense(embedding_dim)(inputs)
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(e)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
else:
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(inputs)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
# add additional hidden layers
for _ in range(num_hidden_layer - 1):
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(x)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
x = Dense(num_output)(x)
preds = Activation('softmax')(x)
model = Model(inputs=inputs, outputs=preds)
model.compile(optimizer=Adam(lr=learning_rate), loss=loss)
return model
# Initial variables
reg1_ = 0
reg2_ = 0
lrs_ = 0.1
ac_in_ = 'selu'
ac_hid_ = 'elu'
ac_out_ = 'relu'
# Define the layer which will be used and give them a name. This is useful for
# accessing the weights and biases.
input_layer = Dense(5,
input_dim=5,
activation=ac_in_,
kernel_regularizer=regularizers.l1_l2(0, 0))
#hidden_layer1 = Dense(3, activation = ac_hid_,
# kernel_regularizer = regularizers.l1_l2(0,0))
hidden_layer2 = Dense(10,
activation=ac_hid_,
kernel_regularizer=regularizers.l1_l2(0, 0))
# hidden_layer3 = Dense(5, activation = ac_hid_,
# kernel_regularizer = regularizers.l1_l2(0,0))
hidden_layer4 = Dense(100,
activation=ac_hid_,
kernel_regularizer=regularizers.l1_l2(0, 0))
hidden_layer5 = Dense(100,
activation=ac_hid_,
kernel_regularizer=regularizers.l1_l2(0, 0))
hidden_layer6 = Dense(100,
activation=ac_hid_,
def twoBlocksDCNN(self):
"""
:param in_channels: int, number of input channel
:param in_shape: int, dim of the input image
:return: Model, TwoPathCNN compiled
"""
input = Input(shape=(65, 65, 4))
# localPath
locPath = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input)
locPath = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(locPath)
locPath = Dropout(self.dropout_rate)(locPath)
locPath = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(locPath)
locPath = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(locPath)
locPath = Dropout(self.dropout_rate)(locPath)
# globalPath
globPath = Conv2D(160, (13, 13), strides=1, padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input)
globPath = Dropout(self.dropout_rate)(globPath)
# concatenation of the two path
path = Concatenate(axis=-1)([locPath, globPath])
# output layer
cnn1 = Conv2D(5, (21, 21), padding='valid', activation='softmax', use_bias=True)(path)
#second CNN
input_cnn2 = Input(shape=(33, 33, 4))
conc_input = Concatenate(axis=-1)([input_cnn2, cnn1])
locPath2 = Conv2D(64, (7, 7), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(conc_input)
locPath2 = MaxPooling2D(pool_size=(4, 4), strides=1, padding='valid')(locPath2)
locPath2 = Dropout(self.dropout_rate)(locPath2)
locPath2 = Conv2D(64, (3, 3), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(locPath2)
locPath2 = MaxPooling2D(pool_size=(2, 2), strides=1, padding='valid')(locPath2)
locPath2 = Dropout(self.dropout_rate)(locPath2)
# globalPath
globPath2 = Conv2D(160, (13, 13), padding='valid', activation='relu', use_bias=True,
kernel_regularizer=regularizers.l1_l2(self.l1_rate, self.l2_rate),
kernel_constraint=max_norm(2.),
bias_constraint=max_norm(2.))(input_cnn2)
globPath2 = Dropout(self.dropout_rate)(globPath2)
# concatenation of the two path
path2 = Concatenate(axis=-1)([locPath2, globPath2])
# output layer
output = Conv2D(5, (21, 21), strides=1, padding='valid', activation='softmax', use_bias=True)(path2)
#compiling model
model = Model(inputs=[input, input_cnn2], outputs=output)
sgd = SGD(lr=self.learning_rate, momentum=self.momentum_rate, decay=self.decay_rate, nesterov=False)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
print 'DCNN done!'
return model
def validate(X,Y,epoch):
dimension=X.shape[1]
print('Starting Execution of CV')
kfold = KFold(n_splits=5, shuffle=True)
cvscores = []
trainingscores =[]
split=0
best_lr = 0.01
best_bs = 256
dropout=0.1
initializer='lecun_uniform'
for train, test in kfold.split(X,Y):
model = Sequential()
model.add(Dense(units=96, activation='softsign', input_dim=dimension, kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001)))
model.add(Dropout(dropout))
model.add(Dense(units=96, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001)))
model.add(Dense(units=48, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001)))
model.add(Dense(units=48, activation='softsign', kernel_initializer=initializer,kernel_regularizer=l1_l2(l1=0.001,l2=0.001)))
sgd = SGD(lr=best_lr)
model.add(Dense(units=1, activation='linear'))
model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy'])
# Fit the model
model.fit(X[train], Y[train], epochs=epoch, batch_size=best_bs, verbose=False)
y_pred = model.predict(X[test])
y_train = model.predict(X[train])
y_train = y_train.flatten()
y_pred = y_pred.flatten()
try:
training_error = metrics.mean_absolute_error(Y[train], y_train)
error = metrics.mean_absolute_error(Y[test], y_pred)
trainingscores.append(training_error)
except:
print("Input contains null values. Skipping Config.")
continue
cvscores.append(error)
split=split+1
print("Validation Score: %.2f%% (+/- %.2f%%)" % (np.mean(cvscores), np.std(cvscores)))
print("Training Score: %.2f%% (+/- %.2f%%)" % (np.mean(trainingscores), np.std(trainingscores)))
return
model = Sequential()
model.add(
Conv2D(32,
kernel_size=3,
padding='same',
activation='relu',
input_shape=(32, 32, 3)))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), activation='relu'))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(Flatten())
model.add(Dense(10, activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(Dense(10, activation='softmax'))
#3. 컴파일, 훈련
model.compile(loss='categorical_crossentropy',
optimizer=Adam(1e-4),
metrics=['acc'])
modelpath = './model/cifar100/{epoch:02d}-{val_loss:.4f}.hdf5'
checkpoint = ModelCheckpoint(filepath=modelpath,
monitor='val_loss',
save_best_only=True,
mode='auto')
width_shift_range=0.15,
height_shift_range=0.15,
rescale=0.3,
horizontal_flip=False,
vertical_flip=False)
datagen.fit(train_x)
inp = Input(shape=(96, 100, 1))
x = inp
x = BatchNormalization()(x)
x = Conv2D(filters=4,
strides=1,
kernel_size=(1, 1),
padding="same",
kernel_regularizer=regularizers.l1_l2())(x)
x = Conv2D(filters=4,
strides=1,
kernel_size=(1, 1),
padding="same",
kernel_regularizer=regularizers.l1_l2())(x)
x = MaxPool2D(pool_size=(1, 1))(x)
x = BatchNormalization()(x)
x = Conv2D(filters=32,
strides=1,
kernel_size=(5, 5),
padding="same",
kernel_regularizer=regularizers.l1_l2())(x)
x = Conv2D(filters=32,
strides=1,
X_test_deep = [test[c] for c in deep_cols] Y_test_deep = test[target] X_tr_wd = [X_train_wide] + X_train_deep Y_tr_wd = Y_train_deep # wide or deep is the same here X_te_wd = [X_test_wide] + X_test_deep Y_te_wd = Y_test_deep # wide or deep is the same here #WIDE wide_inp = Input(shape=(X_train_wide.shape[1],), dtype='float32', name='wide_inp') #DEEP deep_inp = merge([ge, xyz, ag, fb, it, cp, cpa], mode='concat') deep_inp = Flatten()(deep_inp) # layer to normalise continous columns with the embeddings deep_inp = BatchNormalization()(deep_inp) deep_inp = Dense(100, activation='relu', kernel_regularizer=l1_l2(l1=0.01, l2=0.01))(deep_inp) deep_inp = Dense(50, activation='relu',name='deep_inp')(deep_inp) #WIDE + DEEP wide_deep_inp = concatenate([wide_inp, deep_inp]) wide_deep_out = Dense(1, activation='sigmoid', name='wide_deep_out')(wide_deep_inp) wide_deep = Model(inputs=[wide_inp, gender, age, xyz_campaign, fb_campaign_id,cpco, cpcoa], outputs=wide_deep_out) wide_deep.compile(optimizer=Adam(lr=0.001),loss='mse', metrics=['accuracy']) wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=50, batch_size=80) # wide_deep.optimizer.lr = 0.001 # wide_deep.fit(X_tr_wd, Y_tr_wd, nb_epoch=5, batch_size=64) results = wide_deep.evaluate(X_te_wd, Y_te_wd) print "\n Results with wide and deep model: %.3f" % results[1]
def model_Pnem4(img_dims):
inputs = Input(shape=(img_dims, img_dims, 3))
# Ist conv block
x = Conv2D(filters=64, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=64, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
#x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = MaxPooling2D(pool_size=(4, 4))(x)
# 2nd conv block
x = Conv2D(filters=128, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=128, kernel_size=(4, 4), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x)
# 3rd conv block
x = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=256, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x)
# Fourth conv block
x = Conv2D(filters=384, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=384, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x)
# fifth conv block
x = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=512, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x)
# sixth conv block
x = Conv2D(filters=1024, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=1024, kernel_size=(3, 3), padding='same', kernel_regularizer=regularizers.l1_l2(l1=0.0001, l2=0.0001))(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(x)
# FC layer
x = Flatten()(x)
x = Dense(units=2048, activation='relu')(x)
x = Dropout(rate=0.4)(x)
x = Dense(units=1024, activation='relu')(x)
x = Dropout(rate=0.3)(x)
x = Dense(units=512, activation='relu')(x)
x = Dropout(rate=0.2)(x)
x = Dense(units=256, activation='relu')(x)
x = Dropout(rate=0.2)(x)
x = Dense(units=128, activation='relu')(x)
x = Dropout(rate=0.1)(x)
# Output layer
output = Dense(units=3, activation='softmax', name='dense_out')(x)
# Creating model and compiling
model = Model(inputs=inputs, outputs=output)
model.summary()
return model
print("y_val ", y_val.shape)
print("partial_y_train ", partial_y_train.shape)
# NN MODEL
# Use of DROPOUT
#model = models.Sequential()
#model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001), activation='relu', input_shape=(10000,)))
#model.add(layers.Dropout(0.5))
#model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001),activation='relu'))
#model.add(layers.Dropout(0.5))
#model.add(layers.Dense(1, activation='sigmoid'))
# Use of REGULARIZATION
model = models.Sequential()
model.add(layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),activation='relu', input_shape=(10000,)))
model.add(layers.Dense(16, kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# REGULARIZERS L1 L2
#regularizers.l1(0.001)
#regularizers.l2(0.001)
regularizers.l1_l2(l1=0.001, l2=0.001)
# OPTIMIZERS
#model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss=losses.binary_crossentropy, metrics=[metrics.binary_accuracy])
model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=['accuracy'])
# FIT / TRAIN model
NumEpochs = 28
model.add(Dropout(0.2))
model.add(Conv1D(filters=100, kernel_size=4, padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=4))
model.add(CuDNNGRU(100, return_sequences=True))
model.add(GlobalMaxPooling1D(input_shape=(25, 100)))
model.add(
Dense(3,
activation='softmax',
activity_regularizer=regularizers.l1_l2(0.01)))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy'])
print(model.summary())
history = model.fit(X_train,
y_train,
batch_size=BATCH,
epochs=10,
validation_data=(X_val, y_val))
model.save_weights('./hate_detector_trained.h5')
# embed_xvals = np.array(embed_xvals)
ut = get_unique_tokens(prep_xvals)
num_words = len(ut)
word_index = buildWordToInt(ut, [])
embed_xvals = prep_feed_model(prep_xvals, word_index)
rawtest_x, rawtest_y = get_test_set()
rawtest_x = myTokenize(rawtest_x)
test_x = prep_feed_model(rawtest_x, word_index)
test_y = intents_to_categorical(rawtest_y)
save_to_json("xval_man_tokens.json", str(word_index))
reg = l2(0.01)
reg_2 = l1_l2()
# embed_init = RandomUniform(seed=313)
embed_init = RandomUniform(seed=15)
# Activity reg deals with outputs
# Embeddings regs deals with hidden values
my_embedding = Embedding(num_words,
embedding_vector_length,
embeddings_initializer=embed_init,
activity_regularizer=reg,
embeddings_constraint=MaxNorm(max_value=2, axis=0),
embeddings_regularizer=None,
input_length=max_review_length,
trainable=True)
folder = "D:\PickledData\\"
# create the MLP and CNN models
mlp = mg.create_mlp(6, regress=False)
cnn = mg.create_cnn(512, 512, 1, regress=False)
# create the input to our final set of layers as the *output* of both
# the MLP and CNN
combinedInput = concatenate([mlp.output, cnn.output])
# our final FC layer head will have two dense layers, the final one
# being our regression head
x = Dense(4, activation="relu")(combinedInput)
comb_output = (Dense(1,activation='linear', kernel_regularizer=regularizers.l1_l2(l1 = 0.001,l2 = 0.001)))(x)
# our final model will accept categorical/numerical data on the MLP
# input and images on the CNN input, outputting a single value
model = Model(inputs=[mlp.input, cnn.input], outputs=comb_output)
adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(loss="mean_absolute_percentage_error", optimizer=adam,metrics=['mae'])
model.summary()
tensorboard = TensorBoard(log_dir = "/logs/mixed_data_3/")
early_stop= EarlyStopping(monitor = 'val_loss', patience = 3,restore_best_weights=True)
model.fit_generator(generator = pd.data_generator(folder,'train',25),
validation_data = pd.data_generator(folder,'test',25),
steps_per_epoch=(240),
print("\n\n----" + log_key + "----", file=log_file)
print("------------------------------------------------")
print("%d) Implementation Model: %s" % (r + 1, dl_model))
print("------------------------------------------------")
start_time = time.time() # START: Training Time Tracker
K.clear_session() # Kills current TF comp-graph & creates a new one
model = Sequential()
if (dl_model == "RLVECN_Link_Pred"):
model.add(
Conv1D(filters=(embed_dim * 4) + 1,
kernel_size=3,
strides=1,
input_shape=(train_X.shape[1], train_X.shape[2]),
kernel_initializer='glorot_uniform',
kernel_regularizer=l1_l2(l1=0.00, l2=0.04),
use_bias=True,
padding='same',
data_format='channels_last'))
model.add(
Conv1D(filters=(embed_dim * 4) + 1,
kernel_size=3,
strides=1,
kernel_initializer='glorot_uniform',
kernel_regularizer=l1_l2(l1=0.00, l2=0.04),
use_bias=True,
padding='same',
data_format='channels_last'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
# NN MODEL
# Use of DROPOUT
#model = models.Sequential()
#model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001), activation='relu', input_shape=(10000,)))
#model.add(layers.Dropout(0.5))
#model.add(layers.Dense(16, kernel_regularizer=regularizers.l1(0.001),activation='relu'))
#model.add(layers.Dropout(0.5))
#model.add(layers.Dense(1, activation='sigmoid'))
# Use of REGULARIZATION
model = models.Sequential()
model.add(
layers.Dense(16,
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),
activation='relu',
input_shape=(10000, )))
model.add(
layers.Dense(16,
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001),
activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# REGULARIZERS L1 L2
#regularizers.l1(0.001)
#regularizers.l2(0.001)
regularizers.l1_l2(l1=0.001, l2=0.001)
# OPTIMIZERS
#model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss=losses.binary_crossentropy, metrics=[metrics.binary_accuracy])
def train_model(training_set, valid_set, hidden_units,
n_epochs=250, batch_size=10, n_prints=25,
actv='relu', optimizer='adam', reg=0.0001,
regtype = None):
if regtype == 'l1':
regularizer = regularizers.l1(reg)
elif regtype == 'l2':
regularizer=regularizers.l2(reg)
elif regtype == 'l1l2':
regularizer=regularizers.l1_l2(reg)
else:
regularizer=None
X_train = training_set[:,:-1]
Y_train = convert_to_onehot(training_set[:,-1].astype('int'), N_OUT)
X_valid = valid_set[:,:-1]
Y_valid = convert_to_onehot(valid_set[:,-1].astype('int'), N_OUT)
model = Sequential()
# adding layers to neural network
for i in range(len(hidden_units) + 1):
if i == 0 :
model.add(Dense(hidden_units[i],
input_dim=N_IN,
activation=actv,
kernel_regularizer=regularizer))
elif i == len(hidden_units):
model.add(Dense(N_OUT, activation='softmax'))
else:
model.add(Dense(hidden_units[i], activation=actv,
kernel_regularizer=regularizer))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
print("\nArchitecture of hidden layers : %s" % str(hidden_units))
train_error = []
test_error = []
for i in range(n_prints):
scores_train = model.evaluate(X_train, Y_train, verbose=0)
scores_test = model.evaluate(X_valid, Y_valid, verbose=0)
print("\nProgress : %d / %d epochs" %(i * n_epochs //n_prints, n_epochs))
print("Train %s: %.2f%%" % (model.metrics_names[1],
scores_train[1]*100))
print("Valid %s: %.2f%%" % (model.metrics_names[1],
scores_test[1]*100))
train_error.append(1 - scores_train[1])
test_error.append(1 - scores_test[1])
# Fit the model
model.fit(X_train, Y_train, epochs=1,
batch_size=batch_size, verbose=0)
print("\nResults")
print("Train %s: %.2f%%" % (model.metrics_names[1],
scores_train[1]*100))
print("Valid %s: %.2f%%" % (model.metrics_names[1],
scores_test[1]*100))
train_error.append(1 - scores_train[1])
test_error.append(1 - scores_test[1])
train_error = np.array(train_error)
test_error = np.array(test_error)
plt.plot(np.arange(n_epochs+1), train_error, 'k', label='Train error')
plt.plot(np.arange(n_epochs+1), test_error, 'k--', label='Test error')
plt.xlabel("Number of training epochs")
plt.ylabel("Classification error")
plt.legend(fancybox=True, shadow=True)
plt.title("Architecture of hidden layers : %s" % str(hidden_units))
archstr = str(hidden_units).strip('[]').replace(' ','_').replace(',','')
#plt.savefig("figures/ecg_%s.png"%archstr)
plt.savefig("figures/sdss_%s_%f.png"%(archstr, reg))
plt.clf()
np.savetxt("log/sdss_train_%s_%f.txt"%(archstr, reg), train_error)
np.savetxt("log/sdss_test_%s_%f.txt"%(archstr, reg), test_error)
case_str_train = "sdss_train_%s_%s_%f"%(archstr, regtype, reg)
case_str_test = "sdss_test_%s_%s_%f"%(archstr, regtype, reg)
log_dict[case_str_train] = (1-scores_train[1])
log_dict[case_str_test] = (1-scores_test[1])
print('y_train :',y_train.shape)
print('y_test :',y_test.shape)
#2. 데이터 정규화
x_train = x_train/255
x_test = x_test/255
print('x_train :',x_train.shape)
print('x_test :',x_test.shape)
#.3 모델구성
model = Sequential()
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu', input_shape=(32,32,3)))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same'))
model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same'))
model.add(Conv2D(128, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(Conv2D(128, kernel_size=3, padding='same', activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=2, padding='same'))
model.add(Flatten())
model.add(Dense(256, activation='relu', kernel_regularizer=l1_l2(0.001)))
model.add(Dense(10, activation='softmax'))
def build_model(input_layer, start_neurons):
# 128 -> 64
conv1 = Conv2D(start_neurons * 1, (3, 3),
activation="relu",
padding="same")(input_layer)
conv1 = Conv2D(start_neurons * 1, (3, 3),
activation="relu",
padding="same")(conv1)
batch1 = BatchNormalization()(conv1)
reg1 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv1)
pool1 = MaxPooling2D((2, 2))(conv1)
pool1 = Dropout(0.25)(pool1)
# 64 -> 32
conv2 = Conv2D(start_neurons * 2, (3, 3),
activation="relu",
padding="same")(pool1)
conv2 = Conv2D(start_neurons * 2, (3, 3),
activation="relu",
padding="same")(conv2)
batch2 = BatchNormalization()(conv2)
reg2 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv2)
pool2 = MaxPooling2D((2, 2))(conv2)
pool2 = Dropout(0.5)(pool2)
# 32 -> 16
conv3 = Conv2D(start_neurons * 4, (3, 3),
activation="relu",
padding="same")(pool2)
conv3 = Conv2D(start_neurons * 4, (3, 3),
activation="relu",
padding="same")(conv3)
batch3 = BatchNormalization()(conv3)
reg3 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv3)
pool3 = MaxPooling2D((2, 2))(conv3)
pool3 = Dropout(0.5)(pool3)
# 16 -> 8
conv4 = Conv2D(start_neurons * 8, (3, 3),
activation="relu",
padding="same")(pool3)
conv4 = Conv2D(start_neurons * 8, (3, 3),
activation="relu",
padding="same")(conv4)
batch4 = BatchNormalization()(conv4)
reg4 = regularizers.l1_l2(l1=0.01, l2=0.01)(conv4)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(0.5)(pool4)
# Middle
convm = Conv2D(start_neurons * 16, (3, 3),
activation="relu",
padding="same")(pool4)
convm = Conv2D(start_neurons * 16, (3, 3),
activation="relu",
padding="same")(convm)
# 8 -> 16
deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(convm)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(0.5)(uconv4)
uconv4 = Conv2D(start_neurons * 8, (3, 3),
activation="relu",
padding="same")(uconv4)
uconv4 = Conv2D(start_neurons * 8, (3, 3),
activation="relu",
padding="same")(uconv4)
# 16 -> 32
deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="same")(uconv4)
uconv3 = concatenate([deconv3, conv3])
uconv3 = Dropout(0.5)(uconv3)
uconv3 = Conv2D(start_neurons * 4, (3, 3),
activation="relu",
padding="same")(uconv3)
uconv3 = Conv2D(start_neurons * 4, (3, 3),
activation="relu",
padding="same")(uconv3)
# 32 -> 64
deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3),
strides=(2, 2),
padding="same")(uconv3)
uconv2 = concatenate([deconv2, conv2])
uconv2 = Dropout(0.5)(uconv2)
uconv2 = Conv2D(start_neurons * 2, (3, 3),
activation="relu",
padding="same")(uconv2)
uconv2 = Conv2D(start_neurons * 2, (3, 3),
activation="relu",
padding="same")(uconv2)
# 64 -> 128
deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="same")(uconv2)
uconv1 = concatenate([deconv1, conv1])
uconv1 = Dropout(0.25)(uconv1)
uconv1 = Conv2D(start_neurons * 1, (3, 3),
activation="relu",
padding="same")(uconv1)
uconv1 = Conv2D(start_neurons * 1, (3, 3),
activation="relu",
padding="same")(uconv1)
output_layer = Conv2D(1, (1, 1), padding="same",
activation="sigmoid")(uconv1)
return output_layer
# Get hdf5 file
# Please read the README_Info in GAN-TransferLearning for information about how to get Blondes64_Transfer.h5
hdf5_file = os.path.join("PATH TO DATASET", "Blondes64_Transfer.h5")
with h5py.File(hdf5_file, "r") as hf:
X_train = hf["data"][()] #[()] makes it read the whole thing
X_train = X_train.astype(np.float32) / 255
#----------------
# HYPERPARAMETERS
#----------------
randomDim = 100
adam = Adam(lr=0.0002, beta_1=0.5)
reg = lambda: l1_l2(l1=1e-7, l2=1e-7)
dropout = 0
# Load the old models
# Please read the README_Info in GAN-TransferLearning for information about how to get the weight-files below
old_discriminator = 'dcgan64_discriminator_transfer.h5'
old_generator = 'dcgan64_generator_transfer.h5'
# There is a strange bug if the optimizer is loaded from last network therefore just delete them
with h5py.File(old_generator, 'a') as f:
if 'optimizer_weights' in f.keys():
del f['optimizer_weights']
with h5py.File(old_discriminator, 'a') as f:
if 'optimizer_weights' in f.keys():
def double_tanh(x):
return (K.tanh(x) * 2)
get_custom_objects().update({'double_tanh': Double_Tanh(double_tanh)})
# Model Generation
model = Sequential()
#check https://machinelearningmastery.com/use-weight-regularization-lstm-networks-time-series-forecasting/
model.add(
LSTM(25,
input_shape=(20, 1),
dropout=0.0,
kernel_regularizer=l1_l2(0.00, 0.00),
bias_regularizer=l1_l2(0.00, 0.00)))
model.add(Dense(1))
model.add(Activation(double_tanh))
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['mse', 'mae'])
#, kernel_regularizer=l1_l2(0,0.1), bias_regularizer=l1_l2(0,0.1),
print(model.metrics_names)
# Fitting the Model
model_scores = {}
Reg = False
d = 'hybrid_LSTM'
if Reg:
def test_arg_l1_reg_and_l2_reg(self, model):
model._regularizer = l1_l2(0.01, 0.01)
self._build_and_assert(model)
def Conv2DClassifierIn1(x_train, y_train, x_test, y_test):
summary = True
verbose = 1
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = {{choice([32, 64, 128, 256, 512])}}
epoch = {{choice([25, 50, 75, 100, 125, 150, 175, 200])}}
conv_block = {{choice(['two', 'three', 'four'])}}
conv1_num = {{choice([8, 16, 32, 64])}}
conv2_num = {{choice([16, 32, 64, 128])}}
conv3_num = {{choice([32, 64, 128])}}
conv4_num = {{choice([32, 64, 128, 256])}}
dense1_num = {{choice([128, 256, 512])}}
dense2_num = {{choice([64, 128, 256])}}
l1_regular_rate = {{uniform(0.00001, 1)}}
l2_regular_rate = {{uniform(0.000001, 1)}}
drop1_num = {{uniform(0.1, 1)}}
drop2_num = {{uniform(0.0001, 1)}}
activator = {{choice(['elu', 'relu', 'tanh'])}}
optimizer = {{choice(['adam', 'rmsprop', 'SGD'])}}
#---------------------------------------------------------------------------------------------------------------
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
loss_type = 'binary_crossentropy'
metrics = ['accuracy']
my_callback = None
# early_stopping = EarlyStopping(monitor='val_loss', patience=4)
# checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5',
# verbose=1,
# save_best_only=True)
# my_callback = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,
# patience=5, min_lr=0.0001)
# build --------------------------------------------------------------------------------------------------------
input_layer = Input(shape=x_train.shape[1:])
conv = layers.Conv2D(conv1_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(input_layer)
conv = layers.Conv2D(conv1_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(conv)
if conv_block == 'two':
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
elif conv_block == 'three':
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv3_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv3_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
elif conv_block == 'four':
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv2_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv3_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv3_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv4_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(pool)
conv = layers.Conv2D(conv4_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size, padding=padding_style)(BatchNorm)
flat = layers.Flatten()(pool)
drop = layers.Dropout(drop1_num)(flat)
dense = layers.Dense(dense1_num,
activation=activator,
kernel_regularizer=regularizers.l1_l2(
l1=l1_regular_rate, l2=l2_regular_rate))(drop)
BatchNorm = layers.BatchNormalization(axis=-1)(dense)
drop = layers.Dropout(drop2_num)(BatchNorm)
dense = layers.Dense(dense2_num,
activation=activator,
kernel_regularizer=regularizers.l1_l2(
l1=l1_regular_rate, l2=l2_regular_rate))(drop)
output_layer = layers.Dense(len(np.unique(y_train)),
activation='softmax')(dense)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
# train(self):
class_weights = class_weight.compute_class_weight('balanced',
np.unique(y_train),
y_train.reshape(-1))
class_weights_dict = dict(enumerate(class_weights))
model.compile(
optimizer=optimizer,
loss=loss_type,
metrics=metrics # accuracy
)
result = model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epoch,
verbose=verbose,
callbacks=my_callback,
validation_data=(x_test, y_test),
shuffle=True,
class_weight=class_weights_dict)
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def get_regularizer(l1=0.01, l2=0.01):
return l1_l2(l1=l1, l2=l2)
def build_model(self, n_features):
"""
The method builds a new member of the ensemble and returns it.
"""
# derived parameters
self.hyperparameters['n_members'] = self.hyperparameters[
'n_segments'] * self.hyperparameters['n_members_segment']
# initialize optimizer and early stopping
self.optimizer = Adam(lr=self.hyperparameters['lr'],
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
amsgrad=False)
self.es = EarlyStopping(monitor=f'val_{self.loss_name}',
min_delta=0.0,
patience=self.hyperparameters['patience'],
verbose=1,
mode='min',
restore_best_weights=True)
inputs = Input(shape=(n_features, ))
h = GaussianNoise(self.hyperparameters['noise_in'],
name='noise_input')(inputs)
for i in range(self.hyperparameters['layers']):
h = Dense(self.hyperparameters['neurons'],
activation=self.hyperparameters['activation'],
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_hidden'],
self.hyperparameters['l2_hidden']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name=f'hidden_{i}')(h)
h = Dropout(self.hyperparameters['dropout'],
name=f'hidden_dropout_{i}')(h)
mu = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_mu'],
self.hyperparameters['l2_mu']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='mu_output')(h)
mu = GaussianNoise(self.hyperparameters['noise_mu'],
name='noise_mu')(mu)
if self.hyperparameters['pdf'] == 'normal' or self.hyperparameters[
'pdf'] == 'skewed':
sigma = Dense(1,
activation='softplus',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_sigma'],
self.hyperparameters['l2_sigma']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='sigma_output')(h)
sigma = GaussianNoise(self.hyperparameters['noise_sigma'],
name='noise_sigma')(sigma)
if self.hyperparameters['pdf'] == 'skewed':
alpha = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_alpha'],
self.hyperparameters['l2_alpha']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='alpha_output')(h)
alpha = GaussianNoise(self.hyperparameters['noise_alpha'],
name='noise_alpha')(alpha)
if self.hyperparameters['pdf'] is None:
outputs = mu
elif self.hyperparameters['pdf'] == 'normal':
outputs = concatenate([mu, sigma])
elif self.hyperparameters['pdf'] == 'skewed':
outputs = concatenate([mu, sigma, alpha])
model = Model(inputs=inputs, outputs=outputs)
return model
from keras.layers import TimeDistributed, Bidirectional
from keras import regularizers
from ..custom_layers.lrn import LRN
fillerMap = {
'constant': 'Constant',
'uniform': 'RandomUniform',
'gaussian': 'RandomNormal',
'xavier': 'glorot_normal',
'msra': 'he_normal'
}
regularizerMap = {
'l1': regularizers.l1(),
'l2': regularizers.l2(),
'l1_l2': regularizers.l1_l2(),
'L1L2': regularizers.l1_l2(),
'None': None
}
constraintMap = {
'max_norm': 'max_norm',
'non_neg': 'non_neg',
'unit_norm': 'unit_norm',
'MaxNorm': 'max_norm',
'NonNeg': 'non_neg',
'UnitNorm': 'unit_norm',
'None': None
}
def PGNN_train_test(optimizer_name, optimizer_val, drop_rate, iteration, n_layers, n_nodes, tr_size, pre_train, use_YPhy, lake_name):
# Hyper-parameters of the training process
# batch_size = int(tr_size/2)
batch_size = 1000
num_epochs = 1000
val_frac = 0.2
patience_val = 100
# Initializing results filename
exp_name = "Pre-train" + optimizer_name + '_drop' + str(drop_rate) + '_nL' + str(n_layers) + '_nN' + str(n_nodes) + '_trsize' + str(tr_size) + '_iter' + str(iteration)
exp_name = exp_name.replace('.','pt')
results_dir = '../../../../results/Lake/'
model_name = results_dir + exp_name + '.h5' # storing the trained model
results_name = results_dir + exp_name + '_results.dat' # storing the results of the model
# Load features (Xc) and target values (Y)
data_dir = '../../../../data/'
filename = lake_name + '.mat'
mat = spio.loadmat(data_dir + filename, squeeze_me=True,
variable_names=['Y','Xc_doy','Modeled_temp'])
Xc = mat['Xc_doy']
Y = mat['Y']
Xc = Xc[:,:-1] # remove Y_phy, physics model outputs
# train and test data
trainX, testX, trainY, testY = train_test_split(Xc, Y, train_size=tr_size/Xc.shape[0],
test_size=tr_size/Xc.shape[0], random_state=42, shuffle=True)
## train and test data
#trainX, trainY = Xc[:tr_size,:], Y[:tr_size]
#testX, testY = Xc[-50:,:], Y[-50:]
dependencies = {'root_mean_squared_error': root_mean_squared_error}
# load the pre-trained model using non-calibrated physics-based model predictions (./data/unlabeled.dat)
loaded_model = load_model(results_dir + pre_train, custom_objects=dependencies)
# Creating the model
model = Sequential()
for layer in np.arange(n_layers):
if layer == 0:
model.add(Dense(n_nodes, activation='relu', input_shape=(np.shape(trainX)[1],)))
else:
model.add(Dense(n_nodes, activation='relu', kernel_regularizer=l1_l2(l1=.00, l2=.00)))
# model.add(Dropout(rate=drop_rate))
model.add(MCDropout(rate=drop_rate))
model.add(Dense(1, activation='linear'))
# pass the weights to all layers but 1st input layer, whose dimensions are updated
for new_layer, layer in zip(model.layers[1:], loaded_model.layers[1:]):
new_layer.set_weights(layer.get_weights())
model.compile(loss='mean_squared_error',
optimizer=optimizer_val,
metrics=[root_mean_squared_error])
early_stopping = EarlyStopping(monitor='val_loss', patience=patience_val,verbose=1)
print('Running...' + optimizer_name)
history = model.fit(trainX, trainY,
batch_size=batch_size,
epochs=num_epochs,
verbose=0,
validation_split=val_frac, callbacks=[early_stopping, TerminateOnNaN()])
test_score = model.evaluate(testX, testY, verbose=1)
print(test_score)
# scale the uniform numbers to original space
# max and min value in each column
max_in_column_Xc = np.max(trainX,axis=0)
min_in_column_Xc = np.min(trainX,axis=0)
# Xc_scaled = (Xc-min_in_column_Xc)/(max_in_column_Xc-min_in_column_Xc)
Xc_org = Xx*(max_in_column_Xc-min_in_column_Xc) + min_in_column_Xc
samples = []
for i in range(int(nsim)):
#print("simulation num:",i)
predictions = model.predict(Xc_org)
samples.append(predictions)
return np.array(samples)
def unet(input_shape=(256, 256, 1), l1=0.00001, l2=0.005):
# Hyper-parameters values
initializer = 'he_normal'
pool_size = (2, 2)
# Compute input shape, receptive field and output shape after softmax activation
input_shape = input_shape
# Activations, regularizers and optimizers
regularizer = l1_l2(l1=l1, l2=l2)
# Architecture definition
# INPUT
x = Input(shape=input_shape, name='V-net_input')
# First block (down)
first_conv = Conv2D(8,
3,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_initial',
padding='same')(x)
tmp = BatchNormalization(axis=3, name='batch_norm_1.1')(first_conv)
tmp = Activation('relu')(tmp)
z1 = Conv2D(8,
3,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_1.1',
padding='same')(tmp)
c11 = Conv2D(8,
1,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_1.1',
padding='same')(x)
end_11 = Add()([z1, c11])
# Second block (down)
tmp = BatchNormalization(axis=3, name='batch_norm_2.1')(end_11)
tmp = Activation('relu')(tmp)
tmp = Conv2D(16,
2,
strides=(2, 2),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='downpool_1',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_2.2')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(16,
3,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_2.1',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_2.3')(tmp)
tmp = Activation('relu')(tmp)
z2 = Conv2D(16,
3,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_2.2',
padding='same')(tmp)
c21 = MaxPooling2D(pool_size=pool_size, name='pool_1')(end_11)
c21 = Conv2D(16,
1,
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_2.1',
padding='same')(c21)
end_21 = Add()([z2, c21])
# Third block (down)
tmp = BatchNormalization(axis=3, name='batch_norm_3.1')(end_21)
tmp = Activation('relu')(tmp)
tmp = Conv2D(32,
2,
strides=(2, 2),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='downpool_2',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_3.2')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(32, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_3.1',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_3.3')(tmp)
tmp = Activation('relu')(tmp)
z3 = Conv2D(32, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_3.2',
padding='same')(tmp)
c31 = MaxPooling2D(pool_size=pool_size, name='pool_2')(end_21)
c31 = Conv2D(32, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_3.1',
padding='same')(c31)
end_31 = Add()([z3, c31])
# Fourth block (down)
tmp = BatchNormalization(axis=3, name='batch_norm_4.1')(end_31)
tmp = Activation('relu')(tmp)
tmp = Conv2D(64, (2, 2),
strides=(2, 2),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='downpool_3',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_4.2')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(64, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_4.1',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_4.3')(tmp)
tmp = Activation('relu')(tmp)
z4 = Conv2D(64, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_4.2',
padding='same')(tmp)
c41 = MaxPooling2D(pool_size=pool_size, name='pool_3')(end_31)
c41 = Conv2D(64, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_4.1',
padding='same')(c41)
end_41 = Add()([z4, c41])
# Fifth block
tmp = BatchNormalization(axis=3, name='batch_norm_5.1')(end_41)
tmp = Activation('relu')(tmp)
tmp = Conv2D(128, (2, 2),
strides=(2, 2),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='downpool_4',
padding='same')(tmp)
tmp = BatchNormalization(axis=4, name='batch_norm_5.2')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(128, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_5.1',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_5.3')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(128, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_5.2',
padding='same')(tmp) # inflection point
c5 = MaxPooling2D(pool_size=pool_size, name='pool_4')(end_41)
c5 = Conv2D(128, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_5',
padding='same')(c5)
end_5 = Add()([tmp, c5])
# Fourth block (up)
tmp = BatchNormalization(axis=3, name='batch_norm_4.4')(end_5)
tmp = Activation('relu')(tmp)
tmp = UpSampling2D(size=pool_size, name='up_4')(tmp)
tmp = Conv2D(64, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_up_4',
padding='same')(tmp)
tmp = Concatenate(axis=3)([tmp, z4])
tmp = BatchNormalization(axis=3, name='batch_norm_4.5')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(64, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_4.3',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_4.6')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(64, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_4.4',
padding='same')(tmp)
c42 = UpSampling2D(size=pool_size, name='up_4conn')(end_5)
c42 = Conv2D(64, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_4.2',
padding='same')(c42)
end_42 = Add()([tmp, c42])
# Third block (up)
tmp = BatchNormalization(axis=3, name='batch_norm_3.4')(end_42)
tmp = Activation('relu')(tmp)
tmp = UpSampling2D(size=pool_size, name='up_3')(tmp)
tmp = Conv2D(32, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_up_3',
padding='same')(tmp)
tmp = Concatenate(axis=3)([tmp, z3])
tmp = BatchNormalization(axis=3, name='batch_norm_3.5')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(32, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_3.3',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_3.6')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(32, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_3.4',
padding='same')(tmp)
c32 = UpSampling2D(size=pool_size, name='up_3conn')(end_42)
c32 = Conv2D(32, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_3.2',
padding='same')(c32)
end_32 = Add()([tmp, c32])
# Second block (up)
tmp = BatchNormalization(axis=3, name='batch_norm_2.4')(end_32)
tmp = Activation('relu')(tmp)
tmp = UpSampling2D(size=pool_size, name='up_2')(tmp)
tmp = Conv2D(16, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_up_2',
padding='same')(tmp)
tmp = Concatenate(axis=3)([tmp, z2])
tmp = BatchNormalization(axis=3, name='batch_norm_2.5')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(16, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_2.3',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_2.6')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(16, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_2.4',
padding='same')(tmp)
c22 = UpSampling2D(size=pool_size, name='up_2conn')(end_32)
c22 = Conv2D(16, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_2.2',
padding='same')(c22)
end_22 = Add()([tmp, c22])
# First block (up)
tmp = BatchNormalization(axis=3, name='batch_norm_1.4')(end_22)
tmp = Activation('relu')(tmp)
tmp = UpSampling2D(size=pool_size, name='up_1')(tmp)
tmp = Conv2D(8, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_up_1',
padding='same')(tmp)
tmp = Concatenate(axis=3)([tmp, z1])
tmp = BatchNormalization(axis=3, name='batch_norm_1.5')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(8, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_1.3',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_1.6')(tmp)
tmp = Activation('relu')(tmp)
tmp = Conv2D(8, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_1.4',
padding='same')(tmp)
c12 = UpSampling2D(size=pool_size, name='up_1_conn')(end_22)
c12 = Conv2D(8, (1, 1),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_conn_1.2',
padding='same')(c12)
end_12 = Add()([tmp, c12])
# Final convolution
tmp = BatchNormalization(axis=3, name='batch_norm_1.7')(end_12)
tmp = Activation('relu')(tmp)
tmp = Conv2D(8, (3, 3),
kernel_initializer=initializer,
kernel_regularizer=regularizer,
name='conv_pre_softmax',
padding='same')(tmp)
tmp = BatchNormalization(axis=3, name='batch_norm_pre_softmax')(tmp)
in_softmax = Activation('relu')(tmp)
classification = Conv2D(313, (1, 1, 1),
kernel_initializer=initializer,
name='final_convolution_1x1x1')(in_softmax)
y = softmax(classification)
model = Model(inputs=x, outputs=y)
return model
def build_keras_model(args, modelEpsilon, input_shape, hiddenShrinkage, M , y , M_validation = None, y_validation = None) :
hLayerCount = args.hidCount
k.set_image_dim_ordering('tf') # otherwise it would give 'negative dimension size' for maxpool operations
#k.image_dim_ordering()
BNEnabled = int(args.bnorm) == 1
decay_Enabled = int(args.lr_decay) == 1
shrinkage = hiddenShrinkage
# k.set_image_dim_ordering('tf')
tf.set_random_seed(args.randomSeed) ## tf runs off a different random generator, let it be random for now, to be able to see if it really is the random init that is causing the 'zero' results?
if args.optimizer == 1 :
optimizer=Adam(lr=args.learnRate, epsilon=modelEpsilon, beta_1=args.momentum, beta_2=0.999, decay=args.lr_decay) # for float16, otherwise we get NaNs
else :
optimizer = SGD(lr=args.learnRate, momentum=args.momentum, decay=args.lr_decay)
# , beta_1=0.99, since the new Batchnorm logic this isn't needed
loss = 'mean_squared_error'
accMetrc = 'mae'
# Set up the base model.
model = Sequential()
lastLayerSize = args.firstLayerSize #lastLayerSize_MAX
# Input = knet_main.knnLayer( myNet,np.array([-1]), knet_main.LAYER_SUBTYPE_INPUT)
w_reg = l1_l2(l1=0.0, l2=hiddenShrinkage)
# if conv was enabled we then do NOT regularize stuff at the first FC layer as we only want to regularize by h2 once
if args.convLayers > 0 :
lastOutput = input_shape[0] # for conv nets it is channels last format for Tensorflow, so the first element of the input shape is the actual number of SNPs
print("Adding "+str(args.convLayers)+" conv layers, with initial input dimension: " + str(lastOutput), flush=True)
currentNumFilters= args.convFilters
currentStride = 3
filter_size = 5 # as it turns out it is not actually a problem if the conv outputs something that isn't an integer, so we just need to downsample it
model.add(Conv1D(currentNumFilters, filter_size, input_shape=(input_shape),kernel_regularizer=w_reg, kernel_initializer='he_normal' , padding="same", strides=currentStride ))
if BNEnabled : model.add(BatchNormalization())
addActivation(model,args.hidAct)
if args.dropout != -1 : model.add(Dropout(args.dropout))
# k.floatx() # 'float32'
# args.convLayers = 2
lastOutput = (lastOutput - filter_size +2) / currentStride + 1
lastOutput = int(lastOutput) # as these can only be integers
print("filter size : " + str(filter_size), flush=True)
#i=1
shrinkage = 0.0
currentStride = 1
pool_size = 2
for i in range(1, args.convLayers +1) :
# decide on filter size, depending on input, Conv layers must always produce even outputs so that maxpool can half them
filter_size = 3
if lastOutput % 2 != 0 : filter_size = 4 # if the current output is not even, then we have to use a filter size of 4, otherwise we get fractions after the maxpool operation
## currentNumFilters = (i+1) * args.convFilters
currentNumFilters = currentNumFilters // 2
model.add(Conv1D(currentNumFilters, filter_size,kernel_regularizer=None, kernel_initializer='he_normal' , padding="same", strides=currentStride))
if BNEnabled : model.add(BatchNormalization())
addActivation(model,args.hidAct)
if args.dropout != -1 : model.add(Dropout(args.dropout))
lastOutput = (lastOutput - filter_size +2) / currentStride + 1
lastOutput = int(lastOutput) # as these can only be integers
print("filter size affter Conv ("+str(i)+") : " + str(filter_size) + " / output: " + str(lastOutput), flush=True)
lastOutput = (lastOutput - pool_size) / pool_size + 1 # compute what dimensions the conv+maxpool operations are going to leave for the next layer
print("filter size affter Maxpool ("+str(i)+") : " + str(filter_size) + " / output: " + str(lastOutput), flush=True)
model.add(MaxPooling1D()) # the default is 2
model.add(Flatten()) # Flatten the data for input into the plain hidden layer
for i in range(1,hLayerCount+1) : # iterate 1 based, otherwise we will get a reduction after the first layer, no matter the widthReductionRate, as 0 is divisible by anything
if i > 1 or args.convLayers > 0 : shrinkageParam = w_reg # only add regularizer for first layer, subsequent layers will always have none
else : shrinkageParam = None
#if i == (hLayerCount-1) : lastWidth = 2 # enforce so that the last widht is always 2, ie 1 neuron makes it MORE like the other LESS likely
if i == 1: model.add(Dense(lastLayerSize,kernel_regularizer=shrinkageParam, kernel_initializer='he_normal', input_shape = input_shape))
else : model.add(Dense(lastLayerSize,kernel_regularizer=shrinkageParam, kernel_initializer='he_normal'))
if BNEnabled : model.add(BatchNormalization())
addActivation(model,args.hidAct)
if args.dropout != -1 : model.add(Dropout(args.dropout))
print("added layer at depth: " + str(i) + " with width: " + str(lastLayerSize) + " / shrinkage: " + str(shrinkage))
# control the 'fatness' of the network: we reduce the width at a given rate: if this is 1, then at every subsequent layer, if its 2, then every 2nd layer etc
if i % args.widthReductionRate == 0 : lastLayerSize = lastLayerSize // 2
if lastLayerSize < 2 : break # if
#
if hLayerCount == 0 : model.add(Dense(1, kernel_initializer='he_normal',kernel_regularizer=w_reg, input_shape = input_shape))
else : model.add(Dense(1, kernel_initializer='he_normal',kernel_regularizer=shrinkageParam))
if len( y.shape) > 1 : model.add(Activation('softmax'))
# Compile the model.
model.compile(loss=loss, optimizer=optimizer, metrics=[accMetrc])
return(model)
nn_y_train = nn_y_train.reshape(len(nn_y_train), 1)
nn_y_train = onehot_encoder.fit_transform(nn_y_train)
nn_y_validation = nn_y_validation.reshape(len(nn_y_validation), 1)
nn_y_validation = onehot_encoder.fit_transform(nn_y_validation)
test_y_01 = test_y_true.reshape(len(test_y_true), 1)
test_y_01 = onehot_encoder.fit_transform(test_y_01)
# Training neural network
model = Sequential()
# first hidden layer
model.add(
Dense(units=LAYER1_NODES,
input_dim=INPUT_NODES,
activation="relu",
kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2)))
# second hidden layer
model.add(
Dense(units=LAYER2_NODES,
activation='relu',
kernel_regularizer=regularizers.l1_l2(l1=L1, l2=L2)))
# output layer
model.add(Dense(units=OUTPUT_NODES, activation='softmax'))
# compile models with the learning rates set
opt = optimizers.adam(learning_rate=LEARNING_RATE)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
history = model.fit(
for l1 in range(1, 5):
for l2 in range(1, 5):
for l3 in range(1, 5):
Hlayer1 = 128 * l1
Hlayer2 = 128 * l2
Hlayer3 = 128 * l3
model = Sequential([
Dense(256,
input_dim=202,
kernel_regularizer=regularizers.l1(lamda1),
kernel_initializer='random_uniform'),
Activation('relu'),
Dense(Hlayer1,
kernel_regularizer=regularizers.l1_l2(0),
kernel_initializer='random_uniform'),
Activation('relu'),
Dense(Hlayer2,
kernel_regularizer=regularizers.l1_l2(0),
kernel_initializer='random_uniform'),
Activation('relu'),
Dense(Hlayer3,
kernel_regularizer=regularizers.l1_l2(0),
kernel_initializer='random_uniform'),
Activation('relu'),
Dense(51,
kernel_regularizer=regularizers.l1(0),
kernel_initializer='random_uniform'),
Activation('softmax')
])
def PGNN_train_test(optimizer_name, optimizer_val, drop_frac, use_YPhy,
iteration, n_layers, n_nodes, tr_size, lamda, reg,
samp):
# fix_seeds(ss)
# Hyper-parameters of the training process
# batch_size = tr_size
batch_size = 1
num_epochs = 300
val_frac = 0.25
patience_val = 80
# Initializing results filename
exp_name = "DNN_loss" + optimizer_name + '_drop' + str(
drop_frac) + '_usePhy' + str(use_YPhy) + '_nL' + str(
n_layers) + '_nN' + str(n_nodes) + '_trsize' + str(
tr_size) + '_lamda' + str(lamda) + '_iter' + str(iteration)
exp_name = exp_name.replace('.', 'pt')
results_dir = '../results/'
model_name = results_dir + exp_name + '_model.h5' # storing the trained model
if reg == True and samp == 25:
results_name = results_dir + exp_name + '_results_25_regularizer.dat' # storing the results of the model
elif reg == False and samp == 25:
results_name = results_dir + exp_name + '_results_25.dat' # storing the results of the model
elif reg == True and samp == 1519:
results_name = results_dir + exp_name + '_results_1519_regularizer.dat' # storing the results of the model
elif reg == False and samp == 1519:
results_name = results_dir + exp_name + '_results_1519.dat' # storing the results of the model
# Load labeled data
data = np.loadtxt('../data/labeled_data.dat')
# data = np.loadtxt('../data/labeled_data_BK_constw_unique.dat')
# data = np.loadtxt('../data/labeled_data_BK_constw_v2.dat')
x_labeled = data[:, :
2] # -2 because we do not need porosity predictions
y_labeled = data[:, -3:
-1] # dimensionless bond length and porosity measurements
if samp == 25:
data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_25.dat')
x_unlabeled = data[:, :]
elif samp == 1519:
data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_1525.dat')
x_unlabeled = data[:, :]
x_unlabeled1 = x_unlabeled[:1303, :]
x_unlabeled2 = x_unlabeled[-6:, :]
x_unlabeled = np.vstack((x_unlabeled1, x_unlabeled2))
# initial porosity
init_poro = x_unlabeled[:, -1]
x_unlabeled = x_unlabeled[:, :2]
# data = np.loadtxt('../data/unlabeled_data_BK_constw_v2_1519.dat')
# x_unlabeled = data[:1303, :] # 1303 last regular sample: 260, 46
# x_unlabeled_non = x_unlabeled
# normalize dataset with MinMaxScaler
scaler = preprocessing.MinMaxScaler(feature_range=(0.0, 1.0))
# scaler = preprocessing.StandardScaler()
x_labeled = scaler.fit_transform(x_labeled)
# y_labeled = scaler.fit_transform(y_labeled)
x_unlabeled = scaler.fit_transform(x_unlabeled)
# # initial porosity & physics outputs are removed
# x_unlabeled = x_unlabeled[:, :-3]
# train and test data
trainX, trainY = x_labeled[:tr_size, :], y_labeled[:tr_size]
# testX, testY = x_labeled[tr_size:,:], y_labeled[tr_size:]
testX, testY = x_labeled[tr_size:, :], y_labeled[tr_size:]
if use_YPhy == 0:
# Removing the last column from x_unlabeled (corresponding to Y_PHY)
x_unlabeled = x_unlabeled[:, :-1]
# Creating the model
model = Sequential()
for layer in np.arange(n_layers):
if layer == 0:
model.add(
Dense(n_nodes,
activation='relu',
input_shape=(np.shape(trainX)[1], )))
else:
if reg:
model.add(
Dense(n_nodes,
activation='relu',
kernel_regularizer=l1_l2(l1=.001, l2=.001)))
else:
model.add(Dense(n_nodes, activation='relu'))
# model.add(Dropout(rate=drop_frac))
model.add(MCDropout(rate=drop_frac))
model.add(Dense(2, activation='linear'))
# physics-based regularization
uinp_sc = K.constant(value=x_unlabeled) # unlabeled input data
lam1 = K.constant(value=lamda[0]) # regularization hyper-parameter
lam2 = K.constant(value=lamda[1]) # regularization hyper-parameter
lam3 = K.constant(value=lamda[2]) # regularization hyper-parameter
lam4 = K.constant(value=lamda[3]) # regularization hyper-parameter
predictions = model(uinp_sc) # model output at depth i
# porosity = K.relu(predictions[:,1])
phyloss1 = bond(predictions[:, 0]) # physics loss 1
# uinp = K.constant(value=x_unlabeled_non) # unlabeled input data
phyloss2 = poros(init_poro, predictions[:, 1]) # physics loss 1
phyloss3 = strength1(predictions[:, 0], predictions[:, 1])
phyloss4 = strength2(predictions[:, 0], predictions[:, 1])
totloss = combined_loss(
[phyloss1, phyloss2, phyloss3, phyloss4, lam1, lam2, lam3, lam4])
phyloss = phy_loss_mean(
[phyloss1, phyloss2, phyloss3, phyloss4, lam1, lam2, lam3, lam4])
model.compile(loss=totloss,
optimizer=optimizer_val,
metrics=[phyloss, root_mean_squared_error])
early_stopping = EarlyStopping(monitor='val_loss',
patience=patience_val,
verbose=1)
# print('Running...' + optimizer_name)
history = model.fit(trainX,
trainY,
batch_size=batch_size,
epochs=num_epochs,
verbose=0,
validation_split=val_frac,
callbacks=[early_stopping,
TerminateOnNaN()])
# early_stopping = EarlyStopping(monitor='loss', patience=patience_val, verbose=1)
# history = model.fit(trainX, trainY,
# batch_size=batch_size,
# epochs=num_epochs,
# verbose=1,
# callbacks=[early_stopping, TerminateOnNaN()])
# test_score = model.evaluate(testX, testY, verbose=0)
# predictions = model.predict(x_labeled) # model output at depth i
# print(np.sort(predictions[:,0], axis=0))
# predictions = model.predict(x_unlabeled) # model output at depth i
# print(np.sort(predictions[:,0], axis=0))
# print('iter: ' + str(iteration) + ' useYPhy: ' + str(use_YPhy) +
# ' nL: ' + str(n_layers) + ' nN: ' + str(n_nodes) +
# ' lamda1: ' + str(lamda[0]) + ' lamda2: ' + str(lamda[1]) + ' trsize: ' + str(tr_size) +
# ' TestRMSE: ' + str(test_score[2]) + ' PhyLoss: ' + str(test_score[1]), ' TestLoss: ' + str(test_score[0]), "\n")
# # print('iter: ' + str(iteration) + ' TestRMSE: ' + str(test_score[2]) + ' PhyLoss: ' + str(test_score[1]), "\n")
# # model.save(model_name)
# # save results
# results = {'train_loss_1':history.history['loss_1'],
# 'val_loss_1':history.history['val_loss_1'],
# 'train_rmse':history.history['root_mean_squared_error'],
# 'val_rmse':history.history['val_root_mean_squared_error'],
# 'test_rmse':test_score[2],
# 'PhyLoss':test_score[1]}
# results = {'train_loss_1':history.history['loss_1'],
# 'train_rmse':history.history['root_mean_squared_error'],
# 'test_rmse':test_score[2],
# 'PhyLoss':test_score[1]}
# save_obj(results, results_name)
# predictions = model.predict(testX)
# return results, results_name, predictions, testY, test_score[2], trainY
test_score = model.evaluate(testX, testY, verbose=1)
print(test_score)
samples = []
for i in range(int(nsim)):
print("simulation num:", i)
predictions = model.predict(Xx)
predictions = predictions[:, 1]
samples.append(predictions)
return np.array(samples)
# create the MLP and CNN models
mlp = mg.create_mlp(4, regress=False)
cnn = mg.create_cnn(508, 508, 1, regress=False)
# create the input to our final set of layers as the *output* of both
# the MLP and CNN
combinedInput = concatenate([mlp.output, cnn.output])
# our final FC layer head will have two dense layers, the final one
# being our regression head
x = Dense(4, activation="relu")(combinedInput)
#for vector output we have to have many output nodes
out_0 = (Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x)
out_100 = (Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x)
out_200 = (Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x)
out_300 = (Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x)
out_400 = (Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=0.001, l2=0.001)))(x)
# our final model will accept categorical/numerical data on the MLP
# input and images on the CNN input, outputting a single value
class KerasRegularizersTest(keras_parameterized.TestCase,
parameterized.TestCase):
def create_model(self, kernel_regularizer=None, activity_regularizer=None):
model = keras.models.Sequential()
model.add(
keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=kernel_regularizer,
activity_regularizer=activity_regularizer,
input_shape=(DATA_DIM, )))
return model
def get_data(self):
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=10,
test_samples=10,
input_shape=(DATA_DIM, ),
num_classes=NUM_CLASSES)
y_train = np_utils.to_categorical(y_train, NUM_CLASSES)
y_test = np_utils.to_categorical(y_test, NUM_CLASSES)
return (x_train, y_train), (x_test, y_test)
def create_multi_input_model_from(self, layer1, layer2):
input_1 = keras.layers.Input(shape=(DATA_DIM, ))
input_2 = keras.layers.Input(shape=(DATA_DIM, ))
out1 = layer1(input_1)
out2 = layer2(input_2)
out = keras.layers.Average()([out1, out2])
model = keras.models.Model([input_1, input_2], out)
model.add_loss(keras.backend.mean(out2))
model.add_loss(tf.reduce_sum(input_1))
return model
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_kernel_regularization(self, regularizer):
(x_train, y_train), _ = self.get_data()
model = self.create_model(kernel_regularizer=regularizer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly())
self.assertEqual(len(model.losses), 1)
model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
('l2_zero', keras.regularizers.l2(0.)),
])
def test_activity_regularization(self, regularizer):
(x_train, y_train), _ = self.get_data()
model = self.create_model(activity_regularizer=regularizer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly())
self.assertEqual(len(model.losses), 1 if tf.executing_eagerly() else 1)
model.fit(x_train, y_train, batch_size=10, epochs=1, verbose=0)
@keras_parameterized.run_all_keras_modes
@keras_parameterized.run_with_all_model_types
def test_zero_regularization(self):
# Verifies that training with zero regularization works.
x, y = np.ones((10, 10)), np.ones((10, 3))
model = testing_utils.get_model_from_layers([
keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0))
],
input_shape=(10, ))
model.compile('sgd',
'mse',
run_eagerly=testing_utils.should_run_eagerly())
model.fit(x, y, batch_size=5, epochs=1)
def test_custom_regularizer_saving(self):
def my_regularizer(weights):
return tf.reduce_sum(tf.abs(weights))
inputs = keras.Input((10, ))
outputs = keras.layers.Dense(1,
kernel_regularizer=my_regularizer)(inputs)
model = keras.Model(inputs, outputs)
model2 = model.from_config(
model.get_config(),
custom_objects={'my_regularizer': my_regularizer})
self.assertEqual(model2.layers[1].kernel_regularizer, my_regularizer)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_layer(self, regularizer):
dense_layer = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
model = self.create_multi_input_model_from(dense_layer, dense_layer)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly())
self.assertLen(model.losses, 5)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_model(self, regularizer):
dense_layer = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
dummy_model = keras.models.Model(input_tensor,
dense_layer(input_tensor))
model = self.create_multi_input_model_from(dummy_model, dummy_model)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly())
self.assertLen(model.losses, 6)
@keras_parameterized.run_all_keras_modes
@parameterized.named_parameters([
('l1', regularizers.l1()),
('l2', regularizers.l2()),
('l1_l2', regularizers.l1_l2()),
])
def test_regularization_shared_layer_in_different_models(
self, regularizer):
shared_dense = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer,
activity_regularizer=regularizer)
models = []
for _ in range(2):
input_tensor = keras.layers.Input(shape=(DATA_DIM, ))
unshared_dense = keras.layers.Dense(NUM_CLASSES,
kernel_regularizer=regularizer)
out = unshared_dense(shared_dense(input_tensor))
models.append(keras.models.Model(input_tensor, out))
model = self.create_multi_input_model_from(layer1=models[0],
layer2=models[1])
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
run_eagerly=testing_utils.should_run_eagerly())
# We expect to see 9 losses on the model:
# - 2 from the 2 add_loss calls on the outer model.
# - 3 from the weight regularizers on the shared_dense layer, unshared_dense
# in inner model 1, unshared_dense in inner model 2.
# - 4 from activity regularizers on the shared_dense layer.
self.assertLen(model.losses, 9)
def test_deserialization_error(self):
with self.assertRaisesRegex(ValueError,
'Could not interpret regularizer'):
keras.regularizers.get(0)
@parameterized.named_parameters([
('l1', regularizers.l1(l1=None), 0.01),
('l2', regularizers.l2(l2=None), 0.01),
('l1_l2', regularizers.l1_l2(l1=None, l2=None), 0.),
])
def test_default_value_when_init_with_none(self, regularizer,
expected_value):
expected_value = np.asarray(expected_value)
if hasattr(regularizer, 'l1'):
self.assertAllClose(regularizer.l1, expected_value)
if hasattr(regularizer, 'l2'):
self.assertAllClose(regularizer.l2, expected_value)
论文中的损失函数
:param y_true: 真实值
:param y_pred: 预测值
:return: 修订版MAPE
"""
diff = K.square(y_true - y_pred) / K.clip(K.square(y_true), 1, None)
weight = 10
return K.mean(K.square(y_pred - y_true) + weight * diff)
LOSS_NAME_FUNC = {
'mix': metric_loss,
'rmse': metric_rmse,
'mape': metric_mape,
'mae': metric_mae,
'mse': metric_mse
}
def get_loss_func(name):
assert name in LOSS_NAME_FUNC
return LOSS_NAME_FUNC[name]
REGULAR_NAME_FUNC = {'l1': l1(), 'l2': l2(), 'l1_l2': l1_l2(), 'none': None}
def get_regularizer(name):
assert name in REGULAR_NAME_FUNC
return REGULAR_NAME_FUNC[name]
def Conv2DClassifierIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test,
class_weights_dict, obj):
K.clear_session()
CUDA = '2'
summary = False
verbose = 0
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = {{choice([32, 64])}}
# batch_size = 64
# epoch = 2
epoch = {{choice([1, 2])}}
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
activator = 'relu'
regular_rate = (0.001, 0.001)
dropout_rate = 0.1
optimizer = 'adam'
loss_type = 'mse'
# metrics=('accuracy',acc, mcc, mcc_concise, recall_p, recall_n, precision_p, precision_n, tp_Concise,tn_Concise,fp_Concise,fn_Concise, recall_p_Concise,recall_n_Concise,precision_p_Concise,precision_n_Concise)
metrics = ('mae', )
callbacks = None
# config TF-----------------------------------------------------------------------------------------------------
os.environ['CUDA_VISIBLE_DEVICES'] = CUDA
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
# build --------------------------------------------------------------------------------------------------------
# mCNN feature inputs as a whole 2D array
input_layer = Input(shape=x_train.shape[1:])
conv1 = layers.Conv2D(16,
kernel_size,
kernel_initializer=initializer,
activation=activator)(input_layer)
conv2 = layers.Conv2D(32,
kernel_size,
kernel_initializer=initializer,
activation=activator)(conv1)
pool1 = layers.MaxPooling2D(pool_size, padding=padding_style)(conv2)
conv3 = layers.Conv2D(64,
kernel_size,
kernel_initializer=initializer,
activation=activator,
kernel_regularizer=regularizers.l1_l2(
l1=regular_rate[0], l2=regular_rate[1]))(pool1)
conv3_BatchNorm = layers.BatchNormalization(axis=-1)(conv3)
pool2 = layers.MaxPooling2D(pool_size,
padding=padding_style)(conv3_BatchNorm)
conv4 = layers.Conv2D(128,
kernel_size,
kernel_initializer=initializer,
activation=activator,
kernel_regularizer=regularizers.l1_l2(
l1=regular_rate[0], l2=regular_rate[1]))(pool2)
pool3 = layers.MaxPooling2D(pool_size, padding=padding_style)(conv4)
flat = layers.Flatten()(pool3)
dense = layers.Dense(1024, activation=activator)(flat)
dense_BatchNorm = layers.BatchNormalization(axis=-1)(dense)
drop = layers.Dropout(dropout_rate)(dense_BatchNorm)
# output_layer = layers.Dense(len(np.unique(y_train)),activation='softmax')(drop)
output_layer = layers.Dense(1)(drop)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
# train(self):
# class_weights_dict = dict(enumerate(class_weights))
# print(class_weights_dict)
model.compile(
optimizer=optimizer,
loss=loss_type,
metrics=list(metrics) # accuracy
)
K.set_session(tf.Session(graph=model.output.graph))
init = K.tf.global_variables_initializer()
K.get_session().run(init)
result = model.fit(
x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epoch,
verbose=verbose,
callbacks=callbacks,
validation_data=(x_test, ddg_test),
shuffle=True,
)
# print('\n----------History:'
# '\n%s'%result.history)
if obj == 'test_report_reg':
pearson_coeff, std = test_report_reg(model, x_test, ddg_test)
print('\n----------Predict:\npearson_coeff: %s, std: %s' %
(pearson_coeff, std))
objective = pearson_coeff * 2 + std
return {'loss': -objective, 'status': STATUS_OK}
elif obj == 'val_mae':
print(result.history)
validation_mae = np.amax(result.history['val_mean_absolute_error'])
print('Best validation mae of epoch:', validation_mae)
return {'loss': validation_mae, 'status': STATUS_OK}
