def get_optimizer(self, optimizer, learning_rate):
"""Get optimizer by name"""
if optimizer == 'rmsprop':
if learning_rate == None:
return optimizers.RMSprop()
else:
return optimizers.RMSprop(lr=learning_rate)
elif optimizer == 'adam':
if learning_rate == None:
return optimizers.Adam()
else:
return optimizers.Adam(lr=learning_rate)
elif optimizer == 'adamax':
if learning_rate == None:
return optimizers.Adamax()
else:
return optimizers.Adamax(lr=learning_rate)
elif optimizer == 'nadam':
if learning_rate == None:
return optimizers.Nadam()
else:
return optimizers.Nadam(lr=learning_rate)
else:
return None
def __optimizer(opt_name, lr=None):
# type: (str, float) -> optimizers.Optimizer
"""
Given a name and learning rate returns a keras optimizer based on it.
:param opt_name: Name of optimizer to use.
Legal arguments are:
* adam
* nadam
* rmsprop
* adamax
* adagrad
* adadelta
:param lr: Learning rate of an optimizer.
:return: A new optimizer based on given name and learning rate.
"""
opt_name = opt_name.lower()
if lr is None:
if opt_name == 'adam':
return optimizers.Adam()
elif opt_name == 'sgd':
return optimizers.SGD(nesterov=True)
elif opt_name == 'nadam':
return optimizers.Nadam()
elif opt_name == 'rmsprop':
return optimizers.RMSprop()
elif opt_name == 'adamax':
return optimizers.Adamax()
elif opt_name == 'adagrad':
return optimizers.Adagrad()
elif opt_name == 'adadelta':
return optimizers.Adadelta()
else:
if opt_name == 'adam':
return optimizers.Adam(lr=lr)
elif opt_name == 'sgd':
return optimizers.SGD(lr=lr, nesterov=True)
elif opt_name == 'nadam':
return optimizers.Nadam(lr=lr)
elif opt_name == 'rmsprop':
return optimizers.RMSprop(lr=lr)
elif opt_name == 'adamax':
return optimizers.Adamax(lr=lr)
elif opt_name == 'adagrad':
return optimizers.Adagrad(lr=lr)
elif opt_name == 'adadelta':
return optimizers.Adadelta(lr=lr)
raise AttributeError('Invalid name of optimizer given.')
def _init_model(self):
self.regularizer = SSGL_WeightRegularizer(l1_reg=self.alpha * self.lambda_,
l2_reg=(1. - self.alpha) * self.lambda_, groups=self.groups)
self.model = Sequential()
self.model.add(Dense(units=self.hidden_layers[0], input_dim=self.d, activation=self.activation,
kernel_regularizer=self.regularizer))
for n_units in self.hidden_layers[1:]:
self.model.add(Dense(units=n_units, activation=self.activation))
if self.n_classes == 2:
activation = 'sigmoid'
loss = 'binary_crossentropy'
units = 1
else:
activation = 'softmax'
loss = 'categorical_crossentropy'
units = self.n_classes
self.model.add(Dense(units=units, activation=activation))
if self.optimizer == 'adam':
optimizer = optimizers.Adam(lr=self.lr)
elif self.optimizer == 'sgd':
optimizer = optimizers.SGD(lr=self.lr)
elif self.optimizer == 'adamax':
optimizer = optimizers.Adamax(lr=self.lr)
elif self.optimizer == 'adagrad':
optimizer = optimizers.Adagrad(lr=self.lr)
else:
raise ValueError("optimizer must be one of ['adam', 'adagrad', 'adamax', 'sgd'].")
self.model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
def create_cnn():
# Adicione uma camada de entrada
input_layer = layers.Input((70, ))
# Adicione a camada de incorporação de palavras
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Adicione a camada convolucional
conv_layer = layers.Convolution1D(90, 3,
activation="relu")(embedding_layer)
# Adicione a camada de pooling máximo, pega maior valor resltande do mapa de ativação.
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Adicione as camadas de saída
# camada totalmente conectada para normalização dos dados.
output_layer1 = layers.Dropout(0.7)(pooling_layer)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile o modelo
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adamax(),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def get_discriminator():
inputs = Input(shape=(64, 64, 2))
conv_1 = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs)
act_1 = Activation('relu')(conv_1)
pl_1=MaxPooling2D((2, 2), strides=(2, 2))(act_1)
conv_2 = Conv2D(16, (3, 3), strides=(1, 1), padding='same')(pl_1)
act_2 = Activation('relu')(conv_2)
pl_2=MaxPooling2D((2, 2), strides=(2, 2))(act_2)
conv_3 = Conv2D(8, (3, 3), strides=(1, 1), padding='same')(pl_2)
act_3 = Activation('relu')(conv_3)
pl_3=MaxPooling2D((2, 2), strides=(2, 2))(act_3)
fc=Flatten()(pl_3)
fc_2=Dense(40)(fc)
act_4=Activation('relu')(fc_2)
fc_3=Dense(25)(act_4)
act_5=Activation('relu')(fc_3)
fc_4=Dense(10)(act_5)
act_6=Activation('relu')(fc_4)
fc_5=Dense(1)(act_6)
act_7=Activation('sigmoid')(fc_5)
rmsprop = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
sgd = optimizers.SGD(lr=0.01, clipvalue=0.5)
adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
adagrad = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
adamax = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
nadam = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
model = Model(inputs=[inputs], outputs=[act_7])
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
return model
def get_optimizer(optimizer_params):
"""
Parse optimizer parameters.
Input : dict with key
'type' -> optimizer type
other -> optimizer legit parameters
"""
optimizer_param = optimizer_params.copy()
if 'type' in optimizer_param:
method = optimizer_param.pop('type', None)
else:
raise Exception('You must provide an optimization method')
if method == 'adadelta':
optim_fn = kopt.Adadelta(**optimizer_param)
elif method == 'adagrad':
optim_fn = kopt.Adagrad(**optimizer_param)
elif method == 'adam':
optim_fn = kopt.Adam(**optimizer_param)
elif method == 'adamax':
optim_fn = kopt.Adamax(**optimizer_param)
elif method == 'sgd':
optim_fn = kopt.SGD(**optimizer_param)
elif method == 'rmsprop':
optim_fn = kopt.RMSprop(**optimizer_param)
else:
raise Exception('Unknown optimization method: "%s"' % method)
return optim_fn
def _init_model(self):
self.regularizer = SSGL_WeightRegularizer(l1_reg=self.alpha * self.lambda_,
l2_reg=(1. - self.alpha) * self.lambda_, groups=self.groups)
self.model = Sequential()
self.model.add(Dense(units=self.hidden_layers[0], input_dim=self.d, activation=self.activation,
kernel_regularizer=self.regularizer))
for n_units in self.hidden_layers[1:]:
self.model.add(Dense(units=n_units, activation=self.activation))
self.model.add(Dense(units=1))
if self.optimizer == 'adam':
optimizer = optimizers.Adam(lr=self.lr)
elif self.optimizer == 'sgd':
optimizer = optimizers.SGD(lr=self.lr)
elif self.optimizer == 'adamax':
optimizer = optimizers.Adamax(lr=self.lr)
elif self.optimizer == 'adagrad':
optimizer = optimizers.Adagrad(lr=self.lr)
else:
raise ValueError("optimizer must be one of ['adam', 'adagrad', 'adamax', 'sgd'].")
self.model.compile(loss='mean_squared_error', optimizer=optimizer)
def get_optimizer():
clipvalue = 0
clipnorm = 10
if MC.OPTIMIZER == 'rmsprop':
optimizer = opt.RMSprop(lr=0.001,
rho=0.9,
epsilon=1e-06,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'sgd':
optimizer = opt.SGD(lr=0.01,
momentum=0.0,
decay=0.0,
nesterov=False,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'adagrad':
optimizer = opt.Adagrad(lr=0.01,
epsilon=1e-06,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'adadelta':
optimizer = opt.Adadelta(lr=1.0,
rho=0.95,
epsilon=1e-06,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'adam':
optimizer = opt.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'amsgrad':
optimizer = opt.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
clipvalue=clipvalue,
amsgrad=True)
elif MC.OPTIMIZER == 'adamax':
optimizer = opt.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
clipvalue=clipvalue)
elif MC.OPTIMIZER == 'amsgrad':
optimizer = opt.AMSGrad(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
clipnorm=clipnorm,
clipvalue=clipvalue)
return optimizer
def get_optimizer(name, l_r, decay=0.):
if name == 'sgd':
optimizer = optimizers.SGD(lr=l_r,
momentum=0.,
decay=decay,
nesterov=False)
elif name == 'rmsprop':
optimizer = optimizers.RMSprop(lr=l_r, rho=0.9, decay=decay)
elif name == 'adagrad':
optimizer = optimizers.Adagrad(lr=l_r, decay=decay)
elif name == 'adadelta':
optimizer = optimizers.Adadelta(lr=l_r, rho=0.95, decay=decay)
elif name == 'adam':
optimizer = optimizers.Adam(lr=l_r,
beta_1=0.9,
beta_2=0.999,
decay=0.1 * decay)
elif name == 'adamax':
optimizer = optimizers.Adamax(lr=l_r,
beta_1=0.9,
beta_2=0.999,
decay=0.1 * decay)
elif name == 'nadam':
optimizer = optimizers.Nadam(lr=l_r,
beta_1=0.9,
beta_2=0.999,
schedule_decay=0.004)
else:
optimizer = None
return optimizer
def switch_optimazer(arg):
switcher = {
#learning_rate=0.01, momentum=0.0, nesterov=False
1:
optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True),
#learning rate, can be freely tuned >=0)
2:
optimizers.RMSprop(learning_rate=0.001, rho=0.9),
#It is recommended to leave the parameters of this optimizer at their default values.
3:
optimizers.Adagrad(learning_rate=0.01),
#It is recommended to leave the parameters of this optimizer at their default values.
4:
optimizers.Adadelta(learning_rate=1.0, rho=0.95),
5:
optimizers.Adam(learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
amsgrad=False),
6:
optimizers.Adamax(learning_rate=0.002, beta_1=0.9, beta_2=0.999),
#It is recommended to leave the parameters of this optimizer at their default values.
7:
optimizers.Nadam(learning_rate=0.002, beta_1=0.9, beta_2=0.999)
}
return switcher.get(arg, "Invalid number")
def get_optimizer(label):
"""
Obtains a Keras optimizer from a string with its default parameters
Inputs:
- label: string representation of the optimizer
Output:
- optimizer as Keras object
"""
if label == "sgd":
return optimizers.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=True)
if label == "rmsprop":
return optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
if label == "adam":
return optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
if label == 'adamax':
return optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
if label == 'nadam':
return optimizers.Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
return None
def creatModel(self):
K.set_learning_phase(1)
self.model = Sequential()
self.model.add(Reshape((28, 28, 1), input_shape=(784, )))
# add the layer below for an accuracy of 89%.(Training time - over 20 hours)
self.model.add(
Convolution2D(32, (5, 5),
input_shape=(28, 28, 1),
activation='relu',
padding='same',
kernel_constraint=maxnorm(3)))
self.model.add(Convolution2D(32, (5, 5), activation='relu'))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Flatten())
self.model.add(
Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
self.model.add(Dropout(0.5))
self.model.add(Dense(62, activation='softmax'))
opt = optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
self.model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
print(self.model.summary())
return self.model
def cnn_model():
num_of_classes = get_num_of_classes()
model = Sequential()
model.add(
Conv2D(16, (2, 2),
input_shape=(image_x, image_y, 1),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(3, 3), padding='same'))
model.add(Conv2D(64, (5, 5), activation='relu'))
model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding='same'))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(num_of_classes, activation='softmax'))
sgd = optimizers.Adamax(lr=1e-2)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
filepath = "cnn_model_keras2.h5"
checkpoint1 = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint1]
#from keras.utils import plot_model
#plot_model(model, to_file='model.png', show_shapes=True)
return model, callbacks_list
def get_hyperparameter(self):
hyper = dict()
############################
'''
(1) 파라미터 값들을 수정해주세요
'''
hyper['batch_size'] = 16 # 배치 사이즈 16
hyper['epochs'] = 20 # epochs은 최대 20 설정 !!
#hyper['learning_rate'] = 1.0 # 학습률
#hyper['learning_rate'] = 0.1
#hyper['learning_rate'] = 0.01
hyper['learning_rate'] = 0.002
#hyper['learning_rate'] = 0.0002
# 최적화 알고리즘 선택 [sgd, rmsprop, adagrad, adam 등]
#hyper['optimizer'] = optimizers.Adadelta(lr=hyper['learning_rate'], decay=1e-6) # default: SGD
#hyper['optimizer'] = optimizers.sgd(lr=hyper['learning_rate'], decay=1e-6)
#hyper['optimizer'] = optimizers.RMSprop(lr=hyper['learning_rate'], decay=1e-6)
#hyper['optimizer'] = optimizers.Adam(lr=hyper['learning_rate'], decay=1e-6)
#hyper['optimizer'] = optimizers.Nadam(lr=hyper['learning_rate'], decay=1e-6)
#hyper['optimizer'] = optimizers.Adagrad(lr=hyper['learning_rate'], decay=1e-6)
hyper['optimizer'] = optimizers.Adamax(lr=hyper['learning_rate'],
decay=1e-6)
############################
return hyper
def optselect(self):
if self.verbose:
print("Setting training optimizer to: " + self.paramdict["train_optimizer"])
if self.paramdict["train_optimizer"] == "nadam":
self.train_optimizer = optimizers.Nadam(lr=self.paramdict["train_learn_rate"],
beta_1=self.paramdict["nadam_beta_1"],
beta_2=self.paramdict["nadam_beta_2"])
elif self.paramdict["train_optimizer"] == "sgd":
self.train_optimizer = optimizers.SGD(lr=self.paramdict["train_learn_rate"],
momentum=self.paramdict["momentum"], nesterov=True)
elif self.paramdict["train_optimizer"] == "rmsprop":
self.train_optimizer = optimizers.RMSprop(lr=self.paramdict["train_learn_rate"])
elif self.paramdict["train_optimizer"] == "adagrad":
self.train_optimizer = optimizers.Adagrad(lr=self.paramdict["train_learn_rate"])
elif self.paramdict["train_optimizer"] == "adadelta":
self.train_optimizer = optimizers.Adadelta(lr=self.paramdict["train_learn_rate"])
elif self.paramdict["train_optimizer"] == "adam":
self.train_optimizer = optimizers.Adam(lr=self.paramdict["train_learn_rate"])
elif self.paramdict["train_optimizer"] == "adamax":
self.train_optimizer = optimizers.Adamax(lr=self.paramdict["train_learn_rate"])
else:
self.train_optimizer = optimizers.Nadam()
if self.verbose:
print("Setting fine tuning optimizer to: " + self.paramdict["fine_tune_optimizer"])
if self.paramdict["fine_tune_optimizer"] == "nadam":
self.fine_tune_optimizer = optimizers.Nadam(lr=self.paramdict["fine_tune_learn_rate"],
beta_1=self.paramdict["nadam_beta_1"],
beta_2=self.paramdict["nadam_beta_2"])
elif self.paramdict["fine_tune_optimizer"] == "sgd":
self.fine_tune_optimizer = optimizers.SGD(lr=self.paramdict["fine_tune_learn_rate"],
momentum=self.paramdict["momentum"], nesterov=True)
elif self.paramdict["fine_tune_optimizer"] == "rmsprop":
self.fine_tune_optimizer = optimizers.RMSprop(lr=self.paramdict["fine_tune_learn_rate"])
elif self.paramdict["fine_tune_optimizer"] == "adagrad":
self.fine_tune_optimizer = optimizers.Adagrad(lr=self.paramdict["fine_tune_learn_rate"])
elif self.paramdict["fine_tune_optimizer"] == "adadelta":
self.fine_tune_optimizer = optimizers.Adadelta(lr=self.paramdict["fine_tune_learn_rate"])
elif self.paramdict["fine_tune_optimizer"] == "adam":
self.fine_tune_optimizer = optimizers.Adam(lr=self.paramdict["fine_tune_learn_rate"])
elif self.paramdict["fine_tune_optimizer"] == "adamax":
self.fine_tune_optimizer = optimizers.Adamax(lr=self.paramdict["fine_tune_learn_rate"])
else:
self.fine_tune_optimizer = optimizers.Nadam()
if self.verbose:
print("Optimizers set!")
def get_optimizer(optimizer_name, kwargs):
# Принимает строку и словарь с параметрами для оптимизатора
# По соответствию с этими данными возвращает оптимизатор с заданными пар-ми
# Если оптимизатора с таким названием нет, выводит сообщ. и возвр. дефолтный
name = optimizer_name.lower()
if name == "adam":
return optimizers.Adam(lr=kwargs.get('lr', 0.001),
beta_1=kwargs.get('beta_1', 0.9),
beta_2=kwargs.get('beta_2', 0.999),
epsilon=kwargs.get('epsilon', 0),
decay=kwargs.get('decay', 0.0))
elif name == "sgd":
return optimizers.SGD(lr=kwargs.get('lr', 0.01),
momentum=kwargs.get('momentum', 0.0),
decay=kwargs.get('decay', 0.0),
nesterov=kwargs.get('nesterov', False))
elif name == "rmsprop":
return optimizers.RMSprop(lr=kwargs.get('lr', 0.001),
rho=kwargs.get('rho', 0.9),
epsilon=kwargs.get('epsilon', 0),
decay=kwargs.get('decay', 0.0))
elif name == "adagard":
return optimizers.Adagard(lr=kwargs.get('lr', 0.01),
epsilon=kwargs.get('epsilon', 0),
decay=kwargs.get('decay', 0.0))
elif name == "adadelta":
return optimizers.Adadelta(lr=kwargs.get('lr', 1.0),
rho=kwargs.get('rho', 0.95),
epsilon=kwargs.get('epsilon', 0),
decay=kwargs.get('decay', 0.0))
elif name == "adamax":
return optimizers.Adamax(lr=kwargs.get('lr', 0.002),
beta_1=kwargs.get('beta_1', 0.9),
beta_2=kwargs.get('beta_2', 0.999),
epsilon=kwargs.get('epsilon', 0),
decay=kwargs.get('decay', 0.0))
elif name == "nadam":
return optimizers.Nadam(lr=kwargs.get('lr', 0.002),
beta_1=kwargs.get('beta_1', 0.9),
beta_2=kwargs.get('beta_2', 0.999),
epsilon=kwargs.get('epsilon', 0),
schedule_decay=kwargs.get(
'schedule_decay', 0.004))
else:
print("No such optimizer ({}) in the Keras Library!"
" Returning default Adam.".format(name))
return optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=0,
decay=0.0)
def load_model(self, path=None):
if path is None: path = self.log_path
# load json and create model
json_file = open(path + "/model_topology.json", 'r')
loaded_model_json = json_file.read()
json_file.close()
self.model = model_from_json(loaded_model_json)
# load weights into new model
self.model.load_weights(path + "/model_weights.h5")
# Load model optimizer
opti_file = open(path + "/model_optimizer.bin", 'rb')
self.optimizer, self.learning_rate = pickle.load(opti_file)
# Optimizer
if self.optimizer == 1:
opti = optimizers.Adam(lr=self.learning_rate,
epsilon=None,
decay=0.0)
elif self.optimizer == 2:
opti = optimizers.SGD(lr=self.learning_rate,
decay=1e-6,
momentum=0.9,
nesterov=True)
elif self.optimizer == 3:
opti = optimizers.RMSprop(lr=self.learning_rate,
rho=0.9,
epsilon=None,
decay=0.0)
elif self.optimizer == 4:
opti = optimizers.Adagrad(lr=self.learning_rate,
epsilon=None,
decay=0.0)
elif self.optimizer == 5:
opti = optimizers.Adadelta(lr=self.learning_rate,
rho=0.95,
epsilon=None,
decay=0.0)
elif self.optimizer == 6:
opti = optimizers.Adamax(lr=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
elif self.optimizer == 7:
opti = optimizers.Nadam(lr=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
# compile model
self.model.compile(loss='mean_squared_error',
optimizer=opti,
metrics=['mse'])
self.model.predict(
np.array([[
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
]]))
print("Loaded model from disk")
def get_optimizer(args):
# clipvalue = 0
# clipnorm = 10
# clipnorm = 0
if args.algorithm == 'rmsprop':
# optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue)
# optimizer = opt.RMSprop(lr=0.01)
if args.learning_rate > 0:
optimizer = opt.RMSprop(lr=args.learning_rate)
else:
optimizer = opt.RMSprop()
elif args.algorithm == 'sgd':
# optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue)
if args.learning_rate > 0:
optimizer = opt.SGD(lr=args.learning_rate)
else:
optimizer = opt.SGD()
elif args.algorithm == 'adagrad':
# optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue)
if args.learning_rate > 0:
optimizer = opt.Adagrad(lr=args.learning_rate)
else:
optimizer = opt.Adagrad()
elif args.algorithm == 'adadelta':
# optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue)
if args.learning_rate > 0:
optimizer = opt.Adadelta(lr=args.learning_rate)
else:
optimizer = opt.Adadelta()
elif args.algorithm == 'adam':
# optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue)
if args.learning_rate > 0:
optimizer = opt.Adam(lr=args.learning_rate)
else:
optimizer = opt.Adam()
elif args.algorithm == 'adamax':
# optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue)
if args.learning_rate > 0:
optimizer = opt.Adamax(lr=args.learning_rate)
else:
optimizer = opt.Adamax()
return optimizer
def get_optimizer(name, lr=None):
if name == 'Adam':
if lr is not None:
return optimizers.Adam(lr=lr)
else:
return optimizers.Adam()
elif name == 'SGD':
if lr is not None:
return optimizers.SGD(lr=lr)
else:
return optimizers.SGD()
elif name == 'RMSprop':
if lr is not None:
return optimizers.RMSprop(lr=lr)
else:
return optimizers.RMSprop()
elif name == 'Adagrad':
if lr is not None:
return optimizers.Adagrad(lr=lr)
else:
return optimizers.Adagrad()
elif name == 'Adadelta':
if lr is not None:
return optimizers.Adadelta(lr=lr)
else:
return optimizers.Adadelta()
elif name == 'Adamax':
if lr is not None:
return optimizers.Adamax(lr=lr)
else:
return optimizers.Adamax()
elif name == 'Nadam':
if lr is not None:
return optimizers.Nadam(lr=lr)
else:
return optimizers.Nadam()
else:
raise ValueError('"{}" was not valid optimizer.'.format(name))
def evaluate(self, x_test, y_test, batch=20):
#return metrics for evaluation.
adamax = optimizers.Adamax(lr=0.01)
self.model.compile(
loss='binary_crossentropy', #categorical_crossentropy
optimizer=adamax, #'adadelta',
metrics=['accuracy'])
score, acc = self.model.evaluate(x_test, y_test, batch_size=batch)
return {'score': score, 'acc': acc}
def compile(self,
optimizer,
loss,
metrics=None,
loss_weights=None,
sample_weight_mode=None,
**options):
if isinstance(optimizer, str):
opts = {
'sgd':
opt.SGD(lr=options.get('lr', .01),
decay=options.get('decay', 1e-6),
momentum=options.get('momentum', 0.9),
nesterov=True,
clipnorm=options.get('clipnorm', 0)),
'rmsprop':
opt.RMSprop(lr=options.get('lr', .001)),
'adadelta':
opt.Adadelta(lr=options.get('lr', 1.)),
'adagrad':
opt.Adagrad(lr=options.get('lr', .01)),
'adam':
opt.Adam(lr=options.get('lr', .001)),
'nadam':
opt.Nadam(lr=options.get('lr', .002),
clipnorm=options.get('clipnorm', 0)),
'adamax':
opt.Adamax(lr=options.get('lr', .002))
}
optimizer = opts[optimizer]
mode = options.get('mode', None)
if K.backend() == 'theano' and mode:
modes = {
'NaNGuardMode':
NanGuardMode(nan_is_error=True,
inf_is_error=True,
big_is_error=True),
'MonitorMode':
MonitorMode(pre_func=inspect_inputs,
post_func=inspect_outputs),
}
mode = modes[mode]
super(Model, self).compile(loss=loss,
loss_weights=loss_weights,
optimizer=optimizer,
metrics=metrics,
sample_weight_mode=sample_weight_mode,
mode=mode)
else:
super(Model, self).compile(loss=loss,
loss_weights=loss_weights,
optimizer=optimizer,
sample_weight_mode=sample_weight_mode,
metrics=metrics)
def create_base_network(input_dim, nb_classes):
'''Base network to be shared (eq. to feature extraction).
'''
sgd = optimizers.SGD(lr=0.01, clipnorm=1.)
sgd = optimizers.SGD(lr=0.01, momentum=0.05, decay=0.0, nesterov=True)
rmsprop = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
adagrad = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
adam = optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
adamax = optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
nadam = optimizers.Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
N_nodes = input_dim
r_droupout = 0.05
model_base = Sequential()
model_base.add(Dense(N_nodes, input_shape=(input_dim, )))
model_base.add(Activation('relu'))
model_base.add(Dropout(r_droupout))
#N_nodes = int(np.floor(N_nodes/2))
model_base.add(Dense(N_nodes))
model_base.add(Activation('relu'))
model_base.add(Dropout(r_droupout))
#N_nodes = int(np.floor(N_nodes/2))
model_base.add(Dense(N_nodes))
model_base.add(Activation('relu'))
model_base.add(Dropout(r_droupout))
#N_nodes = int(np.floor(N_nodes/2))
model_base.add(Dense(N_nodes))
model_base.add(Activation('relu'))
model_base.add(Dropout(r_droupout))
N_nodes = int(np.floor(N_nodes / 2))
model_base.add(Dense(N_nodes))
model_base.add(Activation('relu'))
model_base.add(Dropout(r_droupout))
N_nodes = int(np.floor(N_nodes / 2))
model_base.add(Dense(N_nodes))
model_base.add(Activation('relu'))
model_base.add(Dense(nb_classes))
model_base.add(Activation('softmax'))
model_base.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
#model_base.load_weights('model_base.h5')
return model_base
class optimizer(Enum):
SGD = optimizers.SGD()
ADAM = optimizers.Adagrad()
SGD = optimizers.SGD()
RMSprop = optimizers.RMSprop()
Adagrad = optimizers.Adagrad()
Adadelta = optimizers.Adadelta()
Adamax = optimizers.Adamax()
Nadam = optimizers.Nadam()
def baseline_model():
model = Sequential()
model.add(Dense(100, input_dim=56))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(4, activation='softmax'))
opt = optimizers.Adamax(learning_rate=0.003)
model.compile(loss='categorical_crossentropy', optimizer=opt)
return model
def optimizer(self, optimizerType = 'rmsprop', default = True, learningrate = 1e4, learningrate_decay = 0.05):
if default:
if optimizerType == 'rmsprop':
opt = optimizers.RMSprop(lr = 0.001, rho = 0.9, epsilon = None, decay = 0.0)
if optimizerType == 'sgd':
opt = optimizers.SGD(lr = 0.01, momentum = 0.0, decay = 0.0, nesterov = False)
if optimizerType == 'Adagrad':
opt = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0)
if optimizerType == 'Adam':
opt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
if optimizerType == 'Adamax':
opt = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
if optimizerType == 'Nadam':
opt = optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
else:
if optimizerType == 'rmsprop':
opt = optimizers.RMSprop(lr = learningrate, rho = 0.9, epsilon = None, decay = 0.0)
if optimizerType == 'sgd':
opt = optimizers.SGD(lr = learningrate, decay = learningrate_decay, momentum = 0.9, nesterov = True)
if optimizerType == 'Adagrad':
opt = optimizers.Adagrad(lr = learningrate, epsilon=None, decay = learningrate_decay)
if optimizerType == 'Adam':
opt = optimizers.Adam(lr = learningrate, beta_1=0.9, beta_2=0.999, epsilon=None, decay = learningrate_decay, amsgrad=False)
if optimizerType == 'Adamax':
opt = optimizers.Adamax(lr = learningrate, beta_1=0.9, beta_2=0.999, epsilon=None, decay = learningrate_decay)
if optimizerType == 'Nadam':
opt = optimizers.Nadam(lr = learningrate, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay = learningrate_decay)
return opt
def create_TMPNN(inputDim=2, outputDim=2, order=3):
''' Creates polynomial neural network based on Taylor map'''
model = Sequential()
model.add(TaylorMap(output_dim = outputDim, order=order,
input_shape = (inputDim,)
#weights_regularizer=lambda W: dim2_order3(0.009, W),
))
opt = optimizers.Adamax(clipvalue=1e-3)
model.compile(loss='mean_squared_error', optimizer=opt)
return model
def create_model(x_train, y_train, x_test, y_test):
d1 = {{choice([64, 128, 256, 512, 1024, 2048])}}
d2 = {{choice([64, 128, 256, 512, 1024, 2048])}}
d3 = {{choice([64, 128, 256, 512, 1024, 2048])}}
d4 = {{choice([64, 128, 256, 512, 1024, 2048])}}
act = {{choice(['relu', 'sigmoid', 'tanh'])}}
bn = {{choice([0, 1])}}
c1 = {{uniform(0, 0.4)}}
c2 = {{uniform(0, 0.4)}}
c3 = {{uniform(0, 0.4)}}
opt = {{
choice([
optimizers.Adamax(),
optimizers.RMSprop(),
optimizers.Adam(),
optimizers.SGD()
])
}}
batch = {{choice([64, 128, 256, 512, 1024, 2048])}}
model = Sequential()
model.add(Dense(d1, activation=act, input_shape=(2, )))
if bn == 1:
model.add(BatchNormalization())
#model.add(BatchNormalization()) BN
#model.add(Dense(1024, activation='relu', kernel_regularizer=regularizers.l2(0.01))) L2
#model.add(Dropout(0.2))
model.add(Dense(d2, activation=act))
model.add(Dropout(c1))
if bn == 1:
model.add(BatchNormalization())
model.add(Dense(d3, activation=act))
model.add(Dropout(c2))
if bn == 1:
model.add(BatchNormalization())
model.add(Dense(d4, activation=act))
model.add(Dropout(c3))
if bn == 1:
model.add(BatchNormalization())
model.add(Dense(2))
import keras.backend as K
model.compile(loss='mse', optimizer=opt)
result = model.fit(x_train,
y_train,
validation_data=(x_test, y_test),
epochs=100,
batch_size=batch,
verbose=0)
return {
'loss': result.history['val_loss'][-1],
'status': STATUS_OK,
'model': model
}
def create_model(self, optimizer='rmsprop', init= 'glorot_uniform', activation='relu', hidden_layers=1,
neurons = 8, lr = 0.01, weight_constraint = 1, momentum = 0.2, dropout_rate = 0.5, decay = 0.0):
import tensorflow as tf
from keras import models
from keras import layers
from keras.layers import Dropout
from keras import optimizers
from keras.constraints import maxnorm
from keras import backend
lr = lr
weight_constraint = weight_constraint
momentum = momentum
dropout_rate = dropout_rate
auc_roc = self.as_keras_metric(tf.metrics.auc)
recall = self.as_keras_metric(tf.metrics.recall)
# create model
model = models.Sequential()
model.add(layers.Dense(neurons, input_dim=self.input_dim, kernel_initializer=init,
activation=activation, kernel_constraint=maxnorm(weight_constraint)))
model.add(Dropout(dropout_rate))
for i in range(hidden_layers):
# Add one hidden layer
model.add(layers.Dense(neurons, kernel_initializer=init, activation=activation,
kernel_constraint=maxnorm(weight_constraint)))
model.add(Dropout(dropout_rate))
model.add(layers.Dense(1, kernel_initializer=init, activation='sigmoid',
kernel_constraint=maxnorm(weight_constraint)))
# optimizer 'rmsprop', 'adam', 'sgd', 'adagrad', 'adadelta', 'adamax', 'nadam'
if optimizer == 'rmsprop':
optimizer_func = optimizers.RMSprop(lr=lr)
elif optimizer == 'adam':
optimizer_func = optimizers.Adam(lr=lr, decay= decay)
elif optimizer == 'sgd':
optimizer_func = optimizers.SGD(lr=lr, momentum = momentum)
elif optimizer == 'adagrad':
optimizer_func = optimizers.Adagrad(lr=lr)
elif optimizer == 'adadelta':
optimizer_func = optimizers.Adadelta(lr=lr)
elif optimizer == 'adamax':
optimizer_func = optimizers.Adamax(lr=lr)
elif optimizer == 'nadam':
optimizer_func = optimizers.Nadam(lr=lr)
else:
optimizer_func = 'rmsprop'
# Compile model
model.compile(loss='binary_crossentropy', optimizer=optimizer_func, metrics=[auc_roc])
return model
def build_model(lstm_layer_lhs, lstm_layer_rhs, input_sequence_1,
input_sequence_2, features_input):
features_dense = BatchNormalization()(features_input)
features_dense = Dense(200, activation="relu")(features_dense)
features_dense = Dropout(FLAGS.layer_dropout)(features_dense)
# Square difference
addition = add([lstm_layer_lhs, lstm_layer_rhs])
minus_lstm_layer_rhs = Lambda(lambda x: -x)(lstm_layer_rhs)
merged = add([lstm_layer_lhs, minus_lstm_layer_rhs])
merged = multiply([merged, merged])
merged = concatenate([merged, addition])
merged = Dropout(FLAGS.layer_dropout)(merged)
merged = concatenate([merged, features_dense])
merged = BatchNormalization()(merged)
merged = GaussianNoise(0.1)(merged)
merged = Dense(150, activation="relu")(merged)
merged = Dropout(FLAGS.layer_dropout)(merged)
merged = BatchNormalization()(merged)
out = Dense(1, activation="sigmoid")(merged)
model = Model(inputs=[input_sequence_1, input_sequence_2, features_input],
outputs=out)
if FLAGS.optimizer == "adam":
optimizer = optimizers.Adam(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "adadelta":
optimizer = optimizers.Adadelta(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "sgd":
optimizer = optimizers.SGD(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "adagrad":
optimizer = optimizers.Adagrad(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "rmsprop":
optimizer = optimizers.RMSprop(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "adamax":
# It is a variant of Adam based on the infinity norm.
# Default parameters follow those provided in the paper.
# Default lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0
optimizer = optimizers.Adamax(lr=FLAGS.learning_rate)
elif FLAGS.optimizer == "nadam":
# Much like Adam is essentially RMSprop with momentum,
# Nadam is Adam RMSprop with Nesterov momentum.
# Default same as Adamax with ..., schedule_decay=0.004
optimizer = optimizers.Nadam(lr=FLAGS.learning_rate)
else:
optimizer = optimizers.Nadam(lr=FLAGS.learning_rate)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=['accuracy'])
return model
def getOptimizer():
#最好优化器是4
which_optimizer = input(
"please choose a optimizer,1.sgd,2.adam,3 Adagrad，4 Adadelta,5 Adamax,6 Nadam"
)
if which_optimizer == '1':
lr = 0.0001 # input("please input learning rate,I suggestion you can write in somewhere (learing rate):")
lr = float(lr)
decay = 1e-6 # input("please input decay,I suggestion you can write in somewhere (learing rate):")
decay = float(decay)
momentum = 0.9 # input("please input momentun,I suggestion you can write in somewhere (learing rate):")
momentum = float(momentum)
sgd = optimizers.SGD(lr, decay, momentum, nesterov=True)
return sgd
elif which_optimizer == '2':
lr = 0.0001 # input("please input learning rate,I suggestion you can write in somewhere (learing rate):")
lr = float(lr)
decay = 1e-9 # input("please input decay,I suggestion you can write in somewhere (learing rate):")
decay = float(decay)
adam = optimizers.Adam(lr, decay)
return adam
elif which_optimizer == '3':
Adagrad = optimizers.Adagrad(lr=0.001, epsilon=None, decay=0.0)
return Adagrad
elif which_optimizer == '4':
Adadelta = optimizers.Adadelta(lr=1.0,
rho=0.95,
epsilon=None,
decay=0.0)
return Adadelta
elif which_optimizer == '5':
Adamax = optimizers.Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0)
return Adamax
elif which_optimizer == '6':
Nadam = optimizers.Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
return Nadam
elif which_optimizer == '7':
RMSprop = optimizers.RMSprop(lr=0.001,
rho=0.9,
epsilon=None,
decay=0.0)
return RMSprop
else:
print("do you really understand what my mean!!")
sys.exit(0)
