def gen_keras_linear(w, b, amsgrad=False):
model = keras.models.Sequential()
model.add(
keras.layers.Dense(input_shape=(3, ), units=5, weights=[w, b]))
model.compile(optimizer=AdaBound(lr=1e-3,
final_lr=0.1,
amsgrad=amsgrad),
loss='mse')
return model
def test_with_constraint(self):
w, b = self.gen_random_weights()
model = keras.models.Sequential()
model.add(
keras.layers.Dense(
input_shape=(3, ),
units=5,
kernel_constraint=keras.constraints.max_norm(1.0),
weights=[w, b]), )
model.compile(optimizer=AdaBound(lr=1e-3, final_lr=0.1, decay=0.5),
loss='mse')
x = np.random.standard_normal((1, 3))
y = np.dot(x, w) + b
model.train_on_batch(x, y)
def train(self,
xTrain,
yTrain,
epochs,
batch_size,
optimizer,
ada_low_lr=None,
ada_high_lr=None):
# compiles model that was created
if optimizer != 'adabound':
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
else:
self.model.compile(optimizer=AdaBound(lr=ada_low_lr,
final_lr=ada_high_lr),
loss='mean_squared_error')
# fit model to data
self.model.fit(xTrain, yTrain, epochs=epochs, batch_size=batch_size)
def compile_model(self,lr,opt,adversarial_ratio=0):
def inverse_bce(y_true,y_pred):
y_true=y_true*-1+1
return tf.keras.losses.binary_crossentropy(y_true,y_pred)
#Compile model
if opt =='sgd':
optimizer=tf.keras.optimizers.SGD(lr)
elif opt=='adam':
optimizer=tf.keras.optimizers.Adam(lr)
elif opt=='adabound':
optimizer=AdaBound(lr=lr,final_lr=lr*100)
if self.iterative_refinement:
losses="categorical_crossentropy"
elif self.split_components:
if self.adversarial_learning:
losses = {
"CHOSUNG": "categorical_crossentropy",
"JUNGSUNG": "categorical_crossentropy",
"JONGSUNG": "categorical_crossentropy",
'DISC':inverse_bce}
if self.fit_discriminator:
lossWeights = {"CHOSUNG": 1.0-adversarial_ratio, "JUNGSUNG": 1.0-adversarial_ratio,
"JONGSUNG":1.0-adversarial_ratio,"DISC":3*adversarial_ratio}
else:
lossWeights = {"CHOSUNG": 1.0, "JUNGSUNG": 1.0,"JONGSUNG":1.0,"DISC":0}
else:
losses = {
"CHOSUNG": "categorical_crossentropy",
"JUNGSUNG": "categorical_crossentropy",
"JONGSUNG": "categorical_crossentropy"}
lossWeights = {"CHOSUNG": 1.0, "JUNGSUNG": 1.0,"JONGSUNG":1.0}
else:
losses="categorical_crossentropy"
lossWeights=None
if self.adversarial_learning:
self.model.trainable=True
self.model.get_layer('disc_start').trainable=False
self.model.get_layer('DISC').trainable=False
self.model.compile(optimizer=optimizer, loss=losses,metrics=["accuracy"],loss_weights=lossWeights)
def test_with_plateau(self):
self.reset_seed(0xcafe)
w, b = self.gen_random_weights()
model = keras.models.Sequential()
model.add(
keras.layers.Dense(input_shape=(3, ), units=5, weights=[w, b]), )
model.compile(optimizer=AdaBound(lr=1e-3, final_lr=0.1), loss='mse')
x = np.random.standard_normal((10000, 3))
y = np.dot(x, w) + b
model.fit(
x,
y,
epochs=100,
callbacks=[keras.callbacks.ReduceLROnPlateau(monitor='loss')],
verbose=False)
model_path = os.path.join(tempfile.gettempdir(),
'keras_adabound_plateau.h5')
model.save(model_path)
model = keras.models.load_model(model_path,
custom_objects={'AdaBound': AdaBound})
self.assertGreater(1e-3, float(K.get_value(model.optimizer.lr)))
self.assertEqual(1e-3, model.optimizer.base_lr)
def compile_adversarial_model(self,lr,opt,adversarial_ratio=0):
#build adversarial model for training
input_image=self.model.input
disc_output=self.model.get_layer('DISC')
self.discriminator=tf.keras.models.Model(self.model.input,disc_output.output)
for l in self.model.layers:
l.trainable=False
self.model.get_layer('disc_start').trainable=True
self.model.get_layer('DISC').trainable=True
lr=lr*adversarial_ratio*3
if opt =='sgd':
optimizer=tf.keras.optimizers.SGD(lr)
elif opt=='adam':
optimizer=tf.keras.optimizers.Adam(lr)
elif opt=='adabound':
optimizer=AdaBound(lr=lr,final_lr=lr*100)
self.discriminator.compile(optimizer=optimizer,loss='binary_crossentropy')
def __init__(self):
""" 训练初始化 """
# 构建模型网络
self.backbone = FLAGS.model_backbone # 网络类型
self.input_shape = FLAGS.input_shape
self.output_shapes = FLAGS.output_shapes
model = Classifier.build(self.backbone, self.input_shape,
self.output_shapes, FLAGS.output_names)
# 训练模型: cpu，gpu 或 多gpu
if FLAGS.gpu_mode == MultiLabelClassifier.GPU_MODE and FLAGS.gpu_num > 1:
self.model = keras.utils.multi_gpu_model(model, gpus=FLAGS.gpu_num)
else:
self.model = model
self.model.summary()
self.history = None
# 加载预训练模型（若有）
self.checkpoint_path = FLAGS.checkpoint_path
if self.checkpoint_path is None:
self.checkpoint_path = 'models/'
if os.path.isfile(self.checkpoint_path):
if os.path.exists(self.checkpoint_path):
self.model.load_weights(self.checkpoint_path)
logging.info('加载模型成功！')
else:
self.checkpoint_path = os.path.dirname(self.checkpoint_path)
if os.path.isdir(self.checkpoint_path):
if not os.path.exists(self.checkpoint_path):
os.makedirs(self.checkpoint_path)
latest = tf.train.latest_checkpoint(self.checkpoint_path)
if latest is not None:
self.model.load_weights(latest)
logging.info('加载模型成功！')
logging.info(latest)
else:
self.checkpoint_path = os.path.dirname(self.checkpoint_path)
self.checkpoint_path = os.path.join(self.checkpoint_path,
FLAGS.checkpoint_name)
# 设置训练过程中的回调函数
tensorboard = keras.callbacks.TensorBoard(
log_dir=FLAGS.tensorboard_dir)
cp_callback = keras.callbacks.ModelCheckpoint(self.checkpoint_path,
save_weights_only=True,
verbose=1,
period=FLAGS.ckpt_period)
es_callback = keras.callbacks.EarlyStopping(
monitor='loss',
min_delta=FLAGS.stop_min_delta,
patience=FLAGS.stop_patience,
verbose=0,
mode='min')
lr_callback = keras.callbacks.LearningRateScheduler(FLAGS.lr_func)
# from utils.logger_callback import NBatchProgbarLogger
# log_callback = NBatchProgbarLogger(display=FLAGS.logger_batch)
self.callbacks = [
tensorboard,
cp_callback,
es_callback,
lr_callback,
]
# 设置模型优化方法
self.loss_function = list()
for _ in self.output_shapes:
loss_function = MyLoss(
self.model,
is_label_smoothing=FLAGS.is_label_smoothing,
is_focal_loss=FLAGS.is_focal_loss,
is_gradient_harmonized=FLAGS.is_gradient_harmonized
).categorical_crossentropy
self.loss_function.append(loss_function)
optimizer = keras.optimizers.SGD(lr=FLAGS.init_lr,
momentum=0.95,
nesterov=True)
if FLAGS.optimizer == 'adam':
optimizer = keras.optimizers.Adam(lr=FLAGS.init_lr,
amsgrad=True) # 用AMSGrad
elif FLAGS.optimizer == 'adabound':
from keras_adabound import AdaBound
optimizer = AdaBound(lr=1e-3, final_lr=0.1)
elif FLAGS.optimizer == 'radam':
from utils.radam import RAdam
optimizer = RAdam(lr=1e-3)
# 由于是多标签分类损失，最终答应的损失信息为：
# loss: 44.6420 - char_1_loss: 7.8428 - char_2_loss: 5.8357 - char_3_loss: 4.5361 - char_4_loss: 4.7954
# - char_5_loss: 4.1554 - char_6_loss: 4.6104 - char_7_loss: 5.5645 - char_8_loss: 0.6412
# - char_num_loss: 0.7639 - plate_color_loss: 4.1163
# char_1_loss等为对应标签的平均样本损失，loss=所有标签平均样本损失 + 权重 * 罚项（正则项）误差(model.losses)
self.model.compile(optimizer=optimizer,
loss=self.loss_function,
loss_weights=FLAGS.loss_weights)
# 设置模型训练参数
self.mini_batch = FLAGS.batch_size
self.epoch = FLAGS.epoch
lr = (lr_max - lr_min) *\
lr_exp_decay**(epoch - lr_rampup_epochs\
- lr_sustain_epochs) + lr_min
return lr
return lrfn
#optimizer = tf.keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#optimizer = tf.keras.optimizers.Adam(LR_INIT)
# model.compile(loss=tf.keras.losses.CategoricalCrossentropy(), # label_smoothing=0.1
# optimizer = 'adam',
# metrics=["accuracy"])
model.compile(optimizer=AdaBound(lr=1e-3, final_lr=0.1),
loss=tf.keras.losses.CategoricalCrossentropy())
lr_shedule_fn = build_lrfn()
#lr_callback = keras.callbacks.LearningRateScheduler(lr_shedule_fn, verbose=1)
cb_early_stopper = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
patience=5)
cb_checkpointer = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
monitor='val_loss',
save_best_only=True,
mode='auto')
logger = keras.callbacks.CSVLogger(f'./logs/{dataset_name}_log.csv',
separator=',',
append=False)
# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state = 42)
#X_train_all, X_test_all, y_train_all, y_test_all = train_test_split(X_all, y_all, test_size = 0.1, random_state = 42)
# create model
model = Sequential()
model.add(Dense(256, input_dim=2, kernel_initializer= 'he_normal', activation= 'relu', kernel_regularizer= regularizers.l2(0.000001)))
model.add(BatchNormalization())
model.add(Dense(16, kernel_initializer= 'he_normal', activation= 'relu', kernel_regularizer= regularizers.l2(0.000001)))
model.add(BatchNormalization())
model.add(Dense(1, kernel_initializer= 'glorot_normal', activation= 'sigmoid'))
#set the optimizer
optm = AdaBound()
from keras.optimizers import adam
opt=adam(lr=0.00001)
#Define AUC score as a metric
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
# Compile model
model.compile(loss= 'binary_crossentropy' , optimizer= opt , metrics=[auc, km.precision(), 'acc'])
# Early stoping