def scheduler(hparams: dict, dataset: DataSet):
if hparams['scheduler'] is 'constant':
return LearningRateScheduler(lambda epocs: hparams['learning_rate'],
verbose=False)
if hparams['scheduler'] is 'linear_decay':
return LearningRateScheduler(
lambda epocs: max(hparams['learning_rate'] *
(10. / (10. + epocs)), min_lr),
verbose=False)
if hparams['scheduler'].startswith('CyclicLR')\
or hparams['scheduler'] in ["triangular", "triangular2", "exp_range"]:
# DOCS: https://www.datacamp.com/community/tutorials/cyclical-learning-neural-nets
# CyclicLR_triangular, CyclicLR_triangular2, CyclicLR_exp_range
mode = re.sub(r'^CyclicLR_', '', hparams['scheduler'])
# step_size should be epoc multiple between 2 and 8, but multiple of 2 (= full up/down cycle)
if hparams['patience'] <= 6:
whole_cycles = 1 # 1/2 = 0.5 | 6/2 = 3
elif hparams['patience'] <= 12:
whole_cycles = 2 # 8/4 = 2 | 12/4 = 3
elif hparams['patience'] <= 24:
whole_cycles = 3 # 14/6 = 2.3 | 24/6 = 4
elif hparams['patience'] <= 36:
whole_cycles = 4 # 26/8 = 3.25 | 36/8 = 4.5
elif hparams['patience'] <= 48:
whole_cycles = 5 # 28/10 = 2.8 | 48/10 = 4.8
elif hparams['patience'] <= 72:
whole_cycles = 6 # 50/12 = 4.2 | 72/12 = 6
elif hparams['patience'] <= 96:
whole_cycles = 8 # 74/16 = 4.6 | 96/16 = 6
else:
whole_cycles = 12 # 100/24 = 4.2 | 192/24 = 8
return CyclicLR(mode=mode,
step_size=dataset.epoc_size() * (hparams['patience'] /
(2.0 * whole_cycles)),
base_lr=min_lr(hparams),
max_lr=hparams['learning_rate'])
if hparams['scheduler'].startswith('plateau'):
factor = int((re.findall(r'\d+', hparams['scheduler']) +
[10])[0]) # plateau2 || plateau10 (default)
if 'sqrt' in hparams['scheduler']:
patience = math.sqrt(
hparams['patience']) # plateau2_sqrt || plateau10__sqrt
else:
patience = hparams['patience'] / 2.0
return ReduceLROnPlateau(
monitor='val_loss',
factor=1 / factor,
patience=math.floor(patience),
# min_lr = min_lr(hparams),
verbose=False,
)
print("Unknown scheduler: ", hparams)
def nn_model(x_train, x_test, y_train):
lr_scheduler = LearningRateScheduler(nn_scheduler)
folds = KFold(n_splits=10, shuffle=True, random_state=2018)
oof_nn = np.zeros(len(x_train))
predictions_nn = np.zeros(len(x_test))
for fold, (train_idx, val_idx) in enumerate(folds.split(x_train, x_test)):
print('nn fold {}'.format(fold + 1))
k_x_train = x_train[train_idx]
k_y_train = y_train[train_idx]
k_x_val = x_train[val_idx]
k_y_val = y_train[val_idx]
model_nn = tf.keras.Sequential()
model_nn.add(tf.keras.layers.Dense(512, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)))
model_nn.add(tf.keras.layers.Dense(256, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)))
model_nn.add(tf.keras.layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)))
model_nn.add(tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)))
model_nn.add(tf.keras.layers.Dense(1, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)))
model_nn.compile(loss='mean_absolute_error', optimizer=tf.keras.optimizers.Adam(), metrics=['mae'])
model_nn.fit(k_x_train, k_y_train, batch_size=512, epochs=2000, validation_data=(k_x_val, k_y_val),
callbacks=[lr_scheduler])
oof_nn[val_idx] = model_nn.predict(k_x_val).reshape((model_nn.predict(k_x_val).shape[0],))
predictions_nn += model_nn.predict(x_test).reshape((model_nn.predict(k_x_val).shape[0],)) / folds.n_splits
print('nn mae:{:<8.8f}'.format(mean_absolute_error(np.expm1(oof_nn), np.expm1(y_train))))
return np.expm1(predictions_nn), np.expm1(oof_nn)
def LearningRateSchedulerCallBack(self, schedule, verbose=0):
"""LearningRateScheduler CallBack
学習率スケジューラ
Args:
schedule ([type]): エポックインデックスを入力として受け取り（整数、0からインデックス付け）、新しい学習率を出力（float）として返す関数
verbose (int, optional): 0：何も表示しない、1：メッセージを更新します. Defaults to 0.
"""
return LearningRateScheduler(schedule=schedule, verbose=verbose)
def train_model(model: models.Sequential, train_x: np.ndarray,
train_y: np.ndarray):
print("Training Model")
checkpoint = ModelCheckpoint("models/auto_save.h5",
monitor='accuracy',
verbose=1,
save_best_only=True,
mode='auto',
period=2)
lr_scheduler = LearningRateScheduler(learning_rate_scheduler, verbose=1)
history = model.fit([train_x, train_x],
train_y,
batch_size=256,
epochs=50,
verbose=1,
validation_split=TEST_SPLIT,
shuffle=True,
callbacks=[checkpoint, lr_scheduler])
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 5))
ax1.plot(history.history['accuracy'], label='accuracy')
ax1.plot(history.history['val_accuracy'], label='val_accuracy')
ax1.set_xlabel('Epoch')
ax1.set_ylabel('Accuracy')
ax1.legend(loc='lower right')
ax2.plot(history.history['loss'], label='loss')
ax2.plot(history.history['val_loss'], label='val_loss')
ax2.set_xlabel('Epoch')
ax2.set_ylabel('Loss')
ax2.legend(loc='lower right')
plt.savefig("models/training_progress.png")
plt.show()
model.save(MODEL_PATH)
if epoch == 1400:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.1)
print("lr changed to {}".format(lr * 0.1))
if epoch == 1700:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.1)
print("lr changed to {}".format(lr * 0.1))
if epoch == 1900:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.1)
print("lr changed to {}".format(lr * 0.1))
return K.get_value(model.optimizer.lr)
reduce_lr = LearningRateScheduler(scheduler)
kfolder = KFold(n_splits=10, shuffle=True, random_state=2018)
oof_nn = np.zeros(len(x))
predictions_nn = np.zeros(len(x_test))
predictions_train_nn = np.zeros(len(x))
kfold = kfolder.split(x, y)
fold_ = 0
for train_index, vali_index in kfold:
k_x_train = x[train_index]
k_y_train = y[train_index]
k_x_vali = x[vali_index]
k_y_vali = y[vali_index]
model = tf.keras.Sequential()
model.add(
from keras.applications import VGG16
from tensorflow_core.python.keras.callbacks import LearningRateScheduler
from math import pow, floor
from keras.preprocessing.image import ImageDataGenerator
#-------------
def scheduler(epoch):
init_lrate = 0.0001
drop = 0.7
epochs_drop = 4
lrate = init_lrate * pow(drop, floor(1 + epoch) / epochs_drop)
return lrate
change_Lr = LearningRateScheduler(scheduler)
conv_base = VGG16(weights='imagenet',
include_top=False,
input_shape=(256, 256, 3))
conv_base.summary()
base_dir = "D:/document/data/food-11"
train_dir = os.path.join(base_dir, 'training')
validation_dir = os.path.join(base_dir, 'validation')
datagen = ImageDataGenerator(rescale=1. / 255)
BATCH_SIZE = 1000
train_datagen = ImageDataGenerator(
rescale=1. / 255,
model = SequentialCNN(
input_shape=dataset.input_shape(),
output_shape=dataset.output_shape()
)
model.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer=tf.keras.optimizers.RMSprop(learning_rate=config['learning_rate']),
metrics=['accuracy'])
lr_decay = lambda epocs: config['learning_rate'] * (1. / (1. + epocs/100.))
model.fit(
dataset.data['train_X'], dataset.data['train_Y'],
batch_size = config["batch_size"],
epochs = 99999,
verbose = config["verbose"],
validation_data = (dataset.data["valid_X"], dataset.data["valid_Y"]),
callbacks=[
EarlyStopping(monitor='val_loss', mode='min', verbose=True, patience=10),
# CyclicLR(mode='triangular2', step_size=33600*8, base_lr=0.0001, max_lr=0.006),
LearningRateScheduler(lr_decay, verbose=True),
tf.keras.callbacks.TensorBoard(log_dir='../../../logs/convergence', histogram_freq=1), # log metrics
# ConfusionMatrix(model, dataset).confusion_matrix_callback # breaks EarlyStopping = not a class with set_model()
]
)
score = model.evaluate(dataset.data['valid_X'], dataset.data['valid_Y'], verbose=config["verbose"])
print(model_name.ljust(15), "validation:", '| loss:', score[0], '| accuracy:', score[1])
# predict_to_csv( model.predict(dataset.data['test_X']), f'../../../submissions/keras-examples/keras-examples-{model_name}.csv')
print("time:", int(time.time() - timer_start), "s")
def train(self, X_train, X_val, y_train, y_val, X_test, params, save_type='J'):
if 'epochs' in params:
epochs = params['epochs']
else:
epochs = 20
if 'batch_size' in params:
batch_size = params['batch_size']
else:
batch_size = X_train.shape[0] // 10
if 'show_fig' in params:
show_fig = params['show_fig']
else:
show_fig = False
if 'loss' in params:
loss = params['loss']
else:
loss = 'loss'
if 'optimizer' in params:
optimizer = params['optimizer']
else:
optimizer = 'adam'
if 'metrics' in params:
metrics = params['metrics']
if str(type(metrics)).split('\'')[1] == 'str':
metrics = [metrics]
else:
metrics = 'mse'
# 利用回调函数，调整训练过程的学习率
# def scheduler(epoch):
# # 到规定的epoch，学习率减小为原来的1/10
#
# if epoch == 300:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.5)
# print("lr changed to {}".format(lr * 0.5))
# if epoch == 500:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.2)
# print("lr changed to {}".format(lr * 0.2))
# if epoch == 1000:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.5)
# print("lr changed to {}".format(lr * 0.5))
# if epoch == 1500:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.2)
# print("lr changed to {}".format(lr * 0.2))
# if epoch == 2000:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.5)
# print("lr changed to {}".format(lr * 0.5))
# if epoch == 2500:
# lr = K.get_value(model.optimizer.lr)
# K.set_value(model.optimizer.lr, lr * 0.2)
# print("lr changed to {}".format(lr * 0.2))
# return K.get_value(model.optimizer.lr)
def scheduler(epoch):
# 每隔100个epoch，学习率减小为原来的1/10
if epoch % 20 == 0 and epoch != 0:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.6)
print("lr changed to {}".format(lr * 0.6))
return K.get_value(model.optimizer.lr)
reduce_lr = LearningRateScheduler(scheduler)
# 利用回调函数，保存模型
load_model_dir = self.config.path_data.saveModelPath+'/keras_models/%s/' % self.config.modelType
if not os.path.exists(load_model_dir):
os.makedirs(load_model_dir)
load_model_file = load_model_dir + self.modelName + ".hdf5"
save_model_path = load_model_file
checkpoint = keras.callbacks.ModelCheckpoint(save_model_path, monitor='val_loss',
verbose=0, save_best_only=True, mode='min', skipping=1,
save_weights_only=True)
# 利用回调函数，设置早停止机制
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.000001, patience=200, verbose=0,
mode='auto', baseline=None, restore_best_weights=False)
# 确定使用的回调函数
# callback_lists = [checkpoint, reduce_lr, early_stopping]
callback_lists = [checkpoint, reduce_lr]
# 数据由DF转化为array
train_data_x = np.array(X_train)
train_data_y = np.array(y_train)
val_data_x = np.array(X_val)
val_data_y = np.array(y_val)
#########################################################
h = X_train.shape[1]
# 调用现有模型结构
if self.config.modelType == 'Resreg':
model = FC_Resreg().structure(input_shape=(h,))
elif self.config.modelType == 'fc_6Dmodel':
model = FC_6Dmodel().structure(h)
elif self.config.modelType == 'fc_8Dmodel':
model = FC_8Dmodel().structure(h)
else:
print('不存在这类模型：{}'.format(self.config.modelType))
return None
#########################################################
if not self.config.retrain:
# 调用已有模型进行增量训练
load_model_path = save_model_path
if os.path.exists(load_model_path):
try:
model.load_weights(load_model_path)
# 若成功加载前面保存的参数，输出下列信息
print("调用已有模型进行增量训练")
except Exception as e:
print(e)
print('模型可能参数不匹配')
else:
print("不存在已有模型，重新训练")
# 定义损失函数、优化器、评价函数
print(loss, optimizer, metrics)
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
# 训练模型
model.fit(train_data_x, train_data_y, epochs=epochs,
batch_size=batch_size, validation_data=(val_data_x, val_data_y),
callbacks=callback_lists)
# 绘制训练过程中的loss、metrics曲线
lossFig_dir = self.config.path_data.saveFigsPath + '/%s' % self.config.modelType
if not os.path.exists(lossFig_dir):
os.makedirs(lossFig_dir)
lossFig_path = lossFig_dir + '/%s' % self.modelName
metrice_loss_figs(model, lossFig_path, show_fig=show_fig)
# 验证集、测试集预测
val_data_y_pre = model.predict(X_val).reshape((model.predict(X_val).shape[0],))
test_data_y_pre = model.predict(X_test).reshape((model.predict(X_test).shape[0],))
# 计算验证集得分
metrics_name = self.config.metrics_name
myMetrics = defindMetrics.MyMetrics(metrics_name)
val_score = myMetrics.metricsFunc(val_data_y_pre, y_val)
return val_data_y_pre, test_data_y_pre, val_score