def clr_keras_callback(mode=None, base_lr=1e-4, max_lr=1e-3, gamma=0.999994):
""" Creates keras callback for cyclical learning rate. """
# keras_contrib = './keras_contrib/callbacks'
# sys.path.append(keras_contrib)
from cyclical_learning_rate import CyclicLR
if mode == 'trng1':
clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='triangular')
elif mode == 'trng2':
clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='triangular2')
elif mode == 'exp':
clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, mode='exp_range', gamma=gamma) # 0.99994; 0.99999994; 0.999994
return clr
def feature_train_2d():
train_array = range(0, 17928)
train_array = list(train_array)
train_array = shuffle(train_array)
validation_array = list(range(0, 1991))
ckpt_path = 'log/weights-{val_loss:.4f}.hdf5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
model = PoseNet_50(input_shape=(224, 224, 3))
adadelta = optimizers.Adadelta(lr=0.05, rho=0.9, decay=0.0)
# model.load_weights("model_data/weights-0.0753.hdf5")
model.compile(optimizer=adadelta, loss=euc_dist_keras, metrics=['mae'])
#lrate = LearningRateScheduler(step_decay)
clr = CyclicLR(base_lr=float("3.3e-4"),
max_lr=0.001,
step_size=1900,
mode='triangular')
result = model.fit_generator(
generator=pose2d_get_further_train_batch(train_array, 8, True),
steps_per_epoch=2241,
callbacks=[ckpt, clr],
epochs=60000,
verbose=1,
validation_data=pose2d_get_further_train_batch(validation_array, 8,
False),
validation_steps=249,
workers=1)
def train_3d_conv(base_model='', ckpt_model="None"):
img_path = "/media/disk1/human3.6/H36M-images/images/"
pose_path = "/media/disk1/human3.6/Annot/"
ckpt_path = 'log/3d_conv_weights-{val_loss:.4f}.hdf5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
model = make_seq_model(base_model)
adam = optimizers.adam(lr=float("1e-4"))
model.compile(optimizer=adam,
loss=euc_joint_dist_loss,
metrics=[euc_joint_metrics_dist_keras, metrics_pckh])
clr = CyclicLR(base_lr=float("1e-7"),
max_lr=float("1e-4"),
step_size=45288,
mode='triangular')
model.summary()
if ckpt_model != 'None':
model.load_weights(ckpt_model)
result = model.fit_generator(
generator=get_3d_train_batch(img_path, pose_path),
steps_per_epoch=45288,
callbacks=[ckpt, clr],
epochs=60000,
verbose=1,
validation_data=get_3d_Val_batch(img_path, pose_path),
validation_steps=839,
workers=1)
def train_2d():
index_array = range(1, 1901)
index_array = list(index_array)
index_array = shuffle(index_array)
validation_array = list(range(1901, 2001))
ckpt_path = 'log/weights-{val_loss:.4f}.hdf5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
model = PoseNet_50(input_shape=(224, 224, 3))
adadelta = optimizers.Adadelta(lr=0.05, rho=0.9, decay=0.0)
clr = CyclicLR(base_lr=float("3.3e-4"),
max_lr=0.05,
step_size=1900,
mode='triangular')
model.compile(optimizer=adadelta, loss=euc_dist_keras, metrics=['mae'])
model.load_weights('D:/dissertation/log/weights-118.9920.hdf5')
#lrate = LearningRateScheduler(step_decay)
result = model.fit_generator(
generator=pose2d_get_train_batch(index_array, 8),
steps_per_epoch=238,
callbacks=[ckpt, clr],
epochs=60000,
verbose=1,
validation_data=pose2d_get_train_batch(validation_array, 8),
validation_steps=52,
workers=1)
def get_cb_cyclic_lr():
return CyclicLR(base_lr=0.001,
max_lr=0.006,
step_size=2000.,
mode='triangular',
gamma=1.,
scale_fn=None,
scale_mode='cycle')
def train_3d_8s(base_model='', ckpt_model='None'):
train_skip, val_skip = read_skip()
train_array = list(range(1, 16545))
train_array = [i for i in train_array if i not in train_skip]
train_array = shuffle(train_array)
val_array = list(range(1, 4740))
val_array = [i for i in val_array if i not in val_skip]
val_array = shuffle(val_array)
ckpt_path = 'log/3d_weights_8s-{val_loss:.4f}.hdf5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
model, stride = resnet50_8s(input_shape=(224, 224, 3),
model_input=base_model)
#adadelta = optimizers.Adadelta(lr=0.05, rho=0.9, decay=0.0)
adam = optimizers.adam(lr=float("1e-4"))
model.compile(optimizer=adam,
loss=euc_joint_dist_loss,
metrics=[euc_joint_metrics_dist_keras, metrics_pckh])
#lrate = LearningRateScheduler(step_decay)
clr = CyclicLR(base_lr=float("1e-7"),
max_lr=float("1e-4"),
step_size=2069,
mode='triangular')
model.summary()
print(ckpt)
if ckpt_model != 'None':
model.load_weights(ckpt_model)
result = model.fit_generator(
generator=pose3d_get_train_batch(train_array, 8, True),
steps_per_epoch=2069,
callbacks=[ckpt, clr],
epochs=60000,
verbose=1,
validation_data=pose3d_get_train_batch(val_array, 8, False),
validation_steps=593,
workers=1)
def get_callbacks(path_out, sample_size, batch_size):
# early stopping callback
early_stopping = EarlyStopping(monitor='val_loss',
patience=20,
verbose=1,
restore_best_weights=False)
csv_logger = CSVLogger(f'{path_out}loss_history.log')
# model checkpoint callback
# this saves our model architecture + parameters into model.h5
model_checkpoint = ModelCheckpoint(f'{path_out}model.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
min_lr=0.000001,
cooldown=3,
verbose=1)
lr_scale = 1.
clr = CyclicLR(base_lr=0.0003 * lr_scale,
max_lr=0.001 * lr_scale,
step_size=sample_size / batch_size,
mode='triangular2')
stop_on_nan = tensorflow.keras.callbacks.TerminateOnNaN()
callbacks = [
early_stopping, clr, stop_on_nan, csv_logger, model_checkpoint
]
return callbacks
drop = 0.5
epochs_drop = 3.0
lrate = initial_lrate * math.pow(drop, math.floor(
(1 + epoch) / epochs_drop))
return lrate
############
# Hyperparameter for each model
############
#Model 1:
lr_decay = callbacks.LearningRateScheduler(
schedule=lr_scheduler
) #lambda epoch: configuration.lr * (0.9 ** configuration.epochs))
lr = 1e-3
clr_00 = CyclicLR(base_lr=1e-6, max_lr=6e-3, step_size=3200, mode='exp_range')
#Model 2:
#NONE
#from keras_gradient_noise import add_gradient_noise
#opt_noise = add_gradient_noise(optimizers.Adam)
optimizer = optimizers.Adam() #opt_noise(lr, amsgrad=True)
HED = True
losses = {
"o1": loss_functions.cross_entropy_balanced,
"o2": loss_functions.cross_entropy_balanced,
"o3": loss_functions.cross_entropy_balanced,
"o4": loss_functions.cross_entropy_balanced,
"o5": loss_functions.cross_entropy_balanced,
"ofuse": loss_functions.cross_entropy_balanced,
"output": loss_functions.dice_coef_loss,
}
Y = FileInput(opt.input, 'Y')
print('Y.shape', Y.shape)
# print('Z.shape', Z.shape)
# inputs, outputs = create_output_graph()
if opt.simple:
inputs, outputs = create_simple_graph()
else:
inputs, outputs = create_output_graph(n_features=n_features_pf,
n_features_cat=n_features_pf_cat)
lr_scale = 1.
# lr = CyclicLR(base_lr=0.001, max_lr=0.01, step_size=len(Y)/5., mode='triangular2')
clr = CyclicLR(base_lr=0.0003 * lr_scale,
max_lr=0.001 * lr_scale,
step_size=len(Y) / batch_size,
mode='triangular2')
# create the model
model = Model(inputs=inputs, outputs=outputs)
# compile the model
optimizer = optimizers.Adam(lr=1., clipnorm=1.)
# optimizer = optimizers.SGD(lr=0.0001, decay=0., momentum=0., nesterov=False)
# optimizer = AdamW(lr=0.0000, beta_1=0.8, beta_2=0.999, epsilon=None, decay=0., weight_decay=0.000, batch_size=batch_size, samples_per_epoch=int(len(Z)*0.8), epochs=epochs)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mean_absolute_error', 'mean_squared_error'])
# model.compile(loss=custom_loss, optimizer=optimizer,
# metrics=['mean_absolute_error', 'mean_squared_error'])
# print the model summary
#parallel_model = multi_gpu_model(model, gpus=4)
#parallel_model.compile(loss='mean_squared_error',
# optimizer=SGD(lr=0.0001, momentum=0.9),
# metrics=['mae',r2])
model.compile(loss='categorical_crossentropy',
optimizer=SGD(momentum=0.9),
# optimizer=Adam(lr=0.00001),
# optimizer=Adam(),
# optimizer=RMSprop(lr=0.0001),
# optimizer=Adadelta(),
metrics=['acc',tf_auc])
# set up a bunch of callbacks to do work during model training..
clr = CyclicLR(base_lr=0.0000001, max_lr=0.0001,
step_size=2000., mode='triangular2', scale_mode='iterations')
checkpointer = ModelCheckpoint(filepath='Agg_attn_bin.autosave.model.h5', verbose=1, save_weights_only=False, save_best_only=True)
csv_logger = CSVLogger('Agg_attn_bin.training.log')
reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.20, patience=40, verbose=1, mode='auto', min_delta=0.0001, cooldown=3, min_lr=0.000000001)
early_stop = EarlyStopping(monitor='val_loss', patience=200, verbose=1, mode='auto')
#history = parallel_model.fit(X_train, Y_train,
history = model.fit(X_train, Y_train, class_weight=d_class_weights,
batch_size=BATCH,
epochs=EPOCH,
verbose=1,
validation_data=(X_test, Y_test),