def getcallbacks(self, checkpointstr=None, eval_cb=None):
# defaults
lw_sess_tag = 'Part2_RNN' + time.asctime().replace(" ", '')
chkptname = os.getcwd() + '\\checkpoints\\' + (
checkpointstr or 'rnn2_unnamed_model.hdf5')
basedir = os.getcwd() + '\\checkpoints\\'
callbacks = []
modelcheckpoint = ModelCheckpoint(chkptname,
monitor='loss',
verbose=1,
save_best_only=True)
lrop = ReduceLROnPlateau(monitor='loss',
factor=0.8,
patience=8,
verbose=1,
mode='min',
min_delta=1e-8,
cooldown=4,
min_lr=1e-10)
earlystopping = EarlyStopping(monitor='loss',
min_delta=1e-8,
patience=30,
verbose=1,
mode='min')
tboard = TensorBoard(log_dir=basedir,
histogram_freq=1,
batch_size=self.bsize,
write_graph=True,
write_grads=True,
write_images=True)
losswise_ = LosswiseKerasCallback(tag=lw_sess_tag, display_interval=1)
history = History()
if eval_cb == 'tb':
callbacks = [modelcheckpoint, lrop, earlystopping, history, tboard]
elif eval_cb == 'lw':
callbacks = [
modelcheckpoint, lrop, earlystopping, history, losswise_
]
elif eval_cb is None:
callbacks = [modelcheckpoint, lrop, earlystopping, history]
return callbacks
def losswise(self, keyfile, model_to_json, epochs, steps_per_epoch):
keys = json.load(open(keyfile))
api_key = keys["losswise_api_key"]
tag = keys["losswise_tag"]
losswise.set_api_key(api_key)
params = json.loads(model_to_json)
params['steps_per_epoch'] = steps_per_epoch
params['epochs'] = epochs
self.losswise_callback = LosswiseKerasCallback(tag=tag,
params=params
)
ModelCheckpoint(os.path.join(
model_path, '%s.{epoch:06d}-{val_loss:.6f}.hdf5' % model.name),
save_best_only=True)
]
if cfg['save_predictions_during_training']:
callbacks.append(
LambdaCallback(on_epoch_end=lambda epoch, logs: save_prediction_imgs(
test_generator, model, model_path)))
# save model metadata
shutil.copyfile('config.yaml', os.path.join(model_path, 'config_global.yaml'))
shutil.copyfile(os.path.join('region_proposal', 'config.yaml'),
os.path.join(model_path, 'config.yaml'))
if cfg['losswise_api_key']:
callbacks.append(LosswiseKerasCallback(tag='giterdone',
display_interval=1))
history = model.fit_generator(generator=train_generator,
validation_data=test_generator,
epochs=cfg['training_epochs'],
callbacks=callbacks)
with open(os.path.join(model_path, 'training_history'), 'wb') as training_file:
pickle.dump(history.history, training_file)
# load best model and delete others
models_to_delete = glob(os.path.join(model_path, '*hdf5'))[:-1]
[os.remove(mod) for mod in iter(models_to_delete)]
model = load_model(glob(os.path.join(model_path, '*.hdf5'))[0])
# generate final predictions
save_prediction_imgs(test_generator, model, model_path)
def continue_second():
reg_num = 0.00
drop_rate = 0.0
gap_callback = GapCallback(val_img_class_gen, val_steps_per_small_epoch)
# new_xcpetion_model = load_model(data_folder + '2nd_phase_xcpetion_model.h5')
# save_model_to_file(new_xcpetion_model, data_folder + 'xcpetion_model_.json')
# xception_model.save_weights(data_folder + 'chackpoint_second_phase_xcpetion_model.h5')
new_xcpetion_model = load_model_from_file(
data_folder + 'xcpetion_model_dropout_1024.json')
new_xcpetion_model = set_reg_drop(new_xcpetion_model, reg_num, drop_rate)
# predictions = xcpetion_model.get_layer('dense_2')
# dropout = Dropout(0.2, name='gap_2048_dropout', seed=0)
# fc = xcpetion_model.get_layer('dense_1')
# gap = xcpetion_model.get_layer('global_average_pooling2d_1')
# fc_1024_dropout = xcpetion_model.get_layer('fc_1024_dropout')
# x = dropout(gap.output)
# x = fc(x)
# x = fc_1024_dropout(x)
# predictors = predictions(x)
# new_xcpetion_model = Model(inputs=xcpetion_model.input, outputs=predictors)
# save_model_to_file(new_xcpetion_model, data_folder + 'xcpetion_model_dropout_2048_1024.json')
# new_xcpetion_model.save_weights(data_folder + '2nd_phase_xcpetion_weights.h5')
print(new_xcpetion_model.summary())
new_xcpetion_model = load_weights_from_file(
new_xcpetion_model, data_folder + 'xcpetion_model_.json',
data_folder + '2nd_phase_xcpetion_weights.h5')
# new_xcpetion_model.load_weights(data_folder + '2nd_phase_xcpetion_weights.h5', by_name=False)
new_xcpetion_model.compile(optimizer=Adam(lr=0.0002),
loss=square_error,
metrics=['acc',
gap]) # categorical_crossentropy
print('dropout rate: ',
new_xcpetion_model.get_layer('fc_1024_dropout').rate)
for i in range(second_phase_train_reps):
print(i + 1, 'out of ', third_phase_train_reps)
history = new_xcpetion_model.fit_generator(
train_img_class_gen,
steps_per_epoch=steps_per_small_epoch,
epochs=small_epochs,
verbose=1,
validation_data=val_img_class_gen,
validation_steps=val_steps_per_small_epoch,
workers=4,
callbacks=[
LosswiseKerasCallback(tag='keras xcpetion model'), gap_callback
])
print(i)
if i % saves_per_epoch == 0:
print('{} epoch completed'.format(int(i / saves_per_epoch)))
ts = calendar.timegm(time.gmtime())
new_xcpetion_model.save_weights(continue_second_phase_folder +
str(ts) + '_mse_xcpetion_model.h5')
# new_xcpetion_model.save(continue_second_phase_folder + str(ts) + '_xcpetion_model.h5')
save_obj(history.history,
str(ts) + '_xcpetion_mse_history.h5',
folder=continue_second_phase_folder)
new_xcpetion_model.save_weights(data_folder +
'continue_second_phase_xcpetion_model.h5')
def smoothed_third_phase():
# reg_num = 0.001
# drop_num = 0.3
global xcpetion_model, new_xcpetion_model, optimizer
# tensorboard = TensorBoard(log_dir=third_phase_folder + 'tb_logs', batch_size=batch_size)
# xcpetion_model = load_model(data_folder + '2nd_phase_xcpetion_model.h5')
#
# predictions = xcpetion_model.layers[-1]
# fc = xcpetion_model.layers[-2]
# fc.name = 'penultimate_fc'
#
# x = Dropout(rate=drop_num, name='dropout_1')(fc.output)
# predictors = predictions(x)
# new_xcpetion_model = Model(inputs=xcpetion_model.input, outputs=predictors)
#
# for layer in new_xcpetion_model.layers[:trainable_layers_index]:
# layer.trainable = False
# for layer in new_xcpetion_model.layers[trainable_layers_index:]:
# layer.trainable = True
start_lr_num = 0.0001
end_lr_num = 0.00005
lr_nums = list(
np.linspace(start_lr_num, end_lr_num, third_phase_train_reps))
start_reg_num = 0.0001
end_reg_num = 0.005
reg_nums = list(
np.linspace(start_reg_num, end_reg_num, third_phase_train_reps))
# reg_num = 0.001
start_drop_num = 0.05
end_drop_num = 0.5
drop_nums = list(
np.linspace(start_drop_num, end_drop_num, third_phase_train_reps))
# drop_num = 0.3
for i in range(third_phase_train_reps):
print('clear session')
K.clear_session()
print('load new model...')
try:
new_xcpetion_model = load_model(smoothed_third_phase_folder +
str(ts) + '_xcpetion_model.h5')
print('loaded model: ' + str(ts) + '_xcpetion_model.h5')
except:
new_xcpetion_model = load_model(initial_model)
print('loaded model: ' + initial_model)
trainable_layers_ratio = 1 / 3.0
trainable_layers_index = int(
len(new_xcpetion_model.layers) * (1 - trainable_layers_ratio))
lr_num = lr_nums[i]
reg_num = reg_nums[i]
drop_num = drop_nums[i]
print('lr: ', lr_num, '\nregularizer: ', reg_num, '\ndropout: ',
drop_num)
# add regularizers to the convolutional layers
for layer in new_xcpetion_model.layers[trainable_layers_index:]:
if isinstance(layer, keras.layers.convolutional.Conv2D) \
or isinstance(layer, keras.layers.Dense):
layer.kernel_regularizer = regularizers.l2(reg_num)
layer.activity_regularizer = regularizers.l1(reg_num)
elif isinstance(layer, keras.layers.Dropout):
layer.rate = drop_num
optimizer = Adam(lr=lr_num)
new_xcpetion_model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['acc'])
history = new_xcpetion_model.fit_generator(
train_img_class_gen,
steps_per_epoch=steps_per_small_epoch, # steps_per_small_epoch
epochs=small_epochs,
verbose=1, # small_epochs
validation_data=val_img_class_gen,
validation_steps=val_steps_per_small_epoch,
workers=4,
callbacks=[LosswiseKerasCallback(tag='keras xcpetion model')])
print(i, ' out of ', third_phase_train_reps)
if i % saves_per_epoch == 0:
print('{} epoch completed'.format(int(i / saves_per_epoch)))
ts = calendar.timegm(time.gmtime())
new_xcpetion_model.save(smoothed_third_phase_folder + str(ts) +
'_xcpetion_model.h5')
save_obj(history.history,
str(ts) + '_xcpetion_history.h5',
folder=smoothed_third_phase_folder)
new_xcpetion_model.save(data_folder +
'smoothed_3rd_phase_xcpetion_model.h5')
def third_phase():
global xcpetion_model, new_xcpetion_model, optimizer
# tensorboard = TensorBoard(log_dir=third_phase_folder + 'tb_logs', batch_size=batch_size)
xcpetion_model = load_model(data_folder + '2nd_phase_xcpetion_model.h5')
# add regularizers to the convolutional layers
trainable_layers_ratio = 1 / 2.0
trainable_layers_index = int(
len(xcpetion_model.layers) * (1 - trainable_layers_ratio))
for layer in xcpetion_model.layers[:trainable_layers_index]:
layer.trainable = False
for layer in xcpetion_model.layers[trainable_layers_index:]:
layer.trainable = True
for layer in xcpetion_model.layers:
layer.trainable = True
if isinstance(layer, keras.layers.convolutional.Conv2D):
layer.kernel_regularizer = regularizers.l2(0.001)
layer.activity_regularizer = regularizers.l1(0.001)
# add dropout and regularizer to the penultimate Dense layer
predictions = xcpetion_model.layers[-1]
dropout = Dropout(0.3)
fc = xcpetion_model.layers[-2]
fc.kernel_regularizer = regularizers.l2(0.001)
fc.activity_regularizer = regularizers.l1(0.001)
x = dropout(fc.output)
predictors = predictions(x)
new_xcpetion_model = Model(inputs=xcpetion_model.input, outputs=predictors)
optimizer = Adam(lr=0.1234)
start_lr = 0.001
end_lr = 0.00001
step_lr = (end_lr - start_lr) / (third_phase_train_reps - 1)
new_xcpetion_model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['acc'])
for i in range(third_phase_train_reps):
lr = start_lr + step_lr * i
K.set_value(new_xcpetion_model.optimizer.lr, lr)
print(i, 'out of ', third_phase_train_reps, '\nlearning rate ',
K.eval(new_xcpetion_model.optimizer.lr))
history = new_xcpetion_model.fit_generator(
train_img_class_gen,
steps_per_epoch=steps_per_small_epoch,
epochs=small_epochs,
verbose=1,
validation_data=val_img_class_gen,
validation_steps=val_steps_per_small_epoch,
workers=4,
callbacks=[LosswiseKerasCallback(tag='keras xcpetion model')])
print(i)
if i % saves_per_epoch == 0:
print('{} epoch completed'.format(int(i / saves_per_epoch)))
ts = calendar.timegm(time.gmtime())
new_xcpetion_model.save(third_phase_folder + str(ts) +
'_xcpetion_model.h5')
save_obj(history.history,
str(ts) + '_xcpetion_history.h5',
folder=third_phase_folder)
new_xcpetion_model.save(data_folder + '3rd_phase_xcpetion_model.h5')
#Graph of loss & accuracy
graph_loss = session.graph('loss', kind='min', display_interval=1)
graph_accuracy = session.graph('accuracy', kind='max', display_interval=50)
#Running the CNN on the training set
classifier.fit_generator(
training_set,
steps_per_epoch=4000,
epochs=25,
validation_data=test_set, #test_set accuracy
validation_steps=1000,
callbacks=[
LosswiseKerasCallback(tag='Convolutional Classifier',
params={
'dropout': 0.25,
'cnn_size': 256
},
track_git=True,
max_iter=4000)
])
#Tell Losswise you're done
session.done()
#Save classifier model
classifier.save_weights('cat_dog_full_model_weights2.h5')
classifier.save('cat_dog_classifier2.h5')
#######################################################
'''
###Visualise model static
# list all data in history
model.compile(SGD(lr=0.15),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
samples, labels = extract_data('training_set.txt')
training = model.fit(samples,
labels,
batch_size=10,
epochs=20,
validation_split=0.05,
shuffle=True,
verbose=2,
callbacks=[
LosswiseKerasCallback(tag='keras test',
params={'lstm_size': 32},
track_git=True,
display_interval=500),
ReduceLROnPlateau(monitor='val_acc',
patience=5,
factor=0.5,
verbose=1,
min_lr=0.0001),
EarlyStopping(monitor='val_acc',
min_delta=0.0003,
patience=20,
verbose=1)
])
test_samples, test_labels = extract_data('testing_set.txt')
scores = model.evaluate(test_samples, test_labels)
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy', meanError, meanSquared, meanAbsError])
# Training
print('[INFO] Training...')
callbacks = [
ModelCheckpoint(os.path.join(
args['model_dir'],
args['modelname'] + ".{epoch:02d}-{val_loss:.2f}.h5"),
period=5,
verbose=1)
]
if args['losswise_api']:
callbacks.append(
LosswiseKerasCallback(tag=args['modelname'], display_interval=1))
kwargs = {
'verbose': 1,
'epochs': args['epochs'],
'validation_data': test_gen,
'use_multiprocessing': args['workers'] > 1,
'callbacks': callbacks
}
if args['workers'] > 1:
kwargs['workers'] = args['workers']
if not args['disable_class_weighting']:
kwargs['class_weight'] = train_gen.getClassWeights()
def train(self, data_path, **parameters):
x_train, y_train, x_val, y_val = self.load_data(data_path)
def on_epoch_end():
test = self.generate_text(280)
print(test)
def build_model(learning_rate=0.01, n_units=128):
print('Build model...')
model = Sequential()
model.add(
LSTM(n_units,
input_shape=(self.max_sentence_len, self.charset_size)))
model.add(Dense(self.charset_size))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
best_param = {}
do_gridsearch = False
if (do_gridsearch):
# wrap the model in a scikit-learn regressor
sci_model = KerasRegressor(build_fn=build_model)
# define the grid search parameters
learning_rate = [0.01, 0.05, 0.1]
n_units = [64, 128, 256]
epochs = [5]
param_grid = dict(learning_rate=learning_rate,
epochs=epochs,
n_units=n_units)
# fix random seed for reproducibility
seed = 42
numpy.random.seed(seed)
random.seed(seed)
# downsample randomly for gridsearch
if (len(x_train) > 10000):
train_x_d, train_y_d = downsample(x_train, y_train, 10000)
else:
train_x_d = x_train
train_y_d = y_train
grid = GridSearchCV(estimator=sci_model,
param_grid=param_grid,
n_jobs=1,
verbose=100,
cv=3)
grid_result = grid.fit(train_x_d, train_y_d)
# summarize results
best_param = grid_result.best_params_
print("Best: %f using %s" % (grid_result.best_score_, best_param))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
learning_rate = best_param[
'learning_rate'] if 'learning_rate' in best_param else self.learning_rate
n_units = best_param['n_units'] if 'n_units' in best_param else 128
self.model = build_model(learning_rate=learning_rate, n_units=n_units)
all_params = merge_two_dicts(best_param, self.stored_params)
# get the training history after fitting the model
history = self.model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.n_epochs,
validation_data=(x_val, y_val),
callbacks=[
LambdaCallback(on_epoch_end=on_epoch_end),
LosswiseKerasCallback(params=all_params),
EarlyStopping(patience=2)
])
plot_history(history, self.name)