def callbacks(self, model):
opts = self.opts
cbacks = []
cbacks.append(cb.ProgressLogger())
cbacks.append(cb.EarlyStopping(patience=opts.early_stop, verbose=1))
if opts.max_time is not None:
cbacks.append(cb.Timer(opts.max_time * 3600 * 0.8))
h = ModelCheckpoint(pt.join(opts.out_dir, 'model_weights_last.h5'),
save_best_only=False)
cbacks.append(h)
h = ModelCheckpoint(pt.join(opts.out_dir, 'model_weights.h5'),
save_best_only=True,
verbose=1)
cbacks.append(h)
def lr_schedule():
old_lr = model.optimizer.lr.get_value()
new_lr = old_lr * opts.lr_decay
model.optimizer.lr.set_value(new_lr)
print('Learning rate dropped from %g to %g' % (old_lr, new_lr))
h = cb.LearningRateScheduler(lr_schedule, patience=opts.lr_schedule)
cbacks.append(h)
def save_lc():
log = {
'lc.csv': perf_logger.frame(),
'lc_batch.csv': perf_logger.batch_frame()
}
for k, v in log.items():
with open(pt.join(opts.out_dir, k), 'w') as f:
f.write(perf_logs_str(v))
perf_logger = cb.PerformanceLogger(callbacks=[save_lc])
cbacks.append(perf_logger)
return cbacks
def get_callbacks(self):
opts = self.opts
callbacks = []
if opts.val_files:
callbacks.append(
kcbk.EarlyStopping('val_loss' if opts.val_files else 'loss',
patience=opts.early_stopping,
verbose=1))
callbacks.append(
kcbk.ModelCheckpoint(os.path.join(opts.out_dir,
'model_weights_train.h5'),
save_best_only=False))
monitor = 'val_loss' if opts.val_files else 'loss'
callbacks.append(
kcbk.ModelCheckpoint(os.path.join(opts.out_dir,
'model_weights_val.h5'),
monitor=monitor,
save_best_only=True,
verbose=1))
max_time = int(opts.max_time * 3600) if opts.max_time else None
callbacks.append(
cbk.TrainingStopper(max_time=max_time,
stop_file=opts.stop_file,
verbose=1))
def learning_rate_schedule(epoch):
lr = opts.learning_rate * opts.learning_rate_decay**epoch
print('Learning rate: %.3g' % lr)
return lr
callbacks.append(kcbk.LearningRateScheduler(learning_rate_schedule))
def save_lc(epoch, epoch_logs, val_epoch_logs):
logs = {'lc_train.csv': epoch_logs, 'lc_val.csv': val_epoch_logs}
for name, logs in six.iteritems(logs):
if not logs:
continue
logs = pd.DataFrame(logs)
with open(os.path.join(opts.out_dir, name), 'w') as f:
f.write(perf_logs_str(logs))
metrics = OrderedDict()
for metric_funs in six.itervalues(self.metrics):
for metric_fun in metric_funs:
metrics[metric_fun.__name__] = True
metrics = ['loss'] + list(metrics.keys())
self.perf_logger = cbk.PerformanceLogger(
callbacks=[save_lc],
metrics=metrics,
precision=LOG_PRECISION,
verbose=not opts.no_log_outputs)
callbacks.append(self.perf_logger)
if K._BACKEND == 'tensorflow':
callbacks.append(
kcbk.TensorBoard(log_dir=opts.out_dir,
histogram_freq=1,
write_graph=True,
write_images=True))
return callbacks
def get_callbacks(self):
#callbacks, a function in keras, used to view the internal statistics and states during the training process
opts = self.opts
callbacks = [] #define callbacks arguments
if opts.val_files:
callbacks.append(kcbk.EarlyStopping( #top training when a monitored quantity has stopped improving
'val_loss' if opts.val_files else 'loss', #val_loss and loss are the quantity to moniter
patience=opts.early_stopping, #the number of epochs model can tolerate without improve
verbose=1 #verbosity mode
))
#keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=0,
#save_best_only=False, save_weights_only=False, mode='auto', period=1)
callbacks.append(kcbk.ModelCheckpoint( #save model after every epoch
os.path.join(opts.out_dir, 'model_weights_train.h5'), #filepath, save the model
save_best_only=False)) #the lastest best model based on monitered quantity will be overwritten
monitor = 'val_loss' if opts.val_files else 'loss'
callbacks.append(kcbk.ModelCheckpoint(
os.path.join(opts.out_dir, 'model_weights_val.h5'), #filepath
monitor=monitor, #quantity to moniter
save_best_only=True, verbose=1 # latest best model according to the quantity monitored will not be overwritten.
))
max_time = int(opts.max_time * 3600) if opts.max_time else None
callbacks.append(cbk.TrainingStopper( #Stop training after certain time or when file is detected.
max_time=max_time, #Maximum training time in seconds
stop_file=opts.stop_file, #Name of stop file that triggers the end of training when existing
verbose=1 #If `True`, log message when training is stopped.
))
def learning_rate_schedule(epoch): #calcualte learning rate schedule by input rate and decay rate
lr = opts.learning_rate * opts.learning_rate_decay**epoch
print('Learning rate: %.3g' % lr)
return lr
callbacks.append(kcbk.LearningRateScheduler(learning_rate_schedule)) #a function that takes an epoch index as input
#(integer, indexed from 0) and current learning rate and returns a new learning rate as output (float).
def save_lc(epoch, epoch_logs, val_epoch_logs):
logs = {'lc_train.tsv': epoch_logs,
'lc_val.tsv': val_epoch_logs}
for name, logs in six.iteritems(logs): #Returns an iterator over dictionary‘s items.
if not logs:
continue
logs = pd.DataFrame(logs)
with open(os.path.join(opts.out_dir, name), 'w') as f:
f.write(perf_logs_str(logs))
metrics = OrderedDict()
for metric_funs in six.itervalues(self.metrics): #Returns an iterator over dictionary‘s values.
for metric_fun in metric_funs:
metrics[metric_fun.__name__] = True
metrics = ['loss'] + list(metrics.keys())
self.perf_logger = cbk.PerformanceLogger( #Logs performance metrics during training.
callbacks=[save_lc], #List of functions with parameters `epoch`, `epoch_logs`, and
#`val_epoch_logs` that are called at the end of each epoch.
metrics=metrics, #Name of metrics to be logged.
precision=LOG_PRECISION, # Floating point precision. as defined: 4
verbose=not opts.no_log_outputs #If `True`, log performance metrics of individual outputs.
)
callbacks.append(self.perf_logger)
if K._BACKEND == 'tensorflow' and not opts.no_tensorboard:
callbacks.append(kcbk.TensorBoard( #TensorBoard basic visualizations, which is visualization tool
log_dir=opts.out_dir, #the path of the directory where to save the log files to be parsed by TensorBoard.
histogram_freq=0, #frequency (in epochs) at which to compute activation and weight histograms for the layers of the model.
write_graph=True, #whether to visualize the graph in TensorBoard.
write_images=True #whether to write model weights to visualize as image in TensorBoard.
))
return callbacks