def sg_summary_param(tensor, prefix='40. parameters'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([tensor]))
tf.histogram_summary(name, tensor)
def sg_summary_metric(tensor, prefix='20. metric'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/avg', tf.reduce_mean(tensor))
tf.histogram_summary(name, tensor)
def sg_summary_gradient(tensor, gradient, prefix='50. gradient'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([gradient]))
tf.histogram_summary(name, gradient)
def sg_summary_metric(tensor, prefix='20. metric'):
r"""Writes the average of `tensor` (=metric such as accuracy).
"""
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/avg', tf.reduce_mean(tensor))
tf.histogram_summary(name, tensor)
def sg_summary_activation(tensor, prefix='30. activation'):
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
tf.scalar_summary(name + '/norm', tf.global_norm([tensor]))
tf.scalar_summary(
name + '/ratio',
tf.reduce_mean(tf.cast(tf.greater(tensor, 0), tf.sg_floatx)))
tf.histogram_summary(name, tensor)
def sg_summary_gradient(tensor, gradient, prefix='50. gradient'):
r"""Writes the normalized gradient value
Args:
tensor: A `Tensor` variable.
gradient: A `Tensor`. Gradient of `tensor`.
"""
# defaults
prefix = '' if prefix is None else prefix + '/'
# summary name
name = prefix + _pretty_name(tensor)
# summary statistics
with tf.name_scope('summary'):
try:
tf.scalar_summary(name + '/norm', tf.global_norm([gradient]))
tf.histogram_summary(name, gradient)
except:
pass
def wrapper(**kwargs):
opt = tf.sg_opt(kwargs)
# default training options
opt += tf.sg_opt(lr=0.001,
save_dir='asset/train',
max_ep=1000,
ep_size=100000,
save_interval=600,
log_interval=60,
early_stop=True,
lr_reset=False,
eval_metric=[],
max_keep=5,
keep_interval=1,
tqdm=True,
console_log=False)
# make directory if not exist
if not os.path.exists(opt.save_dir + '/log'):
os.makedirs(opt.save_dir + '/log')
if not os.path.exists(opt.save_dir + '/ckpt'):
os.makedirs(opt.save_dir + '/ckpt')
# find last checkpoint
last_file = tf.train.latest_checkpoint(opt.save_dir + '/ckpt')
if last_file:
ep = start_ep = int(last_file.split('-')[1]) + 1
start_step = int(last_file.split('-')[2])
else:
ep = start_ep = 1
start_step = 0
# checkpoint saver
saver = tf.train.Saver(max_to_keep=opt.max_keep,
keep_checkpoint_every_n_hours=opt.keep_interval)
# summary writer
summary_writer = tf.train.SummaryWriter(opt.save_dir + '/log',
graph=tf.get_default_graph())
# add learning rate summary
with tf.name_scope('summary'):
tf.scalar_summary('60. learning_rate/learning_rate',
_learning_rate)
# add evaluation metric summary
for m in opt.eval_metric:
tf.sg_summary_metric(m)
# summary op
summary_op = tf.merge_all_summaries()
# create session
if opt.sess:
sess = opt.sess
else:
# session with multiple GPU support
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# initialize variables
sg_init(sess)
# restore last checkpoint
if last_file:
saver.restore(sess, last_file)
# set learning rate
if start_ep == 1 or opt.lr_reset:
sess.run(_learning_rate.assign(opt.lr))
# logging
tf.sg_info('Training started from epoch[%03d]-step[%d].' %
(start_ep, start_step))
try:
# start data queue runner
with tf.sg_queue_context(sess):
# set session mode to train
tf.sg_set_train(sess)
# loss history for learning rate decay
loss, loss_prev, early_stopped = None, None, False
# time stamp for saving and logging
last_saved = last_logged = time.time()
# epoch loop
for ep in range(start_ep, opt.max_ep + 1):
# show progressbar
if opt.tqdm:
iterator = tqdm(range(opt.ep_size),
desc='train',
ncols=70,
unit='b',
leave=False)
else:
iterator = range(opt.ep_size)
# batch loop
for _ in iterator:
# call train function
batch_loss = func(sess, opt)
# loss history update
if batch_loss is not None:
if loss is None:
loss = np.mean(batch_loss)
else:
loss = loss * 0.9 + np.mean(batch_loss) * 0.1
# saving
if time.time() - last_saved > opt.save_interval:
last_saved = time.time()
saver.save(sess,
opt.save_dir + '/ckpt/model-%03d' % ep,
write_meta_graph=False,
global_step=sess.run(
tf.sg_global_step()))
# logging
if time.time() - last_logged > opt.log_interval:
last_logged = time.time()
# set session mode to infer
tf.sg_set_infer(sess)
# run evaluation op
if len(opt.eval_metric) > 0:
sess.run(opt.eval_metric)
if opt.console_log: # console logging
# log epoch information
tf.sg_info(
'\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s'
% (ep, sess.run(_learning_rate),
sess.run(tf.sg_global_step()),
('NA' if loss is None else '%8.6f' %
loss)))
else: # tensorboard logging
# run logging op
summary_writer.add_summary(
sess.run(summary_op),
global_step=sess.run(tf.sg_global_step()))
# learning rate decay
if opt.early_stop and loss_prev:
# if loss stalling
if loss >= 0.95 * loss_prev:
# early stopping
current_lr = sess.run(_learning_rate)
if current_lr < 5e-6:
early_stopped = True
break
else:
# decrease learning rate by half
sess.run(
_learning_rate.assign(current_lr /
2.))
# update loss history
loss_prev = loss
# revert session mode to train
tf.sg_set_train(sess)
# log epoch information
if not opt.console_log:
tf.sg_info(
'\tEpoch[%03d:lr=%7.5f:gs=%d] - loss = %s' %
(ep, sess.run(_learning_rate),
sess.run(tf.sg_global_step()),
('NA' if loss is None else '%8.6f' % loss)))
if early_stopped:
tf.sg_info('\tEarly stopped ( no loss progress ).')
break
finally:
# save last epoch
saver.save(sess,
opt.save_dir + '/ckpt/model-%03d' % ep,
write_meta_graph=False,
global_step=sess.run(tf.sg_global_step()))
# set session mode to infer
tf.sg_set_infer(sess)
# logging
tf.sg_info('Training finished at epoch[%d]-step[%d].' %
(ep, sess.run(tf.sg_global_step())))
# close session
if opt.sess is None:
sess.close()
