def train_and_eval(model):
for n in range(FLAGS.train_epochs):
tf.compat.v1.logging.info('=' * 30 + ' START EPOCH {} '.format(n + 1) +
'=' * 30 + '\n')
train_data_list = list_files(FLAGS.train_data) # dir to file list
for f in train_data_list:
t0 = time.time()
tf.compat.v1.logging.info('<EPOCH {}>: Start training {}'.format(
n + 1, f))
model.train(input_fn=lambda: input_fn(f, FLAGS.image_train_data,
'train', FLAGS.batch_size),
hooks=None,
steps=None,
max_steps=None,
saving_listeners=None)
tf.compat.v1.logging.info(
'<EPOCH {}>: Finish training {}, take {} mins'.format(
n + 1, f, elapse_time(t0)))
print('-' * 80)
tf.compat.v1.logging.info('<EPOCH {}>: Start evaluating {}'.format(
n + 1, FLAGS.eval_data))
t0 = time.time()
results = model.evaluate(
input_fn=lambda:
input_fn(FLAGS.eval_data, FLAGS.image_eval_data, 'eval', FLAGS.
batch_size),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=None, # latest checkpoint in model_dir is used.
name=None)
tf.compat.v1.logging.info(
'<EPOCH {}>: Finish evaluation {}, take {} mins'.format(
n + 1, FLAGS.eval_data, elapse_time(t0)))
print('-' * 80)
# Display evaluation metrics
for key in sorted(results):
print('{}: {}'.format(key, results[key]))
# every epochs_per_eval test the model (use larger test dataset)
if (n + 1) % FLAGS.epochs_per_eval == 0:
tf.compat.v1.logging.info('<EPOCH {}>: Start testing {}'.format(
n + 1, FLAGS.test_data))
results = model.evaluate(
input_fn=lambda:
input_fn(FLAGS.test_data, FLAGS.image_test_data, 'pred', FLAGS.
batch_size),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=
None, # If None, the latest checkpoint in model_dir is used.
name=None)
tf.compat.v1.logging.info(
'<EPOCH {}>: Finish testing {}, take {} mins'.format(
n + 1, FLAGS.test_data, elapse_time(t0)))
print('-' * 80)
# Display evaluation metrics
for key in sorted(results):
print('{}: {}'.format(key, results[key]))
def dynamic_train(model):
"""Dynamic train mode.
For example:
train_data_files: [0301, 0302, 0303, ...]
train mode:
first take 0301 as train data, 0302 as test data;
then keep training take 0302 as train data, 0303 as test data ...
"""
data_files = list_files(FLAGS.train_data)
data_files.sort()
assert len(data_files) > 1, 'Dynamic train mode need more than 1 data file'
for i in range(len(data_files) - 1):
train_data = data_files[i]
test_data = data_files[i + 1]
tf.compat.v1.logging.info(
'=' * 30 + ' START TRAINING DATA: {} '.format(train_data) +
'=' * 30 + '\n')
for n in range(FLAGS.train_epochs):
t0 = time.time()
tf.compat.v1.logging.info(
'START TRAIN DATA <{}> <EPOCH {}>'.format(train_data, n + 1))
model.train(input_fn=lambda: input_fn(
train_data, FLAGS.image_train_data, 'train', FLAGS.batch_size),
hooks=None,
steps=None,
max_steps=None,
saving_listeners=None)
tf.compat.v1.logging.info(
'FINISH TRAIN DATA <{}> <EPOCH {}> take {} mins'.format(
train_data, n + 1, elapse_time(t0)))
print('-' * 80)
tf.compat.v1.logging.info(
'START EVALUATE TEST DATA <{}> <EPOCH {}>'.format(
test_data, n + 1))
t0 = time.time()
results = model.evaluate(
input_fn=lambda: input_fn(test_data, FLAGS.image_eval_data,
'eval', FLAGS.batch_size),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=None, # latest checkpoint in model_dir is used.
name=None)
tf.compat.v1.logging.info(
'FINISH EVALUATE TEST DATA <{}> <EPOCH {}>: take {} mins'.
format(test_data, n + 1, elapse_time(t0)))
print('-' * 80)
# Display evaluation metrics
for key in sorted(results):
print('{}: {}'.format(key, results[key]))
def train(model):
for n in range(FLAGS.train_epochs):
tf.compat.v1.logging.info('=' * 30 + ' START EPOCH {} '.format(n + 1) + '=' * 30 + '\n')
train_data_list = list_files(FLAGS.train_data) # dir to file list
for f in train_data_list:
t0 = time.time()
tf.compat.v1.logging.info('<EPOCH {}>: Start training {}'.format(n + 1, f))
model.train(
input_fn=lambda: input_fn(f, FLAGS.image_train_data, 'train', FLAGS.batch_size),
hooks=None,
steps=None,
max_steps=None,
saving_listeners=None)
tf.compat.v1.logging.info('<EPOCH {}>: Finish training {}, take {} mins'.format(n + 1, f, elapse_time(t0)))