def train(opts, estimator):
"""Train and eval model in lcoal mode."""
train_input_fn = input_data.build_train_input_fn(opts,
opts.train_data_path)
max_steps = None
if opts.max_train_steps > 0:
max_steps = opts.max_train_steps
tf.logging.info('max_steps = {}'.format(max_steps))
tf.logging.info("Training model ...")
estimator.train(input_fn=train_input_fn, max_steps=max_steps)
tf.logging.info("Train model done\n")
def train_and_eval_in_local_mode(opts):
"""Train and eval model in lcoal mode."""
if opts.preprocess_type == model_keys.PreprocessType.EASY:
build_train_input_fn = input_data.build_easy_train_input_fn(opts)
build_eval_input_fn = input_data.build_easy_eval_input_fn(opts)
elif opts.preprocess_type == model_keys.PreprocessType.VGG:
build_train_input_fn = input_data.build_train_input_fn(opts)
build_eval_input_fn = input_data.build_eval_input_fn(opts)
else:
raise ValueError("Unsurpported preprocess type.")
best_accuracy = 0.0
accuracy_no_increase = 0
global_lr = opts.lr
build_model_fn.set_global_learning_rate(global_lr)
for epoch in range(opts.epoch):
epoch += 1
tf.logging.info("Beginning train model, epoch {} ...".format(epoch))
opts.estimator.train(input_fn=build_train_input_fn,
max_steps=opts.max_train_steps,
hooks=opts.hooks)
tf.logging.info("Train model OK, epoch {}".format(epoch))
tf.logging.info("Beginning evaluate model, epoch {} ...".format(epoch))
result = opts.estimator.evaluate(input_fn=build_eval_input_fn,
hooks=opts.hooks)
tf.logging.info("Evaluate model OK, epoch {}".format(epoch))
if result['accuracy'] > best_accuracy + opts.min_accuracy_increase:
accuracy_no_increase = 0
best_accuracy = result['accuracy']
else:
accuracy_no_increase += 1
if accuracy_no_increase == opts.lr_decay_epoch_when_no_increase:
global_lr *= opts.lr_decay_rate
build_model_fn.set_global_learning_rate(global_lr)
tf.logging.info(
"Accuracy no increase, learning rate decay by {}.".format(
opts.lr_decay_rate))
elif accuracy_no_increase > opts.lr_decay_epoch_when_no_increase:
tf.logging.info("Accuracy no increase, early stopping.")
break
else:
tf.logging.info("Accuracy no increase, try once more.")
return result
def train_and_eval_in_local_mode(opts, estimator, hooks):
"""Train and eval model in lcoal mode."""
build_train_input_fn = input_data.build_train_input_fn(
opts, opts.train_data_path)
build_eval_input_fn = input_data.build_eval_input_fn(
opts, opts.eval_data_path)
num_samples_per_epoch = len(
input_data.read_txt_file(opts.train_data_path, False))
num_steps_per_epoch = num_samples_per_epoch / opts.batch_size
if opts.max_train_steps > 0:
max_steps = opts.max_train_steps
else:
max_steps = opts.epoch * num_steps_per_epoch
tf.logging.info('max_steps = {}'.format(max_steps))
max_steps_without_decrease = int(opts.max_epoches_without_decrease *
num_steps_per_epoch)
early_stopping_min_steps = int(opts.early_stopping_min_epoches *
num_steps_per_epoch)
run_every_steps = int(opts.early_stopping_run_every_epoches *
num_steps_per_epoch)
early_stopping_hook = tf.contrib.estimator.stop_if_no_decrease_hook(
estimator,
"loss",
max_steps_without_decrease=max_steps_without_decrease,
run_every_secs=None,
min_steps=early_stopping_min_steps,
run_every_steps=run_every_steps)
hooks.append(early_stopping_hook)
train_spec = tf.estimator.TrainSpec(input_fn=build_train_input_fn,
max_steps=max_steps,
hooks=hooks)
eval_spec = tf.estimator.EvalSpec(input_fn=build_eval_input_fn,
steps=None,
name='eval',
start_delay_secs=3,
throttle_secs=opts.throttle_secs)
result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
return result
def data_iterator():
input_fn = input_data.build_train_input_fn(opts, opts.train_data_path)
dataset = input_fn()
iterator = dataset.make_initializable_iterator()
return iterator