def create_config(model_dir, hparams=None):
  """Creates config instance."""
  tf.logging.info("model_dir = " + model_dir)
  assert model_dir

  if hparams:
    tf.logging.info("Given override cfg:\n%s" % pprint.pformat(hparams))
  else:
    hparams = dict()

  # Build the default config
  cfg = model.build_config(model_dir=model_dir, data_path=FLAGS.data_path)

  if FLAGS.config_file:
    with tf.gfile.GFile(FLAGS.config_file) as f:
      file_cfg = json.load(f)
      file_cfg = utils.to_config(file_cfg)
    tf.logging.info("Loaded config from file:\n%s" % file_cfg)
    cfg = utils.merge_fixed_structure(cfg, file_cfg)

  # Override from flags
  overrides = dict()
  if FLAGS.config:
    overrides = utils.parse_config_string(FLAGS.config)
    tf.logging.info("Parsed config overrides:\n%s" % overrides)
    cfg = utils.merge_fixed_structure(cfg, overrides)

  if FLAGS.master:
    cfg.master = FLAGS.master

  cfg = utils.merge_fixed_structure(cfg, utils.unflatten_dict(hparams))

  tf.logging.info("Operative config:\n%s" % cfg)

  return cfg
def evaluate(override_cfg, model_dir, continuous=True):
  """Run training and evaluation."""
  tf.logging.info("model_dir = " + model_dir)
  try:
    cfg = _load_config(model_dir)
  except tf.errors.NotFoundError:
    tf.logging.info("Model directory does not exist yet. Creating new config.")
    cfg = model.build_config(model_dir=model_dir, data_path=FLAGS.data_path)
  tf.logging.info(cfg)
  tf.logging.info(override_cfg)
  cfg = utils.merge(cfg, override_cfg)

  cfg.tpu.enable = False
  cfg.dataset.max_length = None

  # Construct inputs and estimator
  _, eval_input = data.build_dataset(cfg.dataset, is_tpu=cfg.tpu.enable)
  estimator = model.get_estimator(**cfg)
  if continuous:
    checkpoints_iterator = tf.contrib.training.checkpoints_iterator(
        cfg.model_dir)
    eval_metrics = None
    for ckpt_path in checkpoints_iterator:
      eval_metrics = estimator.evaluate(
          input_fn=eval_input, checkpoint_path=ckpt_path)
      tf.logging.info(pprint.pformat(eval_metrics))
    return eval_metrics
  else:
    eval_metrics = estimator.evaluate(input_fn=eval_input)
    return eval_metrics