def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
news_config = GroverConfig.from_json_file(FLAGS.config_file)
tf.gfile.MakeDirs(FLAGS.output_dir)
input_files = []
for input_pattern in FLAGS.input_file.split(","):
input_files.extend(tf.gfile.Glob(input_pattern))
tf.logging.info("*** Input Files ***")
for input_file in input_files:
tf.logging.info(" %s" % input_file)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
keep_checkpoint_max=None,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
model_fn = model_fn_builder(
news_config,
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=FLAGS.num_train_steps,
num_warmup_steps=FLAGS.num_warmup_steps,
use_tpu=FLAGS.use_tpu,
)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
eval_batch_size=FLAGS.train_batch_size,
params={'model_dir': FLAGS.output_dir})
tf.logging.info("***** Running training *****")
tf.logging.info(" Batch size = %d", FLAGS.train_batch_size)
train_input_fn = input_fn_builder(input_files=input_files,
seq_length=FLAGS.max_seq_length,
is_training=True)
estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
news_config = GroverConfig.from_json_file(FLAGS.config_file)
tf.gfile.MakeDirs(FLAGS.output_dir)
input_files = []
for input_pattern in FLAGS.input_file.split(","):
input_files.extend(tf.gfile.Glob(input_pattern))
tf.logging.info("*** Input Files ***")
for input_file in input_files:
tf.logging.info(" %s" % input_file)
tpu_cluster_resolver = None
if FLAGS.use_tpu and FLAGS.tpu_name:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.iterations_per_loop,
keep_checkpoint_max=None,
tpu_config=tf.contrib.tpu.TPUConfig(
iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
model_fn = model_fn_builder(news_config,
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=1e-4,
num_train_steps=0,
num_warmup_steps=0,
use_tpu=FLAGS.use_tpu,
)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.batch_size,
eval_batch_size=FLAGS.batch_size,
predict_batch_size=FLAGS.batch_size,
params={'model_dir': FLAGS.output_dir}
)
eval_input_fn = input_fn_builder(
input_files=input_files,
seq_length=FLAGS.max_seq_length,
evaluate_for_fixed_number_of_steps=False,
num_cpu_threads=1,
is_training=False)
result = [x for x in estimator.predict(input_fn=eval_input_fn, yield_single_examples=True)]
cats = sorted(result[0].keys())
result_stack = {cat: np.stack([x[cat] for x in result]) for cat in cats}
with gcloudwriter(os.path.join(FLAGS.output_dir, FLAGS.validation_name)) as tempfile_name:
with h5py.File(tempfile_name, 'w') as h5:
for cat, data in result_stack.items():
dtype2use = np.float16 if cat.endswith(('logprobs', 'top_p_required')) else np.uint16
h5.create_dataset(cat, data=data.astype(dtype2use))
h5.create_dataset('model', data=FLAGS.config_file)
h5.create_dataset('ckpt', data=FLAGS.init_checkpoint)
h5.create_dataset('input_file', data=FLAGS.input_file)
# This gives the perplexity of the entire article. if you want to replicate the results of the paper you
# might need to do something different to extract the ppl of just the body in particular.
ppl_ex = []
for logprobs_i, ids_i in zip(result_stack['gt_logprobs'], result_stack['labels']):
# Omit the first token. Keep in mind input_ids is shifted by 1
start_ind = ind_where(ids_i, target=50265, default_value=0)
end_ind = ind_where(ids_i, target=50266, default_value=ids_i.shape[0] - 1)
ppl_ex.append(logprobs_i[start_ind:end_ind])
ppl_ex = np.concatenate(ppl_ex, 0)
print("Article perplexity is {:.3f}".format(np.exp(-np.mean(ppl_ex))), flush=True)