def decode_from_text_file(estimator,
problem_name,
filename,
hparams,
decode_hp,
decode_to_file=None,
checkpoint_path=None):
"""Compute predictions on entries in filename and write them out."""
if not decode_hp.batch_size:
decode_hp.batch_size = 32
tf.logging.info("decode_hp.batch_size not specified; default=%d" %
decode_hp.batch_size)
# Inputs vocabulary is set to targets if there are no inputs in the problem,
# e.g., for language models where the inputs are just a prefix of targets.
p_hp = hparams.problem_hparams
has_input = "inputs" in p_hp.vocabulary
inputs_vocab_key = "inputs" if has_input else "targets"
inputs_vocab = p_hp.vocabulary[inputs_vocab_key]
targets_vocab = p_hp.vocabulary["targets"]
filename = decoding._add_shard_to_filename(filename, decode_hp)
tf.logging.info("Performing decoding from file (%s)." % filename)
if has_input:
sorted_inputs, sorted_keys = decoding._get_sorted_inputs(
filename, decode_hp.delimiter)
else:
sorted_inputs = decoding._get_language_modeling_inputs(
filename, decode_hp.delimiter, repeat=decode_hp.num_decodes)
sorted_keys = range(len(sorted_inputs))
# If decode_to_file was provided use it as the output filename without change
# (except for adding shard_id if using more shards for decoding).
# Otherwise, use the input filename plus model, hp, problem, beam, alpha.
decode_filename = decode_to_file if decode_to_file else filename
if not decode_to_file:
decode_filename = decoding._decode_filename(decode_filename,
problem_name, decode_hp)
else:
decode_filename = decoding._add_shard_to_filename(
decode_filename, decode_hp)
tf.logging.info("Writing decodes into %s" % decode_filename)
# Check for decoding checkpoint.
decodes = []
shuffle_file_path = decode_filename + '.shuffle.txt'
if tf.gfile.Exists(shuffle_file_path):
with tf.gfile.Open(shuffle_file_path, 'r') as f:
decodes = [line.strip() for line in f.readlines()]
tf.logging.info('Read {} sentences from checkpoint.'.format(
len(decodes)))
all_sorted_inputs = sorted_inputs
# We only need to decode these inputs:
sorted_inputs = sorted_inputs[len(decodes):]
# We don't need to waste computation on empty lines:
num_empty_lines = 0
while sorted_inputs and sorted_inputs[-1] == '':
num_empty_lines += 1
sorted_inputs.pop(-1)
num_sentences = len(sorted_inputs)
num_decode_batches = (num_sentences - 1) // decode_hp.batch_size + 1
if estimator.config.use_tpu:
length = getattr(hparams, "length", 0) or hparams.max_length
batch_ids = []
for line in sorted_inputs:
if has_input:
ids = inputs_vocab.encode(line.strip()) + [1]
else:
ids = targets_vocab.encode(line)
if len(ids) < length:
ids.extend([0] * (length - len(ids)))
else:
ids = ids[:length]
batch_ids.append(ids)
np_ids = np.array(batch_ids, dtype=np.int32)
def input_fn(params):
batch_size = params["batch_size"]
dataset = tf.data.Dataset.from_tensor_slices({"inputs": np_ids})
dataset = dataset.map(
lambda ex:
{"inputs": tf.reshape(ex["inputs"], (length, 1, 1))})
dataset = dataset.batch(batch_size)
return dataset
else:
def input_fn():
input_gen = decoding._decode_batch_input_fn(
num_decode_batches,
sorted_inputs,
inputs_vocab,
decode_hp.batch_size,
decode_hp.max_input_size,
task_id=-1,
has_input=has_input)
gen_fn = decoding.make_input_fn_from_generator(input_gen)
example = gen_fn()
return decoding._decode_input_tensor_to_features_dict(
example, hparams)
result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)
start_time = time.time()
total_time_per_step = 0
total_cnt = 0
def timer(gen):
while True:
try:
start_time = time.time()
item = next(gen)
elapsed_time = time.time() - start_time
yield elapsed_time, item
except StopIteration:
break
writing_mode = 'a' if tf.gfile.Exists(shuffle_file_path) else 'w'
shuffle_file = tf.gfile.Open(shuffle_file_path, writing_mode)
count = 0
for elapsed_time, result in timer(result_iter):
if decode_hp.return_beams:
beam_decodes = []
beam_scores = []
output_beams = np.split(result["outputs"],
decode_hp.beam_size,
axis=0)
scores = None
if "scores" in result:
if np.isscalar(result["scores"]):
result["scores"] = result["scores"].reshape(1)
scores = np.split(result["scores"],
decode_hp.beam_size,
axis=0)
for k, beam in enumerate(output_beams):
tf.logging.info("BEAM %d:" % k)
score = scores and scores[k]
_, decoded_outputs, _ = decoding.log_decode_results(
result["inputs"],
beam,
problem_name,
None,
inputs_vocab,
targets_vocab,
log_results=decode_hp.log_results,
skip_eos_postprocess=decode_hp.skip_eos_postprocess)
beam_decodes.append(decoded_outputs)
if decode_hp.write_beam_scores:
beam_scores.append(score)
if decode_hp.write_beam_scores:
decodes.append("\t".join([
"\t".join([d, "%.2f" % s])
for d, s in zip(beam_decodes, beam_scores)
]))
else:
decodes.append("\t".join(beam_decodes))
else:
_, decoded_outputs, _ = decoding.log_decode_results(
result["inputs"],
result["outputs"],
problem_name,
None,
inputs_vocab,
targets_vocab,
log_results=decode_hp.log_results,
skip_eos_postprocess=decode_hp.skip_eos_postprocess)
decodes.append(decoded_outputs)
# Write decoded text to checkpoint
new_decode = decodes[-1]
shuffle_file.write(new_decode + '\n')
# Flush checkpoint to storage.
count += 1
if count % decode_hp.batch_size == 0:
tf.logging.info('Done {}/{}. Flushing.'.format(
count, len(sorted_inputs)))
shuffle_file.flush()
shuffle_file.close()
shuffle_file = tf.gfile.Open(shuffle_file_path, 'a')
total_time_per_step += elapsed_time
total_cnt += result["outputs"].shape[-1]
for _ in range(num_empty_lines):
decodes.append('')
shuffle_file.write('\n')
# Write the final output to file.
outfile = tf.gfile.Open(decode_filename, "w")
for index in range(len(all_sorted_inputs)):
outfile.write("%s%s" %
(decodes[sorted_keys[index]], decode_hp.delimiter))
outfile.flush()
outfile.close()
# Close and remove checkpoint.
shuffle_file.flush()
shuffle_file.close()
tf.gfile.Remove(shuffle_file_path)
# Print some decoding stats.
duration = time.time() - start_time
if total_cnt:
tf.logging.info("Elapsed Time: %5.5f" % duration)
tf.logging.info(
"Averaged Single Token Generation Time: %5.7f "
"(time %5.7f count %d)" %
(total_time_per_step / total_cnt, total_time_per_step, total_cnt))
if decode_hp.batch_size == 1:
tf.logging.info("Inference time %.4f seconds "
"(Latency = %.4f ms/setences)" %
(duration, 1000.0 * duration / num_sentences))
else:
tf.logging.info("Inference time %.4f seconds "
"(Throughput = %.4f sentences/second)" %
(duration, num_sentences / duration))
def decode_from_dataset(estimator,
problem_name,
hparams,
decode_hp,
decode_to_file=None,
dataset_split=None):
"""Perform decoding from dataset."""
tf.logging.info("Performing local inference from dataset for %s.",
str(problem_name))
shard = decode_hp.shard_id if decode_hp.shards > 1 else None
output_dir = os.path.join(estimator.model_dir, "decode")
tf.gfile.MakeDirs(output_dir)
if decode_hp.batch_size:
hparams.batch_size = decode_hp.batch_size
hparams.use_fixed_batch_size = True
dataset_kwargs = {
"shard": shard,
"dataset_split": dataset_split,
"max_records": decode_hp.num_samples
}
problem = hparams.problem
infer_input_fn = problem.make_estimator_input_fn(
tf.estimator.ModeKeys.PREDICT, hparams, dataset_kwargs=dataset_kwargs)
predictions = estimator.predict(infer_input_fn)
decode_to_file = decode_to_file or decode_hp.decode_to_file
if decode_to_file:
if decode_hp.shards > 1:
decode_filename = decode_to_file + ("%.2d" % decode_hp.shard_id)
else:
decode_filename = decode_to_file
output_filepath = decoding._decode_filename(decode_filename,
problem_name, decode_hp)
parts = output_filepath.split(".")
parts[-1] = "targets"
target_filepath = ".".join(parts)
parts[-1] = "inputs"
input_filepath = ".".join(parts)
parts[-1] = "enc_state"
encoder_state_file_path = ".".join(parts)
input_file = tf.gfile.Open(input_filepath, "w")
problem_hparams = hparams.problem_hparams
has_input = "inputs" in problem_hparams.vocabulary
inputs_vocab_key = "inputs" if has_input else "targets"
inputs_vocab = problem_hparams.vocabulary[inputs_vocab_key]
##### Modified #####
# Encoder outputs list created
encoder_outputs = []
decoded_inputs = []
for num_predictions, prediction in enumerate(predictions):
num_predictions += 1
inputs = prediction["inputs"]
encoder_output = prediction["encoder_outputs"]
decoded_input = inputs_vocab.decode(
decoding._save_until_eos(inputs, False))
encoder_outputs.append(encoder_output)
decoded_inputs.append(decoded_input)
##### Modified #####
# Writing encoder_outputs list to file
if decode_to_file:
for i, (e_output, d_input) in \
enumerate(zip(encoder_outputs, decoded_inputs)):
input_file.write("{}:\t{}".format(
i,
str(d_input) + decode_hp.delimiter))
np.save(encoder_state_file_path, np.array(encoder_outputs))
if (0 <= decode_hp.num_samples <= num_predictions):
break
if decode_to_file:
input_file.close()
decoding.decorun_postdecode_hooks(
decoding.DecodeHookArgs(estimator=estimator,
problem=problem,
output_dir=output_dir,
hparams=hparams,
decode_hparams=decode_hp))
tf.logging.info("Completed inference on %d samples." % num_predictions) # pylint: disable=undefined-loop-variable
def decode_from_file(estimator,
filename,
hparams,
decode_hp,
decode_to_file=None,
checkpoint_path=None):
"""Compute predictions on entries in filename and write them out."""
if not decode_hp.batch_size:
decode_hp.batch_size = 32
tf.logging.info("decode_hp.batch_size not specified; default=%d" %
decode_hp.batch_size)
p_hp = hparams.problem_hparams
has_input = "inputs" in p_hp.vocabulary
inputs_vocab_key = "inputs" if has_input else "targets"
inputs_vocab = p_hp.vocabulary[inputs_vocab_key]
problem_name = FLAGS.problem
tf.logging.info("Performing decoding from a file.")
sorted_inputs, sorted_keys = decoding._get_sorted_inputs(
filename, decode_hp.shards, decode_hp.delimiter)
num_decode_batches = (len(sorted_inputs) - 1) // decode_hp.batch_size + 1
def input_fn():
input_gen = decoding._decode_batch_input_fn(num_decode_batches,
sorted_inputs,
inputs_vocab,
decode_hp.batch_size,
decode_hp.max_input_size)
gen_fn = decoding.make_input_fn_from_generator(input_gen)
example = gen_fn()
return decoding._decode_input_tensor_to_features_dict(example, hparams)
##### Modified #####
# Encoder outputs list created
decoded_inputs = []
encoder_outputs = []
result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)
start_time = time.time()
total_time_per_step = 0
total_cnt = 0
def timer(gen):
while True:
try:
start_time = time.time()
item = next(gen)
elapsed_time = time.time() - start_time
yield elapsed_time, item
except StopIteration:
break
for elapsed_time, result in timer(result_iter):
decoded_input = inputs_vocab.decode(
decoding._save_until_eos(result["inputs"], False))
decoded_inputs.append(decoded_input)
encoder_outputs.append(np.array(result["encoder_outputs"]))
total_time_per_step += elapsed_time
total_cnt += result["outputs"].shape[-1]
tf.logging.info("Elapsed Time: %5.5f" % (time.time() - start_time))
tf.logging.info("Averaged Single Token Generation Time: %5.7f" %
(total_time_per_step / total_cnt))
decoded_inputs.reverse()
encoder_outputs.reverse()
decode_filename = decode_to_file if decode_to_file else filename
if decode_hp.shards > 1:
decode_filename += "%.2d" % decode_hp.shard_id
if not decode_to_file:
decode_filename = decoding._decode_filename(decode_filename,
problem_name, decode_hp)
base = os.path.basename(decode_filename).split('.')
dirname = os.path.dirname(decode_filename)
encode_filename = os.path.join(dirname, '{}{}'.format(base[0], '.npy'))
tf.logging.info("Writing inputs into %s" % decode_filename)
tf.logging.info("Writing encoder outputs into %s" % encode_filename)
print("Writing encoder outputs into %s" % encode_filename)
outfile = tf.gfile.Open(decode_filename, "w")
##### Modified #####
# Writing encoder_outputs list to file
if decode_to_file:
for i, (e_output, d_input) in \
enumerate(zip(encoder_outputs, decoded_inputs)):
outfile.write("{}".format(' '.join([
word for word in str(d_input).strip().split()
if word.strip() != '' and word.strip() != '<unk>'
]) + decode_hp.delimiter))
np.save(encode_filename, np.array(encoder_outputs))
if decode_to_file:
outfile.close()
def decode_from_babi_dataset(estimator,
problem_name,
hparams,
decode_hp,
decode_to_file=None):
"""Perform decoding from dataset."""
tf.logging.info("Performing local inference from dataset for %s.",
str(problem_name))
# We assume that worker_id corresponds to shard number.
shard = decode_hp.shard_id if decode_hp.shards > 1 else None
# Setup decode output directory for any artifacts that may be written out
output_dir = os.path.join(estimator.model_dir, "decode")
tf.gfile.MakeDirs(output_dir)
# If decode_hp.batch_size is specified, use a fixed batch size
if decode_hp.batch_size:
hparams.batch_size = decode_hp.batch_size
hparams.use_fixed_batch_size = True
dataset_kwargs = {
"shard": shard,
"dataset_split": None,
"max_records": decode_hp.num_samples
}
# Build the inference input function
problem = hparams.problem
infer_input_fn = problem.make_estimator_input_fn(
tf.estimator.ModeKeys.PREDICT, hparams, dataset_kwargs=dataset_kwargs)
predictions, output_dirs = [], []
for decode_id in range(decode_hp.num_decodes):
tf.logging.info("Decoding {}".format(decode_id))
# Create decode directory if not in-memory decoding.
if not decode_hp.decode_in_memory:
output_dir = os.path.join(estimator.model_dir, "decode_%05d" % decode_id)
tf.gfile.MakeDirs(output_dir)
output_dirs.append(output_dir)
# Get the predictions as an iterable
predictions = estimator.predict(infer_input_fn)
# Prepare output file writers if decode_to_file passed
decode_to_file = decode_to_file or decode_hp.decode_to_file
if decode_to_file:
if decode_hp.shards > 1:
decode_filename = decode_to_file + ("%.2d" % decode_hp.shard_id)
else:
decode_filename = decode_to_file
output_filepath = decoding._decode_filename(decode_filename, problem_name,
decode_hp)
parts = output_filepath.split(".")
parts[-1] = "decoded"
decoded_filepath = ".".join(parts)
dir = os.path.dirname(decoded_filepath)
if not tf.gfile.Exists(dir):
tf.gfile.MakeDirs(dir)
decoded_file = tf.gfile.Open(decoded_filepath, "w")
problem_hparams = hparams.problem_hparams
story_vocab = problem_hparams.vocabulary[babi_qa.FeatureNames.STORY]
question_vocab = problem_hparams.vocabulary[babi_qa.FeatureNames.STORY]
targets_vocab = problem_hparams.vocabulary["targets"]
for num_predictions, prediction in enumerate(predictions):
num_predictions += 1
story = prediction[babi_qa.FeatureNames.STORY]
question = prediction[babi_qa.FeatureNames.QUESTION]
targets = prediction["targets"]
outputs = prediction["outputs"]
# Log predictions
decoded_outputs = []
decoded_scores = []
if decode_hp.return_beams:
output_beams = np.split(outputs, decode_hp.beam_size, axis=0)
scores = None
if "scores" in prediction:
scores = np.split(prediction["scores"], decode_hp.beam_size, axis=0)
for i, beam in enumerate(output_beams):
tf.logging.info("BEAM %d:" % i)
score = scores and scores[i]
decoded = log_decode_results(
story,
question,
beam,
story_vocab,
question_vocab,
targets_vocab,
identity_output=decode_hp.identity_output,
targets=targets)
decoded_outputs.append(decoded)
if decode_hp.write_beam_scores:
decoded_scores.append(score)
else:
decoded = log_decode_results(
story,
question,
outputs,
story_vocab,
question_vocab,
targets_vocab,
identity_output=decode_hp.identity_output,
targets=targets)
decoded_outputs.append(decoded)
# Write out predictions if decode_to_file passed
if decode_to_file:
for i, (d_story, d_question ,
d_output, d_target) in enumerate(decoded_outputs):
beam_score_str = ""
if decode_hp.write_beam_scores:
beam_score_str = "\t%.2f" % decoded_scores[i]
decoded_file.write('STORY: \n%s\n' % _make_story_pretty(str(d_story)))
decoded_file.write('QUESTION: %s\n' %_remove_pad(str(d_question)))
decoded_file.write('ANSWER: %s\n' % _remove_pad(str(d_target)))
decoded_file.write('OUTPUT: %s\n' % (str(d_output) + beam_score_str + decode_hp.delimiter) )
decoded_file.write('==================================================================\n')
if (decode_hp.num_samples >= 0 and
num_predictions >= decode_hp.num_samples):
break
if decode_to_file:
tf.logging.info("Decoded results are written in: %s" % decoded_filepath)
decoded_file.close()
tf.logging.info("Completed inference on %d samples." % num_predictions) # pylint: disable=undefined-loop-variable
def decode_from_file_search_based(estimator,
filename,
hparams,
decode_hp,
decode_to_file=None,
checkpoint_path=None):
"""Compute predictions on entries in filename and write them out."""
if not decode_hp.batch_size:
decode_hp.batch_size = 32
tf.logging.info(
"decode_hp.batch_size not specified; default=%d" % decode_hp.batch_size)
problem_id = decode_hp.problem_idx
# Inputs vocabulary is set to targets if there are no inputs in the problem,
# e.g., for language models where the inputs are just a prefix of targets.
has_input = "inputs" in hparams.problems[problem_id].vocabulary
inputs_vocab_key = "inputs" if has_input else "targets"
inputs_vocab = hparams.problems[problem_id].vocabulary[inputs_vocab_key]
targets_vocab = hparams.problems[problem_id].vocabulary["targets"]
problem_name = FLAGS.problems.split("-")[problem_id]
tf.logging.info("Performing decoding from a file.")
sorted_inputs, sorted_keys = _get_sorted_inputs(filename, decode_hp.shards,
decode_hp.delimiter)
num_decode_batches = (len(sorted_inputs) - 1) // decode_hp.batch_size + 1
data_dir = '/'.join(filename.split('/')[:-1])
table_path = os.path.join(data_dir, '../../search_engine/big_table.txt')
he_search_path = os.path.join(data_dir, 'he.search.txt')
en_search_path = os.path.join(data_dir, 'en.search.txt')
searcher = Searcher(table_path, he_search_path)
translator = Translator(data_dir, he_search_path)
def input_fn():
input_gen = _decode_batch_input_fn_search_based(
problem_id, num_decode_batches, sorted_inputs, inputs_vocab, targets_vocab,
decode_hp.batch_size, decode_hp.max_input_size, searcher, translator,
hparams.problems[problem_id])
gen_fn = make_input_fn_from_generator(input_gen)
example = gen_fn()
return _decode_input_tensor_to_features_dict(example, hparams)
decodes = []
result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)
for result in result_iter:
if decode_hp.return_beams:
beam_decodes = []
beam_scores = []
output_beams = np.split(result["outputs"], decode_hp.beam_size, axis=0)
scores = None
if "scores" in result:
scores = np.split(result["scores"], decode_hp.beam_size, axis=0)
for k, beam in enumerate(output_beams):
tf.logging.info("BEAM %d:" % k)
score = scores and scores[k]
_, decoded_outputs, _ = log_decode_results(result["inputs"], beam,
problem_name, None,
inputs_vocab, targets_vocab)
beam_decodes.append(decoded_outputs)
if decode_hp.write_beam_scores:
beam_scores.append(score)
if decode_hp.write_beam_scores:
decodes.append("\t".join(
["\t".join([d, "%.2f" % s]) for d, s
in zip(beam_decodes, beam_scores)]))
else:
decodes.append("\t".join(beam_decodes))
else:
_, decoded_outputs, _ = log_decode_results(
result["inputs"], result["outputs"], problem_name,
None, inputs_vocab, targets_vocab)
decodes.append(decoded_outputs)
# Reversing the decoded inputs and outputs because they were reversed in
# _decode_batch_input_fn
sorted_inputs.reverse()
decodes.reverse()
# If decode_to_file was provided use it as the output filename without change
# (except for adding shard_id if using more shards for decoding).
# Otherwise, use the input filename plus model, hp, problem, beam, alpha.
decode_filename = decode_to_file if decode_to_file else filename
if decode_hp.shards > 1:
decode_filename += "%.2d" % decode_hp.shard_id
if not decode_to_file:
decode_filename = _decode_filename(decode_filename, problem_name, decode_hp)
tf.logging.info("Writing decodes into %s" % decode_filename)
outfile = tf.gfile.Open(decode_filename, "w")
for index in range(len(sorted_inputs)):
outfile.write("%s%s" % (decodes[sorted_keys[index]], decode_hp.delimiter))
def decode_from_file_fn(estimator,
filename,
hparams,
decode_hp,
decode_to_file=None,
checkpoint_path=None):
"""Compute predictions on entries in filename and write them out."""
if not decode_hp.batch_size:
decode_hp.batch_size = 32
tf.logging.info(
"decode_hp.batch_size not specified; default=%d" % decode_hp.batch_size)
# Inputs vocabulary is set to targets if there are no inputs in the problem,
# e.g., for language models where the inputs are just a prefix of targets.
p_hp = hparams.problem_hparams
has_input = "inputs" in p_hp.vocabulary
inputs_vocab_key = "inputs" if has_input else "targets"
inputs_vocab = p_hp.vocabulary[inputs_vocab_key]
targets_vocab = p_hp.vocabulary["targets"]
problem_name = FLAGS.problem
filename = decoding._add_shard_to_filename(filename, decode_hp)
tf.logging.info("Performing decoding from file (%s)." % filename)
if has_input:
sorted_inputs, sorted_keys = decoding._get_sorted_inputs(
filename, decode_hp.delimiter)
else:
sorted_inputs = decoding._get_language_modeling_inputs(
filename, decode_hp.delimiter, repeat=decode_hp.num_decodes)
sorted_keys = range(len(sorted_inputs))
num_sentences = len(sorted_inputs)
num_decode_batches = (num_sentences - 1) // decode_hp.batch_size + 1
if estimator.config.use_tpu:
length = getattr(hparams, "length", 0) or hparams.max_length
batch_ids = []
for line in sorted_inputs:
if has_input:
ids = inputs_vocab.encode(line.strip()) + [1]
else:
ids = targets_vocab.encode(line)
if len(ids) < length:
ids.extend([0] * (length - len(ids)))
else:
ids = ids[:length]
batch_ids.append(ids)
np_ids = np.array(batch_ids, dtype=np.int32)
def input_fn(params):
batch_size = params["batch_size"]
dataset = tf.data.Dataset.from_tensor_slices({"inputs": np_ids})
dataset = dataset.map(
lambda ex: {"inputs": tf.reshape(ex["inputs"], (length, 1, 1))})
dataset = dataset.batch(batch_size)
return dataset
else:
def input_fn():
input_gen = decoding._decode_batch_input_fn(
num_decode_batches, sorted_inputs,
inputs_vocab, decode_hp.batch_size,
decode_hp.max_input_size,
task_id=decode_hp.multiproblem_task_id, has_input=has_input)
gen_fn = decoding.make_input_fn_from_generator(input_gen)
example = gen_fn()
return decoding._decode_input_tensor_to_features_dict(example, hparams, decode_hp)
decodes = []
result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)
start_time = time.time()
total_time_per_step = 0
total_cnt = 0
def timer(gen):
while True:
try:
start_time = time.time()
item = next(gen)
elapsed_time = time.time() - start_time
yield elapsed_time, item
except StopIteration:
break
for elapsed_time, result in timer(result_iter):
if decode_hp.return_beams:
beam_decodes = []
beam_scores = []
output_beams = np.split(result["outputs"], decode_hp.beam_size, axis=0)
scores = None
if "scores" in result:
if np.isscalar(result["scores"]):
result["scores"] = result["scores"].reshape(1)
scores = np.split(result["scores"], decode_hp.beam_size, axis=0)
for k, beam in enumerate(output_beams):
tf.logging.info("BEAM %d:" % k)
score = scores and scores[k]
_, decoded_outputs, _ = decoding.log_decode_results(
result["inputs"],
beam,
problem_name,
None,
inputs_vocab,
targets_vocab,
log_results=decode_hp.log_results,
skip_eos_postprocess=decode_hp.skip_eos_postprocess)
beam_decodes.append(decoded_outputs)
if decode_hp.write_beam_scores:
beam_scores.append(score)
if decode_hp.write_beam_scores:
decodes.append("\t".join([
"\t".join([d, "%.2f" % s])
for d, s in zip(beam_decodes, beam_scores)
]))
else:
decodes.append("\t".join(beam_decodes))
else:
_, decoded_outputs, _ = decoding.log_decode_results(
result["inputs"],
result["outputs"],
problem_name,
None,
inputs_vocab,
targets_vocab,
log_results=decode_hp.log_results,
skip_eos_postprocess=decode_hp.skip_eos_postprocess)
decodes.append(decoded_outputs)
total_time_per_step += elapsed_time
total_cnt += result["outputs"].shape[-1]
duration = time.time() - start_time
tf.logging.info("Elapsed Time: %5.5f" % duration)
tf.logging.info("Averaged Single Token Generation Time: %5.7f "
"(time %5.7f count %d)" %
(total_time_per_step / total_cnt,
total_time_per_step, total_cnt))
if decode_hp.batch_size == 1:
tf.logging.info("Inference time %.4f seconds "
"(Latency = %.4f ms/setences)" %
(duration, 1000.0*duration/num_sentences))
else:
tf.logging.info("Inference time %.4f seconds "
"(Throughput = %.4f sentences/second)" %
(duration, num_sentences/duration))
# If decode_to_file was provided use it as the output filename without change
# (except for adding shard_id if using more shards for decoding).
# Otherwise, use the input filename plus model, hp, problem, beam, alpha.
decode_filename = decode_to_file if decode_to_file else filename
if not decode_to_file:
decode_filename = decoding._decode_filename(decode_filename, problem_name, decode_hp)
else:
decode_filename = decoding._add_shard_to_filename(decode_filename, decode_hp)
tf.logging.info("Writing decodes into %s" % decode_filename)
outfile = tf.gfile.Open(decode_filename, "w")
for index in range(len(sorted_inputs)):
special_chars = ["\a", "\n", "\f", "\r", "\b"]
output = decodes[sorted_keys[index]]
for c in special_chars:
output = output.replace(c, ' ')
try:
outfile.write("%s%s" % (output, decode_hp.delimiter))
except:
outfile.write("%s" % decode_hp.delimiter)
outfile.flush()
outfile.close()