def parser(input_dic):
"""Parser for string dict."""
inputs = parsing_ops.encode(
tf.reshape(input_dic["inputs"], [1]), max_input_len, vocab_filename,
encoder_type)
targets = parsing_ops.encode(
tf.reshape(input_dic["targets"], [1]), max_target_len, vocab_filename,
encoder_type)
inputs = utils.add_length_bucket_id(inputs, targets, length_bucket_size,
length_bucket_start_id,
length_bucket_max_id)
if add_task_id:
inputs = utils.add_task_id(inputs, task_start_id + input_dic["task_id"])
return {"inputs": inputs, "targets": targets}
def serving_input_fn(params):
"""Returns expected input spec for exported savedmodels."""
inputs_ph = tf.placeholder(
dtype=tf.string, shape=[params.batch_size], name="inputs")
inputs = public_parsing_ops.encode(inputs_ph, params.max_input_len,
params.vocab_filename, params.encoder_type,
params.length_bucket_size > 0)
inputs = tf.reshape(inputs, [params.batch_size, params.max_input_len])
features = {"inputs": inputs}
return tf.estimator.export.ServingInputReceiver(
features=features, receiver_tensors=inputs_ph)
def parser(input_dic):
"""Parser for string dict."""
if parser_strategy not in [
"random", "lead", "rouge", "greedy_rouge", "continuous_rouge", "hybrid",
"none", "dynamic_rouge"
]:
raise ValueError("Invalid parser_strategy. Got %s." % parser_strategy)
if parser_rouge_metric_type not in ["precision", "recall", "F"]:
raise ValueError("Invalid parser_rouge_metric_type. ",
"Got %s." % parser_rouge_metric_type)
if parser_rouge_compute_option not in ["standard", "deduplicate", "log"]:
raise ValueError("Invalid parser_rouge_compute_options. ",
"Got %s." % parser_rouge_compute_option)
supervised = input_dic["supervised"]
pretrain_inputs, pretrain_targets, pretrain_masked_inputs = pretrain_parsing_ops.sentence_mask_and_encode(
input_dic[input_feature], max_input_len, max_target_len,
max_total_words, parser_strategy, parser_masked_sentence_ratio,
parser_masked_words_ratio, parser_mask_word_options_prob,
parser_mask_sentence_options_prob, vocab_filename, encoder_type,
parser_rouge_ngrams_size, parser_rouge_metric_type,
parser_rouge_stopwords_filename, parser_rouge_compute_option,
parser_rouge_noise_ratio, parser_dynamic_mask_min_ratio,
shift_special_token_id)
supervised_inputs = parsing_ops.encode(
tf.reshape(input_dic["inputs"], [1]), max_input_len, vocab_filename,
encoder_type)
supervised_targets = parsing_ops.encode(
tf.reshape(input_dic["targets"], [1]), max_target_len, vocab_filename,
encoder_type)
inputs = tf.cond(pred=supervised, true_fn=lambda: supervised_inputs,
false_fn=lambda: pretrain_inputs)
targets = tf.cond(pred=supervised, true_fn=lambda: supervised_targets,
false_fn=lambda: pretrain_targets)
masked_inputs = tf.cond(pred=supervised, true_fn=lambda: supervised_inputs,
false_fn=lambda: pretrain_masked_inputs)
inputs, targets, masked_inputs = utils.filter_by_length(
[inputs, targets, masked_inputs],
min_len_list=[None, pretrain_target_filter_min, None])
inputs = utils.add_length_bucket_id(inputs, targets, length_bucket_size,
length_bucket_start_id,
length_bucket_max_id)
masked_inputs = utils.add_length_bucket_id(masked_inputs, targets,
length_bucket_size,
length_bucket_start_id,
length_bucket_max_id)
if add_task_id:
inputs = utils.add_task_id(inputs, task_start_id + input_dic["task_id"])
masked_inputs = utils.add_task_id(masked_inputs,
task_start_id + input_dic["task_id"])
output_dic = {
"inputs": inputs,
"targets": targets,
"masked_inputs": masked_inputs
}
return output_dic
def test_tf_encode(self, encoder_type):
string = tf.constant(["the quick brown fox.", "the quick brown\n"])
self.assertAllEqual(
parsing_ops.encode(string, 10, _SPM_VOCAB, encoder_type),
public_parsing_ops.encode(string, 10, _SPM_VOCAB, encoder_type))