def process(self, example) -> Iterator[tf.train.Example]:
converter = data_utils.WikiHopTFExampleConverter(
tokenizer=self._tokenizer,
global_seq_len=FLAGS.global_seq_len,
long_seq_len=FLAGS.long_seq_len,
max_num_sentences=FLAGS.max_num_sentences)
yield converter.convert_single_example(example=example)
def get_serialized_tf_example(wikihop_example: data_utils.WikiHopExample,
tokenizer: tokenization.FullTokenizer,
long_seq_len: int = 4096,
global_seq_len: int = 430,
max_num_sentences: int = 200) -> Text:
"""Returns serialized TF example from the given json example."""
converter = data_utils.WikiHopTFExampleConverter(
tokenizer=tokenizer,
long_seq_len=long_seq_len,
global_seq_len=global_seq_len,
max_num_sentences=max_num_sentences)
tf_example = converter.convert_single_example(example=wikihop_example)
return tf_example.SerializeToString()