def process(self, entry, errors, debug_info=None):
"""Converts TyDi entries into serialized tf examples.
Args:
entry: "TyDi entries", dicts as returned by `create_entry_from_json`.
errors: A list that this function appends to if errors are created. A
non-empty list indicates problems.
debug_info: A dict of information that may be useful during debugging.
These elements should be used for logging and debugging only. For
example, we log how the text was tokenized into WordPieces.
Yields:
`tf.train.Example` with the features needed for training or inference
(depending on how `is_training` was set in the constructor).
"""
if not debug_info:
debug_info = {}
# convert raw data into TyDiExample
tydi_example = data.to_tydi_example(entry, self.is_training)
debug_info["tydi_example"] = tydi_example
# Converts `TyDiExample`s into `InputFeatures` and sends them to `output_fn
input_features = preprocess.convert_single_example(
tydi_example,
tokenizer=self.tokenizer,
is_training=self.is_training,
max_question_length=self.max_question_length,
max_seq_length=self.max_seq_length,
doc_stride=self.doc_stride,
include_unknowns=self.include_unknowns,
errors=errors,
debug_info=debug_info)
# convert example to features
for input_feature in input_features:
input_feature.example_index = int(entry["id"])
input_feature.unique_id = (input_feature.example_index +
input_feature.doc_span_index)
features = collections.OrderedDict()
features["unique_ids"] = [input_feature.unique_id]
features["example_index"] = [input_feature.example_index]
features["input_ids"] = input_feature.input_ids
features["input_mask"] = input_feature.input_mask
features["segment_ids"] = input_feature.segment_ids
features["language_id"] = [input_feature.language_id]
if self.is_training:
features["start_positions"] = [input_feature.start_position]
features["end_positions"] = [input_feature.end_position]
features["answer_types"] = [input_feature.answer_type]
else:
features["wp_start_offset"] = input_feature.wp_start_offset
features["wp_end_offset"] = input_feature.wp_end_offset
self.feature_lst.append(features)
return self.feature_lst
def read_tydi_examples(input_file, is_training, max_passages, max_position,
fail_on_invalid, open_fn):
"""Read a TyDi json file into a list of `TyDiExample`.
Delegates to `preproc.create_entry_from_json` to add special tokens to
input and handle character offset tracking.
Args:
input_file: Path or glob to input JSONL files to be read (possibly gzipped).
is_training: Should we create training samples? (as opposed to eval
samples).
max_passages: See FLAGS.max_passages.
max_position: See FLAGS.max_position.
fail_on_invalid: Should we immediately stop processing if an error is
encountered?
open_fn: A function that returns a file object given a path. Usually
`tf_io.gopen`; could be standard Python `open` if using this module
outside Tensorflow.
Yields:
`TyDiExample`s
"""
input_paths = glob.glob(input_file)
if not input_paths:
raise ValueError("No paths matching glob '{}'".format(input_file))
non_valid_count = 0
n = 0
for path in input_paths:
logging.info("Reading: %s", path)
with open_fn(path) as input_file:
logging.info(path)
for line in input_file:
json_dict = json.loads(line, object_pairs_hook=collections.OrderedDict)
entry = create_entry_from_json(
json_dict,
max_passages=max_passages,
max_position=max_position,
fail_on_invalid=fail_on_invalid)
if entry:
tydi_example = data.to_tydi_example(entry, is_training)
n += 1
yield tydi_example
else:
if fail_on_invalid:
raise ValueError("Found invalid example.")
non_valid_count += 1
if n == 0:
raise ValueError(
"No surviving examples from input_file '{}'".format(input_file))
logging.info("*** # surviving examples %d ***", n)
logging.info("*** # pruned examples %d ***", non_valid_count)
def read_tydi_examples(input_file,
is_training=False,
max_passages=45,
max_position=45):
input_paths = glob.glob(input_file)
n = 0
for path in input_paths:
logging.info("Reading: %s", path)
with open(path) as input_file:
logging.info(path)
for line in input_file:
json_dict = json.loads(
line, object_pairs_hook=collections.OrderedDict)
entry = create_entry_from_json(
json_dict,
max_passages=max_passages,
max_position=max_position
# fail_on_invalid=fail_on_invalid
)
if entry:
tydi_example = data.to_tydi_example(entry, is_training)
n += 1
yield tydi_example