def __init__(
self,
data_file: str,
tokenizer: object,
doc_stride: int,
max_query_length: int,
max_seq_length: int,
version_2_with_negative: bool,
num_samples: int,
mode: str,
use_cache: bool,
):
self.tokenizer = tokenizer
self.version_2_with_negative = version_2_with_negative
self.processor = SquadProcessor(data_file=data_file, mode=mode)
self.mode = mode
if mode not in [TRAINING_MODE, EVALUATION_MODE, INFERENCE_MODE]:
raise ValueError(
f"mode should be either {TRAINING_MODE}, {EVALUATION_MODE}, {INFERENCE_MODE} but got {mode}"
)
self.examples = self.processor.get_examples()
vocab_size = getattr(tokenizer, "vocab_size", 0)
cached_features_file = (
data_file
+ '_cache'
+ '_{}_{}_{}_{}_{}_{}_{}'.format(
mode,
tokenizer.name,
str(vocab_size),
str(max_seq_length),
str(doc_stride),
str(max_query_length),
str(num_samples),
)
)
# check number of samples. Should be either -1 not to limit or positive number
if num_samples == 0:
raise ValueError(
f"num_samples has to be positive or -1 (to use the entire dataset), however got {num_samples}."
)
elif num_samples > 0:
self.examples = self.examples[:num_samples]
if use_cache and os.path.exists(cached_features_file):
logging.info(f"loading from {cached_features_file}")
with open(cached_features_file, "rb") as reader:
self.features = pickle.load(reader)
else:
logging.info(f"Preprocessing data.")
self.features = convert_examples_to_features(
examples=self.examples,
tokenizer=tokenizer,
max_seq_length=max_seq_length,
doc_stride=doc_stride,
max_query_length=max_query_length,
has_groundtruth=mode != INFERENCE_MODE,
)
if use_cache:
master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
if master_device:
logging.info(" Saving train features into cached file %s", cached_features_file)
with open(cached_features_file, "wb") as writer:
pickle.dump(self.features, writer)
class SquadDataset(Dataset):
"""
Creates SQuAD dataset for Question Answering.
Args:
data_file (str): train.*.json eval.*.json or test.*.json.
tokenizer (obj): Tokenizer object, e.g. AutoTokenizer.
version_2_with_negative (bool): True if training should allow
unanswerable questions.
doc_stride (int): When splitting up a long document into chunks,
how much stride to take between chunks.
max_query_length (iny): All training files which have a duration less
than min_duration are dropped. Can't be used if the `utt2dur` file
does not exist. Defaults to None.
max_seq_length (int): All training files which have a duration more
than max_duration are dropped. Can't be used if the `utt2dur` file
does not exist. Defaults to None.
num_samples: number of samples you want to use for the dataset.
If -1, use all dataset. Useful for testing.
mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between
training, evaluation and inference dataset.
use_cache (bool): Caches preprocessed data for future usage
"""
def __init__(
self,
data_file: str,
tokenizer: object,
doc_stride: int,
max_query_length: int,
max_seq_length: int,
version_2_with_negative: bool,
num_samples: int,
mode: str,
use_cache: bool,
):
self.tokenizer = tokenizer
self.version_2_with_negative = version_2_with_negative
self.processor = SquadProcessor(data_file=data_file, mode=mode)
self.mode = mode
if mode not in [TRAINING_MODE, EVALUATION_MODE, INFERENCE_MODE]:
raise ValueError(
f"mode should be either {TRAINING_MODE}, {EVALUATION_MODE}, {INFERENCE_MODE} but got {mode}"
)
self.examples = self.processor.get_examples()
vocab_size = getattr(tokenizer, "vocab_size", 0)
cached_features_file = (
data_file
+ '_cache'
+ '_{}_{}_{}_{}_{}_{}_{}'.format(
mode,
tokenizer.name,
str(vocab_size),
str(max_seq_length),
str(doc_stride),
str(max_query_length),
str(num_samples),
)
)
# check number of samples. Should be either -1 not to limit or positive number
if num_samples == 0:
raise ValueError(
f"num_samples has to be positive or -1 (to use the entire dataset), however got {num_samples}."
)
elif num_samples > 0:
self.examples = self.examples[:num_samples]
if use_cache and os.path.exists(cached_features_file):
logging.info(f"loading from {cached_features_file}")
with open(cached_features_file, "rb") as reader:
self.features = pickle.load(reader)
else:
logging.info(f"Preprocessing data.")
self.features = convert_examples_to_features(
examples=self.examples,
tokenizer=tokenizer,
max_seq_length=max_seq_length,
doc_stride=doc_stride,
max_query_length=max_query_length,
has_groundtruth=mode != INFERENCE_MODE,
)
if use_cache:
master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
if master_device:
logging.info(" Saving train features into cached file %s", cached_features_file)
with open(cached_features_file, "wb") as writer:
pickle.dump(self.features, writer)
def __len__(self):
return len(self.features)
def __getitem__(self, idx):
feature = self.features[idx]
if self.mode == INFERENCE_MODE:
return (
np.array(feature.input_ids),
np.array(feature.segment_ids),
np.array(feature.input_mask),
np.array(feature.unique_id),
)
else:
return (
np.array(feature.input_ids),
np.array(feature.segment_ids),
np.array(feature.input_mask),
np.array(feature.unique_id),
np.array(feature.start_position),
np.array(feature.end_position),
)
def get_predictions(
self,
unique_ids: List[int],
start_logits: List[List[float]],
end_logits: List[List[float]],
n_best_size: int,
max_answer_length: int,
do_lower_case: bool,
version_2_with_negative: bool,
null_score_diff_threshold: float,
):
example_index_to_features = collections.defaultdict(list)
unique_id_to_pos = {}
for index, unique_id in enumerate(unique_ids):
unique_id_to_pos[unique_id] = index
for feature in self.features:
example_index_to_features[feature.example_index].append(feature)
_PrelimPrediction = collections.namedtuple(
"PrelimPrediction", ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]
)
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
scores_diff_json = collections.OrderedDict()
for (example_index, example) in enumerate(self.examples):
# finish this loop if we went through all batch examples
if example_index >= len(unique_ids):
break
features = example_index_to_features[example_index]
prelim_predictions = []
# keep track of the minimum score of null start+end of position 0
# large and positive
score_null = 1000000
# the paragraph slice with min null score
min_null_feature_index = 0
# start logit at the slice with min null score
null_start_logit = 0
# end logit at the slice with min null score
null_end_logit = 0
for (feature_index, feature) in enumerate(features):
pos = unique_id_to_pos[feature.unique_id]
start_indexes = get_best_indexes(start_logits[pos], n_best_size)
end_indexes = get_best_indexes(end_logits[pos], n_best_size)
# if we could have irrelevant answers,
# get the min score of irrelevant
if version_2_with_negative:
feature_null_score = start_logits[pos][0] + end_logits[pos][0]
if feature_null_score < score_null:
score_null = feature_null_score
min_null_feature_index = feature_index
null_start_logit = start_logits[pos][0]
null_end_logit = end_logits[pos][0]
for start_index in start_indexes:
for end_index in end_indexes:
# We could hypothetically create invalid predictions,
# e.g., predict that the start of the span is in the
# question. We throw out all invalid predictions.
if start_index >= len(feature.tokens):
continue
if end_index >= len(feature.tokens):
continue
if start_index not in feature.token_to_orig_map:
continue
if end_index not in feature.token_to_orig_map:
continue
if not feature.token_is_max_context.get(start_index, False):
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
prelim_predictions.append(
_PrelimPrediction(
feature_index=feature_index,
start_index=start_index,
end_index=end_index,
start_logit=start_logits[pos][start_index],
end_logit=end_logits[pos][end_index],
)
)
if version_2_with_negative:
prelim_predictions.append(
_PrelimPrediction(
feature_index=min_null_feature_index,
start_index=0,
end_index=0,
start_logit=null_start_logit,
end_logit=null_end_logit,
)
)
prelim_predictions = sorted(prelim_predictions, key=lambda x: (x.start_logit + x.end_logit), reverse=True)
_NbestPrediction = collections.namedtuple("NbestPrediction", ["text", "start_logit", "end_logit"])
seen_predictions = {}
nbest = []
for pred in prelim_predictions:
if len(nbest) >= n_best_size:
break
feature = features[pred.feature_index]
if pred.start_index > 0: # this is a non-null prediction
tok_tokens = feature.tokens[pred.start_index : (pred.end_index + 1)]
orig_doc_start = feature.token_to_orig_map[pred.start_index]
orig_doc_end = feature.token_to_orig_map[pred.end_index]
orig_tokens = example.doc_tokens[orig_doc_start : (orig_doc_end + 1)]
tok_text = " ".join(tok_tokens)
# De-tokenize WordPieces that have been split off.
tok_text = tok_text.replace(" ##", "")
tok_text = tok_text.replace("##", "")
# Clean whitespace
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
orig_text = " ".join(orig_tokens)
final_text = get_final_text(tok_text, orig_text, do_lower_case)
if final_text in seen_predictions:
continue
seen_predictions[final_text] = True
else:
final_text = ""
seen_predictions[final_text] = True
nbest.append(_NbestPrediction(text=final_text, start_logit=pred.start_logit, end_logit=pred.end_logit))
# if we didn't include the empty option in the n-best, include it
if version_2_with_negative:
if "" not in seen_predictions:
nbest.append(_NbestPrediction(text="", start_logit=null_start_logit, end_logit=null_end_logit))
# In very rare edge cases we could only
# have single null pred. We just create a nonce prediction
# in this case to avoid failure.
if len(nbest) == 1:
nbest.insert(0, _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))
# In very rare edge cases we could have no valid predictions. So we
# just create a nonce prediction in this case to avoid failure.
if not nbest:
nbest.append(_NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0))
assert len(nbest) >= 1
total_scores = []
best_non_null_entry = None
for entry in nbest:
total_scores.append(entry.start_logit + entry.end_logit)
if not best_non_null_entry:
if entry.text:
best_non_null_entry = entry
probs = _compute_softmax(total_scores)
nbest_json = []
for (i, entry) in enumerate(nbest):
output = collections.OrderedDict()
output["question"] = example.question_text
output["text"] = entry.text
output["probability"] = probs[i]
output["start_logit"] = (
entry.start_logit if isinstance(entry.start_logit, float) else list(entry.start_logit)
)
output["end_logit"] = entry.end_logit if isinstance(entry.end_logit, float) else list(entry.end_logit)
nbest_json.append(output)
assert len(nbest_json) >= 1
all_predictions[example.qas_id] = collections.OrderedDict()
all_predictions[example.qas_id]["question"] = example.question_text
if not version_2_with_negative:
all_predictions[example.qas_id]["text"] = nbest_json[0]["text"]
else:
# predict "" iff the null score -
# the score of best non-null > threshold
score_diff = score_null - best_non_null_entry.start_logit - (best_non_null_entry.end_logit)
scores_diff_json[example.qas_id] = score_diff
if score_diff > null_score_diff_threshold:
all_predictions[example.qas_id]["text"] = ""
else:
all_predictions[example.qas_id]["text"] = best_non_null_entry.text
all_nbest_json[example.qas_id] = nbest_json
return all_predictions, all_nbest_json, scores_diff_json
def evaluate_predictions(
self,
all_predictions: Dict[str, str],
no_answer_probs: Optional[float] = None,
no_answer_probability_threshold: float = 1.0,
):
qas_id_to_has_answer = {
example.qas_id: bool(example.answers) for example in self.examples[: len(all_predictions)]
}
has_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if has_answer]
no_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if not has_answer]
if no_answer_probs is None:
no_answer_probs = {k: 0.0 for k in all_predictions}
exact, f1 = self.get_raw_scores(all_predictions)
exact_threshold = apply_no_ans_threshold(
exact, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold
)
f1_threshold = apply_no_ans_threshold(
f1, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold
)
evaluation = make_eval_dict(exact_threshold, f1_threshold)
if has_answer_qids:
has_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=has_answer_qids)
merge_eval(evaluation, has_ans_eval, "HasAns")
if no_answer_qids:
no_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=no_answer_qids)
merge_eval(evaluation, no_ans_eval, "NoAns")
if no_answer_probs:
find_all_best_thresh(evaluation, all_predictions, exact, f1, no_answer_probs, qas_id_to_has_answer)
return evaluation["best_exact"], evaluation["best_f1"]
def get_raw_scores(self, preds: Dict[str, Dict[str, str]]):
"""
Computes the exact and f1 scores from the examples
and the model predictions
"""
exact_scores = {}
f1_scores = {}
for example in self.examples:
qas_id = example.qas_id
gold_answers = [answer["text"] for answer in example.answers if normalize_answer(answer["text"])]
if not gold_answers:
# For unanswerable questions,
# only correct answer is empty string
gold_answers = [""]
if qas_id not in preds:
logging.warning("Missing prediction for %s" % qas_id)
continue
prediction = preds[qas_id]
exact_scores[qas_id] = max(exact_match_score(a, prediction['text']) for a in gold_answers)
f1_scores[qas_id] = max(f1_score(a, prediction['text']) for a in gold_answers)
return exact_scores, f1_scores
def evaluate(
self,
unique_ids: List[str],
start_logits: List[List[float]],
end_logits: List[List[float]],
n_best_size: int,
max_answer_length: int,
do_lower_case: bool,
version_2_with_negative: bool,
null_score_diff_threshold: float,
):
(all_predictions, all_nbest_json, scores_diff_json) = self.get_predictions(
unique_ids,
start_logits,
end_logits,
n_best_size,
max_answer_length,
do_lower_case,
version_2_with_negative,
null_score_diff_threshold,
)
exact_match, f1 = self.evaluate_predictions(all_predictions)
return exact_match, f1, all_predictions, all_nbest_json