def test_normalize_empty_tokenizations():
text = ""
space_tokenized = text.split(" ")
tokenizer = AlbertTokenizer.from_pretrained("albert-base-v1")
target_tokenized = tokenizer.tokenize(text)
with pytest.raises(ValueError):
tn.normalize_tokenizations(space_tokenized, target_tokenized,
tokenizer)
def test_space_tokenization_tokenization_normalization(hf_tokenizer, hf_model):
text = "Jeff Immelt chose to focus on the incomprehensibility of accounting rules ."
space_tokenized = text.split(" ")
tokenizer = hf_tokenizer.from_pretrained(hf_model)
target_tokenized = tokenizer.tokenize(text)
normed_space_tokenized, normed_target_tokenized = tn.normalize_tokenizations(
space_tokenized, target_tokenized, tokenizer)
assert "".join(normed_space_tokenized) == "".join(normed_target_tokenized)
def test_space_tokenization_and_xlm_uncased_tokenization_normalization():
text = "Jeff Immelt chose to focus on the incomprehensibility of accounting rules ."
space_tokenized = text.split(" ")
tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048")
target_tokenized = tokenizer.tokenize(text)
normed_space_tokenized, normed_target_tokenized = tn.normalize_tokenizations(
space_tokenized, target_tokenized, tokenizer)
assert "".join(normed_space_tokenized) == "".join(normed_target_tokenized)
def test_space_tokenization_and_unusual_roberta_tokenization_normalization():
text = (
"As a practitioner of ethnic humor from the old days on the Borscht Belt , live "
"television and the nightclub circuit , Mr. Mason instinctively reached for the "
"vernacular .")
space_tokenized = text.split(" ")
tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
target_tokenized = tokenizer.tokenize(text)
# note 1: exposing the target tokenization to highlight an unusual tokenization:
# " vernacular" -> 'Ġ', 'vern', 'acular' (the usual pattern suggests 'Ġvern', 'acular')
assert target_tokenized == [
"As",
"Ġa",
"Ġpractitioner",
"Ġof",
"Ġethnic",
"Ġhumor",
"Ġfrom",
"Ġthe",
"Ġold",
"Ġdays",
"Ġon",
"Ġthe",
"ĠB",
"ors",
"cht",
"ĠBelt",
"Ġ,",
"Ġlive",
"Ġtelevision",
"Ġand",
"Ġthe",
"Ġnightclub",
"Ġcircuit",
"Ġ,",
"ĠMr",
".",
"ĠMason",
"Ġinstinctively",
"Ġreached",
"Ġfor",
"Ġthe",
"Ġ",
"vern",
"acular",
"Ġ.",
]
normed_space_tokenized, normed_target_tokenized = tn.normalize_tokenizations(
space_tokenized, target_tokenized, tokenizer)
# note: 2: the assert below shows that even with the unusual tokenization (see note 1 above),
# after normalization the space tokenization and the target tokenization match.
assert "".join(normed_space_tokenized) == "".join(normed_target_tokenized)
def tokenize(self, tokenizer):
space_tokenization = self.text.split()
target_tokenization = tokenizer.tokenize(self.text)
normed_space_tokenization, normed_target_tokenization = normalize_tokenizations(
space_tokenization, target_tokenization, tokenizer)
aligner = retokenize.TokenAligner(normed_space_tokenization,
normed_target_tokenization)
target_span = aligner.project_span(self.span[0], self.span[1])
return TokenizedExample(
guid=self.guid,
tokens=target_tokenization,
span=target_span,
span_text=" ".join(
target_tokenization[target_span[0]:target_span[1]]),
label_ids=[self.task.LABEL_TO_ID[label] for label in self.labels],
label_num=len(self.task.LABELS),
)
def tokenize_span(tokenizer, sentence: str, char_span: ExclusiveSpan):
"""Tokenizes sentence and projects char_span to token span.
Args:
tokenizer (transformers.PreTrainedTokenizer): Tokenizer used
sentence (str): Sentence to be tokenized
char_span (ExclusiveSpan): character indexed span for sentence
Returns:
sentence_target_tokenization (List[str]): tokenized sentence
target_span (ExclusiveSpane): token span for sentence
"""
span_start_idx = len(sentence[:char_span.start].split())
# If the first word in a span starts with punctuation, the first word will
# erroneously be split into two strings by .split().
# ie: 'takeaway' -> ["'", "takeaway"]
# For span alignment, we start the list index at the punctuation.
if (span_start_idx != 0) and (sentence[:(char_span.start)][-1]
in string.punctuation):
span_start_idx = span_start_idx - 1
span_text = sentence[char_span.start:char_span.end]
sentence_space_tokenization = sentence.split()
sentence_target_tokenization = tokenizer.tokenize(sentence)
(
sentence_normed_space_tokenization,
sentence_normed_target_tokenization,
) = normalize_tokenizations(sentence_space_tokenization,
sentence_target_tokenization,
tokenizer)
span_start_char = len(" ".join(
sentence_normed_space_tokenization[:span_start_idx]))
span_text_char = len(span_text)
aligner = retokenize.TokenAligner(
sentence_normed_space_tokenization,
sentence_normed_target_tokenization)
target_span = ExclusiveSpan(*aligner.project_char_to_token_span(
span_start_char, span_start_char + span_text_char))
return sentence_target_tokenization, target_span
def tokenize(self, tokenizer):
space_tokenization = self.text.split()
target_tokenization = tokenizer.tokenize(self.text)
normed_space_tokenization, normed_target_tokenization = normalize_tokenizations(
space_tokenization, target_tokenization, tokenizer)
aligner = retokenize.TokenAligner(normed_space_tokenization,
normed_target_tokenization)
span1_token_count = len(self.span1_text.split())
span2_token_count = len(self.span2_text.split())
target_span1 = ExclusiveSpan(
*aligner.project_token_span(self.span1_idx, self.span1_idx +
span1_token_count))
target_span2 = ExclusiveSpan(
*aligner.project_token_span(self.span2_idx, self.span2_idx +
span2_token_count))
return TokenizedExample(
guid=self.guid,
tokens=target_tokenization,
span1_span=target_span1,
span2_span=target_span2,
span1_text=self.span1_text,
span2_text=self.span2_text,
label_id=WSCTask.LABEL_TO_ID[self.label],
)