def normalize(self,
text: str,
verbose: bool = False,
punct_pre_process: bool = False,
punct_post_process: bool = False) -> str:
"""
Main function. Normalizes tokens from written to spoken form
e.g. 12 kg -> twelve kilograms
Args:
text: string that may include semiotic classes
verbose: whether to print intermediate meta information
punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ]
punct_post_process: whether to normalize punctuation
Returns: spoken form
"""
original_text = text
if punct_pre_process:
text = pre_process(text)
text = text.strip()
if not text:
if verbose:
print(text)
return text
text = pynini.escape(text)
tagged_lattice = self.find_tags(text)
tagged_text = self.select_tag(tagged_lattice)
if verbose:
print(tagged_text)
self.parser(tagged_text)
tokens = self.parser.parse()
tags_reordered = self.generate_permutations(tokens)
for tagged_text in tags_reordered:
tagged_text = pynini.escape(tagged_text)
verbalizer_lattice = self.find_verbalizer(tagged_text)
if verbalizer_lattice.num_states() == 0:
continue
output = self.select_verbalizer(verbalizer_lattice)
if punct_post_process:
# do post-processing based on Moses detokenizer
if self.processor:
output = self.processor.moses_detokenizer.detokenize(
[output], unescape=False)
output = post_process_punct(input=original_text,
normalized_text=output)
else:
print(
"NEMO_NLP collection is not available: skipping punctuation post_processing"
)
return output
raise ValueError()
def normalize(
self,
text: str,
n_tagged: int,
punct_post_process: bool = True,
verbose: bool = False,
) -> str:
"""
Main function. Normalizes tokens from written to spoken form
e.g. 12 kg -> twelve kilograms
Args:
text: string that may include semiotic classes
n_tagged: number of tagged options to consider, -1 - to get all possible tagged options
punct_post_process: whether to normalize punctuation
verbose: whether to print intermediate meta information
Returns:
normalized text options (usually there are multiple ways of normalizing a given semiotic class)
"""
assert (
len(text.split()) < 500
), "Your input is too long. Please split up the input into sentences, or strings with fewer than 500 words"
original_text = text
text = pre_process(text) # to handle []
text = text.strip()
if not text:
if verbose:
print(text)
return text
text = pynini.escape(text)
if self.lm:
if self.lang not in ["en"]:
raise ValueError(f"{self.lang} is not supported in LM mode")
if self.lang == "en":
try:
lattice = rewrite.rewrite_lattice(
text, self.tagger.fst_no_digits)
except pynini.lib.rewrite.Error:
lattice = rewrite.rewrite_lattice(text, self.tagger.fst)
lattice = rewrite.lattice_to_nshortest(lattice, n_tagged)
tagged_texts = [(x[1], float(x[2]))
for x in lattice.paths().items()]
tagged_texts.sort(key=lambda x: x[1])
tagged_texts, weights = list(zip(*tagged_texts))
else:
if n_tagged == -1:
if self.lang == "en":
try:
tagged_texts = rewrite.rewrites(
text, self.tagger.fst_no_digits)
except pynini.lib.rewrite.Error:
tagged_texts = rewrite.rewrites(text, self.tagger.fst)
else:
tagged_texts = rewrite.rewrites(text, self.tagger.fst)
else:
if self.lang == "en":
try:
tagged_texts = rewrite.top_rewrites(
text,
self.tagger.fst_no_digits,
nshortest=n_tagged)
except pynini.lib.rewrite.Error:
tagged_texts = rewrite.top_rewrites(text,
self.tagger.fst,
nshortest=n_tagged)
else:
tagged_texts = rewrite.top_rewrites(text,
self.tagger.fst,
nshortest=n_tagged)
# non-deterministic Eng normalization uses tagger composed with verbalizer, no permutation in between
if self.lang == "en":
normalized_texts = tagged_texts
else:
normalized_texts = []
for tagged_text in tagged_texts:
self._verbalize(tagged_text, normalized_texts, verbose=verbose)
if len(normalized_texts) == 0:
raise ValueError()
if punct_post_process:
# do post-processing based on Moses detokenizer
if self.processor:
normalized_texts = [
self.processor.detokenize([t]) for t in normalized_texts
]
normalized_texts = [
post_process_punct(input=original_text, normalized_text=t)
for t in normalized_texts
]
if self.lm:
return normalized_texts, weights
normalized_texts = set(normalized_texts)
return normalized_texts
def normalize(self,
text: str,
verbose: bool = False,
punct_pre_process: bool = False,
punct_post_process: bool = False) -> str:
"""
Main function. Normalizes tokens from written to spoken form
e.g. 12 kg -> twelve kilograms
Args:
text: string that may include semiotic classes
verbose: whether to print intermediate meta information
punct_pre_process: whether to perform punctuation pre-processing, for example, [25] -> [ 25 ]
punct_post_process: whether to normalize punctuation
Returns: spoken form
"""
assert (
len(text.split()) < 500
), "Your input is too long. Please split up the input into sentences, or strings with fewer than 500 words"
original_text = text
if punct_pre_process:
text = pre_process(text)
text = text.strip()
if not text:
if verbose:
print(text)
return text
text = pynini.escape(text)
tagged_lattice = self.find_tags(text)
tagged_text = self.select_tag(tagged_lattice)
if verbose:
print(tagged_text)
self.parser(tagged_text)
tokens = self.parser.parse()
split_tokens = self._split_tokens_to_reduce_number_of_permutations(
tokens)
output = ""
for s in split_tokens:
tags_reordered = self.generate_permutations(s)
verbalizer_lattice = None
for tagged_text in tags_reordered:
tagged_text = pynini.escape(tagged_text)
verbalizer_lattice = self.find_verbalizer(tagged_text)
if verbalizer_lattice.num_states() != 0:
break
if verbalizer_lattice is None:
raise ValueError(
f"No permutations were generated from tokens {s}")
output += ' ' + self.select_verbalizer(verbalizer_lattice)
output = SPACE_DUP.sub(' ', output[1:])
if self.lang == "en" and hasattr(self, 'post_processor'):
output = self.post_process(output)
if punct_post_process:
# do post-processing based on Moses detokenizer
if self.processor:
output = self.processor.moses_detokenizer.detokenize(
[output], unescape=False)
output = post_process_punct(input=original_text,
normalized_text=output)
else:
print(
"NEMO_NLP collection is not available: skipping punctuation post_processing"
)
return output
def normalize(
self,
text: str,
n_tagged: int,
punct_post_process: bool = True,
verbose: bool = False,
) -> str:
"""
Main function. Normalizes tokens from written to spoken form
e.g. 12 kg -> twelve kilograms
Args:
text: string that may include semiotic classes
n_tagged: number of tagged options to consider, -1 - to get all possible tagged options
punct_post_process: whether to normalize punctuation
verbose: whether to print intermediate meta information
Returns:
normalized text options (usually there are multiple ways of normalizing a given semiotic class)
"""
original_text = text
if self.lang == "en":
text = pre_process(text)
text = text.strip()
if not text:
if verbose:
print(text)
return text
text = pynini.escape(text)
if n_tagged == -1:
if self.lang == "en":
try:
tagged_texts = rewrite.rewrites(text,
self.tagger.fst_no_digits)
except pynini.lib.rewrite.Error:
tagged_texts = rewrite.rewrites(text, self.tagger.fst)
else:
tagged_texts = rewrite.rewrites(text, self.tagger.fst)
else:
if self.lang == "en":
try:
tagged_texts = rewrite.top_rewrites(
text, self.tagger.fst_no_digits, nshortest=n_tagged)
except pynini.lib.rewrite.Error:
tagged_texts = rewrite.top_rewrites(text,
self.tagger.fst,
nshortest=n_tagged)
else:
tagged_texts = rewrite.top_rewrites(text,
self.tagger.fst,
nshortest=n_tagged)
# non-deterministic Eng normalization uses tagger composed with verbalizer, no permutation in between
if self.lang == "en":
normalized_texts = tagged_texts
else:
normalized_texts = []
for tagged_text in tagged_texts:
self._verbalize(tagged_text, normalized_texts, verbose=verbose)
if len(normalized_texts) == 0:
raise ValueError()
if punct_post_process:
# do post-processing based on Moses detokenizer
if self.processor:
normalized_texts = [
self.processor.detokenize([t]) for t in normalized_texts
]
normalized_texts = [
post_process_punct(input=original_text, normalized_text=t)
for t in normalized_texts
]
normalized_texts = set(normalized_texts)
return normalized_texts
def _infer(self,
sents: List[str],
inst_directions: List[str],
processed=False):
"""
Main function for Inference
If the 'joint' mode is used, "sents" will include both spoken and written forms on each input sentence,
and "inst_directions" will include both constants.INST_BACKWARD and constants.INST_FORWARD
Args:
sents: A list of input texts.
inst_directions: A list of str where each str indicates the direction of the corresponding instance \
(i.e., constants.INST_BACKWARD for ITN or constants.INST_FORWARD for TN).
processed: Set to True when used with TextNormalizationTestDataset, the data is already tokenized with moses,
repetitive moses tokenization could lead to the number of tokens and class span mismatch
Returns:
tag_preds: A list of lists where the inner list contains the tag predictions from the tagger for each word in the input text.
output_spans: A list of lists where each list contains the decoded semiotic spans from the decoder for an input text.
final_outputs: A list of str where each str is the final output text for an input text.
"""
original_sents = [s for s in sents]
# Separate into words
if not processed:
sents = [input_preprocessing(x, lang=self.lang) for x in sents]
sents = [self.decoder.processor.tokenize(x).split() for x in sents]
else:
sents = [x.split() for x in sents]
# Tagging
# span_ends included, returns index wrt to words in input without auxiliary words
tag_preds, nb_spans, span_starts, span_ends = self.tagger._infer(
sents, inst_directions)
output_spans = self.decoder._infer(sents, nb_spans, span_starts,
span_ends, inst_directions)
# Prepare final outputs
final_outputs = []
for ix, (sent, tags) in enumerate(zip(sents, tag_preds)):
try:
cur_words, jx, span_idx = [], 0, 0
cur_spans = output_spans[ix]
while jx < len(sent):
tag, word = tags[jx], sent[jx]
if constants.SAME_TAG in tag:
cur_words.append(word)
jx += 1
else:
jx += 1
cur_words.append(cur_spans[span_idx])
span_idx += 1
while jx < len(sent) and tags[
jx] == constants.I_PREFIX + constants.TRANSFORM_TAG:
jx += 1
if processed:
# for Class-based evaluation, don't apply Moses detokenization
cur_output_str = " ".join(cur_words)
else:
# detokenize the output with Moses and fix punctuation marks to match the input
# for interactive inference or inference from a file
cur_output_str = self.decoder.processor.detokenize(
cur_words)
cur_output_str = post_process_punct(
input=original_sents[ix],
normalized_text=cur_output_str)
final_outputs.append(cur_output_str)
except IndexError:
logging.warning(
f"Input sent is too long and will be skipped - {original_sents[ix]}"
)
final_outputs.append(original_sents[ix])
return tag_preds, output_spans, final_outputs