class TestWord:
inverse_normalizer = InverseNormalizer()
@parameterized.expand(
parse_test_case_file(
'data_inverse_text_normalization/test_cases_word.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input,
verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='lower_cased')
@parameterized.expand(
parse_test_case_file('data_text_normalization/test_cases_word.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
class TestWhitelist:
@parameterized.expand(
parse_test_case_file(
'data_inverse_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = inverse_normalize(test_input, verbose=False)
assert pred == expected
@parameterized.expand(
parse_test_case_file(
'data_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = normalize(test_input, verbose=False)
assert pred == expected
class TestDate:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(
parse_test_case_file(
'data_inverse_text_normalization/test_cases_date.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input,
verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(
input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(
parse_test_case_file('data_text_normalization/test_cases_date.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_uncased(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(
test_input, n_tagged=100)
assert expected in pred_non_deterministic
normalizer_uppercased = Normalizer(
input_case='cased') if PYNINI_AVAILABLE else None
cases_uppercased = {
"Aug. 8": "august eighth",
"8 Aug.": "the eighth of august",
"aug. 8": "august eighth"
}
@parameterized.expand(cases_uppercased.items())
@pytest.mark.skipif(
not PYNINI_AVAILABLE,
reason=
"`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_cased(self, test_input, expected):
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(
test_input, n_tagged=100)
assert expected in pred_non_deterministic
class TestCardinal:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_cardinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_cardinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
class TestOrdinal:
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_ordinal.txt'))
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = inverse_normalize(test_input, verbose=False)
assert pred == expected
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_ordinal.txt'))
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = normalize(test_input, verbose=False)
assert pred == expected
class TestWhitelist:
inverse_normalizer = InverseNormalizer()
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='lower_cased')
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
normalizer_uppercased = Normalizer(input_case='cased')
cases_uppercased = {"Dr. Evil": "doctor Evil", "No. 4": "number four", "dr. Evil": "dr. Evil", "no. 4": "no. four"}
@parameterized.expand(cases_uppercased.items())
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_cased(self, test_input, expected):
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
assert pred == expected
