class TestWord: inverse_normalizer = InverseNormalizer() @parameterized.expand( parse_test_case_file( 'data_inverse_text_normalization/test_cases_word.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected normalizer = Normalizer(input_case='lower_cased') @parameterized.expand( parse_test_case_file('data_text_normalization/test_cases_word.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected
class TestWhitelist: @parameterized.expand( parse_test_case_file( 'data_inverse_text_normalization/test_cases_whitelist.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = inverse_normalize(test_input, verbose=False) assert pred == expected @parameterized.expand( parse_test_case_file( 'data_text_normalization/test_cases_whitelist.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): pred = normalize(test_input, verbose=False) assert pred == expected
class TestDate: inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None @parameterized.expand( parse_test_case_file( 'data_inverse_text_normalization/test_cases_date.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None normalizer_with_audio = NormalizerWithAudio( input_case='cased') if PYNINI_AVAILABLE else None @parameterized.expand( parse_test_case_file('data_text_normalization/test_cases_date.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm_uncased(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected pred_non_deterministic = self.normalizer_with_audio.normalize( test_input, n_tagged=100) assert expected in pred_non_deterministic normalizer_uppercased = Normalizer( input_case='cased') if PYNINI_AVAILABLE else None cases_uppercased = { "Aug. 8": "august eighth", "8 Aug.": "the eighth of august", "aug. 8": "august eighth" } @parameterized.expand(cases_uppercased.items()) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason= "`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm_cased(self, test_input, expected): pred = self.normalizer_uppercased.normalize(test_input, verbose=False) assert pred == expected pred_non_deterministic = self.normalizer_with_audio.normalize( test_input, n_tagged=100) assert expected in pred_non_deterministic
class TestCardinal: inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_cardinal.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100) assert expected in pred_non_deterministic
class TestOrdinal: @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_ordinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = inverse_normalize(test_input, verbose=False) assert pred == expected @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_ordinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): pred = normalize(test_input, verbose=False) assert pred == expected
class TestWhitelist: inverse_normalizer = InverseNormalizer() @parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_whitelist.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected normalizer = Normalizer(input_case='lower_cased') @parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_whitelist.txt')) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected normalizer_uppercased = Normalizer(input_case='cased') cases_uppercased = {"Dr. Evil": "doctor Evil", "No. 4": "number four", "dr. Evil": "dr. Evil", "no. 4": "no. four"} @parameterized.expand(cases_uppercased.items()) @pytest.mark.skipif( not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh" ) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_norm_cased(self, test_input, expected): pred = self.normalizer_uppercased.normalize(test_input, verbose=False) assert pred == expected