class TestBPEIsolateGlossariesMethod(unittest.TestCase): def setUp(self): amock = mock.MagicMock() amock.readline.return_value = 'something' glossaries = ['like', 'Manuel', 'USA'] self.bpe = BPE(amock, glossaries=glossaries) def _run_test_case(self, test_case): orig, expected = test_case out = self.bpe._isolate_glossaries(orig) self.assertEqual(out, expected) def test_multiple_glossaries(self): orig = 'wordlikeUSAwordManuelManuelwordUSA' exp = ['word', 'like', 'USA', 'word', 'Manuel', 'Manuel', 'word', 'USA'] test_case = (orig, exp) self._run_test_case(test_case)
class TestBPEIsolateGlossariesMethod(unittest.TestCase): def setUp(self): amock = mock.MagicMock() amock.readline.return_value = 'something' glossaries = ['like', 'Manuel', 'USA'] self.bpe = BPE(amock, glossaries=glossaries) def _run_test_case(self, test_case): orig, expected = test_case out = self.bpe._isolate_glossaries(orig) self.assertEqual(out, expected) def test_multiple_glossaries(self): orig = 'wordlikeUSAwordManuelManuelwordUSA' exp = [ 'word', 'like', 'USA', 'word', 'Manuel', 'Manuel', 'word', 'USA' ] test_case = (orig, exp) self._run_test_case(test_case)
class TestRegexIsolateGlossaries(unittest.TestCase): def setUp(self): amock = mock.MagicMock() amock.readline.return_value = 'something' glossaries = ["<country>\w*</country>", "<name>\w*</name>", "\d+"] self.bpe = BPE(amock, glossaries=glossaries) def _run_test_case(self, test_case): orig, expected = test_case out = self.bpe._isolate_glossaries(orig) self.assertEqual(out, expected) def test_regex_glossaries(self): orig = 'wordlike<country>USA</country>word10001word<name>Manuel</name>word<country>USA</country>' exp = ['wordlike', '<country>USA</country>', 'word', '10001', 'word', '<name>Manuel</name>', 'word', '<country>USA</country>'] test_case = (orig, exp) self._run_test_case(test_case)