def test_provide_oov006(self): cinfo = MeCabOovPlugin.CategoryInfo() cinfo.type_ = CategoryType.KANJI cinfo.is_invoke = False cinfo.is_group = False cinfo.length = 6 self.plugin.categories[CategoryType.KANJI] = cinfo mock_inputtext.set_category_type(0, 3, CategoryType.KANJI) nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False) self.assertEqual(3, len(nodes)) n = nodes[0] self.assertEqual('あ', n.get_word_info().surface) self.assertEqual(1, n.get_word_info().length()) self.assertEqual(1, n.get_word_info().pos_id) n = nodes[1] self.assertEqual('あい', n.get_word_info().surface) self.assertEqual(2, n.get_word_info().length()) self.assertEqual(1, n.get_word_info().pos_id) n = nodes[2] self.assertEqual('あいう', n.get_word_info().surface) self.assertEqual(3, n.get_word_info().length()) self.assertEqual(1, n.get_word_info().pos_id) nodes = self.plugin.provide_oov(self.mocked_input_text, 0, True) self.assertEqual(0, len(nodes))
def test_read_oov_with_category_not_in_character_property(self): input_ = os.path.join(self.test_dir, 'test.txt') with open(input_, 'w') as wf: wf.write("ALPHA,1,2,3,補助記号,一般,*,*,*,*\n") plugin = MeCabOovPlugin() plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo() with self.assertRaises(ValueError) as cm: plugin.read_oov(input_, mock_grammar.mocked_grammar) self.assertEqual('`ALPHA` is undefined at line 1', cm.exception.args[0])
def test_read_oov_with_undefined_type(self): input_ = os.path.join(self.test_dir, 'test.txt') with open(input_, 'w') as wf: wf.write("FOO,1,2,3,補助記号,一般,*,*,*,*\n") plugin = MeCabOovPlugin() plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo() with self.assertRaises(ValueError) as cm: plugin.read_oov(input_, mock_grammar.mocked_grammar) self.assertEqual('`FOO` is invalid type at line 1', cm.exception.args[0])
def test_read_oov_with_too_few_columns(self): input_ = os.path.join(self.test_dir, 'test.txt') with open(input_, 'w') as wf: wf.write("DEFAULT,1,2,3\n") plugin = MeCabOovPlugin() plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo() with self.assertRaises(ValueError) as cm: plugin.read_oov(input_, mock_grammar.mocked_grammar) self.assertEqual('invalid format at line 1', cm.exception.args[0])
def test_provide_oov_without_oov_list(self): cinfo = MeCabOovPlugin.CategoryInfo() cinfo.type_ = CategoryType.HIRAGANA cinfo.is_invoke = False cinfo.is_group = True cinfo.length = 0 self.plugin.categories[CategoryType.HIRAGANA] = cinfo mock_inputtext.set_category_type(0, 3, CategoryType.HIRAGANA) nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False) self.assertEqual(0, len(nodes))
def test_read_oov(self): oov = os.path.join(self.test_dir, 'test.txt') with open(oov, 'w') as wf: wf.write("DEFAULT,1,2,3,補助記号,一般,*,*,*,*\n") wf.write("DEFAULT,3,4,5,補助記号,一般,*,*,*,*\n") plugin = MeCabOovPlugin() plugin.categories[CategoryType.DEFAULT] = MeCabOovPlugin.CategoryInfo() plugin.read_oov(oov, mock_grammar.mocked_grammar) self.assertEqual(1, len(plugin.oov_list)) self.assertEqual(2, len(plugin.oov_list[CategoryType.DEFAULT])) self.assertEqual(1, plugin.oov_list[CategoryType.DEFAULT][0].left_id) self.assertEqual(2, plugin.oov_list[CategoryType.DEFAULT][0].right_id) self.assertEqual(3, plugin.oov_list[CategoryType.DEFAULT][0].cost) self.assertEqual(0, plugin.oov_list[CategoryType.DEFAULT][0].pos_id)
def test_provide_oov100(self): cinfo = MeCabOovPlugin.CategoryInfo() cinfo.type_ = CategoryType.KANJI cinfo.is_invoke = True cinfo.is_group = False cinfo.length = 0 self.plugin.categories[CategoryType.KANJI] = cinfo mock_inputtext.set_category_type(0, 3, CategoryType.KANJI) nodes = self.plugin.provide_oov(self.mocked_input_text, 0, False) self.assertEqual(0, len(nodes)) nodes = self.plugin.provide_oov(self.mocked_input_text, 0, True) self.assertEqual(0, len(nodes))