def test_simplified_user_dictionary(self):
# create user dictionary from csv
user_dic = UserDictionary(user_dict=os.path.join(
parent_dir, 'tests/user_simpledic.csv'),
enc='utf8',
type='simpledic',
connections=connections)
self.assertEqual(1, len(user_dic.lookup('東京スカイツリー'.encode('utf8'))))
# save compiled dictionary
dic_dir = os.path.join(parent_dir, 'tests/userdic_simple')
user_dic.save(to_dir=os.path.join(parent_dir, 'tests/userdic_simple'))
self.assertTrue(
os.path.exists(os.path.join(dic_dir, FILE_USER_FST_DATA)))
self.assertTrue(
os.path.exists(os.path.join(dic_dir, FILE_USER_ENTRIES_DATA)))
# load compiled dictionary
compiled_user_dic = CompiledUserDictionary(dic_dir,
connections=connections)
self.assertEqual(
1, len(compiled_user_dic.lookup('とうきょうスカイツリー駅'.encode('utf8'))))
# -*- coding: utf-8 -*-
from janome.tokenizer import Tokenizer
from janome.dic import UserDictionary
import sysdic
print('Compile user dictionary (MeCab IPADIC format)')
user_dict = UserDictionary("user_ipadic.csv", "utf8", "ipadic", sysdic.connections)
user_dict.save("/tmp/userdic")
t = Tokenizer("/tmp/userdic")
for token in t.tokenize(u'東京スカイツリーへのお越しは、東武スカイツリーライン「とうきょうスカイツリー駅」が便 利です。'):
print(token)
print('')
print('Compile user dictionary (simplified format)')
user_dict = UserDictionary("user_simpledic.csv", "utf8", "simpledic", sysdic.connections)
user_dict.save("/tmp/userdic_simple")
t = Tokenizer("/tmp/userdic_simple")
for token in t.tokenize(u'東京スカイツリーへのお越しは、東武スカイツリーライン「とうきょうスカイツリー駅」が便 利です。'):
print(token)
from janome.dic import UserDictionary
from janome import sysdic
user_dict = UserDictionary('neologd.csv', 'utf8', 'ipadic', sysdic.connections)
user_dict.save('neologd')
# -*- coding: utf-8 -*-
from janome.tokenizer import Tokenizer
from janome.dic import UserDictionary
from janome import sysdic
print('Compile user dictionary (MeCab IPADIC format)')
user_dict = UserDictionary("user_ipadic.csv", "utf8", "ipadic", sysdic.connections)
user_dict.save("/tmp/userdic")
t = Tokenizer("/tmp/userdic")
for token in t.tokenize(u'東京スカイツリーへのお越しは、東武スカイツリーライン「とうきょうスカイツリー駅」が便 利です。'):
print(token)
print('')
print('Compile user dictionary (simplified format)')
user_dict = UserDictionary("user_simpledic.csv", "utf8", "simpledic", sysdic.connections)
user_dict.save("/tmp/userdic_simple")
t = Tokenizer("/tmp/userdic_simple")
for token in t.tokenize(u'東京スカイツリーへのお越しは、東武スカイツリーライン「とうきょうスカイツリー駅」が便 利です。'):
print(token)