def load(filename=config.CORE_BIGRAM_NAME):
start = time.time()
logger.info(u"开始加载核心二元语法词表")
import os
if os.path.exists(filename + config.DICT_BIN_EXT):
return BiGramTable.load_bin(filename + config.DICT_BIN_EXT)
else:
table = BiGramTable.build(filename)
import cPickle as Pickle
with open(filename + config.DICT_BIN_EXT, 'w') as f:
Pickle.dump(table, f)
return table
logger.info(u"加载核心二元语法词表完毕，耗时%s", time.time() - start)
def load(filename=config.CORE_BIGRAM_NAME):
start = time.time()
logger.info(u"开始加载核心二元语法词表")
import os
if os.path.exists(filename + config.DICT_BIN_EXT):
return BiGramTable.load_bin(filename + config.DICT_BIN_EXT)
else:
table = BiGramTable.build(filename)
import cPickle as Pickle
with open(filename + config.DICT_BIN_EXT, 'w') as f:
Pickle.dump(table, f)
return table
logger.info(u"加载核心二元语法词表完毕，耗时%s", time.time() - start)
def __init__():
logger.info("字符类型对应表开始加载 %s", config.CHAR_TYPE_PATH)
start = time.time()
byte_array = ByteArray.load_from_file(config.CHAR_TYPE_PATH)
if byte_array is None:
import sys
logger.error("字符类型对应表加载失败：" + config.CHAR_TYPE_PATH)
sys.exit(-1)
else:
while byte_array.has_more():
b = byte_array.next_ushort()
e = byte_array.next_ushort()
t = byte_array.next_uchar()
for i in range(b, e + 1):
char_type[i] = t
logger.info("字符类型对应表加载成功，耗时 %s s", (time.time() - start))