class EnumItem(Enum): def __init__(self): Enum.__init__(self) self.initdict = {} self.labelMap = TreeMap({}) # Treemap()对象 def getFrequency(self, label): frequency = self.labelMap.get(label) if frequency is None: return 0 return frequency def init1(self, label, frequency): self.initdict[label] = frequency self.labelMap = TreeMap(self.initdict) return self def init2(self, *args): """ 创建一个条目,其标签频次都是1,各标签由参数指定 :param args: :return: """ for label in args: self.initdict[label] = 1 self.labelMap = TreeMap(self.initdict) return self def init3(self, initdict): self.initdict = initdict self.labelMap = TreeMap(self.initdict) return self def init4(self, initdict): self.initdict = initdict self.labelMap = TreeMap(self.initdict) return self def containsLabel(self, label): return label in self.labelMap.result.keys() @staticmethod def create(param): if param is None: return None array = param.split(' ') return EnumItem.create1(array) @staticmethod def create1(param): if len(param) % 2 == 0: return None natureCount = (len(param) - 1) / 2 entries = [None] * natureCount for i in range(natureCount): entries[i] = {param[1 + 2 * i]: int(param[2 + 2 * i])} return {param[0]: entries}
def load(self, path): if self.loadDat(CoreBiGramTableDictionary.datPath): return True # Treemap对象 map = TreeMap({}) # map = dict() try: br = open(path, 'r') line = "" total = 0 maxWordId = CoreDictionary.trie.size1() line_num = 1 while 1: line = br.readline().strip("\n\r\t ") if not line: break params = re.split(' ', line) twoWord = params[0].split("@") a = twoWord[0] idA = CoreDictionary.trie.exactMatchSearch(a) if idA == -1: continue b = twoWord[1] idB = CoreDictionary.trie.exactMatchSearch(b) if idB == -1: continue freq = int(params[1]) biMap = map.get(idA) if biMap is None: biMap = TreeMap({}) biMap.put(int(idB), freq) map.put(int(idA), biMap) total += 2 line_num += 1 for k, v in map.items(): map.put(k, v.sort_long()) map.sort_long() br.close() CoreBiGramTableDictionary.start = [int()] * (maxWordId + 1) # total是连续的个数*2 CoreBiGramTableDictionary.pair = [int()] * total offset = 0 for i in range(maxWordId): bMap = map.get(i) if bMap is not None: for k, v in bMap.items(): index = offset << 1 CoreBiGramTableDictionary.pair[index] = k CoreBiGramTableDictionary.pair[index + 1] = v offset += 1 CoreBiGramTableDictionary.start[i + 1] = offset self.logger.info("二元词典读取完毕:%s") except IOError, e: self.logger("二元词典%s不存在或读取错误!%s" % (path, e)) return False
class State(object): def __init__(self): # 模式串的长度,也是这个状态的深度 self.depth = int() # 只要这个状态可达,则记录模式串 self.emits = None # goto 表,也称转移函数。根据字符串的下一个字符转移到下一个状态 self.success = TreeMap({}) # 在双数组中的对应下标 self.index = int() # fail 函数,如果没有匹配到,则跳转到此状态。 self.failure = None def init1(self, depth): """ 构造深度为depth的节点 :param depth: :return: """ self.depth = depth return self def isAcceptable(self): """ 是否是终止状态 :return: """ return self.depth > 0 and self.emits is not None def getDepth(self): """ 获取节点深度 :return: """ return self.depth def getLargestValueId(self): """ 获取最大的值 :return: """ if self.emits is None or len(self.emits) == 0: return None return iter(self.emits).next() def addEmit(self, keyword): """ 添加一个匹配到的模式串(这个状态对应着这个模式串) :param keyword: :return: """ if self.emits is None: # self.emits是倒序排列的treeset # this.emits = new TreeSet<Integer>(Collections.reverseOrder()); self.emits = set() self.emits = set(tuple(self.emits)) self.emits.add(keyword) self.emits = sorted(self.emits, reverse=True) def addEmit1(self, emits): """ 添加一些匹配到的模式串 :param emits: :return: """ for emit in emits: self.addEmit(emit) def nextStateIgnoreRootState(self, character): return self.nextState1(character, True) def addState(self, character): character = character.encode('utf-8') nextState = self.nextStateIgnoreRootState(character) if nextState is None: nextState = State().init1(self.depth + 1) self.success.result[character] = nextState self.success = TreeMap(inputDict=self.success.result).sort() return nextState def getSuccess(self): """ 获取goto表 :return: """ return self.success def setIndex(self, index): self.index = index def getIndex(self): return self.index def getStates(self): return self.success.values() def setFailure(self, failState, fail): """ 设置failure状态 :param failState: :param fail: :return: """ self.failure = failState fail[self.index] = failState.index def getEmit(self): """ 获取这个节点代表的模式串(们) :return: """ if self.emits is None: return set() else: return self.emits def getTransitions(self): return set(self.success.keys()) def nextState(self, character): """ 按照character转移,根节点转移失败会返回自己(永远不会返回null) :param character: :return: """ return self.nextState1(character, False) def nextState1(self, character, ignoreRootState): """ 转移到下一个状态 :param character:希望按此字符转移 :param ignoreRootState:是否忽略根节点,如果是根节点自己调用则应该是true,否则为false :return:转移结果 """ nextState = self.success.get(character) if not ignoreRootState and nextState is None and self.depth == 0: nextState = self return nextState def getFailure(self): """ 获取failure状态 :return: """ return self.failure