def compile(self, grammar_type="regex"): """ 根据文法类型进行编译, 产生dfa. regex 表示 正则表达式, regular 表示 正规文法 :param grammar: 文法类型 :return: """ if grammar_type == 'regex': nfas = [] for le in self.lexs: # print le nfas.append(Regex.compile_nfa(le[1], extend=True, type=le[0])) nfa = NFA.combine(*nfas) self.lex_dfa = nfa.convert_dfa(copy_meta=["type"]) return elif grammar_type == "regular": """ 本来没有想到会做三型文法解析, 由于parser里也有文法解析.. 此处应该跟那边合并.. """ nfas = [] grammar = defaultdict(list) g_in, g_out = defaultdict(int), defaultdict(int) all_symbol = set() for l_hand, r_hand in self.lexs: l_hand = l_hand[1:-1] r_hands = [[x[1:-1] for x in r.strip().split()] for r in r_hand.split('|')] for hand in r_hands: for h in hand: g_in[h] += 1 all_symbol.add(h) g_out[l_hand] += 1 all_symbol.add(l_hand) grammar[l_hand].extend(r_hands) grammar['limit'] = [[' '], ['\t'], ['\n']] ter, not_ter = [], [] for sym in all_symbol: if g_in[sym] == 0: not_ter.append(sym) if g_out[sym] == 0: ter.append(sym) # print ter, not_ter nfas = [] for token_type in not_ter: nfa = NFA() nfa.start = NFANode(r_name=token_type) end_node = NFANode(type=token_type) end_node.end = True nfa.end = {end_node} vis = {token_type: nfa.start} def get_node(name): if name in vis: return vis[name] vis[name] = NFANode(r_name=name) return vis[name] que = Queue() que.put(token_type) while not que.empty(): t = que.get() node = get_node(t) if node.meta.get('vis', 0) > 0: continue node.meta['vis'] = node.meta.get('vis', 0) + 1 for r_hand in grammar[t]: node.next.setdefault(r_hand[0], set()) if len(r_hand) == 2: node.next[r_hand[0]].add(get_node(r_hand[1])) que.put(r_hand[1]) else: node.next[r_hand[0]].add(end_node) nfas.append(nfa) nfa = NFA.combine(*nfas) self.lex_dfa = nfa.convert_dfa(copy_meta=["type"]) return
def compile_nfa(pattern): """ :param pattern: 正则 :return: NFA """ # print 'compile nfa [%s]' % (pattern, ) assert isinstance(pattern, str) if is_base(pattern): if pattern in Regex.meta_bases: pattern = pattern[1:] nfa = NFA() enode = NFANode() enode.end = True nfa.start.next[pattern] = {enode} nfa.end.add(enode) return nfa for i in range(1, len(pattern)): s1, s2 = pattern[:i], pattern[i+1:] if pattern[i]=='|' and is_regex(s1) and is_regex(s2): nfa1, nfa2 = map(compile_nfa, [s1, s2]) nfa = NFA() nfa.start.next["ep"] = set() nfa.start.next["ep"].update([nfa1.start, nfa2.start]) enode = NFANode() enode.end = True nfa.end.add(enode) for node in nfa1.end | nfa2.end: if "ep" not in node.next: node.next["ep"] = set() node.next["ep"].add(enode) node.end = False nfa1.end, nfa2.end = set(), set() return nfa for i in range(1, len(pattern)): s1, s2 = pattern[:i], pattern[i:] if is_regex(s1) and is_regex(s2): # print 'rs 连接型' nfa1, nfa2 = map(compile_nfa, [s1, s2]) nfa = NFA() snode = nfa.start enode = NFANode() enode.end = True nfa.end = {enode} for node in nfa1.end: node.end = False if "ep" not in node.next: node.next["ep"] = set() node.next["ep"].add(nfa2.start) for node in nfa2.end: node.end = False if "ep" not in node.next: node.next["ep"] = set() node.next["ep"].add(enode) snode.next["ep"] = {nfa1.start} #虽然我觉得nfa.start = {nfa1.start} 也可以 , 还是按照教材把 return nfa if pattern[-1] == '*' and is_regex(pattern[:-1]): # print 'r* 型' nfa0 = compile_nfa(pattern[:-1]) nfa = NFA() snode = nfa.start enode = NFANode() enode.end = True nfa.end.add(enode) snode.next["ep"] = {enode, nfa0.start} for node in nfa0.end: if "ep" not in node.next: node.next["ep"] = set() node.next["ep"].update([nfa0.start, enode]) node.end = False nfa0.end = set() return nfa elif pattern[-1] == '+' and is_regex(pattern[:-1]): # print 'r+型' nfa0 = compile_nfa(pattern[:-1]) for node in nfa0.end: if "ep" not in node.next: node.next["ep"] = set() node.next["ep"].add(nfa0.start) return nfa0 elif pattern[-1] == '?' and is_regex(pattern[:-1]): # print 'r?型' nfa0 = compile_nfa(pattern[:-1]) if "ep" not in nfa0.start.next: nfa0.start.next["ep"] = set() nfa0.start.next["ep"].update(nfa0.end) return nfa0 elif pattern[-1] == ')' and pattern[0] == '(' and is_regex(pattern): # print '(r)型' return compile_nfa(pattern[1:-1]) else: print 'Excuse me? What a f*****g regex exp?' #print Regex._cache #raise RegexError() raise Exception()
def get_node(name): if name in vis: return vis[name] vis[name] = NFANode(r_name=name) return vis[name]