Beispiel #1
0
    def compile(self, grammar_type="regex"):
        """
        根据文法类型进行编译, 产生dfa. regex 表示 正则表达式, regular 表示 正规文法
        :param grammar: 文法类型
        :return:
        """
        if grammar_type == 'regex':
            nfas = []
            for le in self.lexs:
                # print le
                nfas.append(Regex.compile_nfa(le[1], extend=True, type=le[0]))
            nfa = NFA.combine(*nfas)
            self.lex_dfa = nfa.convert_dfa(copy_meta=["type"])
            return
        elif grammar_type == "regular":
            """
            本来没有想到会做三型文法解析, 由于parser里也有文法解析.. 此处应该跟那边合并..
            """
            nfas = []
            grammar = defaultdict(list)
            g_in, g_out = defaultdict(int), defaultdict(int)
            all_symbol = set()
            for l_hand, r_hand in self.lexs:
                l_hand = l_hand[1:-1]
                r_hands = [[x[1:-1] for x in r.strip().split()]
                           for r in r_hand.split('|')]
                for hand in r_hands:
                    for h in hand:
                        g_in[h] += 1
                        all_symbol.add(h)
                g_out[l_hand] += 1
                all_symbol.add(l_hand)
                grammar[l_hand].extend(r_hands)
            grammar['limit'] = [[' '], ['\t'], ['\n']]
            ter, not_ter = [], []
            for sym in all_symbol:
                if g_in[sym] == 0:
                    not_ter.append(sym)
                if g_out[sym] == 0:
                    ter.append(sym)
            # print ter, not_ter
            nfas = []
            for token_type in not_ter:
                nfa = NFA()
                nfa.start = NFANode(r_name=token_type)
                end_node = NFANode(type=token_type)
                end_node.end = True
                nfa.end = {end_node}
                vis = {token_type: nfa.start}

                def get_node(name):
                    if name in vis:
                        return vis[name]
                    vis[name] = NFANode(r_name=name)
                    return vis[name]

                que = Queue()
                que.put(token_type)
                while not que.empty():
                    t = que.get()
                    node = get_node(t)
                    if node.meta.get('vis', 0) > 0:
                        continue
                    node.meta['vis'] = node.meta.get('vis', 0) + 1
                    for r_hand in grammar[t]:
                        node.next.setdefault(r_hand[0], set())
                        if len(r_hand) == 2:
                            node.next[r_hand[0]].add(get_node(r_hand[1]))
                            que.put(r_hand[1])
                        else:
                            node.next[r_hand[0]].add(end_node)
                nfas.append(nfa)
            nfa = NFA.combine(*nfas)
            self.lex_dfa = nfa.convert_dfa(copy_meta=["type"])
            return
Beispiel #2
0
def compile_nfa(pattern):
    """
    :param pattern: 正则
    :return: NFA
    """
    # print 'compile nfa [%s]' % (pattern, )
    assert isinstance(pattern, str)
    if is_base(pattern):
        if pattern in Regex.meta_bases:
            pattern = pattern[1:]
        nfa = NFA()
        enode = NFANode()
        enode.end = True
        nfa.start.next[pattern] = {enode}
        nfa.end.add(enode)
        return nfa
    for i in range(1, len(pattern)):
        s1, s2 = pattern[:i], pattern[i+1:]
        if pattern[i]=='|' and is_regex(s1) and is_regex(s2):
            nfa1, nfa2 = map(compile_nfa, [s1, s2])
            nfa = NFA()
            nfa.start.next["ep"] = set()
            nfa.start.next["ep"].update([nfa1.start, nfa2.start])
            enode = NFANode()
            enode.end = True
            nfa.end.add(enode)
            for node in nfa1.end | nfa2.end:
                if "ep" not in node.next:
                    node.next["ep"] = set()
                node.next["ep"].add(enode)
                node.end = False
            nfa1.end, nfa2.end = set(), set()
            return nfa
    for i in range(1, len(pattern)):
        s1, s2 = pattern[:i], pattern[i:]
        if is_regex(s1) and is_regex(s2):
            # print 'rs 连接型'
            nfa1, nfa2 = map(compile_nfa, [s1, s2])
            nfa = NFA()
            snode = nfa.start
            enode = NFANode()
            enode.end = True
            nfa.end = {enode}
            for node in nfa1.end:
                node.end = False
                if "ep" not in node.next:
                    node.next["ep"] = set()
                node.next["ep"].add(nfa2.start)
            for node in nfa2.end:
                node.end = False
                if "ep" not in node.next:
                    node.next["ep"] = set()
                node.next["ep"].add(enode)
            snode.next["ep"] = {nfa1.start}   #虽然我觉得nfa.start = {nfa1.start} 也可以 , 还是按照教材把
            return nfa
    if pattern[-1] == '*' and is_regex(pattern[:-1]):
        # print 'r* 型'
        nfa0 = compile_nfa(pattern[:-1])
        nfa = NFA()
        snode = nfa.start
        enode = NFANode()
        enode.end = True
        nfa.end.add(enode)
        snode.next["ep"] = {enode, nfa0.start}
        for node in nfa0.end:
            if "ep" not in node.next:
                node.next["ep"] = set()
            node.next["ep"].update([nfa0.start, enode])
            node.end = False
        nfa0.end = set()
        return nfa
    elif pattern[-1] == '+' and is_regex(pattern[:-1]):
        # print 'r+型'
        nfa0 = compile_nfa(pattern[:-1])
        for node in nfa0.end:
            if "ep" not in node.next:
                node.next["ep"] = set()
            node.next["ep"].add(nfa0.start)
        return nfa0
    elif pattern[-1] == '?' and is_regex(pattern[:-1]):
        # print 'r?型'
        nfa0 = compile_nfa(pattern[:-1])
        if "ep" not in nfa0.start.next:
            nfa0.start.next["ep"] = set()
        nfa0.start.next["ep"].update(nfa0.end)
        return nfa0
    elif pattern[-1] == ')' and pattern[0] == '(' and is_regex(pattern):
        # print '(r)型'
        return compile_nfa(pattern[1:-1])
    else:
        print 'Excuse me? What a f*****g regex exp?'
        #print Regex._cache
        #raise RegexError()
        raise Exception()
Beispiel #3
0
 def get_node(name):
     if name in vis:
         return vis[name]
     vis[name] = NFANode(r_name=name)
     return vis[name]
Beispiel #4
0
    def compile(self, grammar_type="regex"):
        """
        根据文法类型进行编译, 产生dfa. regex 表示 正则表达式, regular 表示 正规文法
        :param grammar: 文法类型
        :return:
        """
        if grammar_type == 'regex':
            nfas = []
            for le in self.lexs:
                # print le
                nfas.append(Regex.compile_nfa(le[1], extend=True, type=le[0]))
            nfa = NFA.combine(*nfas)
            self.lex_dfa = nfa.convert_dfa(copy_meta=["type"])
            return
        elif grammar_type == "regular":
            """
            本来没有想到会做三型文法解析, 由于parser里也有文法解析.. 此处应该跟那边合并..
            """
            nfas = []
            grammar = defaultdict(list)
            g_in, g_out = defaultdict(int), defaultdict(int)
            all_symbol = set()
            for l_hand, r_hand in self.lexs:
                l_hand = l_hand[1:-1]
                r_hands = [[x[1:-1] for x in r.strip().split()] for r in r_hand.split('|')]
                for hand in r_hands:
                    for h in hand:
                        g_in[h] += 1
                        all_symbol.add(h)
                g_out[l_hand] += 1
                all_symbol.add(l_hand)
                grammar[l_hand].extend(r_hands)
            grammar['limit'] = [[' '], ['\t'], ['\n']]
            ter, not_ter = [], []
            for sym in all_symbol:
                if g_in[sym] == 0:
                    not_ter.append(sym)
                if g_out[sym] == 0:
                    ter.append(sym)
            # print ter, not_ter
            nfas = []
            for token_type in not_ter:
                nfa = NFA()
                nfa.start = NFANode(r_name=token_type)
                end_node = NFANode(type=token_type)
                end_node.end = True
                nfa.end = {end_node}
                vis = {token_type: nfa.start}

                def get_node(name):
                    if name in vis:
                        return vis[name]
                    vis[name] = NFANode(r_name=name)
                    return vis[name]

                que = Queue()
                que.put(token_type)
                while not que.empty():
                    t = que.get()
                    node = get_node(t)
                    if node.meta.get('vis', 0) > 0:
                        continue
                    node.meta['vis'] = node.meta.get('vis', 0) + 1
                    for r_hand in grammar[t]:
                        node.next.setdefault(r_hand[0], set())
                        if len(r_hand) == 2:
                            node.next[r_hand[0]].add(get_node(r_hand[1]))
                            que.put(r_hand[1])
                        else:
                            node.next[r_hand[0]].add(end_node)
                nfas.append(nfa)
            nfa = NFA.combine(*nfas)
            self.lex_dfa = nfa.convert_dfa(copy_meta=["type"])
            return