Beispiel #1
0
def test_match():
    pytest.skip("deprecated")
    plexer = PriorityLexer(code)
    assert plexer.matches("13", "INT")
    assert not plexer.matches("13a", "INT")

    assert plexer.matches("variable_", "VAR")
    assert not plexer.matches("not variable", "VAR")
Beispiel #2
0
def test_plexer():
    pytest.skip("deprecated")
    plexer = PriorityLexer(code)
    assert plexer.get_priority("IF") == 0
    assert plexer.get_priority("[0-9]+") == 2

    assert plexer.get_cls("IF") == "KEYWORD"
    assert plexer.get_cls("[0-9]+") == "INT"
 def __init__(self, rules, language=""):
     self.indentation_based = False
     self.language = language
     if rules.startswith("%"):
         config_line = rules.splitlines()[0]     # get first line
         self.parse_config(config_line[1:])      # remove %
         rules = "\n".join(rules.splitlines()[1:]) # remove config line
     pl = PriorityLexer(rules)
     self.regexlist = pl.rules
     self.compiled_regexes = {}
     for regex in self.regexlist:
         self.compiled_regexes[regex] = re.compile(regex)
    def createDFA(self, rules):
        # lex lexing rules
        pl = PriorityLexer(rules)
        rules = sorted(pl.rules.items(), key=lambda node: node[1][0]) # sort by priority

        # create lexer automaton from rules
        regexs = []
        names = []
        for regex, _ in rules:
            name = pl.rules[regex][1]
            regexs.append(regex)
            names.append(name)
        self.lexer = Lexer(zip(names, regexs))
Beispiel #5
0
def test_match():
    pytest.skip("deprecated")
    plexer = PriorityLexer(code)
    assert plexer.matches("13", "INT")
    assert not plexer.matches("13a", "INT")

    assert plexer.matches("variable_", "VAR")
    assert not plexer.matches("not variable", "VAR")
Beispiel #6
0
def test_plexer():
    pytest.skip("deprecated")
    plexer = PriorityLexer(code)
    assert plexer.get_priority("IF") == 0
    assert plexer.get_priority("[0-9]+") == 2

    assert plexer.get_cls("IF") == "KEYWORD"
    assert plexer.get_cls("[0-9]+") == "INT"
Beispiel #7
0
    def createDFA(self, rules):
        # lex lexing rules
        pl = PriorityLexer(rules)
        rules = sorted(pl.rules.items(), key=lambda node: node[1][0]) # sort by priority

        # create lexer automaton from rules
        regexs = []
        names = []
        for k, _ in rules:
            regex = k
            name = pl.rules[k][1]
            r = parse_regex(regex)
            regexs.append(r)
            names.append(name)
        self.lexer = Lexer(regexs, names)