def test_match(): pytest.skip("deprecated") plexer = PriorityLexer(code) assert plexer.matches("13", "INT") assert not plexer.matches("13a", "INT") assert plexer.matches("variable_", "VAR") assert not plexer.matches("not variable", "VAR")
def test_plexer(): pytest.skip("deprecated") plexer = PriorityLexer(code) assert plexer.get_priority("IF") == 0 assert plexer.get_priority("[0-9]+") == 2 assert plexer.get_cls("IF") == "KEYWORD" assert plexer.get_cls("[0-9]+") == "INT"
def __init__(self, rules, language=""): self.indentation_based = False self.language = language if rules.startswith("%"): config_line = rules.splitlines()[0] # get first line self.parse_config(config_line[1:]) # remove % rules = "\n".join(rules.splitlines()[1:]) # remove config line pl = PriorityLexer(rules) self.regexlist = pl.rules self.compiled_regexes = {} for regex in self.regexlist: self.compiled_regexes[regex] = re.compile(regex)
def createDFA(self, rules): # lex lexing rules pl = PriorityLexer(rules) rules = sorted(pl.rules.items(), key=lambda node: node[1][0]) # sort by priority # create lexer automaton from rules regexs = [] names = [] for regex, _ in rules: name = pl.rules[regex][1] regexs.append(regex) names.append(name) self.lexer = Lexer(zip(names, regexs))
def createDFA(self, rules): # lex lexing rules pl = PriorityLexer(rules) rules = sorted(pl.rules.items(), key=lambda node: node[1][0]) # sort by priority # create lexer automaton from rules regexs = [] names = [] for k, _ in rules: regex = k name = pl.rules[k][1] r = parse_regex(regex) regexs.append(r) names.append(name) self.lexer = Lexer(regexs, names)