def test_transitions(self): lg = LexerGenerator() lg.add('NUMBER', r'\d+') lg.add('ADD', r'\+') lg.add('COMMENT_START', r'\(#', transition='push', target='comment') lg.ignore(r'\s+') comment = lg.add_state('comment') comment.add('COMMENT_START', r'\(#', transition='push', target='comment') comment.add('COMMENT_END', r'#\)', transition='pop') comment.add('COMMENT', r'([^(#]|#(?!\))|\)(?!#))+') l = lg.build() stream = l.lex('(# this is (# a nested comment #)#) 1 + 1 (# 1 # 1 #)') t = stream.next() assert t.name == 'COMMENT_START' assert t.value == '(#' t = stream.next() assert t.name == 'COMMENT' assert t.value == ' this is ' t = stream.next() assert t.name == 'COMMENT_START' assert t.value == '(#' t = stream.next() assert t.name == 'COMMENT' assert t.value == ' a nested comment ' t = stream.next() assert t.name == 'COMMENT_END' assert t.value == '#)' t = stream.next() assert t.name == 'COMMENT_END' assert t.value == '#)' t = stream.next() assert t.name == 'NUMBER' assert t.value == '1' t = stream.next() assert t.name == 'ADD' assert t.value == '+' t = stream.next() assert t.name == 'NUMBER' assert t.value == '1' t = stream.next() assert t.name == 'COMMENT_START' assert t.value == '(#' t = stream.next() assert t.name == 'COMMENT' assert t.value == ' 1 # 1 ' t = stream.next() assert t.name == 'COMMENT_END' assert t.value == '#)'
def test_stacked_lexer(self): lg = LexerGenerator() lg.add('NUMBER', r'\d+') lg.add('ADD', r'\+') lg.add('COMMENT_START', r'\(#', transition='push', target='comment') lg.ignore(r'\s+') comment = lg.add_state('comment') comment.add('COMMENT_START', r'\(#', transition='push', target='comment') comment.add('COMMENT_END', r'#\)', transition='pop') comment.add('COMMENT', r'([^(#]|#(?!\))|\)(?!#))+') l = lg.build() def f(): stream = l.lex('(# this is (# a nested comment #)#) 1 + 1 (# 1 # 1 #)') t = stream.next() if t.name != 'COMMENT_START': return -1 if t.value != '(#': return -2 t = stream.next() if t.name != 'COMMENT': return -3 if t.value != ' this is ': return -4 t = stream.next() if t.name != 'COMMENT_START': return -5 if t.value != '(#': return -6 t = stream.next() if t.name != 'COMMENT': return -7 if t.value != ' a nested comment ': return -8 t = stream.next() if t.name != 'COMMENT_END': return -9 if t.value != '#)': return -10 t = stream.next() if t.name != 'COMMENT_END': return -11 if t.value != '#)': return -12 t = stream.next() if t.name != 'NUMBER': return -13 if t.value != '1': return -14 t = stream.next() if t.name != 'ADD': return -15 if t.value != '+': return -16 t = stream.next() if t.name != 'NUMBER': return -17 if t.value != '1': return -18 t = stream.next() if t.name != 'COMMENT_START': return -19 if t.value != '(#': return -20 t = stream.next() if t.name != 'COMMENT': return -21 if t.value != ' 1 # 1 ': return -22 t = stream.next() if t.name != 'COMMENT_END': return -23 if t.value != '#)': return -24 try: stream.next() except StopIteration: pass else: return -25 return 0 assert self.run(f, []) == 0