def tokenize(readline): """Tokenizer for the quasiquotes language extension. Parameters ---------- readline : callable A callable that returns the next line to tokenize. Yields ------ t : TokenInfo The token stream. """ tok_stream = PeekableIterator(default_tokenize(readline)) for t in tok_stream: if t == with_tok: try: sp, dol, name, col, nl, indent = tok_stream.peek(6) except ValueError: continue if (sp == spaceerror_tok and dol == dollar_tok and col == col_tok and nl == nl_tok and indent.type == INDENT): # pull the items out of the stream. tuple(islice(tok_stream, None, 6)) yield from quote_stmt_tokenizer(name, t, tok_stream) continue elif t == left_bracket_tok: try: dol, name, pipe = tok_stream.peek(3) except ValueError: continue if dol == dollar_tok and pipe == pipe_tok: tuple(islice(tok_stream, None, 3)) yield from quote_expr_tokenizer(name, t, tok_stream) continue yield t
def tokenize(readline): tok_stream = PeekableIterator(default_tokenize(readline)) for t in tok_stream: if t != with_tok: yield t continue try: at, name, col, nl, indent = tok_stream.peek(5) except ValueError: continue if (at != at_tok or col != col_tok or nl != nl_tok or indent.type != INDENT): continue # pull the items out of the stream. tuple(islice(tok_stream, None, 5)) ls = [] append = ls.append prev_line = name.start[0] stack = 1 for u in tok_stream.lookahead_iter(): if u.type == INDENT: stack += 1 elif u.type == DEDENT: stack -= 1 if not stack: break if u.start[0] > prev_line: prev_line = u.start[0] append(u.line) end = t.end[0], t.start[1] + len(name.string) yield name._replace(start=t.start, end=end, line='<line>') open_end = end[0], end[1] + 1 yield TokenInfo( type=OP, string='(', start=end, end=open_end, line='<line>', ) if len(ls) == 1: str_end = open_end[0], open_end[1] + len(ls[-1]) + 2 else: str_end = open_end[0] + len(ls) - 1, len(ls[-1]) + 2 yield TokenInfo( type=STRING, string=repr(dedent(''.join(ls))), start=open_end, end=str_end, line='<line>', ) close_end = str_end[0], str_end[1] + 1 yield TokenInfo( type=OP, string=')', start=str_end, end=close_end, line='<line>', ) nl_end = close_end[0], close_end[1] + 1 yield TokenInfo( type=NEWLINE, string='\n', start=close_end, end=nl_end, line='<line>', ) yield TokenInfo( type=NL, string='\n', start=(nl_end[0] + 1, 0), end=(nl_end[0] + 1, 1), line='<line>', )