def tokenize(readline):
"""Tokenizer for the quasiquotes language extension.
Parameters
----------
readline : callable
A callable that returns the next line to tokenize.
Yields
------
t : TokenInfo
The token stream.
"""
tok_stream = PeekableIterator(default_tokenize(readline))
for t in tok_stream:
if t == with_tok:
try:
sp, dol, name, col, nl, indent = tok_stream.peek(6)
except ValueError:
continue
if (sp == spaceerror_tok and dol == dollar_tok and
col == col_tok and nl == nl_tok and indent.type == INDENT):
# pull the items out of the stream.
tuple(islice(tok_stream, None, 6))
yield from quote_stmt_tokenizer(name, t, tok_stream)
continue
elif t == left_bracket_tok:
try:
dol, name, pipe = tok_stream.peek(3)
except ValueError:
continue
if dol == dollar_tok and pipe == pipe_tok:
tuple(islice(tok_stream, None, 3))
yield from quote_expr_tokenizer(name, t, tok_stream)
continue
yield t
def tokenize(readline):
tok_stream = PeekableIterator(default_tokenize(readline))
for t in tok_stream:
if t != with_tok:
yield t
continue
try:
at, name, col, nl, indent = tok_stream.peek(5)
except ValueError:
continue
if (at != at_tok or col != col_tok or
nl != nl_tok or indent.type != INDENT):
continue
# pull the items out of the stream.
tuple(islice(tok_stream, None, 5))
ls = []
append = ls.append
prev_line = name.start[0]
stack = 1
for u in tok_stream.lookahead_iter():
if u.type == INDENT:
stack += 1
elif u.type == DEDENT:
stack -= 1
if not stack:
break
if u.start[0] > prev_line:
prev_line = u.start[0]
append(u.line)
end = t.end[0], t.start[1] + len(name.string)
yield name._replace(start=t.start, end=end, line='<line>')
open_end = end[0], end[1] + 1
yield TokenInfo(
type=OP,
string='(',
start=end,
end=open_end,
line='<line>',
)
if len(ls) == 1:
str_end = open_end[0], open_end[1] + len(ls[-1]) + 2
else:
str_end = open_end[0] + len(ls) - 1, len(ls[-1]) + 2
yield TokenInfo(
type=STRING,
string=repr(dedent(''.join(ls))),
start=open_end,
end=str_end,
line='<line>',
)
close_end = str_end[0], str_end[1] + 1
yield TokenInfo(
type=OP,
string=')',
start=str_end,
end=close_end,
line='<line>',
)
nl_end = close_end[0], close_end[1] + 1
yield TokenInfo(
type=NEWLINE,
string='\n',
start=close_end,
end=nl_end,
line='<line>',
)
yield TokenInfo(
type=NL,
string='\n',
start=(nl_end[0] + 1, 0),
end=(nl_end[0] + 1, 1),
line='<line>',
)
def tokenize(readline):
tok_stream = PeekableIterator(default_tokenize(readline))
for t in tok_stream:
if t != with_tok:
yield t
continue
try:
at, name, col, nl, indent = tok_stream.peek(5)
except ValueError:
continue
if (at != at_tok or col != col_tok or nl != nl_tok
or indent.type != INDENT):
continue
# pull the items out of the stream.
tuple(islice(tok_stream, None, 5))
ls = []
append = ls.append
prev_line = name.start[0]
stack = 1
for u in tok_stream.lookahead_iter():
if u.type == INDENT:
stack += 1
elif u.type == DEDENT:
stack -= 1
if not stack:
break
if u.start[0] > prev_line:
prev_line = u.start[0]
append(u.line)
end = t.end[0], t.start[1] + len(name.string)
yield name._replace(start=t.start, end=end, line='<line>')
open_end = end[0], end[1] + 1
yield TokenInfo(
type=OP,
string='(',
start=end,
end=open_end,
line='<line>',
)
if len(ls) == 1:
str_end = open_end[0], open_end[1] + len(ls[-1]) + 2
else:
str_end = open_end[0] + len(ls) - 1, len(ls[-1]) + 2
yield TokenInfo(
type=STRING,
string=repr(dedent(''.join(ls))),
start=open_end,
end=str_end,
line='<line>',
)
close_end = str_end[0], str_end[1] + 1
yield TokenInfo(
type=OP,
string=')',
start=str_end,
end=close_end,
line='<line>',
)
nl_end = close_end[0], close_end[1] + 1
yield TokenInfo(
type=NEWLINE,
string='\n',
start=close_end,
end=nl_end,
line='<line>',
)
yield TokenInfo(
type=NL,
string='\n',
start=(nl_end[0] + 1, 0),
end=(nl_end[0] + 1, 1),
line='<line>',
)
