def handle_nl(self, token):
if self.paren_level > 0:
return
yield token
indent_str = token.rsplit("\n", 1)[1] # Tabs and spaces
indent = indent_str.count(" ") + indent_str.count("\t") * self.tab_len
if indent > self.indent_level[-1]:
self.indent_level.append(indent)
yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
else:
last_pop = None
while indent < self.indent_level[-1]:
last_pop = self.indent_level.pop()
yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token)
if indent != self.indent_level[-1]:
if indent > self.indent_level[-1]:
self.indent_level.append(last_pop)
yield Token.new_borrow_pos(
"_DOUBLE_DEDENT", indent_str, token
)
else:
assert indent == self.indent_level[-1], "%s != %s" % (
indent,
self.indent_level[-1],
)
def feed_eof(self, last_token=None):
"""Feed a '$END' Token. Borrows from 'last_token' if given."""
eof = Token.new_borrow_pos(
'$END', '', last_token
) if last_token is not None else self.lexer_thread._Token(
'$END', '', 0, 1, 1)
return self.feed_token(eof)
def process(self, stream):
for token in stream:
if token.type in self.open_bracket_types:
self.context.nest_bracket()
yield token
elif token.type in self.close_bracket_types:
self.context.unnest_bracket()
yield token
elif token.type == "CONTINUATION_MARKER":
if not self.context.is_bracketed():
self.context.start_continuation()
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos(
"UNBRACKETED_CONTINUATION_MARKER", token, token)
else:
yield token
elif token.type == "NEWLINE_AND_MAYBE_INDENT":
for t in self.handle_newline_and_maybe_indent(token):
yield t
else:
yield token
while self.context.current_indent() > 0:
self.context.pop_indent()
yield Token("_UNBRACKETED_DECREASED_INDENT_OUTSIDE_CONTINUATION",
"")
if not self.context.in_continuation():
yield Token("_UNBRACKETED_ALIGNED_INDENT_OUTSIDE_CONTINUATION",
"")
def handle_newline_and_maybe_indent(self, token):
new_indent = token[1:]
if (not self.context.is_bracketed()
and not self.context.starting_continuation()
and not self.context.in_continuation()
and len(new_indent) == self.context.current_indent()):
yield Token.new_borrow_pos(
"_UNBRACKETED_ALIGNED_INDENT_OUTSIDE_CONTINUATION", token,
token)
elif (not self.context.is_bracketed()
and not self.context.starting_continuation()
and len(new_indent) > self.context.current_indent()):
self.context.push_indent(amount=len(new_indent),
is_continuation=False)
yield Token.new_borrow_pos(
"_UNBRACKETED_INCREASED_INDENT_WITHOUT_CONTINUATION_MARKER",
token, token)
elif (not self.context.is_bracketed()
and not self.context.starting_continuation()
and not self.context.in_continuation()
and len(new_indent) < self.context.current_indent()):
if len(new_indent) > self.context.previous_indent():
# partial unindent
self.context.pop_indent()
yield Token.new_borrow_pos(
"_UNBRACKETED_DECREASED_INDENT_OUTSIDE_CONTINUATION",
token, token)
self.context.push_indent(amount=len(new_indent),
is_continuation=True)
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos("IMPLICIT_CONTINUATION_INDENT",
token, token)
else:
self.context.pop_indent()
yield Token.new_borrow_pos(
"_UNBRACKETED_DECREASED_INDENT_OUTSIDE_CONTINUATION", "",
token)
if not self.context.in_continuation():
yield Token(
"_UNBRACKETED_ALIGNED_INDENT_OUTSIDE_CONTINUATION", "")
if len(new_indent) < self.context.current_indent():
# More than one level
for t in self.handle_newline_and_maybe_indent(token):
yield t
elif (self.context.is_bracketed()):
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos("BRACKETED_NEWLINE", token, token)
elif (not self.context.is_bracketed()
and self.context.starting_continuation()
and len(new_indent) > self.context.current_indent()):
self.context.push_indent(amount=len(new_indent),
is_continuation=True)
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos(
"UNBRACKETED_INCREASED_INDENT_AFTER_CONTINUATION_MARKER",
token, token)
elif (not self.context.is_bracketed()
and self.context.in_continuation()
and len(new_indent) == self.context.current_indent()):
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos(
"UNBRACKETED_ALIGNED_INDENT_INSIDE_CONTINUATION", token, token)
elif (not self.context.is_bracketed()
and not self.context.starting_continuation()
and self.context.in_continuation()
and len(new_indent) < self.context.current_indent()):
if len(new_indent) > self.context.previous_indent():
# partial unindent
self.context.pop_indent()
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos(
"UNBRACKETED_DECREASED_INDENT_INSIDE_CONTINUATION", token,
token)
self.context.push_indent(amount=len(new_indent),
is_continuation=True)
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos("IMPLICIT_CONTINUATION_INDENT",
token, token)
else:
self.context.pop_indent()
# The following yield could be omitted -- the token is
# removed by the post-lex filter
yield Token.new_borrow_pos(
"UNBRACKETED_DECREASED_INDENT_INSIDE_CONTINUATION", token,
token)
yield Token("_UNBRACKETED_ALIGNED_INDENT_OUTSIDE_CONTINUATION",
"")
if len(new_indent) < self.context.current_indent():
# More than one level
for t in self.handle_newline_and_maybe_indent(token):
yield t
else:
yield token
