def _handle_token(self, token: TokenInfo):
ws_start = ws_end = None
last_token = self.last_token
if last_token is TokenInfo.fake or last_token.type == tk.ENCODING:
pass
elif last_token.is_WS_NL: # NEWLINE or CONTINUE
assert token.start[0] == last_token.end[0] + 1
if token.start[1] != 0:
ws_start = (token.start[0], 0)
ws_end = token.start
elif last_token.end != token.start:
assert last_token.end[0] == token.start[0]
ws_start = last_token.end
ws_end = token.start
if ws_start is not None:
whitespace_token = TokenInfo(
tk.WHITESPACE,
token.line[ws_start[1]:ws_end[1]],
ws_start,
ws_end,
token.line,
)
yield whitespace_token
self.last_token = token
yield token
self.action = actions.Default(dont_store=True)
def _handle_token(self, token):
if token.type == tk.ENCODING:
return
self._store_constant(token)
if self.newlined:
new_whitespace, action = self._process_leading_whitespace(token)
if action == REPLACE:
token = new_whitespace
elif action == INSERT:
yield new_whitespace
self.last_leading_whitespace = new_whitespace.string
self.newlined = False
if token.annotation == A.DECL_LPAR:
self.scopes.append(Scope(self.last_leading_whitespace))
elif token.annotation == A.DECL_RPAR:
self.scopes.pop()
if token.annotation in (A.BODY_LSQB, A.CLS_BODY_LSQB):
self.scopes[-1].indent_level += 1
if token.annotation in (A.LAST_NL_BEFORE_RSQB, ):
self.scopes[-1].indent_level -= 1
if token.is_NL:
self.newlined = True
yield token
self.action = actions.Default(dont_store=True)
def r(ctx: Context, token: TokenInfo):
if ctx.pop_op()[0].annotation == A.CLS_BODY_LSQB:
ctx.pop_state()
ctx.push_state(State.EXPECT_SUBCLS_DOT)
sentinel = TokenInfo.new_sentinel_before(token, A.STMT_END)
ctx.push_ret(sentinel)
token.annotation = A.CLS_BODY_RSQB
ctx.push_ret(token)
return actions.Default(dont_store=True)
def r(ctx: Context, token: TokenInfo):
sentinel = TokenInfo.new_sentinel_before(token, A.STMT_END)
ctx.push_ret(sentinel)
token.annotation = A.STMT_COMMA
ctx.push_ret(token)
sentinel = TokenInfo.new_sentinel_after(token, A.STMT_START)
ctx.push_ret(sentinel)
return actions.Default(dont_store=True)
def r(ctx: Context, token: TokenInfo):
ctx.pop_state()
ctx.push_state(State.IN_LBDX_BODY_LIST)
token.annotation = A.BODY_LSQB
ctx.push_op(token)
ctx.push_ret(token)
sentinel = TokenInfo.new_sentinel_after(token, A.STMT_START)
ctx.push_ret(sentinel)
return actions.Default(dont_store=True)
def _annotate_clause_declarer(ctx: Context):
if ctx.cache is None:
return actions.Default()
if not isinstance(ctx.cache, list):
ctx.error()
length = len(ctx.cache)
if length == 1:
ctx.cache[0].annotation = A.CLS_DECL
elif length == 2:
ctx.cache[0].annotation = A.CLS_DOT
ctx.cache[1].annotation = A.CLS_DECL
else:
ctx.error()
def r(ctx: Context, token: TokenInfo):
if ctx.last_op[0].annotation not in (A.BODY_LSQB, A.CLS_BODY_LSQB):
return
sentinel = TokenInfo.new_sentinel_before(token, A.STMT_END)
ctx.push_ret(sentinel)
token.annotation = A.STMT_COMMA
ctx.push_ret(token)
sentinel = TokenInfo.new_sentinel_after(token, A.STMT_START)
ctx.push_ret(sentinel)
return actions.Default(dont_store=True)
def _handle_token(self, token: TokenInfo):
if token.annotation == A.DECL_LPAR:
self.scope_stack.append(token)
elif token.annotation == A.DECL_RPAR:
self.scope_stack.pop()
elif not self.scope_stack:
return
if token.is_NL:
self.newlined = True
return
if token.is_WS:
if self.newlined:
self.leading = True
if not self.buffering:
self.action = actions.StartBuffer()
self.newlined = False
return
if not token.is_CMT:
if self.buffering:
self.action = actions.StopBuffer()
self.leading = False
self.newlined = False
return
if not self.buffering and not self.newlined:
yield TokenInfo(tk.WHITESPACE, " ")
yield token
self.action = actions.Default(dont_store=True)
return
if self.buffering:
if any("\\" in x.string for x in self.buffer) or self.leading:
self.action = actions.StopBuffer()
else:
yield TokenInfo(tk.WHITESPACE, " ")
yield token
self.action = actions.StopBuffer(dont_store=True,
dont_yield_buffer=True)
self.leading = False
self.newlined = False
return
self.newlined = False
def r(ctx: Context, token: TokenInfo):
ctx.pop_state()
sentinel = TokenInfo.new_sentinel_before(token, A.STMT_END)
ctx.push_ret(sentinel)
token.annotation = A.STMT_COMMA
ctx.push_ret(token)
sentinel = TokenInfo.new_sentinel_after(token, A.STMT_START)
ctx.push_ret(sentinel)
if ctx.is_buffering():
return actions.StopBuffer(dont_store=True)
return actions.Default(dont_store=True)
def r(ctx: Context, token: TokenInfo):
if ctx.is_buffering():
return actions.StopBuffer(dont_consume=True)
ctx.pop_state()
ctx.push_state(State.IN_LBDX_CLS_BODY)
token.annotation = A.CLS_BODY_LSQB
ctx.push_op(token)
_annotate_clause_declarer(ctx)
ctx.cache = None
ctx.push_ret(token)
sentinel = TokenInfo.new_sentinel_after(token, A.STMT_START)
ctx.push_ret(sentinel)
return actions.Default(dont_store=True)
def _handle_token(self, token):
if self.last_token.annotation in SUPPRESS_WHITESPACE_AFTER and token.is_WS_NL:
self.action = actions.Default(dont_store=True)
return ()
def r(ctx: Context, token: TokenInfo):
ctx.pop_state()
if ctx.is_buffering():
return actions.StopBuffer(dont_consume=True)
return actions.Default(dont_consume=True)
def _handle_token(self, token: TokenInfo):
if token.annotation in ANNOTATIONS_TO_DROP:
self.action = actions.Default(dont_store=True)
return ()
