def lexer(code: StringIO, *, fname:str=None) -> StringIO: out = TokenIO() for ptkn, ctkn in zip_prev(TokenIO(code), STARTMARKER): if is_pow(ptkn, ctkn): out.write_token(ctkn, override='**__PYSH_POW__**') else: out.write_token(ctkn) return out
def lexer(code: StringIO) -> StringIO: out = TokenIO() for ptkn, ctkn in zip_prev(TokenIO(code), STARTMARKER): if ctkn.type == STRING and ptkn.type == NAME and ptkn.string == '_': if ctkn.start == ptkn.end: out.write_token(ctkn, override='[r' + ctkn.string + ']') continue out.write_token(ctkn) return out
def lexer(code: StringIO) -> StringIO: out = TokenIO() tokens = TokenIO(code).iter_tokens() for ptkn, ctkn in zip_prev(tokens, STARTMARKER): assert isinstance(ptkn, TokenInfo) #XXX Mypy stuff if ctkn.type == STRING and ptkn.type == NAME and ptkn.string == '_': if ctkn.start == ptkn.end: out.write_token(ctkn, override='[r' + ctkn.string + ']') continue out.write_token(ctkn) return out
def lexer(code: StringIO) -> StringIO: out = TokenIO() for ptkn, ctkn in zip_prev(TokenIO(code), STARTMARKER): if is_prefix(ctkn): continue # defer until next token if is_prefix(ptkn): if ctkn.type == STRING and ctkn.start == ptkn.end: out.write_token(ptkn, override='__PYSH_RESTRING__(r' + ctkn.string + ')') continue else: out.write_token(ptkn) out.write_token(ctkn) return out
def lexer(code: StringIO) -> StringIO: out = TokenIO() tokens = TokenIO(code).iter_tokens() for ptkn, ctkn in zip_prev(tokens, STARTMARKER): assert isinstance(ptkn, TokenInfo) #XXX Mypy stuff if is_prefix(ctkn): continue # defer until next token if is_prefix(ptkn): if ctkn.type == STRING and ctkn.start == ptkn.end: out.write_token(ptkn, override='__PYSH_RESTRING__(r' + ctkn.string + ')') continue else: out.write_token(ptkn) out.write_token(ctkn) return out
def tokens_with_space(tokens): for ptkn, ctkn in zip_prev(tokens): if ptkn and ctkn.type != ENDMARKER and ctkn.start > ptkn.end: yield WsTokenInfo(WS, ' ', ptkn.end, ctkn.start, ptkn.line) yield ctkn