def process(self, stream: t.Iterator[lark.Token]) -> t.Iterator[Token]:
number_tokens: t.Optional[t.List[lark.Token]] = None
for token in stream:
try:
if not number_tokens:
number_tokens = []
self._english_parser.parse(number_tokens + [token])
number_tokens.append(token)
continue
except ValueError:
if number_tokens:
v = self._english_parser.parse(number_tokens)
yield Token.new_borrow_pos(
"UNSIGNED_INTEGER"
if isinstance(v, int) else "UNSIGNED_REAL",
str(self._english_parser.parse(number_tokens)),
number_tokens[0])
number_tokens = None
if not number_tokens:
try:
# Roman numerals must be at least 2 characters long,
# otherwise we can't use 'i' or 'x'...
if not len(str(token).strip()) < 2:
yield Token.new_borrow_pos(
"INTEGER", str(numeral.roman2int(token)), token)
continue
except ValueError:
pass
except NotImplementedError:
pass
yield token
def process(self, stream):
token = None
for token in stream:
yield token
if token is None:
yield Token(self.token_type, "<EOF>")
else:
yield Token.new_borrow_pos(self.token_type, "<EOF>", token)
def process(self, stream):
for tok in stream:
if tok.type == 'TYPEDEF_NAME':
if not self.fixer.contains(tok.value):
# demote to identifier
yield Token.new_borrow_pos('IDENT', tok.value, tok)
else:
yield tok
else:
yield tok
def ESCAPED_CHAR(self, token):
return Token.new_borrow_pos(
token.type,
{
"\\t": "\t",
"\\n": "\n",
"\\r": "\r",
'\\"': '"',
"\\'": "'",
"\\\\": "\\",
}[str(token)],
token,
)
def _process(self, stream: Iterable[Token]) -> Iterable[Token]:
stack: Deque[Token] = deque([])
last_line = 0
for token in stream:
if token.type == "_NL":
# +1 because columns in lark start at 1.
column = len(token.value.split("\n")[-1]) + 1
while stack and column <= stack[-1].column:
tk = stack.pop()
yield Token.new_borrow_pos(self.BLOCK_END_type, tk.value,
token)
if token.type in self.BLOCK_BEGIN_types:
stack.append(token)
yield token
else:
yield token
last_line = token.end_line or token.line
while stack:
token = stack.pop()
t = Token.new_borrow_pos(self.BLOCK_END_type, token.value, token)
t.line = last_line + 1
yield t
def tok_to_int(tok):
"Converte o valor de 'tok'=string em numero inteiro"
return Token.new_borrow_pos(tok.type, int(tok), tok)
def string_num2int(tok):
return Token.new_borrow_pos(tok.type, int(tok), tok)
def HEX_NUM(self, token):
return Token.new_borrow_pos(token.type, int(token, 16), token)
def FLOAT(self, token):
return Token.new_borrow_pos(token.type, float(token), token)
def INT(self, token):
return Token.new_borrow_pos(token.type, int(token), token)
def get_original_text(self, tree):
# TODO: Once the issue is resolved delete the line setting pos_in_stream
# Reference: https://github.com/lark-parser/lark/issues/587
tree.meta.pos_in_stream = tree.meta.start_pos
original = self.spec[tree.meta.start_pos:tree.meta.end_pos]
return Token.new_borrow_pos("PYTHON_CODE", original, tree.meta)
def update_token(token, value):
# token.update is not released yet (lark v7.8)
return Token.new_borrow_pos(token.type, value, token)
