def scan_word(self, yy: Scanner):
text = yy.matched_text().lower(
) # By this the language is made caseless.
if text in self.RESERVED_WORDS:
if text in self.MONTHS: yy.token('month', self.MONTHS[text])
else:
yy.token(
text, yy.current_span()
) # Sometimes the location of a keyword is used for error reporting.
else:
yy.token("id", Name(text, yy.current_span()))
def scan_letter_escape(self, yy: Scanner):
letter = yy.matched_text()[1]
codepoint = 7 + 'abtnvfr'.index(letter)
yy.token('TEXT', AST.Constant(chr(codepoint), yy.current_span(),
'TEXT'))
def scan_imaginary(self, yy: interfaces.Scanner):
yy.token('number', float(yy.matched_text()[:-1]) * 1j)
def scan_punctuation(self, yy: interfaces.Scanner):
yy.token(yy.matched_text())
def scan_unicode_escape(self, yy: interfaces.Scanner):
yy.token('character', chr(int(yy.matched_text()[2:], 16)))
def scan_escaped_literal(self, yy: interfaces.Scanner):
yy.token('character', yy.matched_text()[1])
def scan_enter_string(self, yy: interfaces.Scanner):
yy.enter('in_string')
yy.token(yy.matched_text())
def scan_integer(self, yy: interfaces.Scanner):
yy.token('number', int(yy.matched_text()))
def scan_hex_integer(self, yy: Scanner):
yy.token(
'INTEGER',
AST.Constant(int(yy.matched_text()[1:], 16), yy.current_span(),
'HEX'))
def scan_integer(self, yy: Scanner):
yy.token(
'INTEGER',
AST.Constant(int(yy.matched_text()), yy.current_span(), 'INT'))
def scan_sigil(self, yy: Scanner, kind: str):
# Like scan_string but removes the first char from the semantic value.
name = AST.Name(yy.matched_text()[1:], yy.current_span())
yy.token(kind, AST.Sigil(name, kind))
def scan_escape_literal(self, yy: Scanner):
yy.token(
'TEXT',
AST.Constant(yy.matched_text()[1:], yy.current_span(), 'TEXT'))
def scan_string(self, yy: Scanner, kind):
# Pick up the token as a constant of string type, not an identifier.
# Like a bareword in ancient perl?
yy.token(kind, AST.Constant(yy.matched_text(), yy.current_span(),
kind))
def scan_name(self, yy: Scanner, kind):
# sort of like an identifier: We need to capture the span in case of semantic error.
yy.token(kind, AST.Name(yy.matched_text(), yy.current_span()))
def scan_token(self, yy: Scanner, kind):
# Concrete tokens that won't contribute to the AST
yy.token(kind, yy.matched_text())
def scan_decimal(self, yy: Scanner):
yy.token(
'DECIMAL',
AST.Constant(float(yy.matched_text()), yy.current_span(), 'DEC'))
def match_integer(yy: Scanner):
# It's sort of assumed you'll be connecting a mini-scanner up to a mini-parser.
# The parser module expects to get (token, value, start, end) quads, but the
# scanner handles the start and end. You just call the `.token(...)` method
# on the parameter, which is a scanning context.
yy.token('number', int(yy.matched_text()))
def scan_delimited(self, yy: Scanner, kind):
yy.token(
kind,
AST.Constant(yy.matched_text()[1:-1], yy.current_span(), 'STRING'))
def scan_reserved_word(self, yy: interfaces.Scanner):
yy.token(yy.matched_text(), self.RESERVED[yy.matched_text()])
def scan_integer(self, yy: Scanner):
yy.token('integer', int(yy.matched_text()))
def scan_stringy_bit(self, yy: interfaces.Scanner):
yy.token('character', yy.matched_text())
def scan_real(self, yy: Scanner):
yy.token('real', float(yy.matched_text()))
def scan_shorthand_escape(self, yy: interfaces.Scanner):
yy.token('character', chr(self.ESCAPES[yy.matched_text()[1]]))
def scan_punctuation(self, yy: Scanner):
yy.token(yy.matched_text(), yy.current_span())
def scan_leave_string(self, yy: interfaces.Scanner):
yy.enter('INITIAL')
yy.token(yy.matched_text())
def scan_sigil(self, yy: Scanner, kind: str):
text = yy.matched_text()[1:].lower()
yy.token(kind, Name(text, yy.current_span()))
def scan_real(self, yy: interfaces.Scanner):
yy.token('number', float(yy.matched_text()))
def scan_string(self, yy: Scanner):
yy.token('string', yy.matched_text()[1:-1])
def scan_variable(self, yy: interfaces.Scanner):
yy.token('variable', yy.matched_text())
def scan_punctuation(self, yy: Scanner):
it = yy.matched_text()
yy.token(it, it)