def scan_word(self, yy: Scanner):
text = yy.matched_text().lower(
) # By this the language is made caseless.
if text in self.RESERVED_WORDS:
if text in self.MONTHS: yy.token('month', self.MONTHS[text])
else:
yy.token(
text, yy.current_span()
) # Sometimes the location of a keyword is used for error reporting.
else:
yy.token("id", Name(text, yy.current_span()))
def scan_sigil(self, yy: Scanner, kind: str):
text = yy.matched_text()[1:].lower()
yy.token(kind, Name(text, yy.current_span()))
def scan_delimited(self, yy: Scanner, kind):
yy.token(
kind,
AST.Constant(yy.matched_text()[1:-1], yy.current_span(), 'STRING'))
def scan_punctuation(self, yy: Scanner):
yy.token(yy.matched_text(), yy.current_span())
def scan_hex_integer(self, yy: Scanner):
yy.token(
'INTEGER',
AST.Constant(int(yy.matched_text()[1:], 16), yy.current_span(),
'HEX'))
def scan_decimal(self, yy: Scanner):
yy.token(
'DECIMAL',
AST.Constant(float(yy.matched_text()), yy.current_span(), 'DEC'))
def scan_sigil(self, yy: Scanner, kind: str):
# Like scan_string but removes the first char from the semantic value.
name = AST.Name(yy.matched_text()[1:], yy.current_span())
yy.token(kind, AST.Sigil(name, kind))
def scan_integer(self, yy: Scanner):
yy.token(
'INTEGER',
AST.Constant(int(yy.matched_text()), yy.current_span(), 'INT'))
def scan_escape_literal(self, yy: Scanner):
yy.token(
'TEXT',
AST.Constant(yy.matched_text()[1:], yy.current_span(), 'TEXT'))
def scan_string(self, yy: Scanner, kind):
# Pick up the token as a constant of string type, not an identifier.
# Like a bareword in ancient perl?
yy.token(kind, AST.Constant(yy.matched_text(), yy.current_span(),
kind))
def scan_name(self, yy: Scanner, kind):
# sort of like an identifier: We need to capture the span in case of semantic error.
yy.token(kind, AST.Name(yy.matched_text(), yy.current_span()))
def scan_letter_escape(self, yy: Scanner):
letter = yy.matched_text()[1]
codepoint = 7 + 'abtnvfr'.index(letter)
yy.token('TEXT', AST.Constant(chr(codepoint), yy.current_span(),
'TEXT'))
def scan_embedded_newline(self, yy: Scanner):
yy.token('TEXT', AST.Constant('\n', yy.current_span(), 'TEXT'))