def read_token(self, start):
''' Read the next symbol out of the text '''
_, position = self.read_whitespace(start)
if position >= self.eof:
return self.make_token(Tokens.EOF, position, 0)
char = self.source[position]
if ( (char < ' ')
and (char != '\t' or char != '\n' or char != '\r')):
syntax_error(source, position,
"invalid character code: {}".format(self.source[position]))
if char == '!': return self.make_token(Tokens.BANG, position)
elif char == '$': return self.make_token(Tokens.DOLLAR, position)
elif char == '(': return self.make_token(Tokens.PAREN_L, position)
elif char == ')': return self.make_token(Tokens.PAREN_R, position)
elif char == '.':
if (self.source[position:position + 3] == '...'):
return self.make_token(Tokens.SPREAD, position, position + 3)
elif char == ':': return self.make_token(Tokens.COLON, position)
elif char == '=': return self.make_token(Tokens.EQUALS, position)
elif char == '@': return self.make_token(Tokens.AT, position)
elif char == '[': return self.make_token(Tokens.BRACKET_L, position)
elif char == ']': return self.make_token(Tokens.BRACKET_R, position)
elif char == '{': return self.make_token(Tokens.BRACE_L, position)
elif char == '|': return self.make_token(Tokens.PIPE, position)
elif char == '}': return self.make_token(Tokens.BRACE_R, position)
elif ('a' <= char <= 'z') or ('A' <= char <= 'Z') or (char == '_'):
return self.read_name(position)
elif '0' <= char <= '9': return self.read_number(position)
elif char == '"': return self.read_string(position)
else: syntax_error(source, position, "unexpected token: {}".format(char))
def unexpected(self):
''' Called when the token stream returns an unexpected
next token.
:throws SyntaxError: With the unexpected token
'''
syntax_error(self.lexer.source, self.curr_token.start,
"unexpected token found {}".format(self.curr_token))
def read_digits(self, start):
''' Read the next digits out of the text '''
position = start
if '0' > self.source[position] > '9':
syntax_error(source, position, "expected digit: {}".format(self.source[position]))
while ( (position < self.eof)
and ('0' <= self.source[position] <= '9')):
position += 1
return start, position
def read_string(self, start):
''' Read the next string out of the text '''
position = start + 1 # pass start quote mark
# TODO handle escaped characters in the quote
while ( (position < self.eof)
and (self.source[position] != '"')):
position += 1
if self.source[position] != '"':
syntax_error(source, position, "unterminated string: {}".format(self.source[position]))
return self.make_token(Tokens.STRING, start, position + 1,
self.source[start + 1:position - 1])
def expect(self, kind):
''' Checks that the current token is of the supplied kind
:param kind: The type of token to expect
:throws SyntaxError: If the wrong token appears
:returns: The expected token
'''
token = self.curr_token
if (token.kind == kind):
self.advance()
return token
syntax_error(self.lexer.source, token.start,
"expected {} token got {}".format(kind, token.kind))
def expect_name(self, value):
''' Checks that the current token of the Tokens.NAME kind
is of the supplied value.
:param value: The value of the name token that is expected
:throws SyntaxError: If the wrong token appears
:returns: The expected token
'''
token = self.curr_token
if (token.kind == Tokens.NAME) and (token.value == value):
self.advance()
return token
syntax_error(self.lexer.source, token.start,
"expected name token({}) token got {}".format(value, token.value))
