def number(self):
number = ''
while self.current_char is not None and self.current_char.isdigit():
number += self.current_char
self.advance()
if self.current_char == '.':
number += self.current_char
self.advance()
while self.current_char is not None and self.current_char.isdigit(
):
number += self.current_char
self.advance()
return Token(FLOAT, float(number))
return Token(INT, int(number))
def num(self, negative=False):
number_value = '' if not negative else '-'
while (self.current_char is not None and self.current_char.isdigit()):
number_value += self.current_char
self.advance()
if self.current_char == '.':
number_value += self.current_char
self.advance()
while (self.current_char is not None
and self.current_char.isdigit()):
number_value += self.current_char
self.advance()
return Token(DOUBLE, float(number_value))
return Token(INT, int(number_value))
def number(self):
number_value = ""
while self.current_char is not None and self.current_char.isdigit():
number_value += self.current_char
self.advance()
if self.current_char == '.':
number_value += self.current_char
self.advance()
while self.current_char is not None and self.current_char.isdigit(
):
number_value += self.current_char
self.advance()
return Token(FLOAT, float(number_value))
else:
return Token(INTEGER, int(number_value))
def function(self):
result = '@'
self.advance()
while self.current_char is not None and (self.current_char.isalnum()
or self.current_char == '_'):
result += self.current_char
self.advance()
return Token(ID, result)
def string(self):
result = ''
self.advance()
while self.current_char is not None and self.current_char != '\'':
result += self.current_char
self.advance()
self.advance()
return Token(STRING, result)
def get_string(self):
string_value = "'"
self.advance()
while self.current_char is not None and self.current_char != "'":
string_value += self.current_char
self.advance()
string_value += self.current_char
self.advance()
return Token(STRING, string_value)
def string(self):
string_value = '"'
prev_char = '"'
self.advance()
while self.current_char is not None and self.current_char != '"':
string_value += self.current_char
prev_char = self.current_char
self.advance()
# if self.current_char == '"' and prev_char != "\\":
string_value += self.current_char
self.advance()
return Token(STRING, string_value)
def get_next_token(self):
while self.current_char is not None:
if self.current_char.isspace():
self.skip_whitespace()
if self.current_char is None:
return Token(EOF, None)
if self.current_char.isdigit():
return Token(INTEGER, self.integer())
if self.current_char.isalpha():
return self._id()
if self.current_char == '`':
self.advance()
return Token(GRACCENT, '`')
if self.current_char == ',':
self.advance()
return Token(COMMA, ',')
if self.current_char == "'":
self.advance()
return Token(APOSTROPHE, "'")
if self.current_char == ';':
self.advance()
return Token(SEMICOLON, ';')
if self.current_char == ':':
self.advance()
return Token(COLON, ':')
if self.current_char == '{':
self.advance()
return Token(LBRACKET, '{')
if self.current_char == '}':
self.advance()
return Token(RBRACKET, '}')
if self.current_char == '.':
self.advance()
return Token(DOT, '.')
if self.current_char == '#':
self.advance()
return Token(HASH, '#')
if self.current_char == '$':
self.advance()
return Token(DOLLAR, '$')
if self.current_char == '?':
self.advance()
return Token(QMARK, '?')
if self.current_char == '%':
self.advance()
return Token(MOD, '%')
if self.current_char == '+':
self.advance()
return Token(PLUS, '+')
if self.current_char == '-':
self.advance()
if self.current_char == '>':
self.advance()
return Token(LARROW, '->')
return Token(MINUS, '-')
if self.current_char == '*':
self.advance()
return Token(MUL, '*')
if self.current_char == '/':
self.advance()
if self.current_char == '/':
self.advance()
return Token(REALDIV, '//')
return Token(DIV, '/')
if self.current_char == '[':
self.advance()
if self.current_char is not None and self.current_char.isalnum() and self.current_char.isupper():
result = ""
while (self.current_char is not None and self.current_char.isalnum()):
result += self.current_char
self.advance()
self.advance()
return Token(TYPE, result)
else:
return Token(LANGLEBRACKET, '[')
if self.current_char == ']':
self.advance()
return Token(RANGLEBRACKET, ']')
if self.current_char == '(':
self.advance()
return Token(LPAREN, '(')
if self.current_char == ')':
self.advance()
return Token(RPAREN, ')')
if self.current_char == '<':
self.advance()
if self.current_char == '=':
self.advance()
return Token(LESS_EQ, '<=')
if self.current_char == '<':
self.advance()
self.advance()
return Token(INITIALIZE, '<<=')
return Token(LESS, '<')
if self.current_char == '>':
self.advance()
if self.current_char == '=':
self.advance()
return Token(GREATHER_EQ, '>=')
return Token(GREATHER, '>')
if self.current_char == '=':
self.advance()
if self.current_char == '=':
self.advance()
return Token(EQUAL, '==')
return Token(ASSIGN, '=')
if self.current_char == '!':
self.advance()
if self.current_char == '=':
self.advance()
return Token(NOT_EQUAL, '!=')
return Token(EMARK, '!')
self.error()
return Token(EOF, None)
def _id(self):
result = ""
while (self.current_char is not None and self.current_char.isalnum()):
result += self.current_char
self.advance()
if result == 'Num' or result == 'Char' or result == 'Bool' or result == 'String' or result == 'IO':
return Token(TYPE, result)
elif result == 'return':
return Token(RETURN, result)
elif result == 'if':
return Token(IF, result)
elif result == 'fi':
return Token(FI, result)
elif result == 'loop':
return Token(LOOP, result)
elif result == 'pool':
return Token(POOL, result)
elif result == 'do':
return Token(DO, result)
elif result == 'else':
return Token(ELSE, result)
elif result == 'for':
return Token(FOR, result)
elif result == 'and':
return Token(AND, result)
elif result == 'or':
return Token(OR, result)
elif result == 'check':
return Token(CHECK, result)
else:
return Token(ID, result)
def _id(self):
result = ""
while self.current_char is not None and (self.current_char.isalnum()
or self.current_char == '_'):
result += self.current_char
self.advance()
if result == 'int':
return Token(TYPE, result)
elif result == 'string':
return Token(TYPE, result)
elif result == 'float':
return Token(TYPE, result)
elif result == 'boolean':
return Token(TYPE, result)
elif result == 'array':
return Token(TYPE, result)
elif result == 'do':
return Token(TYPE, result)
elif result == 'use':
return Token(USE, result)
elif result == 'dec':
return Token(DEC, result)
elif result == 'decfun':
return Token(DECFUN, result)
elif result == 'cond':
return Token(COND, result)
elif result == 'loond':
return Token(LOOND, result)
elif result == 'return':
return Token(RETURN, result)
else:
return Token(ID, result)
def get_next_token(self):
while self.current_char is not None:
if self.current_char.isspace():
self.skip_whitespace()
if self.current_char is None:
return Token(EOF, None)
if self.current_char.isdigit():
return self.number()
if self.current_char.isalpha():
return self._id()
if self.current_char == '$':
return self.variable()
if self.current_char == '@':
return self.function()
if self.current_char == '"':
return self.string()
if self.current_char == ',':
self.advance()
return Token(COMMA, ',')
if self.current_char == ';':
self.advance()
return Token(SEMICOLON, ';')
if self.current_char == ':':
self.advance()
return Token(COLON, ':')
if self.current_char == '{':
self.advance()
return Token(LBRACKET, '{')
if self.current_char == '}':
self.advance()
return Token(RBRACKET, '}')
if self.current_char == '.':
self.advance()
return Token(DOT, '.')
if self.current_char == '#':
self.advance()
return Token(HASH, '#')
if self.current_char == '+':
self.advance()
return Token(PLUS, '+')
if self.current_char == '-':
self.advance()
return Token(MINUS, '-')
if self.current_char == '*':
self.advance()
return Token(MUL, '*')
if self.current_char == '/':
self.advance()
if self.current_char == '/':
self.advance()
return Token(DIV, '//')
return Token(REAL_DIV, '/')
if self.current_char == '%':
self.advance()
return Token(MOD, '%')
if self.current_char == '(':
self.advance()
return Token(LPAREN, '(')
if self.current_char == ')':
self.advance()
return Token(RPAREN, ')')
if self.current_char == '<':
self.advance()
if self.current_char == '=':
self.advance()
return Token(LESS_EQ, '<=')
return Token(LESS, '<')
if self.current_char == '>':
self.advance()
if self.current_char == '=':
self.advance()
return Token(GREATER_EQ, '>=')
return Token(GREATER, '>')
if self.current_char == '=':
self.advance()
if self.current_char == '=':
self.advance()
return Token(EQUAL, '==')
return Token(ASSIGN, '=')
if self.current_char == '!':
self.advance()
if self.current_char == '=':
self.advance()
return Token(NOT_EQUAL, '!=')
return Token(NOT, '!')
if self.current_char == '&':
self.advance()
if self.current_char == '&':
self.advance()
return Token(AND, '&&')
if self.current_char == '|':
self.advance()
if self.current_char == '|':
self.advance()
return Token(AND, '||')
self.error()
return Token(EOF, None)
def _id(self):
result = ''
while self.current_char is not None and self.current_char.isalnum():
result += self.current_char
self.advance()
if result == '***':
return Token(COMMENT, '***')
if result == 'Libraries':
return Token(LIBRARIES, result)
elif result == 'Functions':
return Token(FUNCTIONS, result)
elif result == 'Execution':
return Token(EXECUTION, result)
elif result == 'INT':
return Token(TYPE, result)
elif result == 'STRING':
return Token(TYPE, result)
elif result == 'FLOAT':
return Token(TYPE, result)
elif result == 'ARRAY':
return Token(TYPE, result)
elif result == 'BOOL':
return Token(TYPE, result)
elif result == 'VOID':
return Token(TYPE, result)
elif result == 'AND':
return Token(AND, 'and')
elif result == 'OR':
return Token(OR, 'or')
elif result == 'NOT':
return Token(NOT, 'not')
elif result == 'DIV':
return Token(DIV, '//')
elif result == 'MOD':
return Token(MOD, '%')
elif result == 'COND':
return Token(COND, result)
elif result == 'LOOP':
return Token(LOOP, result)
elif result == 'RETURN':
return Token(RETURN, 'return')
else:
return Token(ID, result)
def get_next_token(self):
while self.current_char is not None:
if self.current_char.isspace():
self.skip_whitespace()
if self.current_char is None:
return Token(EOF, None)
if self.current_char.isdigit():
return self.number()
if self.current_char.isalpha():
return self._id()
if self.current_char == '-':
self.advance()
if self.current_char == '>':
self.advance()
return Token(ARROW, '->')
else:
return Token(MINUS, '-')
if self.current_char == '\'':
return self.string()
if self.current_char == '#':
result = '#'
self.advance()
if self.current_char.isalpha():
result += self._id().value
return Token(ID, result)
if self.current_char == '@':
self.advance()
return Token(MONKEY, '@')
if self.current_char == ':':
self.advance()
return Token(COLON, ':')
if self.current_char == ',':
self.advance()
return Token(COMMA, ',')
if self.current_char == ';':
self.advance()
return Token(SEMICOLON, ';')
if self.current_char == '{':
self.advance()
return Token(LBRACKET, '{')
if self.current_char == '}':
self.advance()
return Token(RBRACKET, '}')
if self.current_char == '.':
self.advance()
return Token(DOT, '.')
if self.current_char == '#':
self.advance()
return Token(HASH, '#')
if self.current_char == '+':
self.advance()
return Token(PLUS, '+')
if self.current_char == '*':
self.advance()
return Token(MUL, '*')
if self.current_char == '/':
self.advance()
return Token(NDIV, '/')
if self.current_char == '(':
self.advance()
return Token(LPAREN, '(')
if self.current_char == ')':
self.advance()
return Token(RPAREN, ')')
if self.current_char == '<':
self.advance()
if self.current_char == '=':
self.advance()
return Token(LESS_EQ, '<=')
return Token(LESS, '<')
if self.current_char == '>':
self.advance()
if self.current_char == '=':
self.advance()
return Token(GREATER_EQ, '>=')
return Token(GREATER, '>')
if self.current_char == '=':
self.advance()
if self.current_char == '=':
self.advance()
return Token(EQUAL, '==')
return Token(ASSIGN, '=')
if self.current_char == '!':
self.advance()
if self.current_char == '=':
self.advance()
return Token(NOT_EQUAL, '!=')
self.error()
self.error()
return Token(EOF, None)
def get_next_token(self):
while self.current_char is not None:
if self.current_char.isspace():
self.skip_whitespace()
# print('------')
# print(self.current_char)
# print('******')
if self.current_char is None:
return Token(EOF, None)
if self.current_char.isdigit():
return Token(INT, self.num())
if self.current_char == "'":
return self.get_string()
if self.current_char.isalpha() or self.current_char == '$':
return self._id()
if self.current_char == ',':
self.advance()
return Token(COMMA, ',')
if self.current_char == ';':
self.advance()
return Token(SEMICOLON, ';')
if self.current_char == ':':
self.advance()
return Token(COLON, ':')
if self.current_char == '{':
self.advance()
return Token(LBRACKET, '{')
if self.current_char == '}':
self.advance()
return Token(RBRACKET, '}')
if self.current_char == '#':
self.advance()
return Token(HASH, '#')
if self.current_char == '+':
self.advance()
if self.current_char == '+':
self.advance()
return Token(PP, '++')
return Token(PLUS, '+')
if self.current_char == '-':
self.advance()
if self.current_char == '-':
self.advance()
return Token(MM, '--')
if self.current_char == '>':
self.advance()
return Token(ASSIGN, '->')
return Token(MINUS, '-')
if self.current_char == '*':
self.advance()
return Token(MUL, '*')
if self.current_char == '/':
self.advance()
return Token(DIV, '/')
if self.current_char == '%':
self.advance()
return Token(MOD, '%')
if self.current_char == '(':
self.advance()
return Token(LPAREN, '(')
if self.current_char == ')':
self.advance()
return Token(RPAREN, ')')
if self.current_char == '\n':
self.advance()
return Token(NEW_LINE, '\n')
if self.current_char == '~':
self.advance()
return Token(THILDE, '~')
if self.current_char == '<':
self.advance()
if self.current_char == '=':
self.advance()
return Token(LESS_EQ, '<=')
return Token(LESS, '<')
if self.current_char == '>':
self.advance()
if self.current_char == '=':
self.advance()
return Token(GREATHER_EQ, '>=')
return Token(GREATHER, '>')
if self.current_char == '=':
self.advance()
if self.current_char == '=':
self.advance()
return Token(EQUAL, '==')
if self.current_char == '!':
self.advance()
if self.current_char == '=':
self.advance()
return Token(NOT_EQUAL, '!=')
self.error()
self.error()
return Token(EOF, None)
def _id(self):
result = ""
while self.current_char is not None and self.current_char.isalnum(
) or self.current_char in ['$', '_']:
result += self.current_char
self.advance()
if result == 'int':
return Token(TYPE, result)
if result == 'string':
return Token(TYPE, result)
if result == 'array':
return Token(TYPE, result)
if result == 'double':
return Token(TYPE, result)
if result == 'function':
return Token(FUNCTION, result)
if result == 'if':
return Token(IF, result)
if result == 'dont_f_stop_until':
return Token(DONT_F_STOP_UNTIL, result)
if result == 'return':
return Token(RETURN, result)
if result == 'import':
return Token(IMPORT, result)
if result == 'do':
return Token(DO, result)
if result == 'end':
return Token(END, result)
if result == 'and':
return Token(AND, result)
if result == 'or':
return Token(OR, result)
else:
return Token(ID, result)