def make_minus_or_arrow(self): tok_type = token.T_MINUS pos_start = self.pos.copy() self.advance() if self.current_char == '>': self.advance() tok_type = token.T_ARROW return token.Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def make_greater_than(self): tok_type = token.T_GT pos_start = self.pos.copy() self.advance() if self.current_char == '=': self.advance() tok_type = token.T_GTE return token.Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def make_divisions(self): tok_type = token.T_DIV pos_start = self.pos.copy() self.advance() if self.current_char == '/': self.advance() tok_type = token.T_INT_DIV return token.Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def make_equals(self): tok_type = token.T_EQ pos_start = self.pos.copy() self.advance() if self.current_char == '=': self.advance() tok_type = token.T_EE return token.Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def make_identifier(self): id_str = '' pos_start = self.pos.copy() while self.current_char != None and self.current_char in LETTERS_DIGITS + '_': id_str += self.current_char self.advance() tok_type = token.T_KEYWORD if id_str in KEYWORDS else token.T_IDENTIFIER return token.Token(tok_type, id_str, pos_start, self.pos)
def make_number(self): num_str = '' dot_count = 0 pos_start = self.pos.copy() while self.current_char != None and self.current_char in DIGITS + '.': if self.current_char == '.': if dot_count == 1: break dot_count += 1 num_str += '.' else: num_str += self.current_char self.advance() if dot_count == 0: return token.Token(token.T_INT, int(num_str), pos_start, self.pos) else: return token.Token(token.T_FLOAT, float(num_str), pos_start, self.pos)
def make_not_equals(self): pos_start = self.pos.copy() self.advance() if self.current_char == '=': self.advance() return token.Token(token.T_NE, pos_start=pos_start, pos_end=self.pos), None self.advance() return None, errors.ExpectedCharError(pos_start, self.pos, "Expected '=' after '!'")
def make_string(self): string = '' pos_start = self.pos.copy() escape_character = False self.advance() escape_characters = {'n': '\n', 't': '\t'} while self.current_char != None and (self.current_char != '"' or escape_character): if escape_character: string += escape_characters.get(self.current_char, self.current_char) else: if self.current_char == '\\': escape_character = True else: string += self.current_char self.advance() escape_character = False self.advance() return token.Token(token.T_STRING, string, pos_start, self.pos)
def make_tokens(self): tokens = [] while self.current_char != None: if self.current_char in ' \t': self.advance() elif self.current_char == '@': _, error = self.skip_comment() if error: return [], error elif self.current_char in ';\n': tokens.append(token.Token(token.T_NEWLINE, pos_start=self.pos)) self.advance() elif self.current_char in DIGITS: tokens.append(self.make_number()) elif self.current_char in LETTERS: tokens.append(self.make_identifier()) elif self.current_char == '"': tokens.append(self.make_string()) elif self.current_char == '+': tokens.append(token.Token(token.T_PLUS, pos_start=self.pos)) self.advance() elif self.current_char == '-': tokens.append(self.make_minus_or_arrow()) elif self.current_char == '*': tokens.append(token.Token(token.T_MUL, pos_start=self.pos)) self.advance() elif self.current_char == '/': tokens.append(self.make_divisions()) elif self.current_char == '%': tokens.append( token.Token(token.T_REMAINDER, pos_start=self.pos)) self.advance() elif self.current_char == '^': tokens.append(token.Token(token.T_POW, pos_start=self.pos)) self.advance() elif self.current_char == '(': tokens.append(token.Token(token.T_LPAREN, pos_start=self.pos)) self.advance() elif self.current_char == ')': tokens.append(token.Token(token.T_RPAREN, pos_start=self.pos)) self.advance() elif self.current_char == '[': tokens.append(token.Token(token.T_LSQUARE, pos_start=self.pos)) self.advance() elif self.current_char == ']': tokens.append(token.Token(token.T_RSQUARE, pos_start=self.pos)) self.advance() elif self.current_char == '{': tokens.append(token.Token(token.T_LCURLY, pos_start=self.pos)) self.advance() elif self.current_char == '}': tokens.append(token.Token(token.T_RCURLY, pos_start=self.pos)) self.advance() elif self.current_char == ':': tokens.append(token.Token(token.T_COLON, pos_start=self.pos)) self.advance() elif self.current_char == ',': tokens.append(token.Token(token.T_COMMA, pos_start=self.pos)) self.advance() elif self.current_char == '!': tok, error = self.make_not_equals() if error: return [], error tokens.append(tok) elif self.current_char == '=': tokens.append(self.make_equals()) elif self.current_char == '<': tokens.append(self.make_less_than()) elif self.current_char == '>': tokens.append(self.make_greater_than()) else: pos_start = self.pos.copy() char = self.current_char self.advance() return [], errors.IllegalCharError(pos_start, self.pos, "'" + char + "'") tokens.append(token.Token(token.T_EOF, pos_start=self.pos)) return tokens, None