def get_tokens(self): t = self.get_token() tokens = [] while t != token.Token('EOF'): tokens.append(t) t = self.get_token() tokens.append(token.Token('EOF')) return tokens
def get_operator(self) -> tok.Token: i = self._index while len(self.code) > i and tok.is_operator( self.code[self._index:i + 1]): i += 1 res = tok.Token(self.code[self._index:i], tok.TokenType.OPERATOR) self._index = i return res
def _process_eof(self): try: self._current_line return None except text_buffer.EOFError: return self._set_current_token_and_skip( token.Token(EOF) )
def _process_eol(self): try: self._current_char return None except text_buffer.EOLError: self._text_storage.newline() return self._set_current_token_and_skip( token.Token(EOL) )
def get_number(self) -> tok.Token: return_type = tok.TokenType.INTEGER start = self._index while not self.is_eof(): if self.cur == '.': if return_type == tok.TokenType.REAL: raise ParseException('숫자 파싱중 에러') return_type = tok.TokenType.REAL elif not str.isdigit(self.cur): break self._index += 1 return tok.Token(self.code[start:self._index], return_type)
def _process_integer(self): regexp = re.compile('\d+') match = regexp.match( self._text_storage.tail ) if not match: return None token_string = match.group() return self._set_current_token_and_skip( token.Token(INTEGER, int(token_string)) )
def _process_name(self): regexp = re.compile('[a-zA-Z_]+') match = regexp.match( self._text_storage.tail ) if not match: return None token_string = match.group() return self._set_current_token_and_skip( token.Token(NAME, token_string) )
def get_paren(self) -> tok.Token: res = tok.Token(self.cur, tok.TokenType.PAREN) self._index += 1 return res
def parse_assignment(self): variable = self._parse_variable() self.lexer.discard(token.Token(clex.LITERAL, '=')) value = self.parse_expression() return AssignmentNode(variable, value)
def _process_literal(self): return self._set_current_token_and_skip( token.Token(LITERAL, self._current_char) )
def __init__(self, toks: List[tok.Token]): self.toks = toks self.toks.append(tok.Token('', tok.TokenType.EOF))