def __parseRepitaIdentAte(self): ident = Variavel(self.__retornaTokenAtual()) # Adiciona uma instrução de atribuição no loop op_soma = Token(op_arit.soma, pos=self.__retornaTokenAtual().retornaPosicao()) valor_soma = Numero(Token(tipo=valores.inteiro, pos=self.__retornaTokenAtual().retornaPosicao(), val=1)) op_bin_soma = OpBin(esq=ident, op=op_soma, dir=valor_soma) var_atrib = AtribuicaoVariavel(ident=self.__retornaTokenAtual(), op=op_soma, val=op_bin_soma) self.__avancaToken() if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.ate: msgErro = f"Espera-se '{palavras_chaves.ate}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'." posErro = self.__retornaTokenAtual().retornaPosicao() self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro)) self.__avancaToken() expr = self.__parseExpr() if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.entao: msgErro = f"Espera-se '{palavras_chaves.entao}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'." posErro = self.__retornaTokenAtual().retornaPosicao() self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro)) self.__avancaToken() instrucoes = Instrucao() while self.__retornaTokenAtual().retornaTipo() not in [palavras_chaves.fim_repita, tipos_tokens.EOF]: instrucoes.adicionaInstrucao(self.__parseInstrucao()) instrucoes.adicionaInstrucao(var_atrib) if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.fim_repita: msgErro = f"Espera-se '{palavras_chaves.fim_repita}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'." posErro = self.__retornaTokenAtual().retornaPosicao() self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro)) self.__avancaToken() return RepitaIdentAte(ident=ident, expr=expr, instrucoes=instrucoes)
def test_scan_currency_round(): '''check that currency is rounded if there are more than two decimals''' scanner = Scanner("100.00 100.420 100.429") assert list(scanner) == [ Token(Type.CURRENCY, 10000), Token(Type.CURRENCY, 10042), Token(Type.CURRENCY, 10043) ]
def test_scan_identifiers(): '''check that non-keywords are identified as identifiers''' scanner = Scanner("equity cash end_of_the_world_fund") assert list(scanner) == [ Token(Type.IDENT, "equity"), Token(Type.IDENT, "cash"), Token(Type.IDENT, "end_of_the_world_fund") ]
def __call__(self, state, string, rownumber): #根据终态、被接受的字符串new出一个Token if state != None: for tk in self.token_list: if tk.split(":")[0] in state.contant: token = Token(string, tk, rownumber) return token else: return Token(string, None, rownumber)
def test_scan_currency(): '''check that currency is identified and stored correctly''' scanner = Scanner("100 100.00 100.42 -123.45") assert list(scanner) == [ Token(Type.CURRENCY, 10000), Token(Type.CURRENCY, 10000), Token(Type.CURRENCY, 10042), Token(Type.CURRENCY, -12345) ]
def retornaTokens(self) -> list: while not self.__EOF(): caracter_atual = self.__retornaCaracterAtual() caracter_posterior = self.__retornaCaracterAtual(1) if caracter_atual in ' \t': self.__avancaColuna() elif caracter_atual in valores.caracteres: tkn_ident = self.__retornaIdentificador() self.__adicionaToken(tkn_ident) elif caracter_atual in [ op_arit.sub, op_arit.soma ] and caracter_posterior in valores.valor_numerico: if caracter_atual == op_arit.sub: self.__avancaColuna() tkn_num = self.__retornaNumero(False) elif caracter_atual == op_arit.soma: self.__avancaColuna() tkn_num = self.__retornaNumero(True) self.__adicionaToken(tkn_num) elif caracter_atual in valores.digitos: tkn_num = self.__retornaNumero() self.__adicionaToken(tkn_num) elif caracter_atual == "\"": tkn_texto = self.__retornaTexto() self.__adicionaToken(tkn_texto) elif caracter_posterior != None and caracter_atual + caracter_posterior == tipos_tokens.comentario: self.__ignoraComentario() elif caracter_atual in op_rel.todos or ( caracter_posterior != None and caracter_atual + caracter_posterior in op_rel.todos): tkn_op_rel = self.__retornaOperadorRelacional() self.__adicionaToken(tkn_op_rel) elif caracter_atual in op_arit.todos: tkn_op_arit = self.__retornaOperadorAritmetico() self.__adicionaToken(tkn_op_arit) elif caracter_atual == '.': pos = self.__retornaPosicaoAtual() tkn_ponto = Token(tipos_tokens.ponto, pos) self.__adicionaToken(tkn_ponto) self.__avancaColuna() elif caracter_atual == ';': pos = self.__retornaPosicaoAtual() tkn_delimitador = Token(tipos_tokens.delimitador, pos) self.__adicionaToken(tkn_delimitador) self.__avancaColuna() elif caracter_atual == ',': pos = self.__retornaPosicaoAtual() tkn_virgula = Token(tipos_tokens.virgula, pos) self.__adicionaToken(tkn_virgula) self.__avancaColuna() else: self.__avancaColuna() self.__adicionaToken( Token(tipos_tokens.EOF, self.__retornaPosicaoAtual())) return self._tokens
def parse_token(self, token: str) -> Token: """converts a string token into the appropriate Token object""" if self._currency_pattern.match(token): match = self._currency_pattern.match(token).group(1) dollars, cents = "{0:.2f}".format(float(match)).split('.') return Token(Type.CURRENCY, int("{0}{1}".format(dollars, cents))) elif self._word_pattern.match(token): match = self._word_pattern.match(token).group(1) return Token(Type.get_type_by_name(match), match) else: raise ValueError(token)
def __retornaNumero(self, sinal_positivo=True) -> Token: valor = '' if sinal_positivo else '-' pos = self.__retornaPosicaoAtual() while not self.__EOF() and self.__retornaCaracterAtual( ) in valores.valor_numerico: valor += self.__retornaCaracterAtual() self.__avancaColuna() try: int(valor) return Token(valores.inteiro, pos, valor) except: return Token(valores.flutuante, pos, valor)
def __retornaOperadorAritmetico(self) -> Token: operador = self.__retornaCaracterAtual() pos = self.__retornaPosicaoAtual() if operador == '+': self.__avancaColuna() return Token(op_arit.soma, pos) elif operador == '-': self.__avancaColuna() return Token(op_arit.sub, pos) elif operador == '/': self.__avancaColuna() return Token(op_arit.div, pos) elif operador == '*': self.__avancaColuna() return Token(op_arit.mult, pos) elif operador == '=': self.__avancaColuna() return Token(op_arit.op_atribuicao, pos) elif operador == '^': self.__avancaColuna() return Token(op_arit.pot, pos) elif operador == '(': self.__avancaColuna() return Token(op_arit.parent_esq, pos) elif operador == ')': self.__avancaColuna() return Token(op_arit.parent_dir, pos)
def test_scan_keywords(): '''check that keywords are scanned correctly, regardless of case''' scanner = Scanner(''' TrAnsFer transfer OPEN BALANCE balance''') toks = [ Token(Type.TRANSFER, "TrAnsFer"), Token(Type.TRANSFER, "transfer"), Token(Type.OPEN, "OPEN"), Token(Type.BALANCE, "BALANCE"), Token(Type.BALANCE, "balance") ] assert [tok for tok in scanner] == toks # iterate a second time assert [tok for tok in scanner] == toks
def __retornaIdentificador(self) -> Token: valor = '' pos = self.__retornaPosicaoAtual() while not self.__EOF() and self.__retornaCaracterAtual( ) in valores.caracteres: valor += self.__retornaCaracterAtual() self.__avancaColuna() if valor == palavras_chaves.senao and not self.__EOF( ) and self.__retornaCaracterAtual() == ' ': self.__avancaColuna() token_senaose = self.__tentaFormarSenaoSe(valor) if token_senaose != None: return token_senaose if valor in palavras_chaves.todas: p_chave = palavras_chaves.retornaPalavraChave(valor) return Token(p_chave, pos) if valor in [valores.verdadeiro, valores.falso]: if valor == valores.verdadeiro: return Token(tipo=valores.verdadeiro, pos=pos, val=valores.verdadeiro) else: return Token(tipo=valores.falso, pos=pos, val=valores.falso) if valor == op_logico.e: return Token(tipo=op_logico.e, pos=pos, val=op_logico.e) elif valor == op_logico.ou: return Token(tipo=op_logico.ou, pos=pos, val=op_logico.ou) elif valor == op_logico.nao: return Token(tipo=op_logico.nao, pos=pos, val=op_logico.nao) return Token(tipos_tokens.identificador, pos, valor)
def __retornaTexto(self) -> Token: valor = '' pos = self.__retornaPosicaoAtual() tipo = valores.texto self.__avancaColuna() while not self.__EOF() and self.__retornaCaracterAtual( ) in valores.valor_texto and self.__retornaCaracterAtual() != "\"": valor += self.__retornaCaracterAtual() self.__avancaColuna() if self.__retornaCaracterAtual() == "\"": self.__avancaColuna() return Token(tipo, pos, valor)
def generate_tokens(self): while self.current_character is not None: if self.current_character in WHITESPACE: self.advance() elif self.current_character == '.' or self.current_character in DIGITS: yield self.generate_number() elif self.current_character == "+": self.advance() yield Token(TokenType.PLUS) elif self.current_character == "-": self.advance() yield Token(TokenType.MINUS) elif self.current_character == "*": self.advance() yield Token(TokenType.MULTIPLY) elif self.current_character == "/": self.advance() yield Token(TokenType.DIVIDE) elif self.current_character == "(": self.advance() yield Token(TokenType.LPAREN) elif self.current_character == ")": self.advance() yield Token(TokenType.RPAREN) else: raise Exception( f"Illegal character '{self.current_character}'")
def __retornaOperadorRelacional(self) -> Token: pos = self.__retornaPosicaoAtual() operador = self.__retornaCaracterAtual() if operador + self.__retornaCaracterAtual(1) in op_rel.todos: self.__avancaColuna() operador = operador + self.__retornaCaracterAtual() self.__avancaColuna() if operador == '>': self.__avancaColuna() return Token(op_rel.maior_que, pos) elif operador == '<': self.__avancaColuna() return Token(op_rel.menor_que, pos) elif operador == '>=': self.__avancaColuna() return Token(op_rel.maior_igual, pos) elif operador == '<=': self.__avancaColuna() return Token(op_rel.menor_igual, pos) elif operador == '==': self.__avancaColuna() return Token(op_rel.igualdade, pos) elif operador == '!=': self.__avancaColuna() return Token(op_rel.desigualdade, pos)
def scan_tokens(self) -> List[Token]: ''' Core of scanner. Consumes source code characters in a loop :return: a list of tokens that are fed to TODO ''' # Each turn of the loop scans a single token while not self._is_at_end(): self.starts_at = self.current self._scan_token() final_token = Token('EOF', '', None, self.line) self.tokens.append(final_token) return self.tokens
def __init__(self): self.account = Account() self.token_ = Token().get_token() self.back_access = BackAccess() self.browser = Browser() self.my_browser = self.browser.get_browser() self.login = Login(self.my_browser) self.home = Home(self.my_browser) self.abnormal = Abnormal(self.my_browser) self.task_follow = TaskFollow(self.my_browser) self.task_host = TaskHost(self.my_browser) self.task_forward = TaskForward(self.my_browser) self.feed_back = Feedback(self.my_browser)
def __get_new_account(self): """ 调用interface的借口,获取新的账号信息 :return: 元组 (int, list, str) """ token = Token().get_token() nick_name = get_host_ip() for i in range(10): code, data, msg = self.__back_access.get_robot_account( token_=token, platform=PLATFORM, nick=nick_name) if code == 0: return code, data, msg elif code == 1: time.sleep(5) continue return 1, [], '取了10次也没有取到'
def __tentaFormarSenaoSe(self, valor): c_percorridas = 0 l_percorridas = 0 palavra = '' pos = self.__retornaPosicaoAtual() while not self.__EOF() and self.__retornaCaracterAtual( ) in palavras_chaves.se: palavra += self.__retornaCaracterAtual() res = self.__avancaColuna() if res == -1: l_percorridas += 1 else: c_percorridas += res if palavra == palavras_chaves.se: return Token(palavras_chaves.senaose, pos) else: # Faz rollback self.__desfazAvancoColunaLinha(c_percorridas, l_percorridas) return None
def generate_number(self): decimal_point_count = 0 number_str: str = self.current_character self.advance() while self.current_character is not None and ( self.current_character == '.' or self.current_character in DIGITS): if self.current_character == ".": decimal_point_count += 1 if decimal_point_count > 1: break number_str += self.current_character self.advance() if number_str.startswith("."): number_str = f"0{number_str}" if number_str.endswith("."): number_str += "0" return Token(TokenType.NUMBER, float(number_str))
def test_equal_keywords(): '''check that Token attributes are equal''' assert Token(Type.TRANSFER, "transfer") != Token(Type.OPEN, "open") assert Token(Type.OPEN, "OPEN") == Token(Type.OPEN, "open") assert Token(Type.BALANCE, "BALANCE") == Token(Type.BALANCE, "balance")
def _add_token(self, token_type, literal=None): text = self.source[self.starts_at:self.current] self.tokens.append(Token(token_type, text, literal, self.line))
def gera_operador(self, token: Token): if token.retornaTipo() == op_arit.soma: return "+" elif token.retornaTipo() == op_arit.sub: return "-" elif token.retornaTipo() == op_arit.mult: return "*" elif token.retornaTipo() == op_arit.div: return "/" elif token.retornaTipo() == op_rel.maior_que: return ">" elif token.retornaTipo() == op_rel.menor_que: return "<" elif token.retornaTipo() == op_rel.maior_igual: return ">=" elif token.retornaTipo() == op_rel.menor_igual: return "<=" elif token.retornaTipo() == op_rel.igualdade: return "==" elif token.retornaTipo() == op_logico.e: return "and" elif token.retornaTipo() == op_logico.ou: return "or" elif token.retornaTipo() == op_logico.nao: return "not"
def __init__(self, token: Token): self._token = token self._val = token.retornaValor()
def get_token(): while char.isspace(): next_char() # end of stream? if not len(char): return Token("$", Tag.END_OF_STREAM, None, True) # parentheses? elif char == "(": next_char() return Token("(", Tag.LEFT_PRT, "(", True) elif char == ")": next_char() return Token(")", Tag.RIGHT_PRT, ")", True) # single-char operator / comments? elif char in symbol_table: # div or comment? if char == "/": next_char() if char == "/": # comment until the end of the line: while char != "\n": next_char() if not len(char): return Token("$", Tag.END_OF_STREAM, None, True) return None elif char == "*": # comment until */: get_next_chr = True while True: if get_next_chr: next_char() get_next_chr = True if not len(char): print_error( ErrorType.LEXICAL, line, "Multiline comment was not closed", "*/ was expected", ) exit(1) if char == "*": next_char() get_next_chr = False if char == "/": next_char() return None else: return Token("/", Tag.DIVIDE, "/", True) else: i = char next_char() return Token(i, symbol_table[i][0], symbol_table[i][1], symbol_table[i][2]) # integer or decimal? elif char.isdigit(): num, is_decimal = get_number() if is_decimal: return Token(num, Tag.DOUBLE, num, False) else: return Token(num, Tag.INTEGER, num, False) # reserved and identifiers? else: lexeme = "" if char.isalnum() or char == "_": lexeme = char else: print_error(ErrorType.LEXICAL, line, "Character not allowed in identifier") exit(1) while True: next_char() if not len(char): if not lexeme: return Token("$", Tag.END_OF_STREAM, None, True) else: return Token(lexeme, Tag.IDENTIFIER, None, False) else: if char.isalnum() or char == "_": lexeme += char if is_reserved(lexeme): next_char() if (char.isspace() or char in symbol_table or char in ")(/*" or not len(char)): return Token( lexeme, symbol_table[lexeme][0], symbol_table[lexeme][1], symbol_table[lexeme][2], ) else: lexeme += char elif (char.isspace() or char in symbol_table or char in ")(/*"): # new identifier: if is_reserved(lexeme): return Token( lexeme, symbol_table[lexeme][0], symbol_table[lexeme][1], symbol_table[lexeme][2], ) else: return Token(lexeme, Tag.IDENTIFIER, None, False) else: print_error(ErrorType.LEXICAL, line, "Identifier has wrong format") exit(1)
def test_token(): '''check that a Token object is properly created''' token = Token(Type.IDENT, "hello") assert token.type == Type.IDENT assert token.value == "hello"
def test_equal_currency_negative(): '''check that negative currency is stored correctly''' assert Token(Type.CURRENCY, -1000) == Token(Type.CURRENCY, -1000) assert Token(Type.CURRENCY, -5000) != Token(Type.CURRENCY, 5000)
def test_equal_currency(): '''check that currency is compared correctly''' assert Token(Type.CURRENCY, 1000) == Token(Type.CURRENCY, 1000) assert Token(Type.CURRENCY, 1000) != Token(Type.CURRENCY, 1001) assert Token(Type.CURRENCY, 1234) == Token(Type.CURRENCY, 1234)
def test_equal_ident(): '''check that Token attributes are equal''' assert Token(Type.IDENT, "Bob") == Token(Type.IDENT, "Bob") assert Token(Type.IDENT, "Bob") != Token(Type.IDENT, "b0b")
def test_str(): '''check that the string representation of a Token object is correct''' id = Token(Type.IDENT, "hello") assert str(id) == '[IDENT: hello]' money = Token(Type.CURRENCY, 10042) assert str(money) == '[CURRENCY: 100.42]'
return self.token.literal def expression_node(self): pass def string(self): el = [] for v in self.elements: el.append(v.string()) out = "[" out += ", ".join(el) out += "]" return out def __str__(self): return "ArrayLiteral(Expression)" if __name__ == "__main__": from token_ import Token, TokenType ls = LetStatement(token=Token(TokenType.LET, "let"), name=Identifier(Token(TokenType.IDENT, "myVar"), "myVar"), value=Identifier(Token(TokenType.IDENT, "anothorVal"), "anothorVal")) p = Program() p.statements.append(ls) print(p.string())