def __parseRepitaIdentAte(self):
ident = Variavel(self.__retornaTokenAtual())
# Adiciona uma instrução de atribuição no loop
op_soma = Token(op_arit.soma, pos=self.__retornaTokenAtual().retornaPosicao())
valor_soma = Numero(Token(tipo=valores.inteiro, pos=self.__retornaTokenAtual().retornaPosicao(), val=1))
op_bin_soma = OpBin(esq=ident, op=op_soma, dir=valor_soma)
var_atrib = AtribuicaoVariavel(ident=self.__retornaTokenAtual(), op=op_soma, val=op_bin_soma)
self.__avancaToken()
if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.ate:
msgErro = f"Espera-se '{palavras_chaves.ate}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'."
posErro = self.__retornaTokenAtual().retornaPosicao()
self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro))
self.__avancaToken()
expr = self.__parseExpr()
if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.entao:
msgErro = f"Espera-se '{palavras_chaves.entao}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'."
posErro = self.__retornaTokenAtual().retornaPosicao()
self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro))
self.__avancaToken()
instrucoes = Instrucao()
while self.__retornaTokenAtual().retornaTipo() not in [palavras_chaves.fim_repita, tipos_tokens.EOF]:
instrucoes.adicionaInstrucao(self.__parseInstrucao())
instrucoes.adicionaInstrucao(var_atrib)
if self.__retornaTokenAtual().retornaTipo() != palavras_chaves.fim_repita:
msgErro = f"Espera-se '{palavras_chaves.fim_repita}' ao invés de '{self.__retornaTokenAtual().retornaValor()}'."
posErro = self.__retornaTokenAtual().retornaPosicao()
self.__registraErro(ErroSintaxe(msg=msgErro, pos=posErro))
self.__avancaToken()
return RepitaIdentAte(ident=ident, expr=expr, instrucoes=instrucoes)
def test_scan_currency_round():
'''check that currency is rounded if there are more than two decimals'''
scanner = Scanner("100.00 100.420 100.429")
assert list(scanner) == [
Token(Type.CURRENCY, 10000),
Token(Type.CURRENCY, 10042),
Token(Type.CURRENCY, 10043)
]
def test_scan_identifiers():
'''check that non-keywords are identified as identifiers'''
scanner = Scanner("equity cash end_of_the_world_fund")
assert list(scanner) == [
Token(Type.IDENT, "equity"),
Token(Type.IDENT, "cash"),
Token(Type.IDENT, "end_of_the_world_fund")
]
def __call__(self, state, string, rownumber): #根据终态、被接受的字符串new出一个Token
if state != None:
for tk in self.token_list:
if tk.split(":")[0] in state.contant:
token = Token(string, tk, rownumber)
return token
else:
return Token(string, None, rownumber)
def test_scan_currency():
'''check that currency is identified and stored correctly'''
scanner = Scanner("100 100.00 100.42 -123.45")
assert list(scanner) == [
Token(Type.CURRENCY, 10000),
Token(Type.CURRENCY, 10000),
Token(Type.CURRENCY, 10042),
Token(Type.CURRENCY, -12345)
]
def retornaTokens(self) -> list:
while not self.__EOF():
caracter_atual = self.__retornaCaracterAtual()
caracter_posterior = self.__retornaCaracterAtual(1)
if caracter_atual in ' \t':
self.__avancaColuna()
elif caracter_atual in valores.caracteres:
tkn_ident = self.__retornaIdentificador()
self.__adicionaToken(tkn_ident)
elif caracter_atual in [
op_arit.sub, op_arit.soma
] and caracter_posterior in valores.valor_numerico:
if caracter_atual == op_arit.sub:
self.__avancaColuna()
tkn_num = self.__retornaNumero(False)
elif caracter_atual == op_arit.soma:
self.__avancaColuna()
tkn_num = self.__retornaNumero(True)
self.__adicionaToken(tkn_num)
elif caracter_atual in valores.digitos:
tkn_num = self.__retornaNumero()
self.__adicionaToken(tkn_num)
elif caracter_atual == "\"":
tkn_texto = self.__retornaTexto()
self.__adicionaToken(tkn_texto)
elif caracter_posterior != None and caracter_atual + caracter_posterior == tipos_tokens.comentario:
self.__ignoraComentario()
elif caracter_atual in op_rel.todos or (
caracter_posterior != None
and caracter_atual + caracter_posterior in op_rel.todos):
tkn_op_rel = self.__retornaOperadorRelacional()
self.__adicionaToken(tkn_op_rel)
elif caracter_atual in op_arit.todos:
tkn_op_arit = self.__retornaOperadorAritmetico()
self.__adicionaToken(tkn_op_arit)
elif caracter_atual == '.':
pos = self.__retornaPosicaoAtual()
tkn_ponto = Token(tipos_tokens.ponto, pos)
self.__adicionaToken(tkn_ponto)
self.__avancaColuna()
elif caracter_atual == ';':
pos = self.__retornaPosicaoAtual()
tkn_delimitador = Token(tipos_tokens.delimitador, pos)
self.__adicionaToken(tkn_delimitador)
self.__avancaColuna()
elif caracter_atual == ',':
pos = self.__retornaPosicaoAtual()
tkn_virgula = Token(tipos_tokens.virgula, pos)
self.__adicionaToken(tkn_virgula)
self.__avancaColuna()
else:
self.__avancaColuna()
self.__adicionaToken(
Token(tipos_tokens.EOF, self.__retornaPosicaoAtual()))
return self._tokens
def parse_token(self, token: str) -> Token:
"""converts a string token into the appropriate Token object"""
if self._currency_pattern.match(token):
match = self._currency_pattern.match(token).group(1)
dollars, cents = "{0:.2f}".format(float(match)).split('.')
return Token(Type.CURRENCY, int("{0}{1}".format(dollars, cents)))
elif self._word_pattern.match(token):
match = self._word_pattern.match(token).group(1)
return Token(Type.get_type_by_name(match), match)
else:
raise ValueError(token)
def __retornaNumero(self, sinal_positivo=True) -> Token:
valor = '' if sinal_positivo else '-'
pos = self.__retornaPosicaoAtual()
while not self.__EOF() and self.__retornaCaracterAtual(
) in valores.valor_numerico:
valor += self.__retornaCaracterAtual()
self.__avancaColuna()
try:
int(valor)
return Token(valores.inteiro, pos, valor)
except:
return Token(valores.flutuante, pos, valor)
def __retornaOperadorAritmetico(self) -> Token:
operador = self.__retornaCaracterAtual()
pos = self.__retornaPosicaoAtual()
if operador == '+':
self.__avancaColuna()
return Token(op_arit.soma, pos)
elif operador == '-':
self.__avancaColuna()
return Token(op_arit.sub, pos)
elif operador == '/':
self.__avancaColuna()
return Token(op_arit.div, pos)
elif operador == '*':
self.__avancaColuna()
return Token(op_arit.mult, pos)
elif operador == '=':
self.__avancaColuna()
return Token(op_arit.op_atribuicao, pos)
elif operador == '^':
self.__avancaColuna()
return Token(op_arit.pot, pos)
elif operador == '(':
self.__avancaColuna()
return Token(op_arit.parent_esq, pos)
elif operador == ')':
self.__avancaColuna()
return Token(op_arit.parent_dir, pos)
def test_scan_keywords():
'''check that keywords are scanned correctly, regardless of case'''
scanner = Scanner('''
TrAnsFer transfer
OPEN BALANCE balance''')
toks = [
Token(Type.TRANSFER, "TrAnsFer"),
Token(Type.TRANSFER, "transfer"),
Token(Type.OPEN, "OPEN"),
Token(Type.BALANCE, "BALANCE"),
Token(Type.BALANCE, "balance")
]
assert [tok for tok in scanner] == toks
# iterate a second time
assert [tok for tok in scanner] == toks
def __retornaIdentificador(self) -> Token:
valor = ''
pos = self.__retornaPosicaoAtual()
while not self.__EOF() and self.__retornaCaracterAtual(
) in valores.caracteres:
valor += self.__retornaCaracterAtual()
self.__avancaColuna()
if valor == palavras_chaves.senao and not self.__EOF(
) and self.__retornaCaracterAtual() == ' ':
self.__avancaColuna()
token_senaose = self.__tentaFormarSenaoSe(valor)
if token_senaose != None:
return token_senaose
if valor in palavras_chaves.todas:
p_chave = palavras_chaves.retornaPalavraChave(valor)
return Token(p_chave, pos)
if valor in [valores.verdadeiro, valores.falso]:
if valor == valores.verdadeiro:
return Token(tipo=valores.verdadeiro,
pos=pos,
val=valores.verdadeiro)
else:
return Token(tipo=valores.falso, pos=pos, val=valores.falso)
if valor == op_logico.e:
return Token(tipo=op_logico.e, pos=pos, val=op_logico.e)
elif valor == op_logico.ou:
return Token(tipo=op_logico.ou, pos=pos, val=op_logico.ou)
elif valor == op_logico.nao:
return Token(tipo=op_logico.nao, pos=pos, val=op_logico.nao)
return Token(tipos_tokens.identificador, pos, valor)
def __retornaTexto(self) -> Token:
valor = ''
pos = self.__retornaPosicaoAtual()
tipo = valores.texto
self.__avancaColuna()
while not self.__EOF() and self.__retornaCaracterAtual(
) in valores.valor_texto and self.__retornaCaracterAtual() != "\"":
valor += self.__retornaCaracterAtual()
self.__avancaColuna()
if self.__retornaCaracterAtual() == "\"":
self.__avancaColuna()
return Token(tipo, pos, valor)
def generate_tokens(self):
while self.current_character is not None:
if self.current_character in WHITESPACE:
self.advance()
elif self.current_character == '.' or self.current_character in DIGITS:
yield self.generate_number()
elif self.current_character == "+":
self.advance()
yield Token(TokenType.PLUS)
elif self.current_character == "-":
self.advance()
yield Token(TokenType.MINUS)
elif self.current_character == "*":
self.advance()
yield Token(TokenType.MULTIPLY)
elif self.current_character == "/":
self.advance()
yield Token(TokenType.DIVIDE)
elif self.current_character == "(":
self.advance()
yield Token(TokenType.LPAREN)
elif self.current_character == ")":
self.advance()
yield Token(TokenType.RPAREN)
else:
raise Exception(
f"Illegal character '{self.current_character}'")
def __retornaOperadorRelacional(self) -> Token:
pos = self.__retornaPosicaoAtual()
operador = self.__retornaCaracterAtual()
if operador + self.__retornaCaracterAtual(1) in op_rel.todos:
self.__avancaColuna()
operador = operador + self.__retornaCaracterAtual()
self.__avancaColuna()
if operador == '>':
self.__avancaColuna()
return Token(op_rel.maior_que, pos)
elif operador == '<':
self.__avancaColuna()
return Token(op_rel.menor_que, pos)
elif operador == '>=':
self.__avancaColuna()
return Token(op_rel.maior_igual, pos)
elif operador == '<=':
self.__avancaColuna()
return Token(op_rel.menor_igual, pos)
elif operador == '==':
self.__avancaColuna()
return Token(op_rel.igualdade, pos)
elif operador == '!=':
self.__avancaColuna()
return Token(op_rel.desigualdade, pos)
def scan_tokens(self) -> List[Token]:
''' Core of scanner. Consumes source code characters in a loop
:return: a list of tokens that are fed to TODO
'''
# Each turn of the loop scans a single token
while not self._is_at_end():
self.starts_at = self.current
self._scan_token()
final_token = Token('EOF', '', None, self.line)
self.tokens.append(final_token)
return self.tokens
def __init__(self):
self.account = Account()
self.token_ = Token().get_token()
self.back_access = BackAccess()
self.browser = Browser()
self.my_browser = self.browser.get_browser()
self.login = Login(self.my_browser)
self.home = Home(self.my_browser)
self.abnormal = Abnormal(self.my_browser)
self.task_follow = TaskFollow(self.my_browser)
self.task_host = TaskHost(self.my_browser)
self.task_forward = TaskForward(self.my_browser)
self.feed_back = Feedback(self.my_browser)
def __get_new_account(self):
"""
调用interface的借口，获取新的账号信息
:return: 元组 （int, list, str）
"""
token = Token().get_token()
nick_name = get_host_ip()
for i in range(10):
code, data, msg = self.__back_access.get_robot_account(
token_=token, platform=PLATFORM, nick=nick_name)
if code == 0:
return code, data, msg
elif code == 1:
time.sleep(5)
continue
return 1, [], '取了10次也没有取到'
def __tentaFormarSenaoSe(self, valor):
c_percorridas = 0
l_percorridas = 0
palavra = ''
pos = self.__retornaPosicaoAtual()
while not self.__EOF() and self.__retornaCaracterAtual(
) in palavras_chaves.se:
palavra += self.__retornaCaracterAtual()
res = self.__avancaColuna()
if res == -1:
l_percorridas += 1
else:
c_percorridas += res
if palavra == palavras_chaves.se:
return Token(palavras_chaves.senaose, pos)
else:
# Faz rollback
self.__desfazAvancoColunaLinha(c_percorridas, l_percorridas)
return None
def generate_number(self):
decimal_point_count = 0
number_str: str = self.current_character
self.advance()
while self.current_character is not None and (
self.current_character == '.'
or self.current_character in DIGITS):
if self.current_character == ".":
decimal_point_count += 1
if decimal_point_count > 1:
break
number_str += self.current_character
self.advance()
if number_str.startswith("."):
number_str = f"0{number_str}"
if number_str.endswith("."):
number_str += "0"
return Token(TokenType.NUMBER, float(number_str))
def test_equal_keywords():
'''check that Token attributes are equal'''
assert Token(Type.TRANSFER, "transfer") != Token(Type.OPEN, "open")
assert Token(Type.OPEN, "OPEN") == Token(Type.OPEN, "open")
assert Token(Type.BALANCE, "BALANCE") == Token(Type.BALANCE, "balance")
def _add_token(self, token_type, literal=None):
text = self.source[self.starts_at:self.current]
self.tokens.append(Token(token_type, text, literal, self.line))
def gera_operador(self, token: Token):
if token.retornaTipo() == op_arit.soma:
return "+"
elif token.retornaTipo() == op_arit.sub:
return "-"
elif token.retornaTipo() == op_arit.mult:
return "*"
elif token.retornaTipo() == op_arit.div:
return "/"
elif token.retornaTipo() == op_rel.maior_que:
return ">"
elif token.retornaTipo() == op_rel.menor_que:
return "<"
elif token.retornaTipo() == op_rel.maior_igual:
return ">="
elif token.retornaTipo() == op_rel.menor_igual:
return "<="
elif token.retornaTipo() == op_rel.igualdade:
return "=="
elif token.retornaTipo() == op_logico.e:
return "and"
elif token.retornaTipo() == op_logico.ou:
return "or"
elif token.retornaTipo() == op_logico.nao:
return "not"
def __init__(self, token: Token):
self._token = token
self._val = token.retornaValor()
def get_token():
while char.isspace():
next_char()
# end of stream?
if not len(char):
return Token("$", Tag.END_OF_STREAM, None, True)
# parentheses?
elif char == "(":
next_char()
return Token("(", Tag.LEFT_PRT, "(", True)
elif char == ")":
next_char()
return Token(")", Tag.RIGHT_PRT, ")", True)
# single-char operator / comments?
elif char in symbol_table:
# div or comment?
if char == "/":
next_char()
if char == "/": # comment until the end of the line:
while char != "\n":
next_char()
if not len(char):
return Token("$", Tag.END_OF_STREAM, None, True)
return None
elif char == "*": # comment until */:
get_next_chr = True
while True:
if get_next_chr:
next_char()
get_next_chr = True
if not len(char):
print_error(
ErrorType.LEXICAL,
line,
"Multiline comment was not closed",
"*/ was expected",
)
exit(1)
if char == "*":
next_char()
get_next_chr = False
if char == "/":
next_char()
return None
else:
return Token("/", Tag.DIVIDE, "/", True)
else:
i = char
next_char()
return Token(i, symbol_table[i][0], symbol_table[i][1],
symbol_table[i][2])
# integer or decimal?
elif char.isdigit():
num, is_decimal = get_number()
if is_decimal:
return Token(num, Tag.DOUBLE, num, False)
else:
return Token(num, Tag.INTEGER, num, False)
# reserved and identifiers?
else:
lexeme = ""
if char.isalnum() or char == "_":
lexeme = char
else:
print_error(ErrorType.LEXICAL, line,
"Character not allowed in identifier")
exit(1)
while True:
next_char()
if not len(char):
if not lexeme:
return Token("$", Tag.END_OF_STREAM, None, True)
else:
return Token(lexeme, Tag.IDENTIFIER, None, False)
else:
if char.isalnum() or char == "_":
lexeme += char
if is_reserved(lexeme):
next_char()
if (char.isspace() or char in symbol_table
or char in ")(/*" or not len(char)):
return Token(
lexeme,
symbol_table[lexeme][0],
symbol_table[lexeme][1],
symbol_table[lexeme][2],
)
else:
lexeme += char
elif (char.isspace() or char in symbol_table
or char in ")(/*"): # new identifier:
if is_reserved(lexeme):
return Token(
lexeme,
symbol_table[lexeme][0],
symbol_table[lexeme][1],
symbol_table[lexeme][2],
)
else:
return Token(lexeme, Tag.IDENTIFIER, None, False)
else:
print_error(ErrorType.LEXICAL, line,
"Identifier has wrong format")
exit(1)
def test_token():
'''check that a Token object is properly created'''
token = Token(Type.IDENT, "hello")
assert token.type == Type.IDENT
assert token.value == "hello"
def test_equal_currency_negative():
'''check that negative currency is stored correctly'''
assert Token(Type.CURRENCY, -1000) == Token(Type.CURRENCY, -1000)
assert Token(Type.CURRENCY, -5000) != Token(Type.CURRENCY, 5000)
def test_equal_currency():
'''check that currency is compared correctly'''
assert Token(Type.CURRENCY, 1000) == Token(Type.CURRENCY, 1000)
assert Token(Type.CURRENCY, 1000) != Token(Type.CURRENCY, 1001)
assert Token(Type.CURRENCY, 1234) == Token(Type.CURRENCY, 1234)
def test_equal_ident():
'''check that Token attributes are equal'''
assert Token(Type.IDENT, "Bob") == Token(Type.IDENT, "Bob")
assert Token(Type.IDENT, "Bob") != Token(Type.IDENT, "b0b")
def test_str():
'''check that the string representation of a Token object is correct'''
id = Token(Type.IDENT, "hello")
assert str(id) == '[IDENT: hello]'
money = Token(Type.CURRENCY, 10042)
assert str(money) == '[CURRENCY: 100.42]'
return self.token.literal
def expression_node(self):
pass
def string(self):
el = []
for v in self.elements:
el.append(v.string())
out = "["
out += ", ".join(el)
out += "]"
return out
def __str__(self):
return "ArrayLiteral(Expression)"
if __name__ == "__main__":
from token_ import Token, TokenType
ls = LetStatement(token=Token(TokenType.LET, "let"),
name=Identifier(Token(TokenType.IDENT, "myVar"),
"myVar"),
value=Identifier(Token(TokenType.IDENT, "anothorVal"),
"anothorVal"))
p = Program()
p.statements.append(ls)
print(p.string())
