def test_create_token(self): tk = Token('(', start=(0, 0)) assert tk.string == '(' assert tk.type == tokenize.OP # Python do not createLPAR tokens. assert tk.start == (0, 0) assert tk.end == (0, 1) assert tk.line == None assert tk == Token(tk)
def process_repetir_command(self, tokens): """ Converts command:: repetir <N> vezes: <BLOCO> or:: repita <N> vezes: <BLOCO> to:: for ___ in range(<N>): <BLOCO> """ matches = [('repetir',), ('repita',), ('vezes',), (NEWLINE,)] iterator = token.token_find(tokens, matches) for idx, match, start, end in iterator: # Send tokens to the beginning of the equivalent "for" loop starttokens = Token.from_strings( tokens[idx].start, 'for', '___', 'in', 'range', '(' ) del tokens[idx] token.insert_tokens_at(tokens, idx, starttokens, end=end) # Matches the 'vezes' token try: idx, match, start, end = next(iterator) except StopIteration: match = [None] if match[0] != 'vezes': raise SyntaxError( 'comando repetir malformado na linha %s.\n' ' Espera comando do tipo\n\n' ' repetir <N> vezes:\n' ' <BLOCO>\n\n' ' Palavra chave "vezes" está faltando!' % (start.lineno) ) else: tokens[idx] = Token(')', start=start) token.displace_tokens(tokens[idx + 1:], -4) return tokens
def insert_a_cada(idx, match, start, end, iterator, tokens): # Matches "a cada" or the end of the line if match == ('a', 'cada'): middletokens = Token.from_strings(start, '+', '1', ',') del tokens[idx:idx + 2] token.insert_tokens_at(tokens, idx, middletokens, end=end) # Proceed to the end of the line idx, match, start, end = next(iterator) if match[0] not in (NEWLINE, ':'): raise SyntaxError('comando malformado na linha %s.\n' ' Espera um ":" no fim do bloco' % (start.lineno)) endtok = Token(')', start=start) token.displace_tokens(tokens[idx:], 1) tokens.insert(idx, endtok) return end_command(idx, match, start, end, iterator, tokens) return None
def test_create_list_of_tokens(self): tk1, tk2, tk3 = Token.from_strings((1, 1), 'x', '=', '42') assert tk1.string == 'x' assert tk2.string == '=' assert tk3.string == '42' assert tk1.start == (1, 1) assert tk2.start == (1, 2) assert tk3.start == (1, 3)
def end_command(idx, match, start, end, iterator, tokens): # Finish command if match[0] in (NEWLINE, ':'): token.displace_tokens(tokens[idx:], 1) endtokens = Token.from_strings(start, '+', '1', ')') token.insert_tokens_at(tokens, idx, endtokens, end=end) # Unexpected token else: raise SyntaxError('comando malformado na linha %s.\n' ' Espera um ":" no fim do bloco' % (start.lineno)) return tokens
def funct_iter(iterator, tokens): for idx, match, start, end in iterator: # Waits for a 'de' token to start processing if match[0] != 'de': continue # Send tokens for the beginning of the equivalent in range(...) # test start = tokens[idx].start starttokens = Token.from_strings(start, 'in', 'range', '(') del tokens[idx] token.insert_tokens_at(tokens, idx, starttokens, end=end) return insert_ate(idx, match, start, end, iterator)
def insert_ate(idx, match, start, end, iterator, tokens): # Matches the 'até' token and insert a comma separator idx, match, start, end = next(iterator) if match[0] in ['até', 'ate']: token.displace_tokens(tokens[idx:], -3) tokens[idx] = Token(',', start=tokens[idx - 1].end) else: raise SyntaxError('comando para cada malformado na linha %s.\n' ' Espera comando do tipo\n\n' ' para cada <x> de <a> até <b>:\n' ' <BLOCO>\n\n' ' Palavra chave "até" está faltando!' % (start.lineno)) idx, match, start, end = next(iterator) return insert_a_cada(idx, match, start, end, iterator, tokens)
def test_token_as_sequence(self): tk = Token('tok', start=(0, 0)) assert len(tk) == 5 assert list(tk) == ['tok', tokenize.NAME, (0, 0), (0, 3), None]
def test_token_repr(self): tk = Token('tok', start=(0, 0)) assert repr(tk) == "NAME('tok')" assert str(tk) == "Token('tok', NAME, (0, 0), (0, 3), None)"
def test_creat_special_tokens(self): assert Token('name', start=(0, 0)).type == tokenize.NAME assert Token('42', start=(0, 0)).type == tokenize.NUMBER assert Token('*', start=(0, 0)).type == tokenize.OP
def process_de_ate_command(self, tokens): """ Converts command:: de <X> até <Y> [a cada <Z>] to:: in range(<X>, <Y> + 1[, <Z>]) """ matches = [('de', ), ('ate', ), ( 'a', 'cada', ), (NEWLINE, ), (':', )] iterator = token.token_find(tokens, matches) for idx, match, start, end in iterator: # Waits for a 'de' token to start processing if match[0] != 'de': continue # Send tokens for the beginning of the equivalent in range(...) # test start = tokens[idx].start starttokens = Token.from_strings(start, 'in', 'range', '(') del tokens[idx] token.insert_tokens_at(tokens, idx, starttokens, end=end) # Matches the 'até' token and insert a comma separator idx, match, start, end = next(iterator) if match[0] in ['até', 'ate']: token.displace_tokens(tokens[idx:], -3) tokens[idx] = Token(',', start=tokens[idx - 1].end) else: raise SyntaxError('comando para cada malformado na linha %s.\n' ' Espera comando do tipo\n\n' ' para cada <x> de <a> até <b>:\n' ' <BLOCO>\n\n' ' Palavra chave "até" está faltando!' % (start.lineno)) idx, match, start, end = next(iterator) # Matches "a cada" or the end of the line if match == ('a', 'cada'): middletokens = Token.from_strings(start, '+', '1', ',') del tokens[idx:idx + 2] token.insert_tokens_at(tokens, idx, middletokens, end=end) # Proceed to the end of the line idx, match, start, end = next(iterator) if match[0] not in (NEWLINE, ':'): raise SyntaxError('comando malformado na linha %s.\n' ' Espera um ":" no fim do bloco' % (start.lineno)) endtok = Token(')', start=start) token.displace_tokens(tokens[idx:], 1) tokens.insert(idx, endtok) # Finish command elif match[0] in (NEWLINE, ':'): token.displace_tokens(tokens[idx:], 1) endtokens = Token.from_strings(start, '+', '1', ')') token.insert_tokens_at(tokens, idx, endtokens, end=end) # Unexpected token else: raise SyntaxError('comando malformado na linha %s.\n' ' Espera um ":" no fim do bloco' % (start.lineno)) return tokens