def test_create_token(self):
tk = Token('(', start=(0, 0))
assert tk.string == '('
assert tk.type == tokenize.OP # Python do not createLPAR tokens.
assert tk.start == (0, 0)
assert tk.end == (0, 1)
assert tk.line == None
assert tk == Token(tk)
def process_repetir_command(self, tokens):
"""
Converts command::
repetir <N> vezes:
<BLOCO>
or::
repita <N> vezes:
<BLOCO>
to::
for ___ in range(<N>):
<BLOCO>
"""
matches = [('repetir',), ('repita',), ('vezes',), (NEWLINE,)]
iterator = token.token_find(tokens, matches)
for idx, match, start, end in iterator:
# Send tokens to the beginning of the equivalent "for" loop
starttokens = Token.from_strings(
tokens[idx].start, 'for', '___', 'in', 'range', '('
)
del tokens[idx]
token.insert_tokens_at(tokens, idx, starttokens, end=end)
# Matches the 'vezes' token
try:
idx, match, start, end = next(iterator)
except StopIteration:
match = [None]
if match[0] != 'vezes':
raise SyntaxError(
'comando repetir malformado na linha %s.\n'
' Espera comando do tipo\n\n'
' repetir <N> vezes:\n'
' <BLOCO>\n\n'
' Palavra chave "vezes" está faltando!' % (start.lineno)
)
else:
tokens[idx] = Token(')', start=start)
token.displace_tokens(tokens[idx + 1:], -4)
return tokens
def insert_a_cada(idx, match, start, end, iterator, tokens):
# Matches "a cada" or the end of the line
if match == ('a', 'cada'):
middletokens = Token.from_strings(start, '+', '1', ',')
del tokens[idx:idx + 2]
token.insert_tokens_at(tokens, idx, middletokens, end=end)
# Proceed to the end of the line
idx, match, start, end = next(iterator)
if match[0] not in (NEWLINE, ':'):
raise SyntaxError('comando malformado na linha %s.\n'
' Espera um ":" no fim do bloco' %
(start.lineno))
endtok = Token(')', start=start)
token.displace_tokens(tokens[idx:], 1)
tokens.insert(idx, endtok)
return end_command(idx, match, start, end, iterator, tokens)
return None
def test_create_list_of_tokens(self):
tk1, tk2, tk3 = Token.from_strings((1, 1), 'x', '=', '42')
assert tk1.string == 'x'
assert tk2.string == '='
assert tk3.string == '42'
assert tk1.start == (1, 1)
assert tk2.start == (1, 2)
assert tk3.start == (1, 3)
def end_command(idx, match, start, end, iterator, tokens):
# Finish command
if match[0] in (NEWLINE, ':'):
token.displace_tokens(tokens[idx:], 1)
endtokens = Token.from_strings(start, '+', '1', ')')
token.insert_tokens_at(tokens, idx, endtokens, end=end)
# Unexpected token
else:
raise SyntaxError('comando malformado na linha %s.\n'
' Espera um ":" no fim do bloco' % (start.lineno))
return tokens
def funct_iter(iterator, tokens):
for idx, match, start, end in iterator:
# Waits for a 'de' token to start processing
if match[0] != 'de':
continue
# Send tokens for the beginning of the equivalent in range(...)
# test
start = tokens[idx].start
starttokens = Token.from_strings(start, 'in', 'range', '(')
del tokens[idx]
token.insert_tokens_at(tokens, idx, starttokens, end=end)
return insert_ate(idx, match, start, end, iterator)
def insert_ate(idx, match, start, end, iterator, tokens):
# Matches the 'até' token and insert a comma separator
idx, match, start, end = next(iterator)
if match[0] in ['até', 'ate']:
token.displace_tokens(tokens[idx:], -3)
tokens[idx] = Token(',', start=tokens[idx - 1].end)
else:
raise SyntaxError('comando para cada malformado na linha %s.\n'
' Espera comando do tipo\n\n'
' para cada <x> de <a> até <b>:\n'
' <BLOCO>\n\n'
' Palavra chave "até" está faltando!' %
(start.lineno))
idx, match, start, end = next(iterator)
return insert_a_cada(idx, match, start, end, iterator, tokens)
def test_token_as_sequence(self):
tk = Token('tok', start=(0, 0))
assert len(tk) == 5
assert list(tk) == ['tok', tokenize.NAME, (0, 0), (0, 3), None]
def test_token_repr(self):
tk = Token('tok', start=(0, 0))
assert repr(tk) == "NAME('tok')"
assert str(tk) == "Token('tok', NAME, (0, 0), (0, 3), None)"
def test_creat_special_tokens(self):
assert Token('name', start=(0, 0)).type == tokenize.NAME
assert Token('42', start=(0, 0)).type == tokenize.NUMBER
assert Token('*', start=(0, 0)).type == tokenize.OP
def process_de_ate_command(self, tokens):
"""
Converts command::
de <X> até <Y> [a cada <Z>]
to::
in range(<X>, <Y> + 1[, <Z>])
"""
matches = [('de', ), ('ate', ), (
'a',
'cada',
), (NEWLINE, ), (':', )]
iterator = token.token_find(tokens, matches)
for idx, match, start, end in iterator:
# Waits for a 'de' token to start processing
if match[0] != 'de':
continue
# Send tokens for the beginning of the equivalent in range(...)
# test
start = tokens[idx].start
starttokens = Token.from_strings(start, 'in', 'range', '(')
del tokens[idx]
token.insert_tokens_at(tokens, idx, starttokens, end=end)
# Matches the 'até' token and insert a comma separator
idx, match, start, end = next(iterator)
if match[0] in ['até', 'ate']:
token.displace_tokens(tokens[idx:], -3)
tokens[idx] = Token(',', start=tokens[idx - 1].end)
else:
raise SyntaxError('comando para cada malformado na linha %s.\n'
' Espera comando do tipo\n\n'
' para cada <x> de <a> até <b>:\n'
' <BLOCO>\n\n'
' Palavra chave "até" está faltando!' %
(start.lineno))
idx, match, start, end = next(iterator)
# Matches "a cada" or the end of the line
if match == ('a', 'cada'):
middletokens = Token.from_strings(start, '+', '1', ',')
del tokens[idx:idx + 2]
token.insert_tokens_at(tokens, idx, middletokens, end=end)
# Proceed to the end of the line
idx, match, start, end = next(iterator)
if match[0] not in (NEWLINE, ':'):
raise SyntaxError('comando malformado na linha %s.\n'
' Espera um ":" no fim do bloco' %
(start.lineno))
endtok = Token(')', start=start)
token.displace_tokens(tokens[idx:], 1)
tokens.insert(idx, endtok)
# Finish command
elif match[0] in (NEWLINE, ':'):
token.displace_tokens(tokens[idx:], 1)
endtokens = Token.from_strings(start, '+', '1', ')')
token.insert_tokens_at(tokens, idx, endtokens, end=end)
# Unexpected token
else:
raise SyntaxError('comando malformado na linha %s.\n'
' Espera um ":" no fim do bloco' %
(start.lineno))
return tokens