def test_extract_tokens(self):
code_node = get_code_node(
"<html>",
" <body>",
" <code>for i in range(0, 1):",
" print \"Hello world\"</code>",
" </body>",
"</html>",
)
tokens = extract_tokens(code_node)
self.assertEqual(
tokens,
['for', 'i', 'in', 'range', '(', '0', ',', '1', ')', ':', '\n', ' ', 'print',
'"Hello world"', '', ''])
def test_accept_unfinished_docstrings(self):
code_node = get_code_node(
"<html>",
" <body>",
" <code>print 'Hello'",
"'''",
"docstring</code>",
" </body>",
"</html>",
)
tokens = extract_tokens(code_node)
self.assertEqual(
tokens,
['print', '\'Hello\'', '\n']
)
def test_get_partial_results_upon_indentation_error(self):
code_node = get_code_node(
"<html>",
" <body>",
" <code>for i in l:",
" pass",
" print 'Hello'",
"</code>",
" </body>",
"</html>",
)
tokens = extract_tokens(code_node)
self.assertEqual(
tokens,
['for', 'i', 'in', 'l', ':', '\n', ' ', 'pass', '\n']
)
