def test_extract_tokens(self): code_node = get_code_node( "<html>", " <body>", " <code>for i in range(0, 1):", " print \"Hello world\"</code>", " </body>", "</html>", ) tokens = extract_tokens(code_node) self.assertEqual( tokens, ['for', 'i', 'in', 'range', '(', '0', ',', '1', ')', ':', '\n', ' ', 'print', '"Hello world"', '', ''])
def test_accept_unfinished_docstrings(self): code_node = get_code_node( "<html>", " <body>", " <code>print 'Hello'", "'''", "docstring</code>", " </body>", "</html>", ) tokens = extract_tokens(code_node) self.assertEqual( tokens, ['print', '\'Hello\'', '\n'] )
def test_get_partial_results_upon_indentation_error(self): code_node = get_code_node( "<html>", " <body>", " <code>for i in l:", " pass", " print 'Hello'", "</code>", " </body>", "</html>", ) tokens = extract_tokens(code_node) self.assertEqual( tokens, ['for', 'i', 'in', 'l', ':', '\n', ' ', 'pass', '\n'] )