def compress_source_map(code, pos_map, jump_map, source_id): linenos = asttokens.LineNumbers(code) compressed_map = f"-1:-1:{source_id}:-;" last_pos = [-1, -1, source_id] for pc in sorted(pos_map)[1:]: current_pos = [-1, -1, source_id] if pos_map[pc]: current_pos[0] = linenos.line_to_offset(*pos_map[pc][:2]) current_pos[1] = linenos.line_to_offset( *pos_map[pc][2:]) - current_pos[0] if pc in jump_map: current_pos.append(jump_map[pc]) for i in range(2, -1, -1): if current_pos[i] != last_pos[i]: last_pos[i] = current_pos[i] elif len(current_pos) == i + 1: current_pos.pop() else: current_pos[i] = "" compressed_map += ":".join(str(i) for i in current_pos) + ";" return compressed_map
def test_linenumbers(self): ln = asttokens.LineNumbers("Hello\nworld\nThis\n\nis\n\na test.\n") self.assertEqual(ln.line_to_offset(1, 0), 0) self.assertEqual(ln.line_to_offset(1, 5), 5) self.assertEqual(ln.line_to_offset(2, 0), 6) self.assertEqual(ln.line_to_offset(2, 5), 11) self.assertEqual(ln.line_to_offset(3, 0), 12) self.assertEqual(ln.line_to_offset(4, 0), 17) self.assertEqual(ln.line_to_offset(5, 0), 18) self.assertEqual(ln.line_to_offset(6, 0), 21) self.assertEqual(ln.line_to_offset(7, 0), 22) self.assertEqual(ln.line_to_offset(7, 7), 29) self.assertEqual(ln.offset_to_line(0), (1, 0)) self.assertEqual(ln.offset_to_line(5), (1, 5)) self.assertEqual(ln.offset_to_line(6), (2, 0)) self.assertEqual(ln.offset_to_line(11), (2, 5)) self.assertEqual(ln.offset_to_line(12), (3, 0)) self.assertEqual(ln.offset_to_line(17), (4, 0)) self.assertEqual(ln.offset_to_line(18), (5, 0)) self.assertEqual(ln.offset_to_line(21), (6, 0)) self.assertEqual(ln.offset_to_line(22), (7, 0)) self.assertEqual(ln.offset_to_line(29), (7, 7)) # Test that out-of-bounds inputs still return something sensible. self.assertEqual(ln.line_to_offset(6, 19), 30) self.assertEqual(ln.line_to_offset(100, 99), 30) self.assertEqual(ln.line_to_offset(2, -1), 6) self.assertEqual(ln.line_to_offset(-1, 99), 0) self.assertEqual(ln.offset_to_line(30), (8, 0)) self.assertEqual(ln.offset_to_line(100), (8, 0)) self.assertEqual(ln.offset_to_line(-100), (1, 0))
def _create_syntax_error_code(builder, input_text, err): """ Returns the text for a function that raises the given SyntaxError and includes the offending code in a commented-out form. In addition, it translates the error's position from builder's output to input_text. """ output_ln = asttokens.LineNumbers(builder.get_text()) input_ln = asttokens.LineNumbers(input_text) # A SyntaxError contains .lineno and .offset (1-based), which we need to translate to offset # within the transformed text, so that it can be mapped back to an offset in the original text, # and finally translated back into a line number and 1-based position to report to the user. An # example is that "$x*" is translated to "return x*", and the syntax error in the transformed # python code (line 2 offset 9) needs to be translated to be in line 2 offset 3. output_offset = output_ln.line_to_offset(err.lineno, err.offset - 1 if err.offset else 0) input_offset = builder.map_back_offset(output_offset) line, col = input_ln.offset_to_line(input_offset) return "%s\nraise %s('%s on line %d col %d')" % ( textbuilder.line_start_re.sub('# ', input_text.rstrip()), type(err).__name__, err.args[0], line, col + 1)
def _generate_ast_error_message(self, node, msg): # This is the location info of the start of the decorated @rule. filename = inspect.getsourcefile(self._func) source_lines, line_number = inspect.getsourcelines(self._func) # The asttokens library is able to keep track of line numbers and column offsets for us -- the # stdlib ast library only provides these relative to each parent node. tokenized_rule_body = asttokens.ASTTokens(self._func_source, tree=self._func_node, filename=filename) start_offset, _ = tokenized_rule_body.get_text_range(node) line_offset, col_offset = asttokens.LineNumbers( self._func_source).offset_to_line(start_offset) node_file_line = line_number + line_offset - 1 # asttokens also very helpfully lets us provide the exact text of the node we want to highlight # in an error message. node_text = tokenized_rule_body.get_text(node) fully_indented_node_col = col_offset + self._orig_indent indented_node_text = '{}{}'.format( # The node text doesn't have any initial whitespace, so we have to add it back. col_offset * ' ', '\n'.join( # We removed the indentation from the original source in order to parse it with the ast # library (otherwise it raises an exception), so we add it back here. '{}{}'.format(self._orig_indent * ' ', l) for l in node_text.split('\n'))) return ("""In function {func_name}: {msg} The invalid statement was: {filename}:{node_line_number}:{node_col} {node_text} The rule defined by function `{func_name}` begins at: {filename}:{line_number}:{orig_indent} {source_lines} """.format( func_name=self._func.__name__, msg=msg, filename=filename, line_number=line_number, orig_indent=self._orig_indent, node_line_number=node_file_line, node_col=fully_indented_node_col, node_text=indented_node_text, # Strip any leading or trailing newlines from the start of the rule body. source_lines=''.join(source_lines).strip('\n')))
def line_numbers(self): return asttokens.LineNumbers(self.text)