def compress_source_map(code, pos_map, jump_map, source_id):
linenos = asttokens.LineNumbers(code)
compressed_map = f"-1:-1:{source_id}:-;"
last_pos = [-1, -1, source_id]
for pc in sorted(pos_map)[1:]:
current_pos = [-1, -1, source_id]
if pos_map[pc]:
current_pos[0] = linenos.line_to_offset(*pos_map[pc][:2])
current_pos[1] = linenos.line_to_offset(
*pos_map[pc][2:]) - current_pos[0]
if pc in jump_map:
current_pos.append(jump_map[pc])
for i in range(2, -1, -1):
if current_pos[i] != last_pos[i]:
last_pos[i] = current_pos[i]
elif len(current_pos) == i + 1:
current_pos.pop()
else:
current_pos[i] = ""
compressed_map += ":".join(str(i) for i in current_pos) + ";"
return compressed_map
def test_linenumbers(self):
ln = asttokens.LineNumbers("Hello\nworld\nThis\n\nis\n\na test.\n")
self.assertEqual(ln.line_to_offset(1, 0), 0)
self.assertEqual(ln.line_to_offset(1, 5), 5)
self.assertEqual(ln.line_to_offset(2, 0), 6)
self.assertEqual(ln.line_to_offset(2, 5), 11)
self.assertEqual(ln.line_to_offset(3, 0), 12)
self.assertEqual(ln.line_to_offset(4, 0), 17)
self.assertEqual(ln.line_to_offset(5, 0), 18)
self.assertEqual(ln.line_to_offset(6, 0), 21)
self.assertEqual(ln.line_to_offset(7, 0), 22)
self.assertEqual(ln.line_to_offset(7, 7), 29)
self.assertEqual(ln.offset_to_line(0), (1, 0))
self.assertEqual(ln.offset_to_line(5), (1, 5))
self.assertEqual(ln.offset_to_line(6), (2, 0))
self.assertEqual(ln.offset_to_line(11), (2, 5))
self.assertEqual(ln.offset_to_line(12), (3, 0))
self.assertEqual(ln.offset_to_line(17), (4, 0))
self.assertEqual(ln.offset_to_line(18), (5, 0))
self.assertEqual(ln.offset_to_line(21), (6, 0))
self.assertEqual(ln.offset_to_line(22), (7, 0))
self.assertEqual(ln.offset_to_line(29), (7, 7))
# Test that out-of-bounds inputs still return something sensible.
self.assertEqual(ln.line_to_offset(6, 19), 30)
self.assertEqual(ln.line_to_offset(100, 99), 30)
self.assertEqual(ln.line_to_offset(2, -1), 6)
self.assertEqual(ln.line_to_offset(-1, 99), 0)
self.assertEqual(ln.offset_to_line(30), (8, 0))
self.assertEqual(ln.offset_to_line(100), (8, 0))
self.assertEqual(ln.offset_to_line(-100), (1, 0))
def _create_syntax_error_code(builder, input_text, err):
"""
Returns the text for a function that raises the given SyntaxError and includes the offending
code in a commented-out form. In addition, it translates the error's position from builder's
output to input_text.
"""
output_ln = asttokens.LineNumbers(builder.get_text())
input_ln = asttokens.LineNumbers(input_text)
# A SyntaxError contains .lineno and .offset (1-based), which we need to translate to offset
# within the transformed text, so that it can be mapped back to an offset in the original text,
# and finally translated back into a line number and 1-based position to report to the user. An
# example is that "$x*" is translated to "return x*", and the syntax error in the transformed
# python code (line 2 offset 9) needs to be translated to be in line 2 offset 3.
output_offset = output_ln.line_to_offset(err.lineno, err.offset - 1 if err.offset else 0)
input_offset = builder.map_back_offset(output_offset)
line, col = input_ln.offset_to_line(input_offset)
return "%s\nraise %s('%s on line %d col %d')" % (
textbuilder.line_start_re.sub('# ', input_text.rstrip()),
type(err).__name__, err.args[0], line, col + 1)
def _generate_ast_error_message(self, node, msg):
# This is the location info of the start of the decorated @rule.
filename = inspect.getsourcefile(self._func)
source_lines, line_number = inspect.getsourcelines(self._func)
# The asttokens library is able to keep track of line numbers and column offsets for us -- the
# stdlib ast library only provides these relative to each parent node.
tokenized_rule_body = asttokens.ASTTokens(self._func_source,
tree=self._func_node,
filename=filename)
start_offset, _ = tokenized_rule_body.get_text_range(node)
line_offset, col_offset = asttokens.LineNumbers(
self._func_source).offset_to_line(start_offset)
node_file_line = line_number + line_offset - 1
# asttokens also very helpfully lets us provide the exact text of the node we want to highlight
# in an error message.
node_text = tokenized_rule_body.get_text(node)
fully_indented_node_col = col_offset + self._orig_indent
indented_node_text = '{}{}'.format(
# The node text doesn't have any initial whitespace, so we have to add it back.
col_offset * ' ',
'\n'.join(
# We removed the indentation from the original source in order to parse it with the ast
# library (otherwise it raises an exception), so we add it back here.
'{}{}'.format(self._orig_indent * ' ', l)
for l in node_text.split('\n')))
return ("""In function {func_name}: {msg}
The invalid statement was:
{filename}:{node_line_number}:{node_col}
{node_text}
The rule defined by function `{func_name}` begins at:
{filename}:{line_number}:{orig_indent}
{source_lines}
""".format(
func_name=self._func.__name__,
msg=msg,
filename=filename,
line_number=line_number,
orig_indent=self._orig_indent,
node_line_number=node_file_line,
node_col=fully_indented_node_col,
node_text=indented_node_text,
# Strip any leading or trailing newlines from the start of the rule body.
source_lines=''.join(source_lines).strip('\n')))
def line_numbers(self):
return asttokens.LineNumbers(self.text)
