def _extract_range_jedi_0_11_1(self, definition):
from parso.utils import split_lines
# get the scope range
try:
if definition.type in ['class', 'function']:
tree_name = definition._name.tree_name
scope = tree_name.get_definition()
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
# get the lines
code = scope.get_code(include_prefix=False)
lines = split_lines(code)
# trim the lines
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name.tree_name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def _extract_range_jedi_0_11_1(self, definition):
from parso.utils import split_lines
# get the scope range
try:
if definition.type in ['class', 'function']:
tree_name = definition._name.tree_name
scope = tree_name.get_definition()
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
# get the lines
code = scope.get_code(include_prefix=False)
lines = split_lines(code)
# trim the lines
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name.tree_name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def get_start_pos_of_prefix(self):
previous_leaf = self.get_previous_leaf()
if previous_leaf is None:
lines = split_lines(self.prefix)
# + 1 is needed because split_lines always returns at least [''].
return self.line - len(lines) + 1, 0 # It's the first leaf.
return previous_leaf.end_pos
def get_start_pos_of_prefix(self):
previous_leaf = self.get_previous_leaf()
if previous_leaf is None:
lines = split_lines(self.prefix)
# + 1 is needed because split_lines always returns at least [''].
return self.line - len(lines) + 1, 0 # It's the first leaf.
return previous_leaf.end_pos
def close(self):
self._base_node.finish()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
except IndexError:
end_pos = [1, 0]
else:
last_leaf = _skip_dedent_error_leaves(last_leaf)
end_pos = list(last_leaf.end_pos)
lines = split_lines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
if lines[0].startswith(BOM_UTF8_STRING) and end_pos == [1, 0]:
end_pos[1] -= 1
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos),
self.prefix + self._prefix_remainder)
endmarker.parent = self._module
self._module.children.append(endmarker)
def add_to_pos(string):
lines = split_lines(string)
l = len(lines[-1])
if len(lines) > 1:
start_pos[0] += len(lines) - 1
start_pos[1] = l
else:
start_pos[1] += l
def add_to_pos(string):
lines = split_lines(string)
l = len(lines[-1])
if len(lines) > 1:
start_pos[0] += len(lines) - 1
start_pos[1] = l
else:
start_pos[1] += l
def _get_debug_error_message(module, old_lines, new_lines):
current_lines = split_lines(module.get_code(), keepends=True)
current_diff = difflib.unified_diff(new_lines, current_lines)
old_new_diff = difflib.unified_diff(old_lines, new_lines)
import parso
return (
"There's an issue with the diff parser. Please "
"report (parso v%s) - Old/New:\n%s\nActual Diff (May be empty):\n%s" %
(parso.__version__, ''.join(old_new_diff), ''.join(current_diff)))
def __init__(self, file_path, test_count, change_count):
self._path = file_path
with open(file_path) as f:
code = f.read()
self._code_lines = split_lines(code, keepends=True)
self._test_count = test_count
self._code_lines = self._code_lines
self._change_count = change_count
self._file_modifications = []
def test_open_string_literal(each_version, code):
"""
Testing mostly if removing the last newline works.
"""
lines = split_lines(code, keepends=True)
end_pos = (len(lines), len(lines[-1]))
module = parse(code, version=each_version)
assert module.get_code() == code
assert module.end_pos == end_pos == module.children[1].end_pos
def end_pos(self) -> Tuple[int, int]:
lines = split_lines(self.value)
end_pos_line = self.line + len(lines) - 1
# Check for multiline token
if self.line == end_pos_line:
end_pos_column = self.column + len(lines[-1])
else:
end_pos_column = len(lines[-1])
return end_pos_line, end_pos_column
def end_pos(self):
lines = split_lines(self.value)
end_pos_line = self.line + len(lines) - 1
# Check for multiline token
if self.line == end_pos_line:
end_pos_column = self.column + len(lines[-1])
else:
end_pos_column = len(lines[-1])
return end_pos_line, end_pos_column
def __init__(self, file_path, test_count, change_count):
self._path = file_path
with open(file_path) as f:
code = f.read()
self._code_lines = split_lines(code, keepends=True)
self._test_count = test_count
self._code_lines = self._code_lines
self._change_count = change_count
self._file_modifications = []
def _get_debug_error_message(module, old_lines, new_lines):
current_lines = split_lines(module.get_code(), keepends=True)
current_diff = difflib.unified_diff(new_lines, current_lines)
old_new_diff = difflib.unified_diff(old_lines, new_lines)
import parso
return (
"There's an issue with the diff parser. Please "
"report (parso v%s) - Old/New:\n%s\nActual Diff (May be empty):\n%s"
% (parso.__version__, ''.join(old_new_diff), ''.join(current_diff))
)
def initialize(self, code):
logging.debug('differ: initialize')
try:
del cache.parser_cache[self.grammar._hashed][None]
except KeyError:
pass
self.lines = split_lines(code, keepends=True)
self.module = parse(code, diff_cache=True, cache=True)
return self.module
def initialize(self, code):
logging.debug('differ: initialize')
try:
del cache.parser_cache[self.grammar._hashed][None]
except KeyError:
pass
self.lines = split_lines(code, keepends=True)
self.module = parse(code, diff_cache=True, cache=True)
return self.module
def tokenize(
code: str,
*,
version_info: PythonVersionInfo,
start_pos: Tuple[int, int] = (1, 0)
) -> Iterator[PythonToken]:
"""Generate tokens from a the source code (string)."""
lines = split_lines(code, keepends=True)
return tokenize_lines(lines,
version_info=version_info,
start_pos=start_pos)
def get_start_pos_of_prefix(self):
"""
Basically calls :py:meth:`parso.tree.NodeOrLeaf.get_start_pos_of_prefix`.
"""
# TODO it is really ugly that we have to override it. Maybe change
# indent error leafs somehow? No idea how, though.
previous_leaf = self.get_previous_leaf()
if previous_leaf is not None and previous_leaf.type == 'error_leaf' \
and previous_leaf.token_type in ('INDENT', 'DEDENT', 'ERROR_DEDENT'):
previous_leaf = previous_leaf.get_previous_leaf()
if previous_leaf is None: # It's the first leaf.
lines = split_lines(self.prefix)
# + 1 is needed because split_lines always returns at least [''].
return self.line - len(lines) + 1, 0 # It's the first leaf.
return previous_leaf.end_pos
def get_start_pos_of_prefix(self):
"""
Basically calls :py:meth:`parso.tree.NodeOrLeaf.get_start_pos_of_prefix`.
"""
# TODO it is really ugly that we have to override it. Maybe change
# indent error leafs somehow? No idea how, though.
previous_leaf = self.get_previous_leaf()
if previous_leaf is not None and previous_leaf.type == 'error_leaf' \
and previous_leaf.token_type in ('INDENT', 'DEDENT', 'ERROR_DEDENT'):
previous_leaf = previous_leaf.get_previous_leaf()
if previous_leaf is None: # It's the first leaf.
lines = split_lines(self.prefix)
# + 1 is needed because split_lines always returns at least [''].
return self.line - len(lines) + 1, 0 # It's the first leaf.
return previous_leaf.end_pos
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
lines = split_lines(code, keepends=True)
diff_parser = DiffParser(
self.grammar._pgen_grammar,
self.grammar._tokenizer,
self.module,
)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
assert diff_parser._copy_count == copies
#assert diff_parser._parser_count == parsers
assert expect_error_leaves == _check_error_leaves_nodes(new_module)
_assert_valid_graph(new_module)
return new_module
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
lines = split_lines(code, keepends=True)
diff_parser = DiffParser(
self.grammar._pgen_grammar,
self.grammar._tokenizer,
self.module,
)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
assert diff_parser._copy_count == copies
#assert diff_parser._parser_count == parsers
assert expect_error_leaves == _check_error_leaves_nodes(new_module)
_assert_valid_graph(new_module)
return new_module
def _assert_valid_graph(node):
"""
Checks if the parent/children relationship is correct.
This is a check that only runs during debugging/testing.
"""
try:
children = node.children
except AttributeError:
# Ignore INDENT is necessary, because indent/dedent tokens don't
# contain value/prefix and are just around, because of the tokenizer.
if node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS:
assert not node.value
assert not node.prefix
return
# Calculate the content between two start positions.
previous_leaf = _get_previous_leaf_if_indentation(
node.get_previous_leaf())
if previous_leaf is None:
content = node.prefix
previous_start_pos = 1, 0
else:
assert previous_leaf.end_pos <= node.start_pos, \
(previous_leaf, node)
content = previous_leaf.value + node.prefix
previous_start_pos = previous_leaf.start_pos
if '\n' in content or '\r' in content:
splitted = split_lines(content)
line = previous_start_pos[0] + len(splitted) - 1
actual = line, len(splitted[-1])
else:
actual = previous_start_pos[0], previous_start_pos[1] + len(
content)
if content.startswith(BOM_UTF8_STRING) \
and node.get_start_pos_of_prefix() == (1, 0):
# Remove the byte order mark
actual = actual[0], actual[1] - 1
assert node.start_pos == actual, (node.start_pos, actual)
else:
for child in children:
assert child.parent == node, (node, child)
_assert_valid_graph(child)
def close(self):
while self._tos is not None:
self._close_tos()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
end_pos = list(last_leaf.end_pos)
except IndexError:
end_pos = [1, 0]
lines = split_lines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos), self.prefix + self._last_prefix)
endmarker.parent = self._module
self._module.children.append(endmarker)
def close(self):
while self._tos is not None:
self._close_tos()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
end_pos = list(last_leaf.end_pos)
except IndexError:
end_pos = [1, 0]
lines = split_lines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos), self.prefix + self._last_prefix)
endmarker.parent = self._module
self._module.children.append(endmarker)
def _assert_valid_graph(node):
"""
Checks if the parent/children relationship is correct.
This is a check that only runs during debugging/testing.
"""
try:
children = node.children
except AttributeError:
# Ignore INDENT is necessary, because indent/dedent tokens don't
# contain value/prefix and are just around, because of the tokenizer.
if node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS:
assert not node.value
assert not node.prefix
return
# Calculate the content between two start positions.
previous_leaf = _get_previous_leaf_if_indentation(node.get_previous_leaf())
if previous_leaf is None:
content = node.prefix
previous_start_pos = 1, 0
else:
assert previous_leaf.end_pos <= node.start_pos, \
(previous_leaf, node)
content = previous_leaf.value + node.prefix
previous_start_pos = previous_leaf.start_pos
if '\n' in content or '\r' in content:
splitted = split_lines(content)
line = previous_start_pos[0] + len(splitted) - 1
actual = line, len(splitted[-1])
else:
actual = previous_start_pos[0], previous_start_pos[1] + len(content)
assert node.start_pos == actual, (node.start_pos, actual)
else:
for child in children:
assert child.parent == node, (node, child)
_assert_valid_graph(child)
def close(self):
self._base_node.finish()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
except IndexError:
end_pos = [1, 0]
else:
last_leaf = _skip_dedent_error_leaves(last_leaf)
end_pos = list(last_leaf.end_pos)
lines = split_lines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos), self.prefix + self._prefix_remainder)
endmarker.parent = self._module
self._module.children.append(endmarker)
def get_last_line(self, suffix):
line = 0
if self._children_groups:
children_group = self._children_groups[-1]
last_leaf = _get_previous_leaf_if_indentation(
children_group.last_line_offset_leaf)
line = last_leaf.end_pos[0] + children_group.line_offset
# Newlines end on the next line, which means that they would cover
# the next line. That line is not fully parsed at this point.
if _ends_with_newline(last_leaf, suffix):
line -= 1
line += len(split_lines(suffix)) - 1
if suffix and not suffix.endswith('\n') and not suffix.endswith('\r'):
# This is the end of a file (that doesn't end with a newline).
line += 1
if self._node_children:
return max(line, self._node_children[-1].get_last_line(suffix))
return line
def get_last_line(self, suffix):
line = 0
if self._children_groups:
children_group = self._children_groups[-1]
last_leaf = _get_previous_leaf_if_indentation(
children_group.last_line_offset_leaf
)
line = last_leaf.end_pos[0] + children_group.line_offset
# Newlines end on the next line, which means that they would cover
# the next line. That line is not fully parsed at this point.
if _ends_with_newline(last_leaf, suffix):
line -= 1
line += len(split_lines(suffix)) - 1
if suffix and not suffix.endswith('\n') and not suffix.endswith('\r'):
# This is the end of a file (that doesn't end with a newline).
line += 1
if self._node_children:
return max(line, self._node_children[-1].get_last_line(suffix))
return line
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
lines = split_lines(code, keepends=True)
diff_parser = DiffParser(
self.grammar._pgen_grammar,
self.grammar._tokenizer,
self.module,
)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
_assert_valid_graph(new_module)
without_diff_parser_module = parse(code)
_assert_nodes_are_equal(new_module, without_diff_parser_module)
error_node = _check_error_leaves_nodes(new_module)
assert expect_error_leaves == (error_node is not None), error_node
if parsers is not ANY:
assert diff_parser._parser_count == parsers
if copies is not ANY:
assert diff_parser._copy_count == copies
return new_module
def end_pos(self):
lines = split_lines(self.string)
if len(lines) > 1:
return self.start_pos[0] + len(lines) - 1, 0
else:
return self.start_pos[0], self.start_pos[1] + len(self.string)
def test_split_lines_keepends():
assert split_lines('asd\r\n', keepends=True) == ['asd\r\n', '']
assert split_lines('asd\r\n\f', keepends=True) == ['asd\r\n', '\f']
assert split_lines('\fasd\r\n', keepends=True) == ['\fasd\r\n', '']
assert split_lines('', keepends=True) == ['']
assert split_lines('\n', keepends=True) == ['\n', '']
def _parse(self,
code=None,
error_recovery=True,
path=None,
start_symbol=None,
cache=False,
diff_cache=False,
cache_path=None,
file_io=None,
start_pos=(1, 0)):
"""
Wanted python3.5 * operator and keyword only arguments. Therefore just
wrap it all.
start_pos here is just a parameter internally used. Might be static
sometime in the future.
"""
if code is None and path is None and file_io is None:
raise TypeError("Please provide either code or a path.")
if start_symbol is None:
start_symbol = self._start_nonterminal
if error_recovery and start_symbol != 'file_input':
raise NotImplementedError("This is currently not implemented.")
if file_io is None:
if code is None:
file_io = FileIO(path)
else:
file_io = KnownContentFileIO(path, code)
if cache and file_io.path is not None:
module_node = load_module(self._hashed,
file_io,
cache_path=cache_path)
if module_node is not None:
return module_node
if code is None:
code = file_io.read()
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:
if self._diff_parser is None:
raise TypeError("You have to define a diff parser to be able "
"to use this option.")
try:
module_cache_item = parser_cache[self._hashed][file_io.path]
except KeyError:
pass
else:
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
return module_node
new_node = self._diff_parser(
self._pgen_grammar, self._tokenizer,
module_node).update(old_lines=old_lines, new_lines=lines)
try_to_save_module(
self._hashed,
file_io,
new_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return new_node
tokens = self._tokenizer(lines, start_pos=start_pos)
p = self._parser(self._pgen_grammar,
error_recovery=error_recovery,
start_nonterminal=start_symbol)
root_node = p.parse(tokens=tokens)
if cache or diff_cache:
try_to_save_module(
self._hashed,
file_io,
root_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return root_node
def detect_config(
source: Union[str, bytes],
*,
partial: PartialParserConfig,
detect_trailing_newline: bool,
) -> ConfigDetectionResult:
"""
Computes a ParserConfig given the current source code to be parsed and a partial
config.
"""
python_version = partial.parsed_python_version
partial_encoding = partial.encoding
encoding = (
_detect_encoding(source)
if isinstance(partial_encoding, AutoConfig)
else partial_encoding
)
source_str = source if isinstance(source, str) else source.decode(encoding)
partial_default_newline = partial.default_newline
default_newline = (
_detect_default_newline(source_str)
if isinstance(partial_default_newline, AutoConfig)
else partial_default_newline
)
# HACK: The grammar requires a trailing newline, but python doesn't actually require
# a trailing newline. Add one onto the end to make the parser happy. We'll strip it
# out again during cst.Module's codegen.
#
# I think parso relies on error recovery support to handle this, which we don't
# have. lib2to3 doesn't handle this case at all AFAICT.
has_trailing_newline = detect_trailing_newline and bool(
len(source_str) != 0 and NEWLINE_RE.match(source_str[-1])
)
if detect_trailing_newline and not has_trailing_newline:
source_str += default_newline
lines = split_lines(source_str, keepends=True)
tokens = tokenize_lines(lines, python_version)
partial_default_indent = partial.default_indent
if isinstance(partial_default_indent, AutoConfig):
# We need to clone `tokens` before passing it to `_detect_indent`, because
# `_detect_indent` consumes some tokens, mutating `tokens`.
#
# Implementation detail: CPython's `itertools.tee` uses weakrefs to reduce the
# size of its FIFO, so this doesn't retain items (leak memory) for `tokens_dup`
# once `token_dup` is freed at the end of this method (subject to
# GC/refcounting).
tokens, tokens_dup = itertools.tee(tokens)
default_indent = _detect_indent(tokens_dup)
else:
default_indent = partial_default_indent
return ConfigDetectionResult(
config=ParserConfig(
lines=lines,
encoding=encoding,
default_indent=default_indent,
default_newline=default_newline,
has_trailing_newline=has_trailing_newline,
),
tokens=tokens,
)
def test_split_lines(string, expected_result, keepends):
assert split_lines(string, keepends=keepends) == expected_result
def test_split_lines_no_keepends():
assert split_lines('asd\r\n') == ['asd', '']
assert split_lines('asd\r\n\f') == ['asd', '\f']
assert split_lines('\fasd\r\n') == ['\fasd', '']
assert split_lines('') == ['']
assert split_lines('\n') == ['', '']
def tokenize(code: str,
version_info: PythonVersionInfo) -> Generator[Token, None, None]:
lines = split_lines(code, keepends=True)
return tokenize_lines(lines, version_info)
def insert_line_into_code(code, index, line):
lines = split_lines(code, keepends=True)
lines.insert(index, line)
return ''.join(lines)
def update(self, old_lines, new_lines):
'''
The algorithm works as follows:
Equal:
- Assure that the start is a newline, otherwise parse until we get
one.
- Copy from parsed_until_line + 1 to max(i2 + 1)
- Make sure that the indentation is correct (e.g. add DEDENT)
- Add old and change positions
Insert:
- Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
much more.
Returns the new module node.
'''
LOG.debug('diff parser start')
# Reset the used names cache so they get regenerated.
self._module._used_names = None
self._parser_lines_new = new_lines
self._reset()
line_length = len(new_lines)
sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new)
opcodes = sm.get_opcodes()
LOG.debug('diff parser calculated')
LOG.debug('diff: line_lengths old: %s, new: %s' %
(len(old_lines), line_length))
for operation, i1, i2, j1, j2 in opcodes:
LOG.debug('diff code[%s] old[%s:%s] new[%s:%s]', operation, i1 + 1,
i2, j1 + 1, j2)
if j2 == line_length and new_lines[-1] == '':
# The empty part after the last newline is not relevant.
j2 -= 1
if operation == 'equal':
line_offset = j1 - i1
self._copy_from_old_parser(line_offset, i2, j2)
elif operation == 'replace':
self._parse(until_line=j2)
elif operation == 'insert':
self._parse(until_line=j2)
else:
assert operation == 'delete'
# With this action all change will finally be applied and we have a
# changed module.
self._nodes_stack.close()
last_pos = self._module.end_pos[0]
if last_pos != line_length:
current_lines = split_lines(self._module.get_code(), keepends=True)
diff = difflib.unified_diff(current_lines, new_lines)
raise Exception(
"There's an issue (%s != %s) with the diff parser. Please report:\n%s"
% (last_pos, line_length, ''.join(diff)))
LOG.debug('diff parser end')
return self._module
def _copy_nodes(self, working_stack, nodes, until_line, line_offset, prefix=''):
new_nodes = []
new_prefix = ''
for node in nodes:
if node.start_pos[0] > until_line:
break
if node.type == 'endmarker':
break
if node.type == 'error_leaf' and node.token_type in ('DEDENT', 'ERROR_DEDENT'):
break
# TODO this check might take a bit of time for large files. We
# might want to change this to do more intelligent guessing or
# binary search.
if _get_last_line(node) > until_line:
# We can split up functions and classes later.
if _func_or_class_has_suite(node):
new_nodes.append(node)
break
new_nodes.append(node)
if not new_nodes:
return [], working_stack, prefix
tos = working_stack[-1]
last_node = new_nodes[-1]
had_valid_suite_last = False
if _func_or_class_has_suite(last_node):
suite = last_node
while suite.type != 'suite':
suite = suite.children[-1]
suite_tos = _NodesTreeNode(suite)
# Don't need to pass line_offset here, it's already done by the
# parent.
suite_nodes, new_working_stack, new_prefix = self._copy_nodes(
working_stack + [suite_tos], suite.children, until_line, line_offset
)
if len(suite_nodes) < 2:
# A suite only with newline is not valid.
new_nodes.pop()
new_prefix = ''
else:
assert new_nodes
tos.add_child_node(suite_tos)
working_stack = new_working_stack
had_valid_suite_last = True
if new_nodes:
last_node = new_nodes[-1]
if (last_node.type in ('error_leaf', 'error_node') or
_is_flow_node(new_nodes[-1])):
# Error leafs/nodes don't have a defined start/end. Error
# nodes might not end with a newline (e.g. if there's an
# open `(`). Therefore ignore all of them unless they are
# succeeded with valid parser state.
# If we copy flows at the end, they might be continued
# after the copy limit (in the new parser).
# In this while loop we try to remove until we find a newline.
new_prefix = ''
new_nodes.pop()
while new_nodes:
last_node = new_nodes[-1]
if last_node.get_last_leaf().type == 'newline':
break
new_nodes.pop()
if new_nodes:
if not _ends_with_newline(new_nodes[-1].get_last_leaf()) and not had_valid_suite_last:
p = new_nodes[-1].get_next_leaf().prefix
# We are not allowed to remove the newline at the end of the
# line, otherwise it's going to be missing. This happens e.g.
# if a bracket is around before that moves newlines to
# prefixes.
new_prefix = split_lines(p, keepends=True)[0]
if had_valid_suite_last:
last = new_nodes[-1]
if last.type == 'decorated':
last = last.children[-1]
last_line_offset_leaf = last.children[-2].get_last_leaf()
assert last_line_offset_leaf == ':'
else:
last_line_offset_leaf = new_nodes[-1].get_last_leaf()
tos.add_tree_nodes(prefix, new_nodes, line_offset, last_line_offset_leaf)
prefix = new_prefix
self._prefix_remainder = ''
return new_nodes, working_stack, prefix
def parse(self,
code: Union[str, bytes] = None,
*,
error_recovery=True,
path: Union[os.PathLike, str] = None,
start_symbol: str = None,
cache=False,
diff_cache=False,
cache_path: Union[os.PathLike, str] = None,
file_io: FileIO = None) -> _NodeT:
"""
If you want to parse a Python file you want to start here, most likely.
If you need finer grained control over the parsed instance, there will be
other ways to access it.
:param str code: A unicode or bytes string. When it's not possible to
decode bytes to a string, returns a
:py:class:`UnicodeDecodeError`.
:param bool error_recovery: If enabled, any code will be returned. If
it is invalid, it will be returned as an error node. If disabled,
you will get a ParseError when encountering syntax errors in your
code.
:param str start_symbol: The grammar rule (nonterminal) that you want
to parse. Only allowed to be used when error_recovery is False.
:param str path: The path to the file you want to open. Only needed for caching.
:param bool cache: Keeps a copy of the parser tree in RAM and on disk
if a path is given. Returns the cached trees if the corresponding
files on disk have not changed. Note that this stores pickle files
on your file system (e.g. for Linux in ``~/.cache/parso/``).
:param bool diff_cache: Diffs the cached python module against the new
code and tries to parse only the parts that have changed. Returns
the same (changed) module that is found in cache. Using this option
requires you to not do anything anymore with the cached modules
under that path, because the contents of it might change. This
option is still somewhat experimental. If you want stability,
please don't use it.
:param bool cache_path: If given saves the parso cache in this
directory. If not given, defaults to the default cache places on
each platform.
:return: A subclass of :py:class:`parso.tree.NodeOrLeaf`. Typically a
:py:class:`parso.python.tree.Module`.
"""
if code is None and path is None and file_io is None:
raise TypeError("Please provide either code or a path.")
if isinstance(path, str):
path = Path(path)
if isinstance(cache_path, str):
cache_path = Path(cache_path)
if start_symbol is None:
start_symbol = self._start_nonterminal
if error_recovery and start_symbol != 'file_input':
raise NotImplementedError("This is currently not implemented.")
if file_io is None:
if code is None:
file_io = FileIO(path) # type: ignore
else:
file_io = KnownContentFileIO(path, code)
if cache and file_io.path is not None:
module_node = load_module(self._hashed,
file_io,
cache_path=cache_path)
if module_node is not None:
return module_node # type: ignore
if code is None:
code = file_io.read()
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:
if self._diff_parser is None:
raise TypeError("You have to define a diff parser to be able "
"to use this option.")
try:
module_cache_item = parser_cache[self._hashed][file_io.path]
except KeyError:
pass
else:
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
return module_node # type: ignore
new_node = self._diff_parser(
self._pgen_grammar, self._tokenizer,
module_node).update(old_lines=old_lines, new_lines=lines)
try_to_save_module(
self._hashed,
file_io,
new_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return new_node # type: ignore
tokens = self._tokenizer(lines)
p = self._parser(self._pgen_grammar,
error_recovery=error_recovery,
start_nonterminal=start_symbol)
root_node = p.parse(tokens=tokens)
if cache or diff_cache:
try_to_save_module(
self._hashed,
file_io,
root_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return root_node # type: ignore
def _parse(self, code=None, error_recovery=True, path=None,
start_symbol=None, cache=False, diff_cache=False,
cache_path=None, file_io=None, start_pos=(1, 0)):
"""
Wanted python3.5 * operator and keyword only arguments. Therefore just
wrap it all.
start_pos here is just a parameter internally used. Might be public
sometime in the future.
"""
if code is None and path is None and file_io is None:
raise TypeError("Please provide either code or a path.")
if start_symbol is None:
start_symbol = self._start_nonterminal
if error_recovery and start_symbol != 'file_input':
raise NotImplementedError("This is currently not implemented.")
if file_io is None:
if code is None:
file_io = FileIO(path)
else:
file_io = KnownContentFileIO(path, code)
if cache and file_io.path is not None:
module_node = load_module(self._hashed, file_io, cache_path=cache_path)
if module_node is not None:
return module_node
if code is None:
code = file_io.read()
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:
if self._diff_parser is None:
raise TypeError("You have to define a diff parser to be able "
"to use this option.")
try:
module_cache_item = parser_cache[self._hashed][file_io.path]
except KeyError:
pass
else:
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
return module_node
new_node = self._diff_parser(
self._pgen_grammar, self._tokenizer, module_node
).update(
old_lines=old_lines,
new_lines=lines
)
save_module(self._hashed, file_io, new_node, lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return new_node
tokens = self._tokenizer(lines, start_pos)
p = self._parser(
self._pgen_grammar,
error_recovery=error_recovery,
start_nonterminal=start_symbol
)
root_node = p.parse(tokens=tokens)
if cache or diff_cache:
save_module(self._hashed, file_io, root_node, lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return root_node
def tokenize(code, version_info, start_pos=(1, 0)):
"""Generate tokens from a the source code (string)."""
lines = split_lines(code, keepends=True)
return tokenize_lines(lines, version_info, start_pos=start_pos)
def end_pos(self):
lines = split_lines(self.string)
if len(lines) > 1:
return self.start_pos[0] + len(lines) - 1, 0
else:
return self.start_pos[0], self.start_pos[1] + len(self.string)
def test_split_lines(string, expected_result, keepends):
assert split_lines(string, keepends=keepends) == expected_result
def check(code):
tokens = _get_token_list(code)
lines = split_lines(code)
assert tokens[-1].end_pos == (len(lines), len(lines[-1]))
def update(self, old_lines, new_lines):
'''
The algorithm works as follows:
Equal:
- Assure that the start is a newline, otherwise parse until we get
one.
- Copy from parsed_until_line + 1 to max(i2 + 1)
- Make sure that the indentation is correct (e.g. add DEDENT)
- Add old and change positions
Insert:
- Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
much more.
Returns the new module node.
'''
LOG.debug('diff parser start')
# Reset the used names cache so they get regenerated.
self._module._used_names = None
self._parser_lines_new = new_lines
self._reset()
line_length = len(new_lines)
sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new)
opcodes = sm.get_opcodes()
LOG.debug('diff parser calculated')
LOG.debug('diff: line_lengths old: %s, new: %s' % (len(old_lines), line_length))
for operation, i1, i2, j1, j2 in opcodes:
LOG.debug('diff code[%s] old[%s:%s] new[%s:%s]',
operation, i1 + 1, i2, j1 + 1, j2)
if j2 == line_length and new_lines[-1] == '':
# The empty part after the last newline is not relevant.
j2 -= 1
if operation == 'equal':
line_offset = j1 - i1
self._copy_from_old_parser(line_offset, i2, j2)
elif operation == 'replace':
self._parse(until_line=j2)
elif operation == 'insert':
self._parse(until_line=j2)
else:
assert operation == 'delete'
# With this action all change will finally be applied and we have a
# changed module.
self._nodes_stack.close()
last_pos = self._module.end_pos[0]
if last_pos != line_length:
current_lines = split_lines(self._module.get_code(), keepends=True)
diff = difflib.unified_diff(current_lines, new_lines)
raise Exception(
"There's an issue (%s != %s) with the diff parser. Please report:\n%s"
% (last_pos, line_length, ''.join(diff))
)
LOG.debug('diff parser end')
return self._module
def tokenize(code, version_info, start_pos=(1, 0)):
"""Generate tokens from a the source code (string)."""
lines = split_lines(code, keepends=True)
return tokenize_lines(lines, version_info, start_pos=start_pos)
def insert_line_into_code(code, index, line):
lines = split_lines(code, keepends=True)
lines.insert(index, line)
return ''.join(lines)
def _copy_nodes(self, working_stack, nodes, until_line, line_offset, prefix=''):
new_nodes = []
new_prefix = ''
for node in nodes:
if node.start_pos[0] > until_line:
break
if node.type == 'endmarker':
break
if node.type == 'error_leaf' and node.token_type in ('DEDENT', 'ERROR_DEDENT'):
break
# TODO this check might take a bit of time for large files. We
# might want to change this to do more intelligent guessing or
# binary search.
if _get_last_line(node) > until_line:
# We can split up functions and classes later.
if _func_or_class_has_suite(node):
new_nodes.append(node)
break
new_nodes.append(node)
if not new_nodes:
return [], working_stack, prefix
tos = working_stack[-1]
last_node = new_nodes[-1]
had_valid_suite_last = False
if _func_or_class_has_suite(last_node):
suite = last_node
while suite.type != 'suite':
suite = suite.children[-1]
suite_tos = _NodesTreeNode(suite)
# Don't need to pass line_offset here, it's already done by the
# parent.
suite_nodes, new_working_stack, new_prefix = self._copy_nodes(
working_stack + [suite_tos], suite.children, until_line, line_offset
)
if len(suite_nodes) < 2:
# A suite only with newline is not valid.
new_nodes.pop()
new_prefix = ''
else:
assert new_nodes
tos.add_child_node(suite_tos)
working_stack = new_working_stack
had_valid_suite_last = True
if new_nodes:
last_node = new_nodes[-1]
if (last_node.type in ('error_leaf', 'error_node') or
_is_flow_node(new_nodes[-1])):
# Error leafs/nodes don't have a defined start/end. Error
# nodes might not end with a newline (e.g. if there's an
# open `(`). Therefore ignore all of them unless they are
# succeeded with valid parser state.
# If we copy flows at the end, they might be continued
# after the copy limit (in the new parser).
# In this while loop we try to remove until we find a newline.
new_prefix = ''
new_nodes.pop()
while new_nodes:
last_node = new_nodes[-1]
if last_node.get_last_leaf().type == 'newline':
break
new_nodes.pop()
if new_nodes:
if not _ends_with_newline(new_nodes[-1].get_last_leaf()) and not had_valid_suite_last:
p = new_nodes[-1].get_next_leaf().prefix
# We are not allowed to remove the newline at the end of the
# line, otherwise it's going to be missing. This happens e.g.
# if a bracket is around before that moves newlines to
# prefixes.
new_prefix = split_lines(p, keepends=True)[0]
if had_valid_suite_last:
last = new_nodes[-1]
if last.type == 'decorated':
last = last.children[-1]
if last.type in ('async_funcdef', 'async_stmt'):
last = last.children[-1]
last_line_offset_leaf = last.children[-2].get_last_leaf()
assert last_line_offset_leaf == ':'
else:
last_line_offset_leaf = new_nodes[-1].get_last_leaf()
tos.add_tree_nodes(prefix, new_nodes, line_offset, last_line_offset_leaf)
prefix = new_prefix
self._prefix_remainder = ''
return new_nodes, working_stack, prefix
def _convert_token( # noqa: C901: too complex
state: _TokenizeState, curr_token: OrigToken,
next_token: Optional[OrigToken]) -> Token:
ct_type = curr_token.type
ct_string = curr_token.string
ct_start_pos = curr_token.start_pos
if ct_type is _ERRORTOKEN:
raise ParserSyntaxError(
message="invalid token",
encountered=ct_string,
expected=["TOKEN"],
pos=curr_token.start_pos,
lines=state.lines,
)
if ct_type is _ERROR_DEDENT:
raise ParserSyntaxError(
message="inconsistent indentation",
encountered=next_token.string if next_token is not None else None,
expected=["DEDENT"],
pos=curr_token.start_pos,
lines=state.lines,
)
# Compute relative indent changes for indent/dedent nodes
relative_indent: Optional[str] = None
if ct_type is _INDENT:
old_indent = "" if len(state.indents) < 2 else state.indents[-2]
new_indent = state.indents[-1]
relative_indent = new_indent[len(old_indent):]
if next_token is not None:
nt_type = next_token.type
if nt_type is _INDENT:
nt_line, nt_column = next_token.start_pos
state.indents.append(state.lines[nt_line - 1][:nt_column])
elif nt_type is _DEDENT:
state.indents.pop()
whitespace_before = state.previous_whitespace_state
if ct_type is _INDENT or ct_type is _DEDENT or ct_type is _ENDMARKER:
# Don't update whitespace state for these dummy tokens. This makes it possible
# to partially parse whitespace for IndentedBlock footers, and then parse the
# rest of the whitespace in the following statement's leading_lines.
# Unfortunately, that means that the indentation is either wrong for the footer
# comments, or for the next line. We've chosen to allow it to be wrong for the
# IndentedBlock footer and manually override the state when parsing whitespace
# in that particular node.
whitespace_after = whitespace_before
ct_end_pos = ct_start_pos
else:
# Not a dummy token, so update the whitespace state.
# Compute our own end_pos, since parso's end_pos is wrong for triple-strings.
lines = split_lines(ct_string)
if len(lines) > 1:
ct_end_pos = ct_start_pos[0] + len(lines) - 1, len(lines[-1])
else:
ct_end_pos = (ct_start_pos[0], ct_start_pos[1] + len(ct_string))
# Figure out what mode the whitespace parser should use. If we're inside
# parentheses, certain whitespace (e.g. newlines) are allowed where they would
# otherwise not be. f-strings override and disable this behavior, however.
#
# Parso's tokenizer tracks this internally, but doesn't expose it, so we have to
# duplicate that logic here.
pof_stack = state.parenthesis_or_fstring_stack
if ct_type is _FSTRING_START:
pof_stack.append(_FSTRING_STACK_ENTRY)
elif ct_type is _FSTRING_END:
pof_stack.pop()
elif ct_type is _OP:
if ct_string in "([{":
pof_stack.append(_PARENTHESIS_STACK_ENTRY)
elif ct_string in ")]}":
pof_stack.pop()
is_parenthesized = (len(pof_stack) > 0
and pof_stack[-1] == _PARENTHESIS_STACK_ENTRY)
whitespace_after = WhitespaceState(ct_end_pos[0], ct_end_pos[1],
state.indents[-1], is_parenthesized)
# Hold onto whitespace_after, so we can use it as whitespace_before in the next
# node.
state.previous_whitespace_state = whitespace_after
return Token(
ct_type,
ct_string,
ct_start_pos,
ct_end_pos,
whitespace_before,
whitespace_after,
relative_indent,
)
def _parse(self,
code=None,
error_recovery=True,
path=None,
start_symbol=None,
cache=False,
diff_cache=False,
cache_path=None,
start_pos=(1, 0)):
"""
Wanted python3.5 * operator and keyword only arguments. Therefore just
wrap it all.
start_pos here is just a parameter internally used. Might be public
sometime in the future.
"""
if code is None and path is None:
raise TypeError("Please provide either code or a path.")
if start_symbol is None:
start_symbol = self._start_symbol
if error_recovery and start_symbol != 'file_input':
raise NotImplementedError("This is currently not implemented.")
if cache and code is None and path is not None:
# With the current architecture we cannot load from cache if the
# code is given, because we just load from cache if it's not older than
# the latest change (file last modified).
module_node = load_module(self._hashed,
path,
cache_path=cache_path)
if module_node is not None:
return module_node
if code is None:
with open(path, 'rb') as f:
code = f.read()
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:
if self._diff_parser is None:
raise TypeError("You have to define a diff parser to be able "
"to use this option.")
try:
module_cache_item = parser_cache[self._hashed][path]
except KeyError:
pass
else:
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
return module_node
new_node = self._diff_parser(
self._pgen_grammar, self._tokenizer,
module_node).update(old_lines=old_lines, new_lines=lines)
save_module(
self._hashed,
path,
new_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return new_node
tokens = self._tokenizer(lines, start_pos)
p = self._parser(self._pgen_grammar,
error_recovery=error_recovery,
start_symbol=start_symbol)
root_node = p.parse(tokens=tokens)
if cache or diff_cache:
save_module(
self._hashed,
path,
root_node,
lines,
# Never pickle in pypy, it's slow as hell.
pickling=cache and not is_pypy,
cache_path=cache_path)
return root_node
def _copy_nodes(self,
working_stack,
nodes,
until_line,
line_offset,
prefix='',
is_nested=False):
new_nodes = []
added_indents = []
nodes = list(
self._get_matching_indent_nodes(
nodes,
is_new_suite=is_nested,
))
new_prefix = ''
for node in nodes:
if node.start_pos[0] > until_line:
break
if node.type == 'endmarker':
break
if node.type == 'error_leaf' and node.token_type in (
'DEDENT', 'ERROR_DEDENT'):
break
# TODO this check might take a bit of time for large files. We
# might want to change this to do more intelligent guessing or
# binary search.
if _get_last_line(node) > until_line:
# We can split up functions and classes later.
if _func_or_class_has_suite(node):
new_nodes.append(node)
break
try:
c = node.children
except AttributeError:
pass
else:
# This case basically appears with error recovery of one line
# suites like `def foo(): bar.-`. In this case we might not
# include a newline in the statement and we need to take care
# of that.
n = node
if n.type == 'decorated':
n = n.children[-1]
if n.type in ('async_funcdef', 'async_stmt'):
n = n.children[-1]
if n.type in ('classdef', 'funcdef'):
suite_node = n.children[-1]
else:
suite_node = c[-1]
if suite_node.type in ('error_leaf', 'error_node'):
break
new_nodes.append(node)
# Pop error nodes at the end from the list
if new_nodes:
while new_nodes:
last_node = new_nodes[-1]
if (last_node.type in ('error_leaf', 'error_node')
or _is_flow_node(new_nodes[-1])):
# Error leafs/nodes don't have a defined start/end. Error
# nodes might not end with a newline (e.g. if there's an
# open `(`). Therefore ignore all of them unless they are
# succeeded with valid parser state.
# If we copy flows at the end, they might be continued
# after the copy limit (in the new parser).
# In this while loop we try to remove until we find a newline.
new_prefix = ''
new_nodes.pop()
while new_nodes:
last_node = new_nodes[-1]
if last_node.get_last_leaf().type == 'newline':
break
new_nodes.pop()
continue
if len(new_nodes) > 1 and new_nodes[-2].type == 'error_node':
# The problem here is that Parso error recovery sometimes
# influences nodes before this node.
# Since the new last node is an error node this will get
# cleaned up in the next while iteration.
new_nodes.pop()
continue
break
if not new_nodes:
return [], working_stack, prefix, added_indents
tos = working_stack[-1]
last_node = new_nodes[-1]
had_valid_suite_last = False
# Pop incomplete suites from the list
if _func_or_class_has_suite(last_node):
suite = last_node
while suite.type != 'suite':
suite = suite.children[-1]
indent = _get_suite_indentation(suite)
added_indents.append(indent)
suite_tos = _NodesTreeNode(suite,
indentation=_get_indentation(last_node))
# Don't need to pass line_offset here, it's already done by the
# parent.
suite_nodes, new_working_stack, new_prefix, ai = self._copy_nodes(
working_stack + [suite_tos],
suite.children,
until_line,
line_offset,
is_nested=True,
)
added_indents += ai
if len(suite_nodes) < 2:
# A suite only with newline is not valid.
new_nodes.pop()
new_prefix = ''
else:
assert new_nodes
tos.add_child_node(suite_tos)
working_stack = new_working_stack
had_valid_suite_last = True
if new_nodes:
if not _ends_with_newline(new_nodes[-1].get_last_leaf()
) and not had_valid_suite_last:
p = new_nodes[-1].get_next_leaf().prefix
# We are not allowed to remove the newline at the end of the
# line, otherwise it's going to be missing. This happens e.g.
# if a bracket is around before that moves newlines to
# prefixes.
new_prefix = split_lines(p, keepends=True)[0]
if had_valid_suite_last:
last = new_nodes[-1]
if last.type == 'decorated':
last = last.children[-1]
if last.type in ('async_funcdef', 'async_stmt'):
last = last.children[-1]
last_line_offset_leaf = last.children[-2].get_last_leaf()
assert last_line_offset_leaf == ':'
else:
last_line_offset_leaf = new_nodes[-1].get_last_leaf()
tos.add_tree_nodes(
prefix,
new_nodes,
line_offset,
last_line_offset_leaf,
)
prefix = new_prefix
self._prefix_remainder = ''
return new_nodes, working_stack, prefix, added_indents
