def complete(self, text, state):
"""
This complete stuff is pretty weird, a generator would make
a lot more sense, but probably due to backwards compatibility
this is still the way how it works.
The only important part is stuff in the ``state == 0`` flow,
everything else has been copied from the ``rlcompleter`` std.
library module.
"""
if state == 0:
sys.path.insert(0, os.getcwd())
# Calling python doesn't have a path, so add to sys.path.
try:
interpreter = Interpreter(text,
[namespace_module.__dict__])
lines = common.splitlines(text)
position = (len(lines), len(lines[-1]))
name = get_on_completion_name(lines, position)
before = text[:len(text) - len(name)]
completions = interpreter.completions()
finally:
sys.path.pop(0)
self.matches = [
before + c.name_with_symbols for c in completions
]
try:
return self.matches[state]
except IndexError:
return None
def _is_on_comment(leaf, position):
# We might be on a comment.
if leaf.type == 'endmarker':
try:
dedent = leaf.get_previous_leaf()
if dedent.type == 'dedent' and dedent.prefix:
# TODO This is needed because the fast parser uses multiple
# endmarker tokens within a file which is obviously ugly.
# This is so ugly that I'm not even commenting how it exactly
# happens, but let me tell you that I want to get rid of it.
leaf = dedent
except IndexError:
pass
comment_lines = common.splitlines(leaf.prefix)
difference = leaf.start_pos[0] - position[0]
prefix_start_pos = leaf.get_start_pos_of_prefix()
if difference == 0:
indent = leaf.start_pos[1]
elif position[0] == prefix_start_pos[0]:
indent = prefix_start_pos[1]
else:
indent = 0
line = comment_lines[-difference - 1][:position[1] - indent]
return '#' in line
def complete(self, text, state):
"""
This complete stuff is pretty weird, a generator would make
a lot more sense, but probably due to backwards compatibility
this is still the way how it works.
The only important part is stuff in the ``state == 0`` flow,
everything else has been copied from the ``rlcompleter`` std.
library module.
"""
if state == 0:
sys.path.insert(0, os.getcwd())
# Calling python doesn't have a path, so add to sys.path.
try:
logging.debug("Start REPL completion: " + repr(text))
interpreter = Interpreter(text, [namespace_module.__dict__])
lines = common.splitlines(text)
position = (len(lines), len(lines[-1]))
name = get_on_completion_name(interpreter._get_module(), lines, position)
before = text[:len(text) - len(name)]
completions = interpreter.completions()
except:
logging.error("REPL Completion error:\n" + traceback.format_exc())
raise
finally:
sys.path.pop(0)
self.matches = [before + c.name_with_symbols for c in completions]
try:
return self.matches[state]
except IndexError:
return None
def __init__(self, source=None, line=None, column=None, path=None,
encoding='utf-8', source_path=None, source_encoding=None):
if source_path is not None:
warnings.warn("Use path instead of source_path.", DeprecationWarning)
path = source_path
if source_encoding is not None:
warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
encoding = source_encoding
self._orig_path = path
self.path = None if path is None else os.path.abspath(path)
if source is None:
with open(path) as f:
source = f.read()
self.source = common.source_to_unicode(source, encoding)
lines = common.splitlines(self.source)
line = max(len(lines), 1) if line is None else line
if not (0 < line <= len(lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
cache.clear_time_caches()
debug.reset_time()
self._user_context = UserContext(self.source, self._pos)
self._parser = UserContextParser(self.source, path, self._pos, self._user_context)
self._evaluator = Evaluator()
debug.speed('init')
def _is_on_comment(leaf, position):
# We might be on a comment.
if leaf.type == 'endmarker':
try:
dedent = leaf.get_previous_leaf()
if dedent.type == 'dedent' and dedent.prefix:
# TODO This is needed because the fast parser uses multiple
# endmarker tokens within a file which is obviously ugly.
# This is so ugly that I'm not even commenting how it exactly
# happens, but let me tell you that I want to get rid of it.
leaf = dedent
except IndexError:
pass
comment_lines = common.splitlines(leaf.prefix)
difference = leaf.start_pos[0] - position[0]
prefix_start_pos = leaf.get_start_pos_of_prefix()
if difference == 0:
indent = leaf.start_pos[1]
elif position[0] == prefix_start_pos[0]:
indent = prefix_start_pos[1]
else:
indent = 0
line = comment_lines[-difference - 1][:position[1] - indent]
return '#' in line
def _remove_last_newline(node):
endmarker = node.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
prefix = endmarker.prefix
leaf = endmarker.get_previous_leaf()
if prefix:
text = prefix
else:
if leaf is None:
raise ValueError("You're trying to remove a newline from an empty module.")
text = leaf.value
if not text.endswith('\n'):
raise ValueError("There's no newline at the end, cannot remove it.")
text = text[:-1]
if prefix:
endmarker.prefix = text
if leaf is None:
end_pos = (1, 0)
else:
end_pos = leaf.end_pos
lines = splitlines(text, keepends=True)
if len(lines) == 1:
end_pos = end_pos[0], end_pos[1] + len(lines[0])
else:
end_pos = end_pos[0] + len(lines) - 1, len(lines[-1])
endmarker.start_pos = end_pos
else:
leaf.value = text
endmarker.start_pos = leaf.end_pos
def get_completions(self, info):
'''Get Python completions'''
# https://github.com/davidhalter/jedi/blob/master/jedi/utils.py
if jedi is None:
return []
text = info['code']
position = (info['line_num'], info['column'])
interpreter = Interpreter(text, [self.env])
if jedi.__version__ >= LooseVersion('0.10.0'):
lines = common.splitlines(text)
name = get_on_completion_name(interpreter._get_module_node(),
lines, position)
before = text[:len(text) - len(name)]
else:
path = UserContext(text, position).get_path_until_cursor()
path, dot, like = completion_parts(path)
before = text[:len(text) - len(like)]
completions = interpreter.completions()
completions = [before + c.name_with_symbols for c in completions]
self.kernel.log.error(completions)
return [c[info['start']:] for c in completions]
def _rename(names, replace_str):
""" For both rename and inline. """
order = sorted(names,
key=lambda x: (x.module_path, x.line, x.column),
reverse=True)
def process(path, old_lines, new_lines):
if new_lines is not None: # goto next file, save last
dct[path] = path, old_lines, new_lines
dct = {}
current_path = object()
new_lines = old_lines = None
for name in order:
if name.in_builtin_module():
continue
if current_path != name.module_path:
current_path = name.module_path
process(current_path, old_lines, new_lines)
if current_path is not None:
# None means take the source that is a normal param.
with open(current_path) as f:
source = f.read()
new_lines = common.splitlines(common.source_to_unicode(source))
old_lines = new_lines[:]
nr, indent = name.line, name.column
line = new_lines[nr - 1]
new_lines[nr - 1] = line[:indent] + replace_str + \
line[indent + len(name.name):]
process(current_path, old_lines, new_lines)
return dct
def extract(script, new_name):
""" The `args` / `kwargs` params are the same as in `api.Script`.
:param operation: The refactoring operation to execute.
:type operation: str
:type source: str
:return: list of changed lines/changed files
"""
new_lines = common.splitlines(common.source_to_unicode(script.source))
old_lines = new_lines[:]
user_stmt = script._parser.user_stmt()
# TODO care for multiline extracts
dct = {}
if user_stmt:
pos = script._pos
line_index = pos[0] - 1
arr, index = helpers.array_for_pos(user_stmt, pos)
if arr is not None:
start_pos = arr[index].start_pos
end_pos = arr[index].end_pos
# take full line if the start line is different from end line
e = end_pos[1] if end_pos[0] == start_pos[0] else None
start_line = new_lines[start_pos[0] - 1]
text = start_line[start_pos[1]:e]
for l in range(start_pos[0], end_pos[0] - 1):
text += '\n' + l
if e is None:
end_line = new_lines[end_pos[0] - 1]
text += '\n' + end_line[:end_pos[1]]
# remove code from new lines
t = text.lstrip()
del_start = start_pos[1] + len(text) - len(t)
text = t.rstrip()
del_end = len(t) - len(text)
if e is None:
new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:]
e = len(start_line)
else:
e = e - del_end
start_line = start_line[:del_start] + new_name + start_line[e:]
new_lines[start_pos[0] - 1] = start_line
new_lines[start_pos[0]:end_pos[0] - 1] = []
# add parentheses in multiline case
open_brackets = ['(', '[', '{']
close_brackets = [')', ']', '}']
if '\n' in text and not (text[0] in open_brackets and text[-1] ==
close_brackets[open_brackets.index(text[0])]):
text = '(%s)' % text
# add new line before statement
indent = user_stmt.start_pos[1]
new = "%s%s = %s" % (' ' * indent, new_name, text)
new_lines.insert(line_index, new)
dct[script.path] = script.path, old_lines, new_lines
return Refactoring(dct)
def get_completions(self, info):
"""Gets Python completions based on the current cursor position
within the %%init_spark cell.
Based on
https://github.com/Calysto/metakernel/blob/master/metakernel/magics/python_magic.py
Parameters
----------
info : dict
Information about the current caret position
"""
if jedi is None:
return []
text = info['code']
position = (info['line_num'], info['column'])
interpreter = jedi.Interpreter(text, [self.env])
lines = common.splitlines(text)
name = get_on_completion_name(interpreter._get_module_node(), lines,
position)
before = text[:len(text) - len(name)]
completions = interpreter.completions()
completions = [before + c.name_with_symbols for c in completions]
return [c[info['start']:] for c in completions]
def extract(script, new_name):
""" The `args` / `kwargs` params are the same as in `api.Script`.
:param operation: The refactoring operation to execute.
:type operation: str
:type source: str
:return: list of changed lines/changed files
"""
new_lines = common.splitlines(common.source_to_unicode(script.source))
old_lines = new_lines[:]
user_stmt = script._parser.user_stmt()
# TODO care for multiline extracts
dct = {}
if user_stmt:
pos = script._pos
line_index = pos[0] - 1
arr, index = helpers.array_for_pos(user_stmt, pos)
if arr is not None:
start_pos = arr[index].start_pos
end_pos = arr[index].end_pos
# take full line if the start line is different from end line
e = end_pos[1] if end_pos[0] == start_pos[0] else None
start_line = new_lines[start_pos[0] - 1]
text = start_line[start_pos[1]:e]
for l in range(start_pos[0], end_pos[0] - 1):
text += '\n' + l
if e is None:
end_line = new_lines[end_pos[0] - 1]
text += '\n' + end_line[:end_pos[1]]
# remove code from new lines
t = text.lstrip()
del_start = start_pos[1] + len(text) - len(t)
text = t.rstrip()
del_end = len(t) - len(text)
if e is None:
new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:]
e = len(start_line)
else:
e = e - del_end
start_line = start_line[:del_start] + new_name + start_line[e:]
new_lines[start_pos[0] - 1] = start_line
new_lines[start_pos[0]:end_pos[0] - 1] = []
# add parentheses in multiline case
open_brackets = ['(', '[', '{']
close_brackets = [')', ']', '}']
if '\n' in text and not (text[0] in open_brackets and text[-1] ==
close_brackets[open_brackets.index(text[0])]):
text = '(%s)' % text
# add new line before statement
indent = user_stmt.start_pos[1]
new = "%s%s = %s" % (' ' * indent, new_name, text)
new_lines.insert(line_index, new)
dct[script.path] = script.path, old_lines, new_lines
return Refactoring(dct)
def _rename(names, replace_str):
""" For both rename and inline. """
order = sorted(names, key=lambda x: (x.module_path, x.line, x.column),
reverse=True)
def process(path, old_lines, new_lines):
if new_lines is not None: # goto next file, save last
dct[path] = path, old_lines, new_lines
dct = {}
current_path = object()
new_lines = old_lines = None
for name in order:
if name.in_builtin_module():
continue
if current_path != name.module_path:
current_path = name.module_path
process(current_path, old_lines, new_lines)
if current_path is not None:
# None means take the source that is a normal param.
with open(current_path) as f:
source = f.read()
new_lines = common.splitlines(common.source_to_unicode(source))
old_lines = new_lines[:]
nr, indent = name.line, name.column
line = new_lines[nr - 1]
new_lines[nr - 1] = line[:indent] + replace_str + \
line[indent + len(name.name):]
process(current_path, old_lines, new_lines)
return dct
def test_open_string_literal(code):
"""
Testing mostly if removing the last newline works.
"""
lines = splitlines(code, keepends=True)
end_pos = (len(lines), len(lines[-1]))
module = parse(code)
assert module.get_code() == code
assert module.end_pos == end_pos == module.children[1].end_pos
def test_open_string_literal(code):
"""
Testing mostly if removing the last newline works.
"""
lines = splitlines(code, keepends=True)
end_pos = (len(lines), len(lines[-1]))
module = parse(code)
assert module.get_code() == code
assert module.end_pos == end_pos == module.children[1].end_pos
def __call__(self, grammar, source, module_path=None):
pi = parser_cache.get(module_path, None)
if pi is None or not settings.fast_parser:
return ParserWithRecovery(grammar, source, module_path)
parser = pi.parser
d = DiffParser(parser)
new_lines = splitlines(source, keepends=True)
parser.module = parser._parsed = d.update(new_lines)
return parser
def __call__(self, grammar, source, module_path=None):
pi = parser_cache.get(module_path, None)
if pi is None or not settings.fast_parser:
return ParserWithRecovery(grammar, source, module_path)
parser = pi.parser
d = DiffParser(parser)
new_lines = splitlines(source, keepends=True)
parser.module = parser._parsed = d.update(new_lines)
return parser
def main(args):
jedi.set_debug_function(notices=args['--debug'])
with open(args['<file>']) as f:
code = f.read()
grammar = load_grammar()
parser = ParserWithRecovery(grammar, u(code))
code = code + '\na\n' # Add something so the diff parser needs to run.
lines = splitlines(code, keepends=True)
cProfile.runctx('run(parser, lines)', globals(), locals(), sort=args['-s'])
def main(args):
jedi.set_debug_function(notices=args['--debug'])
with open(args['<file>']) as f:
code = f.read()
grammar = load_grammar()
parser = ParserWithRecovery(grammar, u(code))
code = code + '\na\n' # Add something so the diff parser needs to run.
lines = splitlines(code, keepends=True)
cProfile.runctx('run(parser, lines)', globals(), locals(), sort=args['-s'])
def _extract_range(self, definition):
"""Provides the definition range of a given definition
For regular symbols it returns the start and end location of the
characters making up the symbol.
For scoped containers it will return the entire definition of the
scope.
The scope that jedi provides ends with the first character of the next
scope so it's not ideal. For vscode we need the scope to end with the
last character of actual code. That's why we extract the lines that
make up our scope and trim the trailing whitespace.
"""
from jedi import common
from jedi.parser.utils import load_parser
# get the scope range
try:
if definition.type in ['class', 'function'] and hasattr(
definition, '_definition'):
scope = definition._definition
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
end_line = scope.end_pos[0] - 1
end_column = scope.end_pos[1]
# get the lines
path = definition._definition.get_parent_until().path
parser = load_parser(path)
lines = common.splitlines(parser.source)
lines[end_line] = lines[end_line][:end_column]
# trim the lines
lines = lines[start_line:end_line + 1]
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
source_path=None,
source_encoding=None,
sys_path=None):
if source_path is not None:
warnings.warn("Use path instead of source_path.",
DeprecationWarning)
path = source_path
if source_encoding is not None:
warnings.warn("Use encoding instead of source_encoding.",
DeprecationWarning)
encoding = source_encoding
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
try:
with open(path) as f:
source = f.read()
except UnicodeDecodeError:
with open(path, encoding=encoding) as f:
source = f.read()
self._source = common.source_to_unicode(source, encoding)
self._code_lines = common.splitlines(self._source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
if sys_path is None:
venv = os.getenv('VIRTUAL_ENV')
if venv:
sys_path = list(get_venv_path(venv))
self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
debug.speed('init')
def cache_call_signatures(source, user_pos, stmt):
"""This function calculates the cache key."""
index = user_pos[0] - 1
lines = common.splitlines(source)
before_cursor = lines[index][:user_pos[1]]
other_lines = lines[stmt.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = stmt.get_parent_until().path
return None if module_path is None else (module_path, before_bracket, stmt.start_pos)
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
source_path=None,
source_encoding=None,
sys_path=None):
if source_path is not None:
warnings.warn(
"Deprecated since version 0.7. Use path instead of source_path.",
DeprecationWarning,
stacklevel=2)
path = source_path
if source_encoding is not None:
warnings.warn(
"Deprecated since version 0.8. Use encoding instead of source_encoding.",
DeprecationWarning,
stacklevel=2)
encoding = source_encoding
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
self._source = common.source_to_unicode(source, encoding)
self._code_lines = common.splitlines(self._source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
if sys_path is None:
venv = os.getenv('VIRTUAL_ENV')
if venv:
sys_path = list(get_venv_path(venv))
self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
debug.speed('init')
def _extract_range(self, definition):
"""Provides the definition range of a given definition
For regular symbols it returns the start and end location of the
characters making up the symbol.
For scoped containers it will return the entire definition of the
scope.
The scope that jedi provides ends with the first character of the next
scope so it's not ideal. For vscode we need the scope to end with the
last character of actual code. That's why we extract the lines that
make up our scope and trim the trailing whitespace.
"""
from jedi import common
from jedi.parser.utils import load_parser
# get the scope range
try:
if definition.type in ['class', 'function'] and hasattr(definition, '_definition'):
scope = definition._definition
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
end_line = scope.end_pos[0] - 1
end_column = scope.end_pos[1]
# get the lines
path = definition._definition.get_parent_until().path
parser = load_parser(path)
lines = common.splitlines(parser.source)
lines[end_line] = lines[end_line][:end_column]
# trim the lines
lines = lines[start_line:end_line + 1]
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def _is_on_comment(leaf, position):
comment_lines = common.splitlines(leaf.prefix)
difference = leaf.start_pos[0] - position[0]
prefix_start_pos = leaf.get_start_pos_of_prefix()
if difference == 0:
indent = leaf.start_pos[1]
elif position[0] == prefix_start_pos[0]:
indent = prefix_start_pos[1]
else:
indent = 0
line = comment_lines[-difference - 1][:position[1] - indent]
return '#' in line
def _is_on_comment(leaf, position):
comment_lines = common.splitlines(leaf.prefix)
difference = leaf.start_pos[0] - position[0]
prefix_start_pos = leaf.get_start_pos_of_prefix()
if difference == 0:
indent = leaf.start_pos[1]
elif position[0] == prefix_start_pos[0]:
indent = prefix_start_pos[1]
else:
indent = 0
line = comment_lines[-difference - 1][:position[1] - indent]
return '#' in line
def cache_call_signatures(source, user_pos, stmt):
"""This function calculates the cache key."""
index = user_pos[0] - 1
lines = common.splitlines(source)
before_cursor = lines[index][:user_pos[1]]
other_lines = lines[stmt.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = stmt.get_parent_until().path
return None if module_path is None else (module_path, before_bracket,
stmt.start_pos)
def cache_call_signatures(evaluator, call, source, user_pos):
"""This function calculates the cache key."""
index = user_pos[0] - 1
lines = common.splitlines(source)
before_cursor = lines[index][:user_pos[1]]
other_lines = lines[call.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = call.get_parent_until().path
yield None if module_path is None else (module_path, before_bracket, call.start_pos)
yield evaluator.eval_element(call)
def cache_call_signatures(evaluator, call, source, user_pos):
"""This function calculates the cache key."""
index = user_pos[0] - 1
lines = common.splitlines(source)
before_cursor = lines[index][:user_pos[1]]
other_lines = lines[call.start_pos[0]:index]
whole = '\n'.join(other_lines + [before_cursor])
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
module_path = call.get_parent_until().path
yield None if module_path is None else (module_path, before_bracket, call.start_pos)
yield evaluator.eval_element(call)
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
debug.dbg('differ: parse copies=%s parsers=%s', copies, parsers, color='YELLOW')
lines = splitlines(code, keepends=True)
diff_parser = DiffParser(self.grammar, self.module)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
assert diff_parser._copy_count == copies
assert diff_parser._parser_count == parsers
assert expect_error_leaves == _check_error_leaves_nodes(new_module)
_assert_valid_graph(new_module)
return new_module
def get_line(self, line_nr):
if not self._line_cache:
self._line_cache = common.splitlines(self.source)
if line_nr == 0:
# This is a fix for the zeroth line. We need a newline there, for
# the backwards parser.
return u('')
if line_nr < 0:
raise StopIteration()
try:
return self._line_cache[line_nr - 1]
except IndexError:
raise StopIteration()
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
source_path=None,
source_encoding=None,
sys_path=None):
if source_path is not None:
warnings.warn("Use path instead of source_path.",
DeprecationWarning)
path = source_path
if source_encoding is not None:
warnings.warn("Use encoding instead of source_encoding.",
DeprecationWarning)
encoding = source_encoding
self._orig_path = path
self.path = None if path is None else os.path.abspath(path)
if source is None:
with open(path) as f:
source = f.read()
self.source = common.source_to_unicode(source, encoding)
lines = common.splitlines(self.source)
line = max(len(lines), 1) if line is None else line
if not (0 < line <= len(lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
cache.clear_time_caches()
debug.reset_time()
self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
self._user_context = UserContext(self.source, self._pos)
self._parser = UserContextParser(self._grammar, self.source, path,
self._pos, self._user_context,
self._parsed_callback)
if sys_path is None:
venv = os.getenv('VIRTUAL_ENV')
if venv:
sys_path = list(get_venv_path(venv))
self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
debug.speed('init')
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
debug.dbg('differ: parse copies=%s parsers=%s',
copies,
parsers,
color='YELLOW')
lines = splitlines(code, keepends=True)
diff_parser = DiffParser(self.grammar, self.module)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
assert diff_parser._copy_count == copies
assert diff_parser._parser_count == parsers
assert expect_error_leaves == _check_error_leaves_nodes(new_module)
_assert_valid_graph(new_module)
return new_module
def inline(script):
"""
:type script: api.Script
"""
new_lines = common.splitlines(common.source_to_unicode(script.source))
dct = {}
definitions = script.goto_assignments()
with common.ignored(AssertionError):
assert len(definitions) == 1
stmt = definitions[0]._definition
usages = script.usages()
inlines = [
r for r in usages
if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos
]
inlines = sorted(inlines,
key=lambda x: (x.module_path, x.line, x.column),
reverse=True)
expression_list = stmt.expression_list()
# don't allow multiline refactorings for now.
assert stmt.start_pos[0] == stmt.end_pos[0]
index = stmt.start_pos[0] - 1
line = new_lines[index]
replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
replace_str = replace_str.strip()
# tuples need parentheses
if expression_list and isinstance(expression_list[0], pr.Array):
arr = expression_list[0]
if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
replace_str = '(%s)' % replace_str
# if it's the only assignment, remove the statement
if len(stmt.get_defined_names()) == 1:
line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
dct = _rename(inlines, replace_str)
# remove the empty line
new_lines = dct[script.path][2]
if line.strip():
new_lines[index] = line
else:
new_lines.pop(index)
return Refactoring(dct)
def _get_backwards_tokenizer(self, start_pos, line_gen=None):
if line_gen is None:
line_gen = self._backwards_line_generator(start_pos)
token_gen = tokenize.generate_tokens(lambda: next(line_gen))
for typ, tok_str, tok_start_pos, prefix in token_gen:
line = self.get_line(self._line_temp)
# Calculate the real start_pos of the token.
if tok_start_pos[0] == 1:
# We are in the first checked line
column = start_pos[1] - tok_start_pos[1]
else:
column = len(line) - tok_start_pos[1]
# Multi-line docstrings must be accounted for.
first_line = common.splitlines(tok_str)[0]
column -= len(first_line)
# Reverse the token again, so that it is in normal order again.
yield typ, tok_str[::-1], (self._line_temp, column), prefix[::-1]
def inline(script):
"""
:type script: api.Script
"""
new_lines = common.splitlines(common.source_to_unicode(script.source))
dct = {}
definitions = script.goto_assignments()
with common.ignored(AssertionError):
assert len(definitions) == 1
stmt = definitions[0]._definition
usages = script.usages()
inlines = [r for r in usages
if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
reverse=True)
expression_list = stmt.expression_list()
# don't allow multiline refactorings for now.
assert stmt.start_pos[0] == stmt.end_pos[0]
index = stmt.start_pos[0] - 1
line = new_lines[index]
replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
replace_str = replace_str.strip()
# tuples need parentheses
if expression_list and isinstance(expression_list[0], pr.Array):
arr = expression_list[0]
if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
replace_str = '(%s)' % replace_str
# if it's the only assignment, remove the statement
if len(stmt.get_defined_names()) == 1:
line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]
dct = _rename(inlines, replace_str)
# remove the empty line
new_lines = dct[script.path][2]
if line.strip():
new_lines[index] = line
else:
new_lines.pop(index)
return Refactoring(dct)
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
source_path=None,
source_encoding=None):
if source_path is not None:
warnings.warn("Use path instead of source_path.",
DeprecationWarning)
path = source_path
if source_encoding is not None:
warnings.warn("Use encoding instead of source_encoding.",
DeprecationWarning)
encoding = source_encoding
self._orig_path = path
self.path = None if path is None else os.path.abspath(path)
if source is None:
with open(path) as f:
source = f.read()
self.source = common.source_to_unicode(source, encoding)
lines = common.splitlines(self.source)
line = max(len(lines), 1) if line is None else line
if not (0 < line <= len(lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
cache.clear_caches()
debug.reset_time()
self._user_context = UserContext(self.source, self._pos)
self._parser = UserContextParser(self.source, path, self._pos,
self._user_context)
self._evaluator = Evaluator()
debug.speed('init')
def __update(self, lines_new):
"""For now we use this function for better error reporting."""
old_source = self._parser.source
try:
return self._update(lines_new)
except Exception:
# Only log for linux, it's easier.
import sys
if 'linux' in sys.platform:
lines_old = splitlines(old_source, keepends=True)
import traceback
with open('/tmp/jedi_error.log', 'w') as f:
f.write('parser issue, please report:\n%s\n%s\n%s' %
(repr(''.join(lines_old)), repr(
''.join(lines_new)), traceback.format_exc()))
raise Exception(
"There's a parser issue. Please report /tmp/jedi_error.log"
)
raise
def get_line_code(self, before=0, after=0):
"""
Returns the line of code where this object was defined.
:param before: Add n lines before the current line to the output.
:param after: Add n lines after the current line to the output.
:return str: Returns the line(s) of code or an empty string if it's a
builtin.
"""
if self.in_builtin_module():
return ''
path = self._name.get_root_context().py__file__()
parser = load_parser(path)
lines = common.splitlines(parser.source)
line_nr = self._name.start_pos[0]
start_line_nr = line_nr - before
return '\n'.join(lines[start_line_nr:line_nr + after + 1])
def get_line_code(self, before=0, after=0):
"""
Returns the line of code where this object was defined.
:param before: Add n lines before the current line to the output.
:param after: Add n lines after the current line to the output.
:return str: Returns the line(s) of code or an empty string if it's a
builtin.
"""
if self.in_builtin_module():
return ''
path = self._name.get_root_context().py__file__()
parser = load_parser(path)
lines = common.splitlines(parser.source)
line_nr = self._name.start_pos[0]
start_line_nr = line_nr - before
return '\n'.join(lines[start_line_nr:line_nr + after + 1])
def _extract_range_jedi_0_9_0(self, definition):
from jedi import common
from jedi.parser.utils import load_parser
# get the scope range
try:
if definition.type in ['class', 'function'] and hasattr(
definition, '_definition'):
scope = definition._definition
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
end_line = scope.end_pos[0] - 1
end_column = scope.end_pos[1]
# get the lines
path = definition._definition.get_parent_until().path
parser = load_parser(path)
lines = common.splitlines(parser.source)
lines[end_line] = lines[end_line][:end_column]
# trim the lines
lines = lines[start_line:end_line + 1]
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def close(self):
while self._tos is not None:
self._close_tos()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
end_pos = list(last_leaf.end_pos)
except IndexError:
end_pos = [1, 0]
lines = splitlines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos), self.prefix + self._last_prefix)
endmarker.parent = self._module
self._module.children.append(endmarker)
def close(self):
while self._tos is not None:
self._close_tos()
# Add an endmarker.
try:
last_leaf = self._module.get_last_leaf()
end_pos = list(last_leaf.end_pos)
except IndexError:
end_pos = [1, 0]
lines = splitlines(self.prefix)
assert len(lines) > 0
if len(lines) == 1:
end_pos[1] += len(lines[0])
else:
end_pos[0] += len(lines) - 1
end_pos[1] = len(lines[-1])
endmarker = EndMarker('', tuple(end_pos), self.prefix + self._last_prefix)
endmarker.parent = self._module
self._module.children.append(endmarker)
def _extract_range_jedi_0_9_0(self, definition):
from jedi import common
from jedi.parser.utils import load_parser
# get the scope range
try:
if definition.type in ['class', 'function'] and hasattr(definition, '_definition'):
scope = definition._definition
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
end_line = scope.end_pos[0] - 1
end_column = scope.end_pos[1]
# get the lines
path = definition._definition.get_parent_until().path
parser = load_parser(path)
lines = common.splitlines(parser.source)
lines[end_line] = lines[end_line][:end_column]
# trim the lines
lines = lines[start_line:end_line + 1]
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def _remove_last_newline(node):
endmarker = node.children[-1]
# The newline is either in the endmarker as a prefix or the previous
# leaf as a newline token.
prefix = endmarker.prefix
leaf = endmarker.get_previous_leaf()
if prefix:
text = prefix
else:
if leaf is None:
raise ValueError(
"You're trying to remove a newline from an empty module.")
text = leaf.value
if not text.endswith('\n'):
raise ValueError("There's no newline at the end, cannot remove it.")
text = text[:-1]
if prefix:
endmarker.prefix = text
if leaf is None:
end_pos = (1, 0)
else:
end_pos = leaf.end_pos
lines = splitlines(text, keepends=True)
if len(lines) == 1:
end_pos = end_pos[0], end_pos[1] + len(lines[0])
else:
end_pos = end_pos[0] + len(lines) - 1, len(lines[-1])
endmarker.start_pos = end_pos
else:
leaf.value = text
endmarker.start_pos = leaf.end_pos
def _extract_range_jedi_0_10_1(self, definition):
from jedi import common
from jedi.parser.python import parse
# get the scope range
try:
if definition.type in ['class', 'function']:
tree_name = definition._name.tree_name
scope = tree_name.get_definition()
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
# get the lines
code = scope.get_code(include_prefix=False)
lines = common.splitlines(code)
# trim the lines
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name.tree_name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def _extract_range_jedi_0_10_1(self, definition):
from jedi import common
from jedi.parser.python import parse
# get the scope range
try:
if definition.type in ['class', 'function']:
tree_name = definition._name.tree_name
scope = tree_name.get_definition()
start_line = scope.start_pos[0] - 1
start_column = scope.start_pos[1]
# get the lines
code = scope.get_code(include_prefix=False)
lines = common.splitlines(code)
# trim the lines
lines = '\n'.join(lines).rstrip().split('\n')
end_line = start_line + len(lines) - 1
end_column = len(lines[-1]) - 1
else:
symbol = definition._name.tree_name
start_line = symbol.start_pos[0] - 1
start_column = symbol.start_pos[1]
end_line = symbol.end_pos[0] - 1
end_column = symbol.end_pos[1]
return {
'start_line': start_line,
'start_column': start_column,
'end_line': end_line,
'end_column': end_column
}
except Exception as e:
return {
'start_line': definition.line - 1,
'start_column': definition.column,
'end_line': definition.line - 1,
'end_column': definition.column
}
def check(code):
tokens = _get_token_list(code)
lines = splitlines(code)
assert tokens[-1].end_pos == (len(lines), len(lines[-1]))
def initialize(self, code):
debug.dbg('differ: initialize', color='YELLOW')
self.lines = splitlines(code, keepends=True)
parser_cache.pop(None, None)
self.module = parse(code, diff_cache=True, cache=True)
return self.module
def update(self, old_lines, new_lines):
'''
The algorithm works as follows:
Equal:
- Assure that the start is a newline, otherwise parse until we get
one.
- Copy from parsed_until_line + 1 to max(i2 + 1)
- Make sure that the indentation is correct (e.g. add DEDENT)
- Add old and change positions
Insert:
- Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
much more.
Returns the new module node.
'''
debug.speed('diff parser start')
# Reset the used names cache so they get regenerated.
self._module._used_names = None
self._parser_lines_new = new_lines
self._added_newline = False
if new_lines[-1] != '':
# The Python grammar needs a newline at the end of a file, but for
# everything else we keep working with new_lines here.
self._parser_lines_new = list(new_lines)
self._parser_lines_new[-1] += '\n'
self._parser_lines_new.append('')
self._added_newline = True
self._reset()
line_length = len(new_lines)
sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new)
opcodes = sm.get_opcodes()
debug.speed('diff parser calculated')
debug.dbg('diff: line_lengths old: %s, new: %s' % (len(old_lines), line_length))
for operation, i1, i2, j1, j2 in opcodes:
debug.dbg('diff %s old[%s:%s] new[%s:%s]',
operation, i1 + 1, i2, j1 + 1, j2)
if j2 == line_length + int(self._added_newline):
# The empty part after the last newline is not relevant.
j2 -= 1
if operation == 'equal':
line_offset = j1 - i1
self._copy_from_old_parser(line_offset, i2, j2)
elif operation == 'replace':
self._parse(until_line=j2)
elif operation == 'insert':
self._parse(until_line=j2)
else:
assert operation == 'delete'
# With this action all change will finally be applied and we have a
# changed module.
self._nodes_stack.close()
if self._added_newline:
_remove_last_newline(self._module)
# Good for debugging.
if debug.debug_function:
self._enabled_debugging(old_lines, new_lines)
last_pos = self._module.end_pos[0]
if last_pos != line_length:
current_lines = splitlines(self._module.get_code(), keepends=True)
diff = difflib.unified_diff(current_lines, new_lines)
raise Exception(
"There's an issue (%s != %s) with the diff parser. Please report:\n%s"
% (last_pos, line_length, ''.join(diff))
)
debug.speed('diff parser end')
return self._module
def end_pos(self):
lines = splitlines(self.string)
if len(lines) > 1:
return self.start_pos[0] + len(lines) - 1, 0
else:
return self.start_pos[0], self.start_pos[1] + len(self.string)
def end_pos(self):
lines = splitlines(self.string)
if len(lines) > 1:
return self.start_pos[0] + len(lines) - 1, 0
else:
return self.start_pos[0], self.start_pos[1] + len(self.string)
def parse(code=None, path=None, grammar=None, error_recovery=True,
start_symbol='file_input', cache=False, diff_cache=False):
"""
If you want to parse a Python file you want to start here, most likely.
If you need finer grained control over the parsed instance, there will be
other ways to access it.
:param code: A unicode string that contains Python code.
:param path: The path to the file you want to open. Only needed for caching.
:param grammar: A Python grammar file, created with load_grammar. You may
not specify it. In that case it's the current Python version.
:param error_recovery: If enabled, any code will be returned. If it is
invalid, it will be returned as an error node. If disabled, you will
get a ParseError when encountering syntax errors in your code.
:param start_symbol: The grammar symbol that you want to parse. Only
allowed to be used when error_recovery is disabled.
:return: A syntax tree node. Typically the module.
"""
if code is None and path is None:
raise TypeError("Please provide either code or a path.")
if grammar is None:
grammar = load_grammar()
if cache and not code and path is not None:
# In this case we do actual caching. We just try to load it.
module_node = load_module(grammar, path)
if module_node is not None:
return module_node
if code is None:
with open(path, 'rb') as f:
code = source_to_unicode(f.read())
if diff_cache and settings.fast_parser:
try:
module_cache_item = parser_cache[path]
except KeyError:
pass
else:
lines = splitlines(code, keepends=True)
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
save_module(grammar, path, module_node, lines, pickling=False)
return module_node
new_node = DiffParser(grammar, module_node).update(
old_lines=old_lines,
new_lines=lines
)
save_module(grammar, path, new_node, lines, pickling=cache)
return new_node
added_newline = not code.endswith('\n')
lines = tokenize_lines = splitlines(code, keepends=True)
if added_newline:
code += '\n'
tokenize_lines = list(tokenize_lines)
tokenize_lines[-1] += '\n'
tokenize_lines.append('')
tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)
p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol)
root_node = p.parse(tokens=tokens)
if added_newline:
_remove_last_newline(root_node)
if cache or diff_cache:
save_module(grammar, path, root_node, lines, pickling=cache)
return root_node
def check(code):
tokens = _get_token_list(code)
lines = splitlines(code)
assert tokens[-1].end_pos == (len(lines), len(lines[-1]))
def test_splitlines_no_keepends():
assert splitlines('asd\r\n') == ['asd', '']
assert splitlines('asd\r\n\f') == ['asd', '\f']
assert splitlines('\fasd\r\n') == ['\fasd', '']
assert splitlines('') == ['']
assert splitlines('\n') == ['', '']
def parse(code=None, path=None, grammar=None, error_recovery=True,
start_symbol='file_input', cache=False, diff_cache=False):
"""
If you want to parse a Python file you want to start here, most likely.
If you need finer grained control over the parsed instance, there will be
other ways to access it.
:param code: A unicode string that contains Python code.
:param path: The path to the file you want to open. Only needed for caching.
:param grammar: A Python grammar file, created with load_grammar.
:param error_recovery: If enabled, any code will be returned. If it is
invalid, it will be returned as an error node. If disabled, you will
get a ParseError when encountering syntax errors in your code.
:param start_symbol: The grammar symbol that you want to parse. Only
allowed to be used when error_recovery is disabled.
:return: A syntax tree node. Typically the module.
"""
if code is None and path is None:
raise TypeError("Please provide either code or a path.")
if grammar is None:
grammar = load_grammar()
if path is not None:
path = os.path.expanduser(path)
if cache and not code and path is not None:
# In this case we do actual caching. We just try to load it.
module_node = load_module(grammar, path)
if module_node is not None:
return module_node
if code is None:
with open(path, 'rb') as f:
code = source_to_unicode(f.read())
if diff_cache and settings.fast_parser:
try:
module_cache_item = parser_cache[path]
except KeyError:
pass
else:
lines = splitlines(code, keepends=True)
module_node = module_cache_item.node
old_lines = module_cache_item.lines
if old_lines == lines:
save_module(grammar, path, module_node, lines, pickling=False)
return module_node
new_node = DiffParser(grammar, module_node).update(
old_lines=old_lines,
new_lines=lines
)
save_module(grammar, path, new_node, lines, pickling=cache)
return new_node
added_newline = not code.endswith('\n')
lines = tokenize_lines = splitlines(code, keepends=True)
if added_newline:
code += '\n'
tokenize_lines = list(tokenize_lines)
tokenize_lines[-1] += '\n'
tokenize_lines.append('')
tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)
p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol)
root_node = p.parse(tokens=tokens)
if added_newline:
_remove_last_newline(root_node)
if cache or diff_cache:
save_module(grammar, path, root_node, lines, pickling=cache)
return root_node
