def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
h = hash(code)
hashes = [n.hash for n in nodes]
node = None
try:
index = hashes.index(h)
if nodes[index].code != code:
raise ValueError()
except ValueError:
p = Parser(parser_code, self.module_path,
self.user_position, offset=(line_offset, 0),
is_fast_parser=True, top_module=self.module,
no_docstr=no_docstr)
p.module.parent = self.module
else:
if nodes[index] != self.current_node:
offset = int(nodes[0] == self.current_node)
self.current_node.old_children.pop(index - offset)
node = nodes.pop(index)
p = node.parser
m = p.module
m.line_offset += line_offset + 1 - m.start_pos[0]
if self.user_position is not None and \
m.start_pos[0] <= self.user_position[0] <= m.end_pos[0]:
# It's important to take care of the whole user
# positioning stuff, if no reparsing is being done.
p.user_stmt = m.get_statement_for_position(
self.user_position, include_imports=True)
if p.user_stmt:
p.user_scope = p.user_stmt.parent
else:
p.user_scope = self._scan_user_scope(m) or m
return p, node
def test_carriage_return_statements():
source = u(dedent('''
foo = 'ns1!'
# this is a namespace package
'''))
source = source.replace('\n', '\r\n')
stmt = Parser(load_grammar(), source).module.statements[0]
assert '#' not in stmt.get_code()
def test_carriage_return_statements():
source = u(dedent('''
foo = 'ns1!'
# this is a namespace package
'''))
source = source.replace('\n', '\r\n')
stmt = Parser(source).module.statements[0]
assert '#' not in stmt.get_code()
def test_user_statement_on_import():
"""github #285"""
s = "from datetime import (\n" \
" time)"
for pos in [(2, 1), (2, 4)]:
u = Parser(s, user_position=pos).user_stmt
assert isinstance(u, pr.Import)
assert u.defunct is False
assert [str(n) for n in u.get_defined_names()] == ['time']
def test_module():
module = Parser(load_grammar(), u('asdf'), 'example.py').module
name = module.name
assert str(name) == 'example'
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 7)
module = Parser(load_grammar(), u('asdf')).module
name = module.name
assert str(name) == ''
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 0)
def test_module():
module = Parser('asdf', 'example.py', no_docstr=True).module
name = module.name
assert str(name) == 'example'
assert name.start_pos == (0, 0)
assert name.end_pos == (0, 0)
module = Parser('asdf', no_docstr=True).module
name = module.name
assert str(name) == ''
assert name.start_pos == (0, 0)
assert name.end_pos == (0, 0)
def assert_params(param_string, **wanted_dct):
source = dedent('''
def x(%s):
pass
''') % param_string
parser = Parser(load_grammar(), dedent(source))
funcdef = parser.get_parsed_node().subscopes[0]
dct = dict((p.name.value, p.default and p.default.get_code())
for p in funcdef.params)
assert dct == wanted_dct
assert parser.get_parsed_node().get_code() == source
def test_carriage_return_statements():
source = u(
dedent(
"""
foo = 'ns1!'
# this is a namespace package
"""
)
)
source = source.replace("\n", "\r\n")
stmt = Parser(load_grammar(), source).module.statements[0]
assert "#" not in stmt.get_code()
def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
h = hash(code)
hashes = [n.hash for n in nodes]
node = None
try:
index = hashes.index(h)
if nodes[index].code != code:
raise ValueError()
except ValueError:
tokenizer = FastTokenizer(parser_code, line_offset)
p = Parser(parser_code,
self.module_path,
tokenizer=tokenizer,
top_module=self.module,
no_docstr=no_docstr)
p.module.parent = self.module
else:
if nodes[index] != self.current_node:
offset = int(nodes[0] == self.current_node)
self.current_node.old_children.pop(index - offset)
node = nodes.pop(index)
p = node.parser
m = p.module
m.line_offset += line_offset + 1 - m.start_pos[0]
return p, node
def test_explicit_absolute_imports():
"""
Detect modules with ``from __future__ import absolute_import``.
"""
parser = Parser(load_grammar(),
u("from __future__ import absolute_import"), "test.py")
assert parser.module.has_explicit_absolute_import
def test_end_pos(self):
# jedi issue #150
s = u("x()\nx( )\nx( )\nx ( )")
parser = Parser(s)
for i, s in enumerate(parser.module.statements, 3):
for c in s.expression_list():
self.assertEqual(c.execution.end_pos[1], i)
def test_equals(source):
evaluator = Evaluator(load_grammar())
node = Parser(load_grammar(), source, 'eval_input').get_parsed_node()
results = evaluator.eval_element(node)
assert len(results) == 1
first = results.pop()
assert isinstance(first, CompiledObject) and first.obj is True
def _load_faked_module(module):
module_name = module.__name__
if module_name == '__builtin__' and not is_py3:
module_name = 'builtins'
try:
return modules[module_name]
except KeyError:
path = os.path.dirname(os.path.abspath(__file__))
try:
with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
source = f.read()
except IOError:
modules[module_name] = None
return
grammar = load_grammar('grammar3.4')
module = Parser(grammar, unicode(source), module_name).module
modules[module_name] = module
if module_name == 'builtins' and not is_py3:
# There are two implementations of `open` for either python 2/3.
# -> Rename the python2 version (`look at fake/builtins.pym`).
open_func = search_scope(module, 'open')
open_func.children[1] = FakeName('open_python3')
open_func = search_scope(module, 'open_python2')
open_func.children[1] = FakeName('open')
return module
def test_end_pos_one_line(self):
parsed = Parser(load_grammar(), dedent(u('''
def testit():
a = "huhu"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2]
assert tok.end_pos == (3, 14)
def sys_path_with_modifications(evaluator, module):
if module.path is None:
# Support for modules without a path is bad, therefore return the
# normal path.
return list(get_sys_path())
curdir = os.path.abspath(os.curdir)
with common.ignored(OSError):
os.chdir(os.path.dirname(module.path))
result = _check_module(evaluator, module)
result += _detect_django_path(module.path)
# buildout scripts often contain the same sys.path modifications
# the set here is used to avoid duplicate sys.path entries
buildout_paths = set()
for module_path in _get_buildout_scripts(module.path):
try:
with open(module_path, 'rb') as f:
source = f.read()
except IOError:
pass
else:
p = Parser(evaluator.grammar, common.source_to_unicode(source),
module_path)
for path in _check_module(p.module):
if path not in buildout_paths:
buildout_paths.add(path)
result.append(path)
# cleanup, back to old directory
os.chdir(curdir)
return list(result)
def _get_node(self, source, parser_code, line_offset, nodes, no_docstr):
"""
Side effect: Alters the list of nodes.
"""
h = hash(source)
for index, node in enumerate(nodes):
#print('EQ', node, repr(node.source), repr(source))
if node.hash == h and node.source == source:
node.reset_node()
nodes.remove(node)
break
else:
tokenizer = FastTokenizer(parser_code)
self.number_parsers_used += 1
#print('CODE', repr(source))
p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
node = ParserNode(self.module)
end = line_offset + p.module.end_pos[0]
used_lines = self._lines[line_offset:end - 1]
code_part_actually_used = ''.join(used_lines)
node.set_parser(p, code_part_actually_used)
self.current_node.add_node(node, line_offset)
return node
def _get_node(self, source, parser_code, line_offset, nodes):
"""
Side effect: Alters the list of nodes.
"""
indent = len(source) - len(source.lstrip('\t '))
self.current_node = self.current_node.parent_until_indent(indent)
h = hash(source)
for index, node in enumerate(nodes):
if node.hash == h and node.source == source:
node.reset_node()
nodes.remove(node)
break
else:
tokenizer = FastTokenizer(parser_code)
self.number_parsers_used += 1
p = Parser(self._grammar,
parser_code,
self.module_path,
tokenizer=tokenizer)
end = line_offset + p.module.end_pos[0]
used_lines = self._lines[line_offset:end - 1]
code_part_actually_used = ''.join(used_lines)
node = ParserNode(self.module, p, code_part_actually_used)
self.current_node.add_node(node, line_offset)
return node
def _evaluate_for_statement_string(evaluator, string, module):
code = dedent("""
def pseudo_docstring_stuff():
# Create a pseudo function for docstring statements.
%s
""")
if string is None:
return []
for element in re.findall('((?:\w+\.)*\w+)\.', string):
# Try to import module part in dotted name.
# (e.g., 'threading' in 'threading.Thread').
string = 'import %s\n' % element + string
# Take the default grammar here, if we load the Python 2.7 grammar here, it
# will be impossible to use `...` (Ellipsis) as a token. Docstring types
# don't need to conform with the current grammar.
p = Parser(load_grammar(), code % indent_block(string))
try:
pseudo_cls = p.module.subscopes[0]
# First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
# which is also not the last item, because there's a newline.
stmt = pseudo_cls.children[-1].children[-2].children[-2]
except (AttributeError, IndexError):
return []
# Use the module of the param.
# TODO this module is not the module of the param in case of a function
# call. In that case it's the module of the function call.
# stuffed with content from a function call.
pseudo_cls.parent = module
return list(_execute_types_in_stmt(evaluator, stmt))
def test_end_pos(self):
# jedi issue #150
s = "x()\nx( )\nx( )\nx ( )"
parser = Parser(s)
for i, s in enumerate(parser.module.statements, 3):
for c in s.get_commands():
self.assertEqual(c.execution.end_pos[1], i)
def test_get_code():
"""Use the same code that the parser also generates, to compare"""
s = u('''"""a docstring"""
class SomeClass(object, mixin):
def __init__(self):
self.xy = 3.0
"""statement docstr"""
def some_method(self):
return 1
def yield_method(self):
while hasattr(self, 'xy'):
yield True
for x in [1, 2]:
yield x
def empty(self):
pass
class Empty:
pass
class WithDocstring:
"""class docstr"""
pass
def method_with_docstring():
"""class docstr"""
pass
''')
assert Parser(load_grammar(), s).module.get_code() == s
def _evaluate_for_statement_string(evaluator, string, module):
code = dedent("""
def pseudo_docstring_stuff():
'''Create a pseudo function for docstring statements.'''
%s
""")
if string is None:
return []
for element in re.findall('((?:\w+\.)*\w+)\.', string):
# Try to import module part in dotted name.
# (e.g., 'threading' in 'threading.Thread').
string = 'import %s\n' % element + string
p = Parser(code % indent_block(string), no_docstr=True)
pseudo_cls = p.module.subscopes[0]
try:
stmt = pseudo_cls.statements[-1]
except IndexError:
return []
# Use the module of the param.
# TODO this module is not the module of the param in case of a function
# call. In that case it's the module of the function call.
# stuffed with content from a function call.
pseudo_cls.parent = module
definitions = evaluator.eval_statement(stmt)
it = (evaluator.execute(d) for d in definitions)
# TODO Executing tuples does not make sense, people tend to say
# `(str, int)` in a type annotation, which means that it returns a tuple
# with both types.
# At this point we just return the classes if executing wasn't possible,
# i.e. is a tuple.
return list(chain.from_iterable(it)) or definitions
def test_dont_break_imports_without_namespaces():
"""
The code checking for ``from __future__ import absolute_import`` shouldn't
assume that all imports have non-``None`` namespaces.
"""
src = u("from __future__ import absolute_import\nimport xyzzy")
parser = Parser(load_grammar(), src, "test.py")
assert parser.module.has_explicit_absolute_import
def test_end_pos_multi_line(self):
parsed = Parser(load_grammar(), dedent(u('''
def testit():
a = """huhu
asdfasdf""" + "h"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
assert tok.end_pos == (4, 11)
def test_path_from_invalid_sys_path_assignment():
SRC = u("""
import sys
sys.path = 'invalid'""")
p = Parser(SRC)
paths = _check_module(p.module)
assert len(paths) > 0
assert 'invalid' not in paths
def test_sys_path_with_modifications():
SRC = dedent(u("""
import os
"""))
grammar = load_grammar()
p = Parser(grammar, SRC)
p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
paths = sys_path_with_modifications(Evaluator(grammar), p.module)
assert '/tmp/.buildout/eggs/important_package.egg' in paths
def test_path_from_invalid_sys_path_assignment():
SRC = dedent(u("""
import sys
sys.path = 'invalid'"""))
grammar = load_grammar()
p = Parser(grammar, SRC)
paths = _check_module(Evaluator(grammar), p.module)
assert len(paths) > 0
assert 'invalid' not in paths
def _parser(self):
cache.invalidate_star_import_cache(self._path)
if self._use_fast_parser:
parser = FastParser(self._grammar, self._source, self._path)
# Don't pickle that module, because the main module is changing quickly
cache.save_parser(self._path, None, parser, pickling=False)
else:
parser = Parser(self._grammar, self._source, self._path)
return parser
def _get_under_cursor_stmt(self, cursor_txt):
offset = self._line - 1, self._column
r = Parser(cursor_txt, no_docstr=True, offset=offset)
try:
stmt = r.module.statements[0]
except IndexError:
raise NotFoundError()
stmt.parent = self._parser.user_scope
return stmt
def test_end_pos():
s = u(dedent('''
x = ['a', 'b', 'c']
def func():
y = None
'''))
parser = Parser(load_grammar(), s)
scope = parser.module.subscopes[0]
assert scope.start_pos == (3, 0)
assert scope.end_pos == (5, 0)
def check(src, result):
# Python 2 tuple params should be ignored for now.
grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
m = Parser(grammar, u(src)).module
if is_py3:
assert not m.subscopes
else:
# We don't want b and c to be a part of the param enumeration. Just
# ignore them, because it's not what we want to support in the
# future.
assert [str(param.name) for param in m.subscopes[0].params] == result
def test_append_on_non_sys_path():
SRC = u("""
class Dummy(object):
path = []
d = Dummy()
d.path.append('foo')""")
p = Parser(SRC)
paths = _check_module(p.module)
assert len(paths) > 0
assert 'foo' not in paths
def __call__(self, source, module_path=None):
if not settings.fast_parser:
return Parser(source, module_path)
pi = cache.parser_cache.get(module_path, None)
if pi is None or isinstance(pi.parser, Parser):
p = super(CachedFastParser, self).__call__(source, module_path)
else:
p = pi.parser # pi is a `cache.ParserCacheItem`
p.update(source)
return p
def load(buildout_script):
try:
with open(buildout_script, 'rb') as f:
source = common.source_to_unicode(f.read())
except IOError:
debug.dbg('Error trying to read buildout_script: %s', buildout_script)
return
p = Parser(evaluator.grammar, source, buildout_script)
cache.save_parser(buildout_script, p)
return p.module
def definition(correct, correct_start, path):
should_be = set()
for match in re.finditer('(?:[^ ]+)', correct):
string = match.group(0)
parser = Parser(load_grammar(), string, start_symbol='eval_input')
parser.position_modifier.line = self.line_nr
element = parser.get_parsed_node()
element.parent = jedi.api.completion.get_user_scope(
script._get_module(),
(self.line_nr, self.column)
)
results = evaluator.eval_element(element)
if not results:
raise Exception('Could not resolve %s on line %s'
% (match.string, self.line_nr - 1))
should_be |= set(Definition(evaluator, r) for r in results)
# Because the objects have different ids, `repr`, then compare.
should = set(comparison(r) for r in should_be)
return should
def _fix_forward_reference(context, node):
evaled_nodes = context.eval_node(node)
if len(evaled_nodes) != 1:
debug.warning("Eval'ed typing index %s should lead to 1 object, "
" not %s" % (node, evaled_nodes))
return node
evaled_node = list(evaled_nodes)[0]
if isinstance(evaled_node, compiled.CompiledObject) and \
isinstance(evaled_node.obj, str):
try:
p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
start_symbol='eval_input')
new_node = p.get_parsed_node()
except ParseError:
debug.warning('Annotation not parsed: %s' % evaled_node.obj)
return node
else:
module = node.get_parent_until()
new_node.move(module.end_pos[0])
new_node.parent = context.tree_node
return new_node
else:
return node
def test(source, end_pos):
module = Parser(load_grammar(), u(source)).module
assert module.get_code() == source
assert module.end_pos == end_pos
def get_call(self, source):
stmt = Parser(source, no_docstr=True).module.statements[0]
return stmt.get_commands()[0]
def test_newline_positions():
endmarker = Parser(load_grammar(), u('a\n')).module.children[-1]
assert endmarker.end_pos == (2, 0)
new_line = endmarker.get_previous()
assert new_line.start_pos == (1, 1)
assert new_line.end_pos == (2, 0)
def get_call(self, source):
stmt = Parser(u(source), no_docstr=True).module.statements[0]
return stmt.expression_list()[0]
def test_incomplete_list_comprehension():
""" Shouldn't raise an error, same bug as #418. """
s = Parser(u('(1 for def')).module.statements[0]
assert s.expression_list()