def test_equals(source):
evaluator = Evaluator(load_grammar())
node = Parser(load_grammar(), source, 'eval_input').get_parsed_node()
results = evaluator.eval_element(node)
assert len(results) == 1
first = results.pop()
assert isinstance(first, CompiledObject) and first.obj is True
def test_module():
module = Parser(load_grammar(), u('asdf'), 'example.py').module
name = module.name
assert str(name) == 'example'
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 7)
module = Parser(load_grammar(), u('asdf')).module
name = module.name
assert str(name) == ''
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 0)
def test_module():
module = Parser(load_grammar(), u("asdf"), "example.py").module
name = module.name
assert str(name) == "example"
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 7)
module = Parser(load_grammar(), u("asdf")).module
name = module.name
assert str(name) == ""
assert name.start_pos == (1, 0)
assert name.end_pos == (1, 0)
def test_python3_octal():
parser = ParserWithRecovery(load_grammar(), u'0o660')
module = parser.get_parsed_node()
if is_py3:
assert module.children[0].children[0].type == 'number'
else:
assert module.children[0].type == 'error_node'
def test_python2_octal():
parser = ParserWithRecovery(load_grammar(), u'0660')
first = parser.get_parsed_node().children[0]
if is_py3:
assert first.type == 'error_node'
else:
assert first.children[0].type == 'number'
def test_path_from_sys_path_assignment():
SRC = dedent(
u(
"""
#!/usr/bin/python
import sys
sys.path[0:0] = [
'/usr/lib/python3.4/site-packages',
'/home/test/.buildout/eggs/important_package.egg'
]
path[0:0] = [1]
import important_package
if __name__ == '__main__':
sys.exit(important_package.main())"""
)
)
grammar = load_grammar()
p = ParserWithRecovery(grammar, SRC)
paths = _check_module(Evaluator(grammar), p.module)
assert 1 not in paths
assert "/home/test/.buildout/eggs/important_package.egg" in paths
def test_end_pos_one_line(self):
parsed = Parser(load_grammar(), dedent(u('''
def testit():
a = "huhu"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2]
assert tok.end_pos == (3, 14)
def _load_faked_module(module):
module_name = module.__name__
if module_name == '__builtin__' and not is_py3:
module_name = 'builtins'
try:
return modules[module_name]
except KeyError:
path = os.path.dirname(os.path.abspath(__file__))
try:
with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
source = f.read()
except IOError:
modules[module_name] = None
return
grammar = load_grammar(version='3.4')
module = ParserWithRecovery(grammar, unicode(source), module_name).module
modules[module_name] = module
if module_name == 'builtins' and not is_py3:
# There are two implementations of `open` for either python 2/3.
# -> Rename the python2 version (`look at fake/builtins.pym`).
open_func = search_scope(module, 'open')
open_func.children[1] = FakeName('open_python3')
open_func = search_scope(module, 'open_python2')
open_func.children[1] = FakeName('open')
return module
def _evaluate_for_statement_string(evaluator, string, module):
code = dedent("""
def pseudo_docstring_stuff():
# Create a pseudo function for docstring statements.
%s
""")
if string is None:
return []
for element in re.findall('((?:\w+\.)*\w+)\.', string):
# Try to import module part in dotted name.
# (e.g., 'threading' in 'threading.Thread').
string = 'import %s\n' % element + string
# Take the default grammar here, if we load the Python 2.7 grammar here, it
# will be impossible to use `...` (Ellipsis) as a token. Docstring types
# don't need to conform with the current grammar.
p = Parser(load_grammar(), code % indent_block(string))
try:
pseudo_cls = p.module.subscopes[0]
# First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
# which is also not the last item, because there's a newline.
stmt = pseudo_cls.children[-1].children[-2].children[-2]
except (AttributeError, IndexError):
return []
# Use the module of the param.
# TODO this module is not the module of the param in case of a function
# call. In that case it's the module of the function call.
# stuffed with content from a function call.
pseudo_cls.parent = module
return list(_execute_types_in_stmt(evaluator, stmt))
def test_get_code():
"""Use the same code that the parser also generates, to compare"""
s = u(
'''"""a docstring"""
class SomeClass(object, mixin):
def __init__(self):
self.xy = 3.0
"""statement docstr"""
def some_method(self):
return 1
def yield_method(self):
while hasattr(self, 'xy'):
yield True
for x in [1, 2]:
yield x
def empty(self):
pass
class Empty:
pass
class WithDocstring:
"""class docstr"""
pass
def method_with_docstring():
"""class docstr"""
pass
'''
)
assert ParserWithRecovery(load_grammar(), s).module.get_code() == s
def test_end_pos_multi_line(self):
parsed = Parser(load_grammar(), dedent(u('''
def testit():
a = """huhu
asdfasdf""" + "h"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
assert tok.end_pos == (4, 11)
def test_user_statement_on_import():
"""github #285"""
s = u("from datetime import (\n" " time)")
for pos in [(2, 1), (2, 4)]:
p = UserContextParser(load_grammar(), s, None, pos, None).user_stmt()
assert isinstance(p, pt.Import)
assert [str(n) for n in p.get_defined_names()] == ["time"]
def test_round_trip():
source = dedent('''
def x():
"""hahaha"""
func''')
f = FastParser(load_grammar(), u(source))
assert f.get_parsed_node().get_code() == source
def test_dont_break_imports_without_namespaces():
"""
The code checking for ``from __future__ import absolute_import`` shouldn't
assume that all imports have non-``None`` namespaces.
"""
src = u("from __future__ import absolute_import\nimport xyzzy")
parser = ParserWithRecovery(load_grammar(), src, "test.py")
assert parser.module.has_explicit_absolute_import
def test_basic_parsing():
"""Validate the parsing features"""
prs = Parser(load_grammar(), code_basic_features)
diff_code_assert(
code_basic_features,
prs.module.get_code()
)
def test_sys_path_with_modifications():
SRC = dedent(u("""
import os
"""))
grammar = load_grammar()
p = Parser(grammar, SRC)
p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
paths = sys_path_with_modifications(Evaluator(grammar), p.module)
assert '/tmp/.buildout/eggs/important_package.egg' in paths
def test_path_from_invalid_sys_path_assignment():
SRC = dedent(u("""
import sys
sys.path = 'invalid'"""))
grammar = load_grammar()
p = Parser(grammar, SRC)
paths = _check_module(Evaluator(grammar), p.module)
assert len(paths) > 0
assert 'invalid' not in paths
def test_carriage_return_statements():
source = u(dedent('''
foo = 'ns1!'
# this is a namespace package
'''))
source = source.replace('\n', '\r\n')
stmt = Parser(load_grammar(), source).module.statements[0]
assert '#' not in stmt.get_code()
def test_simple():
e = Evaluator(load_grammar())
bltn = compiled.CompiledObject(builtins)
obj = compiled.CompiledObject('_str_', bltn)
upper = e.find_types(obj, 'upper')
assert len(upper) == 1
objs = list(e.execute(upper[0]))
assert len(objs) == 1
assert isinstance(objs[0], representation.Instance)
def test_user_statement_on_import():
"""github #285"""
s = u("from datetime import (\n"
" time)")
for pos in [(2, 1), (2, 4)]:
p = ParserWithRecovery(load_grammar(), s)
stmt = p.module.get_statement_for_position(pos)
assert isinstance(stmt, pt.Import)
assert [str(n) for n in stmt.get_defined_names()] == ['time']
def parse_file(self, file): file_imports = set() with open(file, 'r') as fp: content = fp.read() parser = Parser(load_grammar(), u(content), file) imports = parser.module.imports for imp in imports: for path in imp.paths(): file_imports.update([path[0].value]) return file_imports
def test_fake_loading():
assert isinstance(compiled.create(Evaluator(load_grammar()), next), Function)
string = compiled.builtin.get_subscope_by_name('str')
from_name = compiled._create_from_name(
compiled.builtin,
string,
'__init__'
)
assert isinstance(from_name, Function)
def main(args):
jedi.set_debug_function(notices=args['--debug'])
with open(args['<file>']) as f:
code = f.read()
grammar = load_grammar()
parser = ParserWithRecovery(grammar, u(code))
code = code + '\na\n' # Add something so the diff parser needs to run.
lines = splitlines(code, keepends=True)
cProfile.runctx('run(parser, lines)', globals(), locals(), sort=args['-s'])
def test_end_pos():
s = u(dedent('''
x = ['a', 'b', 'c']
def func():
y = None
'''))
parser = Parser(load_grammar(), s)
scope = parser.module.subscopes[0]
assert scope.start_pos == (3, 0)
assert scope.end_pos == (5, 0)
def check(src, result):
# Python 2 tuple params should be ignored for now.
grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
m = Parser(grammar, u(src)).module
if is_py3:
assert not m.subscopes
else:
# We don't want b and c to be a part of the param enumeration. Just
# ignore them, because it's not what we want to support in the
# future.
assert [str(param.name) for param in m.subscopes[0].params] == result
def assert_params(param_string, **wanted_dct):
source = dedent('''
def x(%s):
pass
''') % param_string
parser = Parser(load_grammar(), dedent(source))
funcdef = parser.get_parsed_node().subscopes[0]
dct = dict((p.name.value, p.default and p.default.get_code())
for p in funcdef.params)
assert dct == wanted_dct
assert parser.get_parsed_node().get_code() == source
def test_append_on_non_sys_path():
SRC = dedent(u("""
class Dummy(object):
path = []
d = Dummy()
d.path.append('foo')"""))
grammar = load_grammar()
p = Parser(grammar, SRC)
paths = _check_module(Evaluator(grammar), p.module)
assert len(paths) > 0
assert 'foo' not in paths
def test_open_parentheses():
func = 'def func():\n a'
p = FastParser(load_grammar(), 'isinstance(\n\n' + func)
# As you can see, the isinstance call cannot be seen anymore after
# get_code, because it isn't valid code.
assert p.module.get_code() == '\n\n' + func
assert p.number_of_splits == 2
assert p.number_parsers_used == 2
cache.save_parser(None, None, p, pickling=False)
# Now with a correct parser it should work perfectly well.
check_fp('isinstance()\n' + func, 1, 2)
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
source_path=None,
source_encoding=None,
sys_path=None):
if source_path is not None:
warnings.warn("Use path instead of source_path.",
DeprecationWarning)
path = source_path
if source_encoding is not None:
warnings.warn("Use encoding instead of source_encoding.",
DeprecationWarning)
encoding = source_encoding
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
try:
with open(path) as f:
source = f.read()
except UnicodeDecodeError:
with open(path, encoding=encoding) as f:
source = f.read()
self._source = common.source_to_unicode(source, encoding)
self._code_lines = common.splitlines(self._source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
if sys_path is None:
venv = os.getenv('VIRTUAL_ENV')
if venv:
sys_path = list(get_venv_path(venv))
self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
debug.speed('init')
def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0):
if number_of_splits is None:
number_of_splits = number_parsers_used
p = FastParser(load_grammar(), u(src))
save_parser(None, p, pickling=False)
assert src == p.module.get_code()
assert p.number_of_splits == number_of_splits
assert p.number_parsers_used == number_parsers_used
assert p.number_of_misses == number_of_misses
return p.module
def test_end_pos_line(self):
# jedi issue #150
s = u("x()\nx( )\nx( )\nx ( )")
parser = Parser(load_grammar(), s)
for i, s in enumerate(parser.module.statements):
assert s.end_pos == (i + 1, i + 3)
def _evaluator():
return Evaluator(load_grammar())
def compare(string):
"""Generates the AST object and then regenerates the code."""
assert Parser(load_grammar(), string).module.get_code() == string
def get_import(self, source):
return ParserWithRecovery(load_grammar(), source).module.imports[0]
def parent(self):
"""
Creating fake statements for the interpreter.
"""
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# TODO this import is wrong. Yields x for x.y.z instead of z
module = __import__(module_name)
parser_path = names
raw_module = get_module(self._value)
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
path = re.sub('c$', '', path)
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path[:-1]).module
if parser_path:
assert len(parser_path) == 1
found = self._evaluator.find_types(mod,
parser_path[0],
search_global=True)
else:
found = [er.wrap(self._evaluator, mod)]
if not found:
debug.warning(
'Possibly an interpreter lookup for Python code failed %s',
parser_path)
if not found:
evaluated = compiled.CompiledObject(obj)
if evaluated == builtins:
# The builtins module is special and always cached.
evaluated = compiled.builtin
found = [evaluated]
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content
])
stmt.parent = self._module
return stmt
def test_started_lambda_stmt():
p = ParserWithRecovery(load_grammar(), u'lambda a, b: a i')
assert p.get_parsed_node().children[0].type == 'error_node'
def test_backslash_dos_style():
grammar = load_grammar()
m = ParserWithRecovery(grammar, u('\\\r\n')).module
assert m
def node(self, request):
parsed = ParserWithRecovery(load_grammar(),
dedent(u(request.param[0])))
request.keywords['expected'] = request.param[1]
return parsed.module.subscopes[0]
def paths(src):
grammar = load_grammar()
stmt = ParserWithRecovery(grammar, unicode(src)).module.statements[0]
return set(sys_path._paths_from_assignment(Evaluator(grammar), stmt))
def check_module_test(code):
grammar = load_grammar()
p = ParserWithRecovery(grammar, code)
module_context = ModuleContext(Evaluator(grammar), p.module)
return _check_module(module_context)
def test_newline_positions():
endmarker = Parser(load_grammar(), u('a\n')).module.children[-1]
assert endmarker.end_pos == (2, 0)
new_line = endmarker.get_previous()
assert new_line.start_pos == (1, 1)
assert new_line.end_pos == (2, 0)
def parent(self):
"""
Creating fake statements for the interpreter.
Here we are trying to link back to Python code, if possible. This means
we try to find the python module for a name (not the builtin).
"""
return mixed.create(self._evaluator, self._value)
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# If we put anything into fromlist, it will just return the
# latest name.
module = __import__(module_name, fromlist=[''])
parser_path = names
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
# Find the corresponding Python file for an interpreted one.
path = re.sub(r'\.pyc$', '.py', path)
if path.endswith('.py'):
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path).module
mod = self._evaluator.wrap(mod)
# We have to make sure that the modules that we import are also
# part of evaluator.modules.
for module_name, module in sys.modules.items():
try:
iterated_path = module.__file__
except AttributeError:
pass
else:
if iterated_path == path:
self._evaluator.modules[module_name] = mod
break
else:
raise NotImplementedError(
'This should not happen, a module '
'should be part of sys.modules.')
if parser_path:
assert len(parser_path) == 1
found = list(
self._evaluator.find_types(mod,
parser_path[0],
search_global=True))
else:
found = [mod]
if not found:
debug.warning(
'Interpreter lookup failed in global scope for %s',
parser_path)
if not found:
evaluated = compiled.create(self._evaluator, obj)
found = [evaluated]
if len(found) > 1:
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''),
content
])
stmt.parent = self._module
return stmt
else:
return found[0]
def test(source, end_pos):
module = ParserWithRecovery(load_grammar(), u(source)).module
assert module.get_code() == source
assert module.end_pos == end_pos
def test_operators():
src = u('5 * 3')
prs = ParserWithRecovery(load_grammar(), src)
diff_code_assert(src, prs.module.get_code())
def test_basic_parsing():
"""Validate the parsing features"""
prs = ParserWithRecovery(load_grammar(), code_basic_features)
diff_code_assert(code_basic_features, prs.module.get_code())
def test_fake_docstr():
assert compiled.create(Evaluator(load_grammar()), next).raw_doc == next.__doc__
def test_no_explicit_absolute_imports():
"""
Detect modules without ``from __future__ import absolute_import``.
"""
parser = Parser(load_grammar(), u("1"), "test.py")
assert not parser.module.has_explicit_absolute_import
def test_incomplete_list_comprehension():
""" Shouldn't raise an error, same bug as #418. """
# With the old parser this actually returned a statement. With the new
# parser only valid statements generate one.
assert ParserWithRecovery(load_grammar(),
u('(1 for def')).module.statements == []
def initialize(self, source):
debug.dbg('differ: initialize', color='YELLOW')
grammar = load_grammar()
self.parser = ParserWithRecovery(grammar, u(source))
return self.parser.module
def get_call(self, source):
# Get the simple_stmt and then the first one.
simple_stmt = ParserWithRecovery(load_grammar(),
u(source)).module.children[0]
return simple_stmt.children[0]
def get_sub(self, source):
return ParserWithRecovery(load_grammar(),
u(source)).module.subscopes[0]
def get_call(self, source):
stmt = Parser(load_grammar(), u(source)).module.statements[0]
return stmt.children[0]
def parse(code, version='3.4'):
code = dedent(code) + "\n\n"
grammar = load_grammar(version=version)
return Parser(grammar, unicode(code), 'file_input').get_parsed_node()