def test_round_trip():
source = dedent('''
def x():
"""hahaha"""
func''')
f = FastParser(load_grammar(), u(source))
assert f.get_parsed_node().get_code() == source
def test_round_trip():
source = dedent('''
def x():
"""hahaha"""
func''')
f = FastParser(load_grammar(), u(source))
assert f.get_parsed_node().get_code() == source
def fp(src):
p = FastParser(u(src))
cache.save_parser(None, None, p, pickling=False)
# TODO Don't change get_code, the whole thing should be the same.
# -> Need to refactor the parser first, though.
assert src == p.module.get_code()[:-1]
def _get_module(self):
cache.invalidate_star_import_cache(self._path)
parser = FastParser(self._grammar, self._source, self.path)
save_parser(self.path, parser, pickling=False)
module = self._evaluator.wrap(parser.module)
imports.add_module(self._evaluator, unicode(module.name), module)
return parser.module
def _parser(self):
cache.invalidate_star_import_cache(self._path)
if self._use_fast_parser:
parser = FastParser(self._grammar, self._source, self._path)
# Don't pickle that module, because the main module is changing quickly
cache.save_parser(self._path, None, parser, pickling=False)
else:
parser = Parser(self._grammar, self._source, self._path)
return parser
def parent(self):
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
class FakeParent(pr.Base):
parent = None # To avoid having no parent for NamePart.
path = None
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
module = __import__(module_name)
fake_name = helpers.FakeName(names, FakeParent())
parser_path = fake_name.names
raw_module = get_module(self._value)
try:
path = module.__file__
except AttributeError:
pass
else:
path = re.sub('c$', '', path)
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(source, path[:-1]).module
if not parser_path:
return mod
found = self._evaluator.eval_call_path(iter(parser_path), mod, None)
if found:
return found[0]
debug.warning('Interpreter lookup for Python code failed %s',
mod)
module = compiled.CompiledObject(raw_module)
if raw_module == builtins:
# The builtins module is special and always cached.
module = compiled.builtin
return compiled.create(self._evaluator, self._value, module, module)
def test_open_parentheses():
func = 'def func():\n a'
p = FastParser(load_grammar(), u('isinstance(\n\n' + func))
# As you can see, the isinstance call cannot be seen anymore after
# get_code, because it isn't valid code.
assert p.module.get_code() == '\n\n' + func
assert p.number_of_splits == 2
assert p.number_parsers_used == 2
cache.save_parser(None, p, pickling=False)
# Now with a correct parser it should work perfectly well.
check_fp('isinstance()\n' + func, 1, 2)
def test_carriage_return_splitting():
source = u(dedent('''
"string"
class Foo():
pass
'''))
source = source.replace('\n', '\r\n')
p = FastParser(load_grammar(), source)
assert [n.value for lst in p.module.names_dict.values() for n in lst] == ['Foo']
def test_carriage_return_splitting():
source = u(dedent('''
"string"
class Foo():
pass
'''))
source = source.replace('\n', '\r\n')
p = FastParser(source)
assert [str(n) for n in p.module.get_defined_names()] == ['Foo']
def check_fp(src, number_parsers_used, number_of_splits=None):
if number_of_splits is None:
number_of_splits = number_parsers_used
p = FastParser(load_grammar(), u(src))
cache.save_parser(None, None, p, pickling=False)
# TODO Don't change get_code, the whole thing should be the same.
# -> Need to refactor the parser first, though.
assert src == p.module.get_code()
assert p.number_of_splits == number_of_splits
assert p.number_parsers_used == number_parsers_used
return p.module
def test_open_parentheses():
func = 'def func():\n a'
code = u('isinstance(\n\n' + func)
p = FastParser(load_grammar(), code)
# As you can see, the part that was failing is still there in the get_code
# call. It is not relevant for evaluation, but still available as an
# ErrorNode.
assert p.module.get_code() == code
assert p.number_of_splits == 2
assert p.number_parsers_used == 2
save_parser(None, p, pickling=False)
# Now with a correct parser it should work perfectly well.
check_fp('isinstance()\n' + func, 1, 2)
def check_fp(src,
number_parsers_used,
number_of_splits=None,
number_of_misses=0):
if number_of_splits is None:
number_of_splits = number_parsers_used
p = FastParser(load_grammar(), u(src))
save_parser(None, p, pickling=False)
assert src == p.module.get_code()
assert p.number_of_splits == number_of_splits
assert p.number_parsers_used == number_parsers_used
assert p.number_of_misses == number_of_misses
return p.module
def parent(self):
parser_path = []
obj = self._value
if inspect.ismodule(obj):
module = obj
else:
try:
o = obj.__objclass__
parser_path.append(
pr.NamePart(obj.__name__, None, (None, None)))
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
parser_path.insert(
0, pr.NamePart(obj.__name__, None, (None, None)))
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
module = __import__(module_name)
raw_module = get_module(self._value)
try:
path = module.__file__
except AttributeError:
pass
else:
path = re.sub('c$', '', path)
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(source, path[:-1]).module
if not parser_path:
return mod
found = self._evaluator.eval_call_path(iter(parser_path), mod,
None)
if found:
return found[0]
debug.warning('Interpreter lookup for Python code failed %s',
mod)
module = compiled.CompiledObject(raw_module)
return compiled.create(self._value, module, module)
def parent(self):
"""
Creating fake statements for the interpreter.
Here we are trying to link back to Python code, if possible. This means
we try to find the python module for a name (not the builtin).
"""
return mixed.create(self._evaluator, self._value)
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# If we put anything into fromlist, it will just return the
# latest name.
module = __import__(module_name, fromlist=[''])
parser_path = names
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
# Find the corresponding Python file for an interpreted one.
path = re.sub(r'\.pyc$', '.py', path)
if path.endswith('.py'):
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path).module
mod = self._evaluator.wrap(mod)
# We have to make sure that the modules that we import are also
# part of evaluator.modules.
for module_name, module in sys.modules.items():
try:
iterated_path = module.__file__
except AttributeError:
pass
else:
if iterated_path == path:
self._evaluator.modules[module_name] = mod
break
else:
raise NotImplementedError(
'This should not happen, a module '
'should be part of sys.modules.')
if parser_path:
assert len(parser_path) == 1
found = list(
self._evaluator.find_types(mod,
parser_path[0],
search_global=True))
else:
found = [mod]
if not found:
debug.warning(
'Interpreter lookup failed in global scope for %s',
parser_path)
if not found:
evaluated = compiled.create(self._evaluator, obj)
found = [evaluated]
if len(found) > 1:
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''),
content
])
stmt.parent = self._module
return stmt
else:
return found[0]
def parent(self):
"""
Creating fake statements for the interpreter.
"""
obj = self._value
parser_path = []
if inspect.ismodule(obj):
module = obj
else:
names = []
try:
o = obj.__objclass__
names.append(obj.__name__)
obj = o
except AttributeError:
pass
try:
module_name = obj.__module__
names.insert(0, obj.__name__)
except AttributeError:
# Unfortunately in some cases like `int` there's no __module__
module = builtins
else:
# TODO this import is wrong. Yields x for x.y.z instead of z
module = __import__(module_name)
parser_path = names
raw_module = get_module(self._value)
found = []
try:
path = module.__file__
except AttributeError:
pass
else:
path = re.sub('c$', '', path)
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
source = source_to_unicode(f.read())
mod = FastParser(load_grammar(), source, path[:-1]).module
if parser_path:
assert len(parser_path) == 1
found = self._evaluator.find_types(mod,
parser_path[0],
search_global=True)
else:
found = [er.wrap(self._evaluator, mod)]
if not found:
debug.warning(
'Possibly an interpreter lookup for Python code failed %s',
parser_path)
if not found:
evaluated = compiled.CompiledObject(obj)
if evaluated == builtins:
# The builtins module is special and always cached.
evaluated = compiled.builtin
found = [evaluated]
content = iterable.AlreadyEvaluated(found)
stmt = pt.ExprStmt([
self,
pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content
])
stmt.parent = self._module
return stmt
def splits(source):
class Mock(FastParser):
def __init__(self, *args):
self.number_of_splits = 0
return tuple(FastParser._split_parts(Mock(None, None), source))
def splits(source):
class Mock(FastParser):
def __init__(self, *args):
self.number_of_splits = 0
return tuple(FastParser._split_parts(Mock(None, None), source))
def parse(grammar, path):
with open(path) as f:
source = f.read()
source = common.source_to_unicode(source)
return FastParser(grammar, source, path)
