def fp(src): p = FastParser(u(src)) cache.save_parser(None, None, p, pickling=False) # TODO Don't change get_code, the whole thing should be the same. # -> Need to refactor the parser first, though. assert src == p.module.get_code()[:-1]
def _get_module(self): cache.invalidate_star_import_cache(self._path) parser = FastParser(self._grammar, self._source, self.path) save_parser(self.path, parser, pickling=False) module = self._evaluator.wrap(parser.module) imports.add_module(self._evaluator, unicode(module.name), module) return parser.module
def test_round_trip(): source = dedent(''' def x(): """hahaha""" func''') f = FastParser(load_grammar(), u(source)) assert f.get_parsed_node().get_code() == source
def _parser(self): cache.invalidate_star_import_cache(self._path) if self._use_fast_parser: parser = FastParser(self._grammar, self._source, self._path) # Don't pickle that module, because the main module is changing quickly cache.save_parser(self._path, None, parser, pickling=False) else: parser = Parser(self._grammar, self._source, self._path) return parser
def parent(self): obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: class FakeParent(pr.Base): parent = None # To avoid having no parent for NamePart. path = None names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: module = __import__(module_name) fake_name = helpers.FakeName(names, FakeParent()) parser_path = fake_name.names raw_module = get_module(self._value) try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(source, path[:-1]).module if not parser_path: return mod found = self._evaluator.eval_call_path(iter(parser_path), mod, None) if found: return found[0] debug.warning('Interpreter lookup for Python code failed %s', mod) module = compiled.CompiledObject(raw_module) if raw_module == builtins: # The builtins module is special and always cached. module = compiled.builtin return compiled.create(self._evaluator, self._value, module, module)
def test_open_parentheses(): func = 'def func():\n a' p = FastParser(load_grammar(), u('isinstance(\n\n' + func)) # As you can see, the isinstance call cannot be seen anymore after # get_code, because it isn't valid code. assert p.module.get_code() == '\n\n' + func assert p.number_of_splits == 2 assert p.number_parsers_used == 2 cache.save_parser(None, p, pickling=False) # Now with a correct parser it should work perfectly well. check_fp('isinstance()\n' + func, 1, 2)
def test_carriage_return_splitting(): source = u(dedent(''' "string" class Foo(): pass ''')) source = source.replace('\n', '\r\n') p = FastParser(source) assert [str(n) for n in p.module.get_defined_names()] == ['Foo']
def test_carriage_return_splitting(): source = u(dedent(''' "string" class Foo(): pass ''')) source = source.replace('\n', '\r\n') p = FastParser(load_grammar(), source) assert [n.value for lst in p.module.names_dict.values() for n in lst] == ['Foo']
def check_fp(src, number_parsers_used, number_of_splits=None): if number_of_splits is None: number_of_splits = number_parsers_used p = FastParser(load_grammar(), u(src)) cache.save_parser(None, None, p, pickling=False) # TODO Don't change get_code, the whole thing should be the same. # -> Need to refactor the parser first, though. assert src == p.module.get_code() assert p.number_of_splits == number_of_splits assert p.number_parsers_used == number_parsers_used return p.module
def test_open_parentheses(): func = 'def func():\n a' code = u('isinstance(\n\n' + func) p = FastParser(load_grammar(), code) # As you can see, the part that was failing is still there in the get_code # call. It is not relevant for evaluation, but still available as an # ErrorNode. assert p.module.get_code() == code assert p.number_of_splits == 2 assert p.number_parsers_used == 2 save_parser(None, p, pickling=False) # Now with a correct parser it should work perfectly well. check_fp('isinstance()\n' + func, 1, 2)
def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0): if number_of_splits is None: number_of_splits = number_parsers_used p = FastParser(load_grammar(), u(src)) save_parser(None, p, pickling=False) assert src == p.module.get_code() assert p.number_of_splits == number_of_splits assert p.number_parsers_used == number_parsers_used assert p.number_of_misses == number_of_misses return p.module
def parent(self): parser_path = [] obj = self._value if inspect.ismodule(obj): module = obj else: try: o = obj.__objclass__ parser_path.append( pr.NamePart(obj.__name__, None, (None, None))) obj = o except AttributeError: pass try: module_name = obj.__module__ parser_path.insert( 0, pr.NamePart(obj.__name__, None, (None, None))) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: module = __import__(module_name) raw_module = get_module(self._value) try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(source, path[:-1]).module if not parser_path: return mod found = self._evaluator.eval_call_path(iter(parser_path), mod, None) if found: return found[0] debug.warning('Interpreter lookup for Python code failed %s', mod) module = compiled.CompiledObject(raw_module) return compiled.create(self._value, module, module)
def parse(grammar, path): with open(path) as f: source = f.read() source = common.source_to_unicode(source) return FastParser(grammar, source, path)
def parent(self): """ Creating fake statements for the interpreter. """ obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # TODO this import is wrong. Yields x for x.y.z instead of z module = __import__(module_name) parser_path = names raw_module = get_module(self._value) found = [] try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path[:-1]).module if parser_path: assert len(parser_path) == 1 found = self._evaluator.find_types(mod, parser_path[0], search_global=True) else: found = [er.wrap(self._evaluator, mod)] if not found: debug.warning( 'Possibly an interpreter lookup for Python code failed %s', parser_path) if not found: evaluated = compiled.CompiledObject(obj) if evaluated == builtins: # The builtins module is special and always cached. evaluated = compiled.builtin found = [evaluated] content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([ self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content ]) stmt.parent = self._module return stmt
def parent(self): """ Creating fake statements for the interpreter. Here we are trying to link back to Python code, if possible. This means we try to find the python module for a name (not the builtin). """ return mixed.create(self._evaluator, self._value) obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # If we put anything into fromlist, it will just return the # latest name. module = __import__(module_name, fromlist=['']) parser_path = names found = [] try: path = module.__file__ except AttributeError: pass else: # Find the corresponding Python file for an interpreted one. path = re.sub(r'\.pyc$', '.py', path) if path.endswith('.py'): with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path).module mod = self._evaluator.wrap(mod) # We have to make sure that the modules that we import are also # part of evaluator.modules. for module_name, module in sys.modules.items(): try: iterated_path = module.__file__ except AttributeError: pass else: if iterated_path == path: self._evaluator.modules[module_name] = mod break else: raise NotImplementedError( 'This should not happen, a module ' 'should be part of sys.modules.') if parser_path: assert len(parser_path) == 1 found = list( self._evaluator.find_types(mod, parser_path[0], search_global=True)) else: found = [mod] if not found: debug.warning( 'Interpreter lookup failed in global scope for %s', parser_path) if not found: evaluated = compiled.create(self._evaluator, obj) found = [evaluated] if len(found) > 1: content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([ self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content ]) stmt.parent = self._module return stmt else: return found[0]