def expand_modules(files_or_modules, black_list): """take a list of files/modules/packages and return the list of tuple (file, module name) which have to be actually checked """ real_stuff = [] for file_or_module_name in files_or_modules: if not any([ re.search(pattern, file_or_module_name) for pattern in black_list ]): real_stuff.append(file_or_module_name) result = [] errors = [] for something in real_stuff: if exists(something): # this is a file or a directory try: modname = '.'.join(modpath_from_file(something)) except ImportError: modname = splitext(basename(something))[0] if isdir(something): filepath = join(something, '__init__.py') else: filepath = something else: # suppose it's a module or package modname = something try: filepath = file_from_modpath(modname.split('.')) if filepath is None: errors.append({'key': 'F0003', 'mod': modname}) continue except (ImportError, SyntaxError), ex: # FIXME p3k : the SyntaxError is a Python bug and should be # removed as soon as possible http://bugs.python.org/issue10588 errors.append({'key': 'F0001', 'mod': modname, 'ex': ex}) continue filepath = normpath(filepath) result.append({ 'path': filepath, 'name': modname, 'basepath': filepath, 'basename': modname }) if not (modname.endswith('.__init__') or modname == '__init__') \ and '__init__.py' in filepath: for subfilepath in get_module_files(dirname(filepath), black_list): if filepath == subfilepath: continue submodname = '.'.join(modpath_from_file(subfilepath)) result.append({ 'path': subfilepath, 'name': submodname, 'basepath': filepath, 'basename': modname })
def astng_from_file(self, filepath, modname=None, fallback=True): """given a module name, return the astng object""" try: filepath = get_source_file(filepath, include_no_ext=True) source = True except NoSourceFile: source = False if modname is None: modname = '.'.join(modpath_from_file(filepath)) if modname in self._cache: return self._cache[modname] if source: try: from logilab.astng.builder import ASTNGBuilder return ASTNGBuilder(self).file_build(filepath, modname) except (SyntaxError, KeyboardInterrupt, SystemExit): raise except Exception, ex: raise if __debug__: print 'error while building astng for', filepath import traceback traceback.print_exc() msg = 'Unable to load module %s (%s)' % (modname, ex) raise ASTNGBuildingException(msg), None, sys.exc_info()[-1]
def astng_from_file(self, filepath, modname=None, fallback=True, source=False): """given a module name, return the astng object""" try: filepath = get_source_file(filepath, include_no_ext=True) source = True except NoSourceFile: pass if modname is None: try: modname = '.'.join(modpath_from_file(filepath)) except ImportError: modname = filepath if modname in self.astng_cache: return self.astng_cache[modname] if source: from logilab.astng.builder import ASTNGBuilder return ASTNGBuilder(self).file_build(filepath, modname) elif fallback and modname: return self.astng_from_module_name(modname) raise ASTNGBuildingException('unable to get astng for file %s' % filepath)
def file_build(self, path, modname=None): """build astng from a source code file (i.e. from an ast) path is expected to be a python source file """ try: stream, encoding, data = open_source_file(path) except IOError as exc: msg = 'Unable to load file %r (%s)' % (path, exc) raise ASTNGBuildingException(msg) except SyntaxError as exc: # py3k encoding specification error raise ASTNGBuildingException(exc) except LookupError as exc: # unknown encoding raise ASTNGBuildingException(exc) # get module name if necessary if modname is None: try: modname = '.'.join(modpath_from_file(path)) except ImportError: modname = splitext(basename(path))[0] # build astng representation node = self.string_build(data, modname, path) node.file_encoding = encoding node.file_stream = stream return node
def _toload_info(path, extrapath, _toload=None): """Return a dictionary of <modname>: <modpath> and an ordered list of (file, module name) to load """ from logilab.common.modutils import modpath_from_file if _toload is None: assert isinstance(path, list) _toload = {}, [] for fileordir in path: if isdir(fileordir) and exists(join(fileordir, '__init__.py')): subfiles = [join(fileordir, fname) for fname in listdir(fileordir)] _toload_info(subfiles, extrapath, _toload) elif fileordir[-3:] == '.py': modpath = modpath_from_file(fileordir, extrapath) # omit '__init__' from package's name to avoid loading that module # once for each name when it is imported by some other object # module. This supposes import in modules are done as:: # # from package import something # # not:: # # from package.__init__ import something # # which seems quite correct. if modpath[-1] == '__init__': modpath.pop() modname = '.'.join(modpath) _toload[0][modname] = fileordir _toload[1].append((fileordir, modname)) return _toload
def expand_modules(files_or_modules, black_list): """take a list of files/modules/packages and return the list of tuple (file, module name) which have to be actually checked """ real_stuff = [] for file_or_module_name in files_or_modules: if not any([re.search(pattern, file_or_module_name) for pattern in black_list]): real_stuff.append(file_or_module_name) result = [] errors = [] for something in real_stuff: if exists(something): # this is a file or a directory try: modname = '.'.join(modpath_from_file(something)) except ImportError: modname = splitext(basename(something))[0] if isdir(something): filepath = join(something, '__init__.py') else: filepath = something else: # suppose it's a module or package modname = something try: filepath = file_from_modpath(modname.split('.')) if filepath is None: errors.append( {'key' : 'F0003', 'mod': modname} ) continue except (ImportError, SyntaxError), ex: # FIXME p3k : the SyntaxError is a Python bug and should be # removed as soon as possible http://bugs.python.org/issue10588 errors.append( {'key': 'F0001', 'mod': modname, 'ex': ex} ) continue filepath = normpath(filepath) result.append( {'path': filepath, 'name': modname, 'basepath': filepath, 'basename': modname} ) if not (modname.endswith('.__init__') or modname == '__init__') \ and '__init__.py' in filepath: for subfilepath in get_module_files(dirname(filepath), black_list): if filepath == subfilepath: continue submodname = '.'.join(modpath_from_file(subfilepath)) result.append( {'path': subfilepath, 'name': submodname, 'basepath': filepath, 'basename': modname} )
def __init__(self, parent, fs_path): super(PackageNode, self).__init__() self.parent = parent self._fs_path = fs_path #gets the module name -- the whole return value of modpath_from_file #is a list containing each element of the dotpath self.name = modpath_from_file(fs_path)[-1]
def __init__(self, parent, fs_path): astroid_object = AstroidManager().ast_from_file(fs_path) super(ModuleNode, self).__init__(parent=parent, astroid_object=astroid_object) self._fs_path = fs_path #gets the module name -- the whole return value of modpath_from_file #is a list containing each element of the dotpath self.name = modpath_from_file(fs_path)[-1]
def __init__(self, project_path): super(ProjectNode, self).__init__() #gets the python 'dotpath' of the project root. If the project root #itself is in the Python path, this will be an empty string self.name = '.'.join(modpath_from_file(project_path)) self.parent = None self.scope = None #the file system (not python) path to the project self._fs_path = project_path
def __init__(self, parent, fs_path): astroid_object = AstroidManager().ast_from_file(fs_path) super(ModuleNode, self).__init__( parent=parent, astroid_object=astroid_object) self._fs_path = fs_path #gets the module name -- the whole return value of modpath_from_file #is a list containing each element of the dotpath self.name = modpath_from_file(fs_path)[-1]
def expand_modules(files_or_modules, black_list): """take a list of files/modules/packages and return the list of tuple (file, module name) which have to be actually checked """ result = [] errors = [] for something in files_or_modules: if exists(something): # this is a file or a directory try: modname = ".".join(modpath_from_file(something)) except ImportError: modname = splitext(basename(something))[0] if isdir(something): filepath = join(something, "__init__.py") else: filepath = something else: # suppose it's a module or package modname = something try: filepath = file_from_modpath(modname.split(".")) if filepath is None: errors.append({"key": "ignored-builtin-module", "mod": modname}) continue except (ImportError, SyntaxError), ex: # FIXME p3k : the SyntaxError is a Python bug and should be # removed as soon as possible http://bugs.python.org/issue10588 errors.append({"key": "fatal", "mod": modname, "ex": ex}) continue filepath = normpath(filepath) result.append({"path": filepath, "name": modname, "isarg": True, "basepath": filepath, "basename": modname}) if not (modname.endswith(".__init__") or modname == "__init__") and "__init__.py" in filepath: for subfilepath in get_module_files(dirname(filepath), black_list): if filepath == subfilepath: continue submodname = ".".join(modpath_from_file(subfilepath)) result.append( {"path": subfilepath, "name": submodname, "isarg": False, "basepath": filepath, "basename": modname} )
def expand_modules(files_or_modules, black_list): """take a list of files/modules/packages and return the list of tuple (file, module name) which have to be actually checked """ result = [] errors = [] for something in files_or_modules: if exists(something): # this is a file or a directory try: modname = '.'.join(modpath_from_file(something)) except ImportError: modname = splitext(basename(something))[0] if isdir(something): filepath = join(something, '__init__.py') else: filepath = something else: # suppose it's a module or package modname = something try: filepath = file_from_modpath(modname.split('.')) if filepath is None: errors.append( {'key' : 'F0003', 'mod': modname} ) continue except ImportError, ex: errors.append( {'key': 'F0001', 'mod': modname, 'ex': ex} ) continue filepath = normpath(filepath) result.append( {'path': filepath, 'name': modname, 'basepath': filepath, 'basename': modname} ) if not (modname.endswith('.__init__') or modname == '__init__') \ and '__init__.py' in filepath: for subfilepath in get_module_files(dirname(filepath), black_list): if filepath == subfilepath: continue submodname = '.'.join(modpath_from_file(subfilepath)) result.append( {'path': subfilepath, 'name': submodname, 'basepath': filepath, 'basename': modname} )
def find_modules(self, exclude_dirs): basepath = osp.dirname(self.code_dir) basedir = osp.basename(basepath) + osp.sep if basedir not in sys.path: sys.path.insert(1, basedir) for filepath in globfind(self.code_dir, '*.py', exclude_dirs): if osp.basename(filepath) in ('setup.py', '__pkginfo__.py'): continue try: module = load_module_from_file(filepath) except: # module might be broken or magic dotted_path = modpath_from_file(filepath) module = type('.'.join(dotted_path), (), {}) # mock it yield module
def expand_modules(files_or_modules, black_list): """take a list of files/modules/packages and return the list of tuple (file, module name) which have to be actually checked """ result = [] errors = [] for something in files_or_modules: if exists(something): # this is a file or a directory try: modname = ".".join(modpath_from_file(something)) except ImportError: modname = splitext(basename(something))[0] if isdir(something): filepath = join(something, "__init__.py") else: filepath = something else: # suppose it's a module or package modname = something try: filepath = file_from_modpath(modname.split(".")) if filepath is None: errors.append({"key": "F0003", "mod": modname}) continue except ImportError, ex: errors.append({"key": "F0001", "mod": modname, "ex": ex}) continue filepath = normpath(filepath) result.append({"path": filepath, "name": modname, "basepath": filepath, "basename": modname}) if not (modname.endswith(".__init__") or modname == "__init__") and "__init__.py" in filepath: for subfilepath in get_module_files(dirname(filepath), black_list): if filepath == subfilepath: continue submodname = ".".join(modpath_from_file(subfilepath)) result.append({"path": subfilepath, "name": submodname, "basepath": filepath, "basename": modname})
def _modname_from_path(path, extrapath=None): modpath = modpath_from_file(path, extrapath) # omit '__init__' from package's name to avoid loading that module # once for each name when it is imported by some other object # module. This supposes import in modules are done as:: # # from package import something # # not:: # # from package.__init__ import something # # which seems quite correct. if modpath[-1] == '__init__': modpath.pop() return '.'.join(modpath)
def _modname_from_path(path, extrapath=None): modpath = modpath_from_file(path, extrapath) # omit '__init__' from package's name to avoid loading that module # once for each name when it is imported by some other object # module. This supposes import in modules are done as:: # # from package import something # # not:: # # from package.__init__ import something # # which seems quite correct. if modpath[-1] == "__init__": modpath.pop() return ".".join(modpath)
def load_django_settings(self, dirname): """try to find project's setting and load it""" curdir = osp.abspath(dirname) previousdir = curdir while not osp.isfile(osp.join(curdir, 'settings.py')) and \ osp.isfile(osp.join(curdir, '__init__.py')): newdir = osp.normpath(osp.join(curdir, os.pardir)) if newdir == curdir: raise AssertionError('could not find settings.py') previousdir = curdir curdir = newdir # late django initialization settings = load_module_from_modpath(modpath_from_file(osp.join(curdir, 'settings.py'))) from django.core.management import setup_environ setup_environ(settings) settings.DEBUG = False self.settings = settings # add settings dir to pythonpath since it's the project's root if curdir not in sys.path: sys.path.insert(1, curdir)
def ast_from_file(self, filepath, modname=None, fallback=True, source=False): """given a module name, return the astroid object""" try: filepath = get_source_file(filepath, include_no_ext=True) source = True except NoSourceFile: pass if modname is None: try: modname = '.'.join(modpath_from_file(filepath)) except ImportError: modname = filepath if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath: return self.astroid_cache[modname] if source: from astroid.builder import AstroidBuilder return AstroidBuilder(self).file_build(filepath, modname) elif fallback and modname: return self.ast_from_module_name(modname) raise AstroidBuildingException('unable to get astroid for file %s' % filepath)
def exec_file(self, filepath, modname): if modname is None: try: modname = '.'.join(modpath_from_file(filepath, self.extrapath)) except ImportError: warn('module for %s can\'t be found, add necessary __init__.py ' 'files to make it importable' % filepath, DeprecationWarning) modname = splitext(basename(filepath))[0] if modname in sys.modules: module = sys.modules[modname] # NOTE: don't test raw equality to avoid .pyc / .py comparisons mpath = realpath(abspath(module.__file__)) fpath = realpath(abspath(filepath)) assert mpath.startswith(fpath), ( modname, filepath, module.__file__) else: fglobals = {} # self.context.copy() fglobals['__file__'] = filepath fglobals['__name__'] = modname package = '.'.join(modname.split('.')[:-1]) if package and not package in sys.modules: __import__(package) with open(filepath) as f: try: code = compile(f.read(), filepath, 'exec') exec(code, fglobals) except: print('exception while reading %s' % filepath, file=sys.stderr) raise fglobals['__file__'] = filepath module = types.ModuleType(str(modname)) module.__dict__.update(fglobals) sys.modules[modname] = module if package: setattr(sys.modules[package], modname.split('.')[-1], module) if basename(filepath) == '__init__.py': # add __path__ to make dynamic loading work as defined in PEP 302 # https://www.python.org/dev/peps/pep-0302/#packages-and-the-role-of-path module.__path__ = [dirname(filepath)] return (modname, module)
path is expected to be a python source file """ try: stream, encoding, data = open_source_file(path) except IOError, exc: msg = 'Unable to load file %r (%s)' % (path, exc) raise AstroidBuildingException(msg) except SyntaxError, exc: # py3k encoding specification error raise AstroidBuildingException(exc) except LookupError, exc: # unknown encoding raise AstroidBuildingException(exc) # get module name if necessary if modname is None: try: modname = '.'.join(modpath_from_file(path)) except ImportError: modname = splitext(basename(path))[0] # build astroid representation node = self.string_build(data, modname, path) node.file_encoding = encoding return node def string_build(self, data, modname='', path=None): """build astroid from source code string and return rebuilded astroid""" module = self._data_build(data, modname, path) self._manager.astroid_cache[module.name] = module # post tree building steps after we stored the module in the cache: for from_node in module._from_nodes: self.add_from_names_to_locals(from_node) # handle delayed assattr nodes
def test_knownValues_modpath_from_file_1(self): with warnings.catch_warnings(record=True) as warns: self.assertEqual(modutils.modpath_from_file(modutils.__file__), ['logilab', 'common', 'modutils']) self.assertIn('you should avoid using modpath_from_file()', [str(w.message) for w in warns])
class ASTNGBuilder: """provide astng building methods """ def __init__(self, manager=None): if manager is None: from logilab.astng import MANAGER as manager self._manager = manager self._module = None self._file = None self._done = None self.rebuilder = TreeRebuilder(manager) self._dyn_modname_map = {'gtk': 'gtk._gtk'} def module_build(self, module, modname=None): """build an astng from a living module instance """ node = None self._module = module path = getattr(module, '__file__', None) if path is not None: path_, ext = splitext(module.__file__) if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'): node = self.file_build(path_ + '.py', modname) if node is None: # this is a built-in module # get a partial representation by introspection node = self.inspect_build(module, modname=modname, path=path) return node def inspect_build(self, module, modname=None, path=None): """build astng from a living module (i.e. using inspect) this is used when there is no python source code available (either because it's a built-in module or because the .py is not available) """ self._module = module if modname is None: modname = module.__name__ node = build_module(modname, module.__doc__) node.file = node.path = path and abspath(path) or path if self._manager is not None: self._manager._cache[modname] = node node.package = hasattr(module, '__path__') self._done = {} self.object_build(node, module) return node def file_build(self, path, modname=None): """build astng from a source code file (i.e. from an ast) path is expected to be a python source file """ try: data = norm_read(path) except IOError, ex: msg = 'Unable to load file %r (%s)' % (path, ex) raise ASTNGBuildingException(msg) self._file = path # get module name if necessary, *before modifying sys.path* if modname is None: try: modname = '.'.join(modpath_from_file(path)) except ImportError: modname = splitext(basename(path))[0] # build astng representation try: sys.path.insert(0, dirname(path)) node = self.string_build(data, modname, path) node.file = abspath(path) finally: self._file = None sys.path.pop(0) return node
def test_knownValues_modpath_from_file_1(self): self.assertEqual(modutils.modpath_from_file(modutils.__file__), ['logilab', 'common', 'modutils'])
path is expected to be a python source file """ try: stream, encoding, data = open_source_file(path) except IOError, exc: msg = 'Unable to load file %r (%s)' % (path, exc) raise ASTNGBuildingException(msg) except SyntaxError, exc: # py3k encoding specification error raise ASTNGBuildingException(exc) except LookupError, exc: # unknown encoding raise ASTNGBuildingException(exc) # get module name if necessary, *before modifying sys.path* if modname is None: try: modname = '.'.join(modpath_from_file(path)) except ImportError: modname = splitext(basename(path))[0] # build astng representation try: sys.path.insert(0, dirname(path)) # XXX (syt) iirk node = self.string_build(data, modname, path) finally: sys.path.pop(0) node.file_encoding = encoding node.file_stream = stream return node def string_build(self, data, modname='', path=None): """build astng from source code string and return rebuilded astng""" module = self._data_build(data, modname, path)
def test_knownValues_modpath_from_file_1(self): self.assertEqual(modutils.modpath_from_file(modutils.__file__), ["logilab", "common", "modutils"])
def test_knownValues_modpath_from_file_2(self): self.assertEqual(modutils.modpath_from_file('unittest_modutils.py', {getcwd(): 'arbitrary.pkg'}), ['arbitrary', 'pkg', 'unittest_modutils'])
def test_knownValues_modpath_from_file_2(self): self.assertEqual( modutils.modpath_from_file("unittest_modutils.py", {getcwd(): "arbitrary.pkg"}), ["arbitrary", "pkg", "unittest_modutils"], )
def test_knownValues_modpath_from_file_2(self): self.assertEqual( modutils.modpath_from_file('unittest_modutils.py', {getcwd(): 'arbitrary.pkg'}), ['arbitrary', 'pkg', 'unittest_modutils'])