Пример #1
0
def get_child_modules(path, fullname):
    """
    Return the suffixes of submodules directly neated beneath of the package
    directory at `path`.

    :param str path:
        Path to the module's source code on disk, or some PEP-302-recognized
        equivalent. Usually this is the module's ``__file__`` attribute, but
        is specified explicitly to avoid loading the module.
    :param str fullname:
        Name of the package we're trying to get child modules for

    :return:
        List of submodule name suffixes.
    """
    mod_path = os.path.dirname(path)
    if mod_path != '':
        return [
            to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])
        ]
    else:
        # we loaded some weird package in memory, so we'll see if it has a custom loader we can use
        loader = pkgutil.find_loader(fullname)
        return [to_text(name)
                for name, _ in loader.iter_modules(None)] if loader else []
Пример #2
0
    def _build_tuple(self, fullname):
        if mitogen.core.is_blacklisted_import(self, fullname):
            raise ImportError('blacklisted')

        if fullname in self._cache:
            return self._cache[fullname]

        path, source, is_pkg = self._finder.get_module_source(fullname)
        if source is None:
            LOG.error('_build_tuple(%r): could not locate source', fullname)
            tup = self._make_negative_response(fullname)
            self._cache[fullname] = tup
            return tup

        if is_pkg:
            pkg_present = get_child_modules(path)
            LOG.debug('_build_tuple(%r, %r) -> %r', path, fullname,
                      pkg_present)
        else:
            pkg_present = None

        if fullname == '__main__':
            source = self.neutralize_main(path, source)
        compressed = mitogen.core.Blob(zlib.compress(source, 9))
        related = [
            to_text(name) for name in self._finder.find_related(fullname)
            if not mitogen.core.is_blacklisted_import(self, name)
        ]
        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
        tup = (to_text(fullname), pkg_present, to_text(path), compressed,
               related)
        self._cache[fullname] = tup
        return tup
Пример #3
0
    def _build_tuple(self, fullname):
        if fullname in self._cache:
            return self._cache[fullname]

        if mitogen.core.is_blacklisted_import(self, fullname):
            raise ImportError('blacklisted')

        path, source, is_pkg = self._finder.get_module_source(fullname)
        if path and is_stdlib_path(path):
            # Prevent loading of 2.x<->3.x stdlib modules! This costs one
            # RTT per hit, so a client-side solution is also required.
            self._log.debug('refusing to serve stdlib module %r', fullname)
            tup = self._make_negative_response(fullname)
            self._cache[fullname] = tup
            return tup

        if source is None:
            # TODO: make this .warning() or similar again once importer has its
            # own logging category.
            self._log.debug('could not find source for %r', fullname)
            tup = self._make_negative_response(fullname)
            self._cache[fullname] = tup
            return tup

        if self.minify_safe_re.search(source):
            # If the module contains a magic marker, it's safe to minify.
            t0 = mitogen.core.now()
            source = mitogen.minify.minimize_source(source).encode('utf-8')
            self.minify_secs += mitogen.core.now() - t0

        if is_pkg:
            pkg_present = get_child_modules(path, fullname)
            self._log.debug('%s is a package at %s with submodules %r',
                            fullname, path, pkg_present)
        else:
            pkg_present = None

        if fullname == '__main__':
            source = self.neutralize_main(path, source)
        compressed = mitogen.core.Blob(zlib.compress(source, 9))
        related = [
            to_text(name)
            for name in self._finder.find_related(fullname)
            if not mitogen.core.is_blacklisted_import(self, name)
        ]
        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
        tup = (
            to_text(fullname),
            pkg_present,
            to_text(path),
            compressed,
            related
        )
        self._cache[fullname] = tup
        return tup
Пример #4
0
    def find(self, fullname):
        """
        See implementation for a description of how this works.
        """
        #if fullname not in sys.modules:
        # Don't attempt this unless a module really exists in sys.modules,
        # else we could return junk.
        #return

        fullname = to_text(fullname)
        modname, search_path, modpath = self._find_sane_parent(fullname)
        while True:
            tup = self._find_one_component(modpath.pop(0), search_path)
            if tup is None:
                return None

            fp, path, (suffix, _, kind) = tup
            if modpath:
                # Still more components to descent. Result must be a package
                if fp:
                    fp.close()
                if kind != imp.PKG_DIRECTORY:
                    LOG.debug('%r: %r appears to be child of non-package %r',
                              self, fullname, path)
                    return None
                search_path = [path]
            elif kind == imp.PKG_DIRECTORY:
                return self._found_package(fullname, path)
            else:
                return self._found_module(fullname, path, fp)
Пример #5
0
    def _find_sane_parent(self, fullname):
        """
        Iteratively search :data:`sys.modules` for the least indirect parent of
        `fullname` that is loaded and contains a :data:`__path__` attribute.

        :return:
            `(parent_name, path, modpath)` tuple, where:

                * `modname`: canonical name of the found package, or the empty
                   string if none is found.
                * `search_path`: :data:`__path__` attribute of the least
                   indirect parent found, or :data:`None` if no indirect parent
                   was found.
                * `modpath`: list of module name components leading from `path`
                   to the target module.
        """
        path = None
        modpath = []
        while True:
            pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
            modpath.insert(0, modname)
            if not pkgname:
                return [], None, modpath

            pkg = sys.modules.get(pkgname)
            path = getattr(pkg, '__path__', None)
            if pkg and path:
                return pkgname.split('.'), path, modpath

            LOG.debug('%r: %r lacks __path__ attribute', self, pkgname)
            fullname = pkgname
Пример #6
0
    def find(self, fullname):
        if fullname not in sys.modules:
            # Don't attempt this unless a module really exists in sys.modules,
            # else we could return junk.
            return

        pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
        pkg = sys.modules.get(pkgname)
        if pkg is None or not hasattr(pkg, '__file__'):
            LOG.debug('%r: %r is not a package or lacks __file__ attribute',
                      self, pkgname)
            return

        pkg_path = [os.path.dirname(pkg.__file__)]
        try:
            fp, path, (suffix, _, kind) = imp.find_module(modname, pkg_path)
        except ImportError:
            e = sys.exc_info()[1]
            LOG.debug('%r: imp.find_module(%r, %r) -> %s', self, modname,
                      [pkg_path], e)
            return None

        if kind == imp.PKG_DIRECTORY:
            return self._found_package(fullname, path)
        else:
            return self._found_module(fullname, path, fp)
Пример #7
0
    def _build_tuple(self, fullname):
        if fullname in self._cache:
            return self._cache[fullname]

        if mitogen.core.is_blacklisted_import(self, fullname):
            raise ImportError('blacklisted')

        path, source, is_pkg = self._finder.get_module_source(fullname)
        if path and is_stdlib_path(path):
            # Prevent loading of 2.x<->3.x stdlib modules! This costs one
            # RTT per hit, so a client-side solution is also required.
            LOG.debug('%r: refusing to serve stdlib module %r', self, fullname)
            tup = self._make_negative_response(fullname)
            self._cache[fullname] = tup
            return tup

        if source is None:
            # TODO: make this .warning() or similar again once importer has its
            # own logging category.
            LOG.debug('_build_tuple(%r): could not locate source', fullname)
            tup = self._make_negative_response(fullname)
            self._cache[fullname] = tup
            return tup

        if is_pkg:
            pkg_present = get_child_modules(path)
            LOG.debug('_build_tuple(%r, %r) -> %r', path, fullname,
                      pkg_present)
        else:
            pkg_present = None

        if fullname == '__main__':
            source = self.neutralize_main(path, source)
        compressed = mitogen.core.Blob(zlib.compress(source, 9))
        related = [
            to_text(name) for name in self._finder.find_related(fullname)
            if not mitogen.core.is_blacklisted_import(self, name)
        ]
        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
        tup = (to_text(fullname), pkg_present, to_text(path), compressed,
               related)
        self._cache[fullname] = tup
        return tup
Пример #8
0
def get_child_modules(path):
    """Return the suffixes of submodules directly neated beneath of the package
    directory at `path`.

    :param str path:
        Path to the module's source code on disk, or some PEP-302-recognized
        equivalent. Usually this is the module's ``__file__`` attribute, but
        is specified explicitly to avoid loading the module.

    :return:
        List of submodule name suffixes.
    """
    it = pkgutil.iter_modules([os.path.dirname(path)])
    return [to_text(name) for _, name, _ in it]
Пример #9
0
 def load_module(self, fullname):
     path, is_pkg = self._by_fullname[fullname]
     source = ansible_mitogen.target.get_small_file(self._context, path)
     code = compile(source, path, 'exec', 0, 1)
     mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
     mod.__file__ = "master:%s" % (path, )
     mod.__loader__ = self
     if is_pkg:
         mod.__path__ = []
         mod.__package__ = str(fullname)
     else:
         mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
     exec(code, mod.__dict__)
     self._loaded.add(fullname)
     return mod
Пример #10
0
    def _get_module_via_parent_enumeration(self, fullname):
        """
        Attempt to fetch source code by examining the module's (hopefully less
        insane) parent package. Required for older versions of
        ansible.compat.six and plumbum.colors.
        """
        if fullname not in sys.modules:
            # Don't attempt this unless a module really exists in sys.modules,
            # else we could return junk.
            return

        pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
        pkg = sys.modules.get(pkgname)
        if pkg is None or not hasattr(pkg, '__file__'):
            return

        pkg_path = os.path.dirname(pkg.__file__)
        try:
            fp, path, ext = imp.find_module(modname, [pkg_path])
            try:
                path = self._py_filename(path)
                if not path:
                    fp.close()
                    return

                source = fp.read()
            finally:
                if fp:
                    fp.close()

            if isinstance(source, mitogen.core.UnicodeType):
                # get_source() returns "string" according to PEP-302, which was
                # reinterpreted for Python 3 to mean a Unicode string.
                source = source.encode('utf-8')
            return path, source, False
        except ImportError:
            e = sys.exc_info()[1]
            LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
Пример #11
0
 def generate_parent_names(self, fullname):
     while '.' in fullname:
         fullname, _, _ = str_rpartition(to_text(fullname), u'.')
         yield fullname
Пример #12
0
def get_child_modules(path):
    it = pkgutil.iter_modules([os.path.dirname(path)])
    return [to_text(name) for _, name, _ in it]