def get_child_modules(path, fullname):
"""
Return the suffixes of submodules directly neated beneath of the package
directory at `path`.
:param str path:
Path to the module's source code on disk, or some PEP-302-recognized
equivalent. Usually this is the module's ``__file__`` attribute, but
is specified explicitly to avoid loading the module.
:param str fullname:
Name of the package we're trying to get child modules for
:return:
List of submodule name suffixes.
"""
mod_path = os.path.dirname(path)
if mod_path != '':
return [
to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])
]
else:
# we loaded some weird package in memory, so we'll see if it has a custom loader we can use
loader = pkgutil.find_loader(fullname)
return [to_text(name)
for name, _ in loader.iter_modules(None)] if loader else []
def _build_tuple(self, fullname):
if mitogen.core.is_blacklisted_import(self, fullname):
raise ImportError('blacklisted')
if fullname in self._cache:
return self._cache[fullname]
path, source, is_pkg = self._finder.get_module_source(fullname)
if source is None:
LOG.error('_build_tuple(%r): could not locate source', fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if is_pkg:
pkg_present = get_child_modules(path)
LOG.debug('_build_tuple(%r, %r) -> %r', path, fullname,
pkg_present)
else:
pkg_present = None
if fullname == '__main__':
source = self.neutralize_main(path, source)
compressed = mitogen.core.Blob(zlib.compress(source, 9))
related = [
to_text(name) for name in self._finder.find_related(fullname)
if not mitogen.core.is_blacklisted_import(self, name)
]
# 0:fullname 1:pkg_present 2:path 3:compressed 4:related
tup = (to_text(fullname), pkg_present, to_text(path), compressed,
related)
self._cache[fullname] = tup
return tup
def _build_tuple(self, fullname):
if fullname in self._cache:
return self._cache[fullname]
if mitogen.core.is_blacklisted_import(self, fullname):
raise ImportError('blacklisted')
path, source, is_pkg = self._finder.get_module_source(fullname)
if path and is_stdlib_path(path):
# Prevent loading of 2.x<->3.x stdlib modules! This costs one
# RTT per hit, so a client-side solution is also required.
self._log.debug('refusing to serve stdlib module %r', fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if source is None:
# TODO: make this .warning() or similar again once importer has its
# own logging category.
self._log.debug('could not find source for %r', fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if self.minify_safe_re.search(source):
# If the module contains a magic marker, it's safe to minify.
t0 = mitogen.core.now()
source = mitogen.minify.minimize_source(source).encode('utf-8')
self.minify_secs += mitogen.core.now() - t0
if is_pkg:
pkg_present = get_child_modules(path, fullname)
self._log.debug('%s is a package at %s with submodules %r',
fullname, path, pkg_present)
else:
pkg_present = None
if fullname == '__main__':
source = self.neutralize_main(path, source)
compressed = mitogen.core.Blob(zlib.compress(source, 9))
related = [
to_text(name)
for name in self._finder.find_related(fullname)
if not mitogen.core.is_blacklisted_import(self, name)
]
# 0:fullname 1:pkg_present 2:path 3:compressed 4:related
tup = (
to_text(fullname),
pkg_present,
to_text(path),
compressed,
related
)
self._cache[fullname] = tup
return tup
def find(self, fullname):
"""
See implementation for a description of how this works.
"""
#if fullname not in sys.modules:
# Don't attempt this unless a module really exists in sys.modules,
# else we could return junk.
#return
fullname = to_text(fullname)
modname, search_path, modpath = self._find_sane_parent(fullname)
while True:
tup = self._find_one_component(modpath.pop(0), search_path)
if tup is None:
return None
fp, path, (suffix, _, kind) = tup
if modpath:
# Still more components to descent. Result must be a package
if fp:
fp.close()
if kind != imp.PKG_DIRECTORY:
LOG.debug('%r: %r appears to be child of non-package %r',
self, fullname, path)
return None
search_path = [path]
elif kind == imp.PKG_DIRECTORY:
return self._found_package(fullname, path)
else:
return self._found_module(fullname, path, fp)
def _find_sane_parent(self, fullname):
"""
Iteratively search :data:`sys.modules` for the least indirect parent of
`fullname` that is loaded and contains a :data:`__path__` attribute.
:return:
`(parent_name, path, modpath)` tuple, where:
* `modname`: canonical name of the found package, or the empty
string if none is found.
* `search_path`: :data:`__path__` attribute of the least
indirect parent found, or :data:`None` if no indirect parent
was found.
* `modpath`: list of module name components leading from `path`
to the target module.
"""
path = None
modpath = []
while True:
pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
modpath.insert(0, modname)
if not pkgname:
return [], None, modpath
pkg = sys.modules.get(pkgname)
path = getattr(pkg, '__path__', None)
if pkg and path:
return pkgname.split('.'), path, modpath
LOG.debug('%r: %r lacks __path__ attribute', self, pkgname)
fullname = pkgname
def find(self, fullname):
if fullname not in sys.modules:
# Don't attempt this unless a module really exists in sys.modules,
# else we could return junk.
return
pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
pkg = sys.modules.get(pkgname)
if pkg is None or not hasattr(pkg, '__file__'):
LOG.debug('%r: %r is not a package or lacks __file__ attribute',
self, pkgname)
return
pkg_path = [os.path.dirname(pkg.__file__)]
try:
fp, path, (suffix, _, kind) = imp.find_module(modname, pkg_path)
except ImportError:
e = sys.exc_info()[1]
LOG.debug('%r: imp.find_module(%r, %r) -> %s', self, modname,
[pkg_path], e)
return None
if kind == imp.PKG_DIRECTORY:
return self._found_package(fullname, path)
else:
return self._found_module(fullname, path, fp)
def _build_tuple(self, fullname):
if fullname in self._cache:
return self._cache[fullname]
if mitogen.core.is_blacklisted_import(self, fullname):
raise ImportError('blacklisted')
path, source, is_pkg = self._finder.get_module_source(fullname)
if path and is_stdlib_path(path):
# Prevent loading of 2.x<->3.x stdlib modules! This costs one
# RTT per hit, so a client-side solution is also required.
LOG.debug('%r: refusing to serve stdlib module %r', self, fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if source is None:
# TODO: make this .warning() or similar again once importer has its
# own logging category.
LOG.debug('_build_tuple(%r): could not locate source', fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if is_pkg:
pkg_present = get_child_modules(path)
LOG.debug('_build_tuple(%r, %r) -> %r', path, fullname,
pkg_present)
else:
pkg_present = None
if fullname == '__main__':
source = self.neutralize_main(path, source)
compressed = mitogen.core.Blob(zlib.compress(source, 9))
related = [
to_text(name) for name in self._finder.find_related(fullname)
if not mitogen.core.is_blacklisted_import(self, name)
]
# 0:fullname 1:pkg_present 2:path 3:compressed 4:related
tup = (to_text(fullname), pkg_present, to_text(path), compressed,
related)
self._cache[fullname] = tup
return tup
def get_child_modules(path):
"""Return the suffixes of submodules directly neated beneath of the package
directory at `path`.
:param str path:
Path to the module's source code on disk, or some PEP-302-recognized
equivalent. Usually this is the module's ``__file__`` attribute, but
is specified explicitly to avoid loading the module.
:return:
List of submodule name suffixes.
"""
it = pkgutil.iter_modules([os.path.dirname(path)])
return [to_text(name) for _, name, _ in it]
def load_module(self, fullname):
path, is_pkg = self._by_fullname[fullname]
source = ansible_mitogen.target.get_small_file(self._context, path)
code = compile(source, path, 'exec', 0, 1)
mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
mod.__file__ = "master:%s" % (path, )
mod.__loader__ = self
if is_pkg:
mod.__path__ = []
mod.__package__ = str(fullname)
else:
mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
exec(code, mod.__dict__)
self._loaded.add(fullname)
return mod
def _get_module_via_parent_enumeration(self, fullname):
"""
Attempt to fetch source code by examining the module's (hopefully less
insane) parent package. Required for older versions of
ansible.compat.six and plumbum.colors.
"""
if fullname not in sys.modules:
# Don't attempt this unless a module really exists in sys.modules,
# else we could return junk.
return
pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
pkg = sys.modules.get(pkgname)
if pkg is None or not hasattr(pkg, '__file__'):
return
pkg_path = os.path.dirname(pkg.__file__)
try:
fp, path, ext = imp.find_module(modname, [pkg_path])
try:
path = self._py_filename(path)
if not path:
fp.close()
return
source = fp.read()
finally:
if fp:
fp.close()
if isinstance(source, mitogen.core.UnicodeType):
# get_source() returns "string" according to PEP-302, which was
# reinterpreted for Python 3 to mean a Unicode string.
source = source.encode('utf-8')
return path, source, False
except ImportError:
e = sys.exc_info()[1]
LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
def generate_parent_names(self, fullname):
while '.' in fullname:
fullname, _, _ = str_rpartition(to_text(fullname), u'.')
yield fullname
def get_child_modules(path):
it = pkgutil.iter_modules([os.path.dirname(path)])
return [to_text(name) for _, name, _ in it]