def checkPluginPath(plugin_filename, module_package):
plugin_filename = os.path.normpath(plugin_filename)
if Options.isShowInclusion():
recursion_logger.info("Checking top level plug-in path %s %s" %
(plugin_filename, module_package))
plugin_info = considerFilename(module_filename=plugin_filename)
if plugin_info is not None:
# File or package makes a difference, handle that
if os.path.isfile(plugin_info[0]) or Importing.isPackageDir(
plugin_info[0]):
checkPluginSinglePath(plugin_filename,
module_package=module_package)
elif os.path.isdir(plugin_info[0]):
for sub_path, sub_filename in listDir(plugin_info[0]):
assert sub_filename != "__init__.py"
if Importing.isPackageDir(sub_path) or sub_path.endswith(
".py"):
checkPluginSinglePath(sub_path, module_package=None)
else:
recursion_logger.warning("Failed to include module from %r." %
plugin_info[0])
else:
recursion_logger.warning("Failed to recurse to directory %r." %
plugin_filename)
def detectPthImportedPackages():
if not hasattr(sys.modules["site"], "getsitepackages"):
return ()
# TODO: Move hard import config to elsewhere.
from nuitka.nodes.ImportNodes import isHardModuleWithoutSideEffect
pth_imports = set()
for prefix in sys.modules["site"].getsitepackages():
if not os.path.isdir(prefix):
continue
for path, filename in listDir(prefix):
if filename.endswith(".pth"):
try:
for line in getFileContentByLine(path, "rU"):
if line.startswith("import "):
if ";" in line:
line = line[:line.find(";")]
for part in line[7:].split(","):
pth_import = part.strip()
if not isHardModuleWithoutSideEffect(
pth_import):
pth_imports.add(pth_import)
except OSError:
recursion_logger.warning(
"Python installation problem, cannot read file '%s'.")
return tuple(sorted(pth_imports))
def _makeModuleBodyFromSyntaxError(exc, module_name, module_filename):
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot follow import to module '%s' because of %r.""" %
(module_name, exc.__class__.__name__))
source_ref = SourceCodeReferences.fromFilename(filename=module_filename)
module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode="compiled",
future_spec=FutureSpec(),
source_ref=source_ref,
)
module_body = makeModuleFrame(
module=module,
statements=(makeRaiseExceptionStatementFromInstance(
source_ref=source_ref, exception=exc), ),
source_ref=source_ref,
)
module_body = makeStatementsSequenceFromStatement(statement=module_body)
module.setChild("body", module_body)
return module
def warnAbout(importing, module_name, parent_package, level, tried_names):
# This probably should not be dealt with here, pylint: disable=too-many-branches
if module_name == "":
return
if not isIgnoreListedImportMaker(importing) and not isIgnoreListedNotExistingModule(
module_name
):
key = module_name, parent_package, level
if key not in warned_about:
warned_about.add(key)
if parent_package is None:
full_name = module_name
else:
full_name = module_name
if Plugins.suppressUnknownImportWarning(importing, full_name):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if _debug_module_finding:
if parent_package is not None:
recursion_logger.warning(
"%s: Cannot find '%s' in package '%s' %s (tried %s)."
% (
importing.getSourceReference().getAsString(),
module_name,
parent_package,
level_desc,
",".join(tried_names),
)
)
else:
recursion_logger.warning(
"%s: Cannot find '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
level_desc,
)
)
def warnAbout(importing, module_name, level, source_ref):
# This probably should not be dealt with here
if module_name == "":
return
if not isIgnoreListedNotExistingModule(
module_name
) and not isIgnoreListedImportMaker(source_ref):
key = module_name, level
if key not in warned_about:
warned_about.add(key)
if Plugins.suppressUnknownImportWarning(
importing=importing, source_ref=source_ref, module_name=module_name
):
return
if level == 0:
level_desc = "as absolute import"
elif level == -1:
level_desc = "as relative or absolute import"
elif level == 1:
level_desc = "%d package level up" % level
else:
level_desc = "%d package levels up" % level
if _debug_module_finding:
if importing.getPackageName() is not None:
recursion_logger.warning(
"%s: Cannot find '%s' in package '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
importing.getPackageName().asString(),
level_desc,
)
)
else:
recursion_logger.warning(
"%s: Cannot find '%s' %s."
% (
importing.getSourceReference().getAsString(),
module_name,
level_desc,
)
)
def _findModuleInPath(module_name):
package_name, module_name = module_name.splitModuleBasename()
if _debug_module_finding:
print("_findModuleInPath: Enter", module_name, "in", package_name)
# The "site" module must be located based on PYTHONPATH before it was
# executed, while we normally search in PYTHONPATH after it was executed,
# and on some systems, that fails.
if package_name is None and module_name == "site":
candidate = os.environ.get("NUITKA_SITE_FILENAME", "")
if candidate:
return candidate
# Free pass for built-in modules, the need not exist.
if package_name is None and imp.is_builtin(module_name):
return None
search_path = getPackageSearchPath(package_name)
if _debug_module_finding:
print("_findModuleInPath: Using search path", search_path, "for",
package_name)
try:
module_filename = _findModuleInPath2(package_name=package_name,
module_name=module_name,
search_path=search_path)
except SyntaxError:
# Warn user, as this is kind of unusual.
recursion_logger.warning(
"%s: Module cannot be imported due to syntax errors.",
module_name if package_name is None else package_name + "." +
module_name,
)
return None
if _debug_module_finding:
print("_findModuleInPath: _findModuleInPath2 gave", module_filename)
return module_filename
def checkPluginFilenamePattern(pattern):
if Options.isShowInclusion():
recursion_logger.info("Checking plug-in pattern '%s':" % pattern)
assert not os.path.isdir(pattern), pattern
found = False
for filename in glob.iglob(pattern):
if filename.endswith(".pyc"):
continue
if not os.path.isfile(filename):
continue
found = True
checkPluginSinglePath(filename, module_package=None)
if not found:
recursion_logger.warning("Didn't match any files against pattern %r." % pattern)
def _makeModuleBodyTooComplex(module_name, module_filename, source_code,
is_package):
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot follow import to import module '%r' ('%r') because code is too complex."""
% (
module_name,
module_filename,
))
module = makeUncompiledPythonModule(
module_name=module_name,
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code, module_filename, "exec", dont_inherit=True)),
is_package=is_package,
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
reason):
from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
from nuitka.tree import Building
module, source_ref, source_code = Building.decideModuleTree(
filename=module_filename,
package=module_package,
is_top=False,
is_main=False,
is_shlib=module_kind == "shlib",
)
if Options.isShowInclusion():
recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" %
(module.getFullName(), module_relpath, reason))
if source_code is not None:
try:
Building.createModuleTree(
module=module,
source_ref=source_ref,
source_code=source_code,
is_main=False,
)
except (SyntaxError, IndentationError) as e:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning(
"""\
Cannot follow import to module %r (%r) because of %r""" %
(module_relpath, module_filename, e.__class__.__name__))
return None, False
except Building.CodeTooComplexCode:
if module_filename not in Importing.warned_about:
Importing.warned_about.add(module_filename)
recursion_logger.warning("""\
Cannot recurse to import module %r (%r) because code is too complex.""" % (
module_relpath,
module_filename,
))
if Options.isStandaloneMode():
module = makeUncompiledPythonModule(
module_name=module.getFullName(),
filename=module_filename,
bytecode=marshal.dumps(
compile(source_code,
module_filename,
"exec",
dont_inherit=True)),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
ModuleRegistry.addUncompiledModule(module)
return None, False
ImportCache.addImportedModule(module)
return module, True
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module_relpath = relpath(plugin_filename)
module, is_added = recurseTo(
module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind=module_kind,
reason=reason,
)
if module:
if not is_added:
recursion_logger.warning(
"Recursed to %s '%s' at '%s' twice." % (
"package"
if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_filename,
))
if not isSameModulePath(module.getFilename(),
plugin_filename):
recursion_logger.warning(
"Duplicate '%s' of '%s' ignored ." % (
plugin_filename,
module.getFilename(),
))
return
if Options.isShowInclusion():
recursion_logger.info("Recursed to '%s' %s" % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonShlibModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
def checkPluginSinglePath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=too-many-branches
# The importing wants these to be unique.
plugin_filename = os.path.abspath(plugin_filename)
if Options.isShowInclusion():
recursion_logger.info("Checking detail plug-in path '%s' '%s':" %
(plugin_filename, module_package))
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
module_name = ModuleName.makeModuleNameInPackage(module_name,
module_package)
if module_kind == "extension" and not Options.isStandaloneMode():
recursion_logger.warning(
"Cannot include '%s' unless using at least standalone mode." %
module_name.asString())
if module_kind is not None:
decision, reason = decideRecursion(
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
extra_recursion=True,
)
if decision:
module = recurseTo(
signal_change=None,
module_filename=plugin_filename,
module_name=module_name,
module_kind=module_kind,
reason=reason,
)
if module:
if Options.isShowInclusion():
recursion_logger.info("Included '%s' as '%s'." % (
module.getFullName(),
module,
))
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if os.path.isdir(package_filename):
# Must be a namespace package.
assert python_version >= 0x300
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = os.path.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
if Options.isShowInclusion():
recursion_logger.info("Package directory '%s'." %
package_dir)
for sub_path, sub_filename in listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(
sub_path) and not os.path.exists(sub_path +
".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif sub_path.endswith(".py"):
checkPluginSinglePath(
sub_path, module_package=module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
elif module.isPythonExtensionModule():
if Options.isStandaloneMode():
ModuleRegistry.addRootModule(module)
else:
recursion_logger.warning(
"Failed to include module from '%s'." % plugin_filename)
