def locateModule(module_name, module_package, source_ref):
from nuitka.importing import Importing
_module_package, module_filename, _finding = Importing.findModule(
source_ref=source_ref, module_name=module_name, parent_package=module_package, level=-1, warn=True
)
return module_filename
def locateModule(importing, module_name, module_package):
from nuitka.importing import Importing
_module_package, module_filename, _finding = Importing.findModule(
importing=importing,
module_name=module_name,
parent_package=module_package,
level=-1,
warn=True)
return module_filename
def _considerImplicitImports(plugin, module):
from nuitka.importing import Importing
result = []
for full_name in plugin.getImplicitImports(module):
if type(full_name) in (tuple, list):
raise NuitkaPluginError(
"Plugin %r needs to be change to only return modules names, not %r"
% (plugin, full_name)
)
full_name = ModuleName(full_name)
try:
_module_package, module_filename, _finding = Importing.findModule(
importing=module,
module_name=full_name,
parent_package=None,
level=-1,
warn=False,
)
module_filename = plugin.locateModule(
importing=module, module_name=full_name
)
except Exception:
plugin.warning(
"Problem locating '%s' for implicit imports of '%s'."
% (module.getFullName(), full_name)
)
raise
if module_filename is None:
if Options.isShowInclusion():
plugin.info(
"Implicit module '%s' suggested for '%s' not found."
% (full_name, module.getFullName())
)
continue
result.append((full_name, module_filename))
if result:
plugin.info(
"Implicit dependencies of module '%s' added '%s'."
% (module.getFullName(), ",".join(r[0] for r in result))
)
return result
def attemptRecursion(self):
# Make sure the package is recursed to.
from nuitka.importing import Recursion
# Return the list of newly added modules.
result = []
if self.package_name is not None and self.package is None:
package_package, package_filename, _finding = \
Importing.findModule(
source_ref = self.getSourceReference(),
module_name = self.package_name,
parent_package = None,
level = 1,
warn = Utils.python_version < 330
)
# TODO: Temporary, if we can't find the package for Python3.3 that
# is semi-OK, maybe.
if Utils.python_version >= 330 and not package_filename:
return []
imported_module, is_added = Recursion.recurseTo(
module_package = package_package,
module_filename = package_filename,
module_relpath = Utils.relpath(package_filename),
module_kind = "py",
reason = "Containing package of recursed module.",
)
self.package = imported_module
if is_added:
result.append(imported_module)
if self.package:
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(self.package)
# print "Recursed to package", self.package_name
result.extend(self.package.attemptRecursion())
return result
def attemptRecursion(self):
# Make sure the package is recursed to.
from nuitka.importing import Recursion
# Return the list of newly added modules.
result = []
if self.package_name is not None and self.package is None:
package_package, package_filename, _finding = \
Importing.findModule(
source_ref = self.getSourceReference(),
module_name = self.package_name,
parent_package = None,
level = 1,
warn = Utils.python_version < 330
)
# TODO: Temporary, if we can't find the package for Python3.3 that
# is semi-OK, maybe.
if Utils.python_version >= 330 and not package_filename:
return []
imported_module, is_added = Recursion.recurseTo(
module_package=package_package,
module_filename=package_filename,
module_relpath=Utils.relpath(package_filename),
module_kind="py",
reason="Containing package of recursed module.",
)
self.package = imported_module
if is_added:
result.append(imported_module)
if self.package:
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(self.package)
# print "Recursed to package", self.package_name
result.extend(self.package.attemptRecursion())
return result
def locateModule(importing, module_name):
"""Provide a filename / -path for a to-be-imported module.
Args:
importing: module object that asked for it (tracing only)
module_name: (str or ModuleName) full name of module
Returns:
filename for module
"""
from nuitka.importing import Importing
_module_package, module_filename, _finding = Importing.findModule(
importing=importing,
module_name=ModuleName(module_name),
parent_package=None,
level=-1,
warn=False,
)
return module_filename
def locateModule(importing, module_name, warn):
""" Provide a filename / -path for a to-be-imported module.
Args:
importing: module object
module_name: (str or ModuleName) full name of module
warn: (bool) True if required module
Returns:
filename for module
"""
from nuitka.importing import Importing
_module_package, module_filename, _finding = Importing.findModule(
importing=importing,
module_name=ModuleName(module_name),
parent_package=None,
level=-1,
warn=warn,
)
return module_filename
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename = filename,
package = None,
is_top = True,
is_main = not Options.shallMakeModule()
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(
path = standalone_dir,
ignore_errors = True
)
makePath(standalone_dir)
deleteFile(
path = getResultFullpath(main_module),
must_exist = False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(
plugin_filename = plugin_filename,
module_package = None
)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(
pattern = pattern
)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing = None,
module_name = package_name,
parent_package = None,
level = 0,
warn = False
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename = package_directory,
module_package = package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing = None,
module_name = module_name,
parent_package = None,
level = 0,
warn = False
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename = module_filename,
module_package = module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def _createNodeTree(filename):
"""Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# Many cases to deal with, pylint: disable=too-many-branches
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
SconsInterface.cleanSconsDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = OutputDirectories.getStandaloneDirectoryPath()
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
# Delete result file, to avoid confusion with previous build and to
# avoid locking issues after the build.
deleteFile(path=OutputDirectories.getResultFullpath(onefile=False),
must_exist=False)
if Options.isOnefileMode():
deleteFile(path=OutputDirectories.getResultFullpath(onefile=True),
must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename,
module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=ModuleName(package_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate package %r you asked to include." %
package_name)
Recursion.checkPluginPath(plugin_filename=package_directory,
module_package=package_package)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=ModuleName(module_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate module %r you asked to include." %
module_name)
Recursion.checkPluginSinglePath(plugin_filename=module_filename,
module_package=module_package)
# Allow plugins to add more modules based on the initial set being complete.
Plugins.onModuleInitialSet()
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=package_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=module_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert package is None or type(package) is ModuleName
assert filename is not None
if is_main and os.path.isdir(filename):
source_filename = os.path.join(filename, "__main__.py")
if not os.path.isfile(source_filename):
sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if os.path.isfile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(filename=filename)
if is_main:
module_name = ModuleName("__main__")
else:
# Derive module name from filename.
module_name = os.path.basename(filename)
if is_shlib:
module_name = module_name.split(".")[0]
elif module_name.endswith(".py"):
module_name = module_name[:-3]
if "." in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" %
(module_name))
sys.exit(2)
module_name = ModuleName.makeModuleNameInPackage(
module_name, package)
if is_shlib:
result = PythonShlibModule(module_name=module_name,
source_ref=source_ref)
source_code = None
else:
source_code = readSourceCodeFromFilename(
module_name=module_name, source_filename=source_filename)
if is_main:
result = PythonMainModule(
main_added=main_added,
mode=decideCompilationMode(False, module_name, source_ref),
future_spec=None,
source_ref=source_ref,
)
checkPythonVersionFromCode(source_code)
else:
mode = decideCompilationMode(is_top, module_name, source_ref)
if (mode == "bytecode" and not is_top
and hasCachedImportedModulesNames(
module_name, source_code)):
optimization_logger.info("%r is included as bytecode." %
(module_name.asString()))
result = UncompiledPythonModule(
module_name=module_name,
filename=filename,
bytecode=demoteSourceCodeToBytecode(
module_name=module_name,
source_code=source_code,
filename=filename,
),
source_ref=source_ref,
user_provided=False,
technical=False,
)
used_modules = OrderedSet()
for used_module_name in getCachedImportedModulesNames(
module_name=module_name, source_code=source_code):
(
_module_package,
module_filename,
_finding,
) = Importing.findModule(
importing=result,
module_name=used_module_name,
parent_package=None,
level=-1,
warn=False,
)
used_modules.add((used_module_name,
os.path.relpath(module_filename)))
result.setUsedModules(used_modules)
# Not used anymore
source_code = None
else:
result = CompiledPythonModule(
module_name=module_name,
is_top=is_top,
mode=mode,
future_spec=None,
source_ref=source_ref,
)
elif Importing.isPackageDir(filename):
if is_top:
module_name = splitPath(filename)[-1]
else:
module_name = os.path.basename(filename)
module_name = ModuleName.makeModuleNameInPackage(module_name, package)
source_filename = os.path.join(filename, "__init__.py")
if not os.path.isfile(source_filename):
source_ref, result = createNamespacePackage(
module_name=module_name,
is_top=is_top,
module_relpath=filename)
source_filename = None
source_code = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename=os.path.abspath(source_filename))
result = CompiledPythonPackage(
module_name=module_name,
is_top=is_top,
mode=decideCompilationMode(is_top, module_name, source_ref),
future_spec=None,
source_ref=source_ref,
)
source_code = readSourceCodeFromFilename(
module_name=module_name, source_filename=source_filename)
else:
sys.stderr.write("%s: can't open file '%s'.\n" %
(os.path.basename(sys.argv[0]), filename))
sys.exit(2)
return result, source_ref, source_code
