def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
if not Options.isExperimental():
return
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
shutil.copytree(
plugin_dir,
Utils.joinpath(
dist_dir,
"qt-plugins"
)
)
return [
(filename, full_name)
for filename in
Utils.getFileList(plugin_dir)
]
return ()
def checkPluginFilenamePattern(pattern):
import sys, glob
debug(
"Checking plug-in pattern '%s':",
pattern,
)
if Utils.isDir(pattern):
sys.exit("Error, pattern cannot be a directory name.")
found = False
for filename in glob.iglob(pattern):
if filename.endswith(".pyc"):
continue
if not Utils.isFile(filename):
continue
found = True
_checkPluginPath(filename, None)
if not found:
warning("Didn't match any files against pattern '%s'." % pattern)
def getPackageDirCandidates(element):
yield Utils.joinpath(element, package_name), False
# Hack for PyWin32. TODO: Move this __path__ extensions to
# plug-in decisions.
if package_name == "win32com":
yield Utils.joinpath(element, "win32comext"), True
def detectPthImportedPackages():
if not hasattr(sys.modules["site"], "getsitepackages"):
return ()
pth_imports = set()
for prefix in sys.modules["site"].getsitepackages():
if not Utils.isDir(prefix):
continue
for path, filename in Utils.listDir(prefix):
if filename.endswith(".pth"):
try:
for line in open(path, "rU"):
if line.startswith("import "):
if ';' in line:
line = line[:line.find(';')]
for part in line[7:].split(','):
pth_imports.add(part.strip())
except OSError:
warning("Python installation problem, cannot read file '%s'.")
return tuple(sorted(pth_imports))
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = Utils.joinpath(
dist_dir,
full_name,
"qt-plugins"
)
shutil.copytree(
plugin_dir,
target_plugin_dir
)
info("Copying all Qt plug-ins to '%s'." % target_plugin_dir)
return [
(filename, full_name)
for filename in
Utils.getFileList(target_plugin_dir)
]
return ()
def detectBinaryDLLs(binary_filename, package_name):
""" Detect the DLLs used by a binary.
Using "ldd" (Linux), "depends.exe" (Windows), or "otool" (MacOS) the list
of used DLLs is retrieved.
"""
if Utils.getOS() in ("Linux", "NetBSD"):
# TODO: FreeBSD may work the same way, not tested.
return _detectBinaryPathDLLsLinuxBSD(
binary_filename = binary_filename
)
elif Utils.getOS() == "Windows":
return _detectBinaryPathDLLsWindows(
binary_filename = binary_filename,
package_name = package_name
)
elif Utils.getOS() == "Darwin":
return _detectBinaryPathDLLsMacOS(
binary_filename = binary_filename
)
else:
# Support your platform above.
assert False, Utils.getOS()
def checkPluginPath(plugin_filename, module_package):
debug(
"Checking top level plug-in path %s %s",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
# File or package makes a difference, handle that
if Utils.isFile(plugin_info[0]) or \
Importing.isPackageDir(plugin_info[0]):
_checkPluginPath(plugin_filename, module_package)
elif Utils.isDir(plugin_info[0]):
for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
assert sub_filename != "__init__.py"
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, None)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
else:
warning("Failed to recurse to directory '%s'.", plugin_filename)
def getResultBasepath(main_module):
assert main_module.isCompiledPythonModule()
if Options.isStandaloneMode():
return Utils.joinpath(
getStandaloneDirectoryPath(main_module), Utils.basename(getTreeFilenameWithSuffix(main_module, ""))
)
else:
return Options.getOutputPath(path=Utils.basename(getTreeFilenameWithSuffix(main_module, "")))
def isPreloadedPackagePath(path):
path = Utils.normcase(path)
for paths in getPreloadedPackagePaths().values():
for element in paths:
if Utils.normcase(element) == path:
return True
return False
def _consider(self, constraint_collection, module_filename, module_package):
assert module_package is None or \
(type(module_package) is str and module_package != "")
module_filename = Utils.normpath(module_filename)
module_name, module_kind = getModuleNameAndKindFromFilename(module_filename)
if module_kind is not None:
decision, reason = decideRecursion(
module_filename = module_filename,
module_name = module_name,
module_package = module_package,
module_kind = module_kind
)
if decision:
module_relpath = Utils.relpath(module_filename)
imported_module, added_flag = recurseTo(
module_package = module_package,
module_filename = module_filename,
module_relpath = module_relpath,
module_kind = module_kind,
reason = reason
)
if added_flag:
constraint_collection.signalChange(
"new_code",
imported_module.getSourceReference(),
"Recursed to module."
)
return imported_module
elif decision is None and module_kind == "py":
if module_package is None:
module_fullpath = module_name
else:
module_fullpath = module_package + '.' + module_name
if module_filename not in self._warned_about and \
module_fullpath not in getModuleWhiteList():
self._warned_about.add(module_filename)
warning(
"""\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--recurse-none (do not warn), \
--recurse-all (recurse to all), \
--recurse-not-to=%(full_path)s (ignore it), \
--recurse-to=%(full_path)s (recurse to it) to change.""" % {
"full_path" : module_fullpath,
"filename" : module_filename
}
)
def getSconsBinaryCall():
""" Return a way to execute Scons.
Using potentially in-line copy if no system Scons is available
or if we are on Windows.
"""
if Utils.getOS() != "Windows" and Utils.isFile("/usr/bin/scons"):
return ["/usr/bin/scons"]
else:
return [getPython2ExePath(), Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")]
def scanStandardLibraryPath(stdlib_dir):
# There is a lot of black-listing here, done in branches, so there
# is many of them, but that's acceptable, pylint: disable=R0912
for root, dirs, filenames in os.walk(stdlib_dir):
import_path = root[len(stdlib_dir):].strip("/\\")
import_path = import_path.replace('\\', '.').replace('/','.')
if import_path == "":
if "site-packages" in dirs:
dirs.remove("site-packages")
if "dist-packages" in dirs:
dirs.remove("dist-packages")
if "test" in dirs:
dirs.remove("test")
if "idlelib" in dirs:
dirs.remove("idlelib")
if "turtledemo" in dirs:
dirs.remove("turtledemo")
if import_path in ("tkinter", "importlib", "ctypes"):
if "test" in dirs:
dirs.remove("test")
if import_path == "lib2to3":
if "tests" in dirs:
dirs.remove("tests")
if Utils.python_version >= 340 and Utils.getOS() == "Windows":
if import_path == "multiprocessing":
filenames.remove("popen_fork.py")
filenames.remove("popen_forkserver.py")
filenames.remove("popen_spawn_posix.py")
if Utils.getOS() == "NetBSD":
if import_path == "xml.sax":
filenames.remove("expatreader.py")
for filename in filenames:
if filename.endswith(".py") and filename not in ignore_modules:
module_name = filename[:-3]
if import_path == "":
yield module_name
else:
yield import_path + '.' + module_name
if Utils.python_version >= 300:
if "__pycache__" in dirs:
dirs.remove("__pycache__")
for dirname in dirs:
if import_path == "":
yield dirname
else:
yield import_path + '.' + dirname
def compileTree(main_module):
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
# Now build the target language code for the whole tree.
makeSourceDirectory(
main_module = main_module
)
if Options.isStandaloneMode():
for module in detectLateImports():
ModuleRegistry.addUncompiledModule(module)
frozen_code = generateBytecodeFrozenCode()
if frozen_code is not None:
writeSourceCode(
filename = Utils.joinpath(
source_dir,
"__frozen.cpp"
),
source_code = frozen_code
)
writeBinaryData(
filename = Utils.joinpath(source_dir, "__constants.bin"),
binary_data = ConstantCodes.stream_data.getBytes()
)
else:
source_dir = getSourceDirectoryPath(main_module)
if not Utils.isFile(Utils.joinpath(source_dir, "__helpers.hpp")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
Tracing.printLine(
"Total memory usage before running scons: {memory}:".format(
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.shallNotDoExecCppCall():
return True, {}
# Run the Scons to build things.
result, options = runScons(
main_module = main_module,
quiet = not Options.isShowScons()
)
return result, options
def getFilenames(module):
base_filename = Utils.joinpath(
source_dir, "module." + module.getFullName() if not module.isInternalModule() else module.getFullName()
)
# Note: Could detect if the file system is cases sensitive in source_dir
# or not, but that's probably not worth the effort. False positives do
# no harm at all.
collision_filename = Utils.normcase(base_filename)
return base_filename, collision_filename
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename = filename,
package = None,
is_top = True,
is_main = not Options.shallMakeModule()
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
shutil.rmtree(standalone_dir, ignore_errors = True)
Utils.makePath(standalone_dir)
Utils.deleteFile(
path = getResultFullpath(main_module),
must_exist = False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(
plugin_filename = plugin_filename,
module_package = None
)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(
pattern = pattern
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
return main_module
def isPackageDir(dirname):
""" Decide if a directory is a package.
Before Python3.3 it's required to have a "__init__.py" file, but then
it became impossible to decide, and for extra fun, there is also the
extra packages provided via "*.pth" file tricks by "site.py" loading.
"""
return Utils.isDir(dirname) and (
Utils.python_version >= 330
or Utils.isFile(Utils.joinpath(dirname, "__init__.py"))
or isPreloadedPackagePath(dirname)
)
def _setupSconsEnvironment():
""" Setup the scons execution environment.
For the scons inline copy on Windows needs to find the library, using
the "SCONS_LIB_DIR" environment variable "NUITKA_SCONS". And for the
target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
Python DLL lives, in case it needs to be copied, and then the
"NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.
"""
# For Python2, avoid unicode working directory.
if Utils.isWin32Windows() and python_version < 300:
if os.getcwd() != os.getcwdu():
os.chdir(getWindowsShortPathName(os.getcwdu()))
if Utils.isWin32Windows():
# On Win32, we use the Python.DLL path for some things. We pass it
# via environment variable
os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()
os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable
# Remove environment variables that can only harm if we have to switch
# major Python versions, these cannot help Python2 to execute scons, this
# is a bit of noise, but helpful.
if python_version >= 300:
if "PYTHONPATH" in os.environ:
old_pythonpath = os.environ["PYTHONPATH"]
del os.environ["PYTHONPATH"]
else:
old_pythonpath = None
if "PYTHONHOME" in os.environ:
old_pythonhome = os.environ["PYTHONHOME"]
del os.environ["PYTHONHOME"]
else:
old_pythonhome = None
yield
if python_version >= 300:
if old_pythonpath is not None:
os.environ["PYTHONPATH"] = old_pythonpath
if old_pythonhome is not None:
os.environ["PYTHONHOME"] = old_pythonhome
if Utils.isWin32Windows():
del os.environ["NUITKA_PYTHON_DLL_PATH"]
del os.environ["NUITKA_PYTHON_EXE_PATH"]
def __init__(self, *args):
QtGui.QDialog.__init__(self, *args)
ui_dir = Utils.dirname(__file__)
ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui")
uic.loadUi(ui_filename, self)
self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection)
self.displayed = None
self.source_code = None
self.model = None
self.moving = None
def getSconsBinaryCall():
""" Return a way to execute Scons.
Using potentially in-line copy if no system Scons is available
or if we are on Windows.
"""
if Utils.getOS() != "Windows":
scons_path = Execution.getExecutablePath("scons")
if scons_path is not None:
return [scons_path]
return [
getPython2ExePath(),
Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")
]
def _detectedPrecompiledFile(filename, module_name, result, is_late):
if filename.endswith(b".pyc"):
if Utils.isFile(filename[:-1]):
return _detectedSourceFile(
filename = filename[:-1],
module_name = module_name,
result = result,
is_late = is_late
)
if Utils.python_version >= 300:
filename = filename.decode("utf-8")
if module_name in module_names:
return
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
(
module_name,
loadCodeObjectData(
precompiled_filename = filename
),
"__init__" in filename,
filename,
is_late
),
)
module_names.add(module_name)
def detectEarlyImports():
if Options.freezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
import_code = "imports = " + repr(sorted(stdlib_modules)) + '\n'\
"for imp in imports:\n" \
" try:\n" \
" __import__(imp)\n" \
" except ImportError:\n" \
" pass\n"
else:
# TODO: Should recursively include all of encodings module.
import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;"
if Utils.getOS() == "Windows":
import_code += "import encodings.mbcs;import encodings.cp437;"
# String method hex depends on it.
if Utils.python_version < 300:
import_code += "import encodings.hex_codec;"
import_code += "import locale;"
result = _detectImports(import_code, False)
debug("Finished detecting early imports.")
return result
def _normalizeModuleFilename(filename):
if python_version >= 300:
filename = filename.replace("__pycache__", "")
suffix = ".cpython-%d.pyc" % (python_version // 10)
if filename.endswith(suffix):
filename = filename[:-len(suffix)] + ".py"
else:
if filename.endswith(".pyc"):
filename = filename[:-3] + ".py"
if Utils.basename(filename) == "__init__.py":
filename = Utils.dirname(filename)
return filename
def buildModuleTree(filename, package, is_top, is_main):
module, source_ref, source_filename = decideModuleTree(
filename = filename,
package = package,
is_top = is_top,
is_main = is_main,
is_shlib = False
)
addImportedModule(
module_relpath = Utils.relpath(filename),
imported_module = module
)
# If there is source code associated (not the case for namespace packages of
# Python3.3 or higher, then read it.
if source_filename is not None:
createModuleTree(
module = module,
source_ref = source_ref,
source_code = readSourceCodeFromFilename(source_filename),
is_main = is_main
)
return module
def buildSconsCommand(quiet, options):
scons_command = getSconsBinaryCall()
if quiet:
scons_command.append("--quiet")
scons_command += [
# The scons file
"-f",
Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),
# Parallel compilation.
"--jobs",
str(Options.getJobLimit()),
# Do not warn about deprecation from Scons
"--warn=no-deprecated",
# Don't load "site_scons" at all.
"--no-site-dir",
]
if Options.isShowScons():
scons_command.append("--debug=explain")
# Option values to provide to scons. Find these in the caller.
for key, value in options.items():
scons_command += [key + '=' + value]
return scons_command
def _getPython2ExePathWindows():
# Shortcuts for the default installation directories, to avoid going to
# registry at all unless necessary. Any Python2 will do for Scons, so it
# can be avoided.
if os.path.isfile(r"c:\Python27\python.exe"):
return r"c:\Python27\python.exe"
elif os.path.isfile(r"c:\Python26\python.exe"):
return r"c:\Python26\python.exe"
# Windows only code, pylint: disable=E0602,F0401,I0021
try:
import _winreg as winreg
except ImportError:
import winreg # lint:ok
for search in ("2.7", "2.6"):
for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
try:
key = winreg.OpenKey(
hkey_branch,
r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
0,
winreg.KEY_READ | arch_key
)
return Utils.joinpath(
winreg.QueryValue(key, ""),
"python.exe"
)
except WindowsError: # @UndefinedVariable
pass
def getPython2ExePath():
""" Find a way to call any Python2. Scons needs it."""
if python_version < 300:
return sys.executable
elif Utils.getOS() == "Windows":
python_exe = _getPython2ExePathWindows()
if python_exe is not None:
return python_exe
else:
sys.exit("""\
Error, while Nuitka is fully Python3 compatible, it needs to find a
Python2 executable under C:\\Python26 or C:\\Python27 to execute
scons which is not yet Python3 compatible.""")
candidate = Execution.getExecutablePath("python2.7")
if candidate is None:
candidate = Execution.getExecutablePath("python2.6")
if candidate is None:
candidate = Execution.getExecutablePath("python2")
# Our weakest bet is that there is no "python3" named "python", but we
# take it, as on some systems it's true.
if candidate is None:
candidate = "python"
return candidate
def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if Utils.isFile(filename[:-1]):
return _detectedSourceFile(
filename = filename[:-1],
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
if module_name in module_names:
return
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
makeUncompiledPythonModule(
module_name = module_name,
bytecode = loadCodeObjectData(
precompiled_filename = filename
),
is_package = "__init__" in filename,
filename = filename,
user_provided = user_provided,
technical = technical
)
)
module_names.add(module_name)
def _detectedSourceFile(filename, module_name, result, is_late):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename = filename,
module_name = "_collections_abc",
result = result,
is_late = is_late
)
source_code = readSourceCodeFromFilename(module_name, filename)
if Utils.python_version >= 300:
filename = filename.decode("utf-8")
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = '\n' + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
Utils.basename(filename),
source_code
)
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
(
module_name,
marshal.dumps(
compile(source_code, filename, "exec")
),
Utils.basename(filename) == "__init__.py",
filename,
is_late
)
)
module_names.add(module_name)
def considerImplicitImports(self, module, signal_change):
""" Consider module imports.
You will most likely want to look at "module.getFullName()" to get
the fully qualified module or package name.
You do not want to overload this method, but rather the things it
calls, as the "signal_change" part of this API is not to be cared
about. Most prominently "getImplicitImports()".
"""
for full_name in self.getImplicitImports(module.getFullName()):
module_name = full_name.split('.')[-1]
module_package = '.'.join(full_name.split('.')[:-1]) or None
module_filename = self.locateModule(
source_ref = module.getSourceReference(),
module_name = module_name,
module_package = module_package,
)
if module_filename is None:
sys.exit(
"Error, implicit module '%s' expected by '%s' not found" % (
module_name,
module.getFullName()
)
)
elif Utils.isDir(module_filename):
module_kind = "py"
elif module_filename.endswith(".py"):
module_kind = "py"
elif module_filename.endswith(".so") or \
module_filename.endswith(".pyd"):
module_kind = "shlib"
else:
assert False, module_filename
# TODO: This should get back to plug-ins, they should be allowed to
# preempt or override the decision.
decision, reason = self.decideRecursion(
module_filename = module_filename,
module_name = module_name,
module_package = module_package,
module_kind = module_kind
)
if decision:
self.recurseTo(
module_package = module_package,
module_filename = module_filename,
module_kind = module_kind,
reason = reason,
signal_change = signal_change
)
full_name = module.getFullName()
if full_name in post_modules:
addUsedModule(post_modules[full_name])
def getSourceDirectoryPath(main_module):
assert main_module.isCompiledPythonModule()
return Options.getOutputPath(
path = Utils.basename(
getTreeFilenameWithSuffix(main_module, ".build")
)
)
def getStandaloneDirectoryPath(main_module):
return Options.getOutputPath(
path = Utils.basename(
getTreeFilenameWithSuffix(main_module, ".dist")
)
)
def getOutputFilename(self):
if self.main_added:
return Utils.dirname(self.getFilename())
else:
return PythonModule.getOutputFilename(self)
def detectEarlyImports():
encoding_names = [
filename[:-3] for _path, filename in listDir(
os.path.dirname(sys.modules["encodings"].__file__))
if filename.endswith(".py") if "__init__" not in filename
]
if Utils.getOS() != "Windows":
for encoding_name in ("mbcs", "cp65001", "oem"):
if encoding_name in encoding_names:
encoding_names.remove(encoding_name)
import_code = ';'.join("import encodings.%s" % encoding_name
for encoding_name in encoding_names)
import_code += ";import locale;"
# For Python3 we patch inspect without knowing if it is used.
if python_version >= 300:
import_code += "import inspect;"
result = _detectImports(command=import_code,
user_provided=False,
technical=True)
if Options.shallFreezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
# Put here ones that should be imported first.
first_ones = ("Tkinter", )
# We have to fight zombie modules in this, some things, e.g. Tkinter
# on newer Python 2.7, comes back after failure without a second error
# being raised, leading to other issues. So we fight it after each
# module that was tried, and prevent re-try by adding a meta path
# based loader that will never load it again, and remove it from the
# "sys.modules" over and over, once it sneaks back. The root cause is
# that extension modules sometimes only raise an error when first
# imported, not the second time around.
# Otherwise this just makes imports of everything so we can see where
# it comes from and what it requires.
import_code = """
imports = %r
failed = set()
class ImportBlocker(object):
def find_module(self, fullname, path = None):
if fullname in failed:
return self
return None
def load_module(self, name):
raise ImportError("%%s has failed before" %% name)
sys.meta_path.insert(0, ImportBlocker())
for imp in imports:
try:
__import__(imp)
except (ImportError, SyntaxError):
failed.add(imp)
for fail in failed:
if fail in sys.modules:
del sys.modules[fail]
""" % sorted(stdlib_modules, key=lambda name: (name not in first_ones, name))
early_names = [module.getFullName() for module in result]
result += [
module for module in _detectImports(
command=import_code, user_provided=False, technical=False)
if module.getFullName() not in early_names
]
debug("Finished detecting early imports.")
return result
Copyright (C) 2016 Kay Hayen."""
import logging
import sys
from optparse import SUPPRESS_HELP, OptionGroup, OptionParser
from nuitka.PythonVersions import (
getSupportedPythonVersions,
getSupportedPythonVersionStr,
python_version_str
)
from nuitka.utils import Utils
# Indicator if we were called as "nuitka-run" in which case we assume some
# other defaults and work a bit different with parameters.
is_nuitka_run = Utils.basename(sys.argv[0]).lower().startswith("nuitka-run")
def getVersion():
return version_string.split()[1][1:]
def getYear():
return int(version_string.split()[4])
if not is_nuitka_run:
usage = "usage: %prog [--module] [--execute] [options] main_module.py"
else:
usage = "usage: %prog [options] main_module.py"
parser = OptionParser(
usage = usage,
version = getVersion()
def decideModuleTree(filename, package, is_shlib, is_top, is_main):
# Many variables, branches, due to the many cases, pylint: disable=R0912,R0915
assert package is None or type(package) is str
assert filename is not None
if is_main and Utils.isDir(filename):
source_filename = Utils.joinpath(filename, "__main__.py")
if not Utils.isFile(source_filename):
sys.stderr.write("%s: can't find '__main__' module in '%s'\n" %
(Utils.basename(sys.argv[0]), filename))
sys.exit(2)
filename = source_filename
main_added = True
else:
main_added = False
if Utils.isFile(filename):
source_filename = filename
source_ref = SourceCodeReferences.fromFilename(filename=filename, )
if is_main:
module_name = "__main__"
else:
module_name = Utils.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
if is_shlib:
module_name = module_name.split('.')[0]
if '.' in module_name:
sys.stderr.write(
"Error, '%s' is not a proper python module name.\n" %
(module_name))
sys.exit(2)
if is_shlib:
result = PythonShlibModule(name=module_name,
package_name=package,
source_ref=source_ref)
elif is_main:
result = PythonMainModule(main_added=main_added,
mode=Plugins.decideCompilation(
"__main__", source_ref),
source_ref=source_ref)
else:
if package is not None:
full_name = package + '.' + module_name
else:
full_name = module_name
result = CompiledPythonModule(name=module_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
source_ref=source_ref)
elif Importing.isPackageDir(filename):
if is_top:
package_name = Utils.splitpath(filename)[-1]
else:
package_name = Utils.basename(filename)
source_filename = Utils.joinpath(filename, "__init__.py")
if not Utils.isFile(source_filename):
source_ref, result = createNamespacePackage(
package_name=package_name, module_relpath=filename)
source_filename = None
else:
source_ref = SourceCodeReferences.fromFilename(
filename=Utils.abspath(source_filename), )
if package is not None:
full_name = package + '.' + package_name
else:
full_name = package_name
result = CompiledPythonPackage(name=package_name,
package_name=package,
mode=Plugins.decideCompilation(
full_name, source_ref),
source_ref=source_ref)
else:
sys.stderr.write("%s: can't open file '%s'.\n" %
(Utils.basename(sys.argv[0]), filename))
sys.exit(2)
if not Options.shallHaveStatementLines():
source_ref = source_ref.atInternal()
return result, source_ref, source_filename
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=R0912
debug("Checking detail plug-in path '%s' '%s':", plugin_filename,
module_package)
module_name, module_kind = Importing.getModuleNameAndKindFromFilename(
plugin_filename)
if module_kind is not None:
decision, _reason = decideRecursion(module_filename=plugin_filename,
module_name=module_name,
module_package=module_package,
module_kind=module_kind)
if decision:
module_relpath = Utils.relpath(plugin_filename)
module, is_added = recurseTo(module_filename=plugin_filename,
module_relpath=module_relpath,
module_package=module_package,
module_kind="py",
reason="Lives in plug-in directory.")
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.", "package"
if module.isCompiledPythonPackage() else "module",
module.getName(), plugin_filename)
if not isSameModulePath(module.getFilename(),
plugin_filename):
warning("Duplicate ignored '%s'.", plugin_filename)
return
debug("Recursed to %s %s %s", module.getName(),
module.getPackage(), module)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug("Package directory %s", package_dir)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_filename)
def isRelevant():
return Utils.getOS() == "Windows" and not Options.shallMakeModule()
def parseArgs():
# singleton with many cases, pylint: disable=global-statement,too-many-branches
global is_nuitka_run, options, positional_args, extra_args
is_nuitka_run, options, positional_args, extra_args = parseOptions()
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
# Standalone mode implies an executable, not importing "site" module, which is
# only for this machine, recursing to all modules, and even including the
# standard library.
if options.is_standalone:
if not options.executable:
sys.exit(
"""\
Error, conflicting options, cannot make standalone module, only executable."""
)
options.recurse_all = True
if Utils.getOS() == "NetBSD":
logging.warning(
"Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported."
)
for any_case_module in getShallFollowModules():
if any_case_module.startswith("."):
bad = True
else:
for char in "/\\:":
if char in any_case_module:
bad = True
break
else:
bad = False
if bad:
sys.exit(
"""\
Error, '--follow-import-to' takes only module names, not directory path '%s'."""
% any_case_module
)
for no_case_module in getShallFollowInNoCase():
if no_case_module.startswith("."):
bad = True
else:
for char in "/\\:":
if char in no_case_module:
bad = True
break
else:
bad = False
if bad:
sys.exit(
"""\
Error, '--nofollow-import-to' takes only module names, not directory path '%s'."""
% no_case_module
)
scons_python = getPythonPathForScons()
if scons_python is not None and not os.path.exists(scons_python):
sys.exit("Error, no such Python binary '%s'." % scons_python)
if options.output_filename is not None and (
isStandaloneMode() or shallMakeModule()
):
sys.exit(
"""\
Error, can only specify output filename for acceleration mode, not for module
mode where filenames are mandatory, and not for standalone where there is a
sane default used inside the dist folder."""
)
for icon_path in [getIconPath()]:
if icon_path is not None and not os.path.exists(icon_path):
sys.exit(
"""\
Error, icon path "%s" does not exist."""
% icon_path
)
def getCodeName(self):
var_name = self.variable_name
var_name = var_name.replace('.', '$')
var_name = Utils.encodeNonAscii(var_name)
return var_name
def parseArgs():
# singleton with many cases, pylint: disable=global-statement,too-many-branches,too-many-statements
global options, positional_args, extra_args
# First, isolate the first non-option arguments.
if is_nuitka_run:
count = 0
for count, arg in enumerate(sys.argv):
if count == 0:
continue
if arg[0] != '-':
break
# Treat "--" as a terminator.
if arg == "--":
count += 1
break
if count > 0:
extra_args = sys.argv[count + 1:]
sys.argv = sys.argv[0:count + 1]
options, positional_args = parser.parse_args()
if shallListPlugins():
from nuitka.plugins.Plugins import listPlugins
listPlugins()
if not positional_args:
parser.print_help()
sys.exit("""
Error, need positional argument with python module or main program.""")
if not options.immediate_execution and len(positional_args) > 1:
parser.print_help()
sys.exit("""
Error, need only one positional argument unless "--run" is specified to
pass them to the compiled program execution.""")
if options.verbose:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
# Standalone mode implies an executable, not importing "site" module, which is
# only for this machine, recursing to all modules, and even including the
# standard library.
if options.is_standalone:
if not options.executable:
sys.exit("""\
Error, conflicting options, cannot make standalone module, only executable.""")
options.recurse_all = True
if Utils.getOS() == "NetBSD":
logging.warning(
"Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported."
)
for any_case_module in getShallFollowModules():
if any_case_module.startswith('.'):
bad = True
else:
for char in "/\\:":
if char in any_case_module:
bad = True
break
else:
bad = False
if bad:
sys.exit(
"""\
Error, '--recurse-to' takes only module names, not directory path '%s'.""" % \
any_case_module
)
for no_case_module in getShallFollowInNoCase():
if no_case_module.startswith('.'):
bad = True
else:
for char in "/\\:":
if char in no_case_module:
bad = True
break
else:
bad = False
if bad:
sys.exit(
"""\
Error, '--recurse-not-to' takes only module names, not directory path '%s'.""" % \
no_case_module
)
scons_python = getPythonPathForScons()
if scons_python is not None and not os.path.exists(scons_python):
sys.exit("Error, no such Python2 binary '%s'." % scons_python)
# Defaults to what you run Nuitka with (currently %s)""" % (
# getSupportedPythonVersionStr(),
# python_version_str
# )
)
if os.name == "nt":
parser.add_option("--python-arch",
action="store",
dest="python_arch",
choices=("x86", "x86_64"),
default=None,
help="""\
Architecture of Python to use. One of "x86" or "x86_64".
Defaults to what you run Nuitka with (currently "%s").""" %
(Utils.getArchitecture()))
parser.add_option("--python-debug",
"--python-dbg",
action="store_true",
dest="python_debug",
default=None,
help="""\
Use debug version or not. Default uses what you are using to run Nuitka, most
likely a non-debug version.""")
parser.add_option("--python-flag",
action="append",
dest="python_flags",
default=[],
help="""\
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = Utils.dirname(Utils.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(
SyntaxErrors.formatOutput(e)
)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(
getSourceDirectoryPath(main_module)
)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(
0,
(binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for module in ModuleRegistry.getDoneModules():
data_files.extend(
Plugins.considerDataFiles(module)
)
for source_filename, target_filename in data_files:
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module),
target_filename
)
Utils.makePath(Utils.dirname(target_filename))
shutil.copy2(
source_filename,
target_filename
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename = filename,
package = None,
is_top = True,
is_main = not Options.shallMakeModule()
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
shutil.rmtree(standalone_dir, ignore_errors = True)
Utils.makePath(standalone_dir)
Utils.deleteFile(
path = getResultFullpath(main_module),
must_exist = False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(
plugin_filename = plugin_filename,
module_package = None
)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(
pattern = pattern
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
if Options.isExperimental():
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def scanStandardLibraryPath(stdlib_dir):
# There is a lot of filtering here, done in branches, so there # is many of
# them, but that's acceptable, pylint: disable=too-many-branches,too-many-statements
for root, dirs, filenames in os.walk(stdlib_dir):
import_path = root[len(stdlib_dir):].strip("/\\")
import_path = import_path.replace("\\", ".").replace("/", ".")
if import_path == "":
if "site-packages" in dirs:
dirs.remove("site-packages")
if "dist-packages" in dirs:
dirs.remove("dist-packages")
if "test" in dirs:
dirs.remove("test")
if "idlelib" in dirs:
dirs.remove("idlelib")
if "turtledemo" in dirs:
dirs.remove("turtledemo")
if "ensurepip" in filenames:
filenames.remove("ensurepip")
if "ensurepip" in dirs:
dirs.remove("ensurepip")
# Ignore "lib-dynload" and "lib-tk" and alike.
dirs[:] = [
dirname for dirname in dirs if not dirname.startswith("lib-")
if dirname != "Tools" if not dirname.startswith("plat-")
]
if import_path in (
"tkinter",
"importlib",
"ctypes",
"unittest",
"sqlite3",
"distutils",
"email",
"bsddb",
):
if "test" in dirs:
dirs.remove("test")
if import_path == "distutils.command":
# Misbehaving and crashing while importing the world.
if "bdist_conda.py" in filenames:
filenames.remove("bdist_conda.py")
if import_path in ("lib2to3", "json", "distutils"):
if "tests" in dirs:
dirs.remove("tests")
if import_path == "asyncio":
if "test_utils.py" in filenames:
filenames.remove("test_utils.py")
if python_version >= 0x340 and Utils.isWin32Windows():
if import_path == "multiprocessing":
filenames.remove("popen_fork.py")
filenames.remove("popen_forkserver.py")
filenames.remove("popen_spawn_posix.py")
if python_version >= 0x300 and Utils.isPosixWindows():
if import_path == "curses":
filenames.remove("has_key.py")
if Utils.getOS() == "NetBSD":
if import_path == "xml.sax":
filenames.remove("expatreader.py")
for filename in filenames:
if filename.endswith(
".py") and filename not in _excluded_stdlib_modules:
module_name = filename[:-3]
if import_path == "":
yield module_name
else:
yield import_path + "." + module_name
if python_version >= 0x300:
if "__pycache__" in dirs:
dirs.remove("__pycache__")
for dirname in dirs:
if import_path == "":
yield dirname
else:
yield import_path + "." + dirname
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
# This is terribly complex, because we check the list of used DLLs
# trying to avoid duplicates, and detecting errors with them not
# being binary identical, so we can report them. And then of course
# we also need to handle OS specifics.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
used_dlls = detectUsedDLLs(
source_dir=source_dir,
standalone_entry_points=standalone_entry_points,
use_cache=not Options.shallNotUseDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
update_cache=not Options.shallNotStoreDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
)
removed_dlls = set()
warned_about = set()
# Fist make checks and remove some.
for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != os.path.basename(
dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
inclusion_logger.info(
"""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))
# Check that if a DLL has the same name, if it's identical, then it's easy.
if haveSameFileContents(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
continue
# For Win32 we can check out file versions.
if Utils.isWin32Windows():
dll_version1 = getWindowsDLLVersion(dll_filename1)
dll_version2 = getWindowsDLLVersion(dll_filename2)
if dll_version2 < dll_version1:
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
solved = True
elif dll_version1 < dll_version2:
del used_dlls[dll_filename1]
removed_dlls.add(dll_filename1)
solved = True
else:
solved = False
if solved:
if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
warned_about.add(dll_name)
inclusion_logger.warning(
"Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
% dll_name)
continue
# So we have conflicting DLLs, in which case we do report the fact.
inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (
dll_name,
dll_filename1,
"\n ".join(sources1),
dll_filename2,
"\n ".join(sources2),
))
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
dll_map = []
for dll_filename, sources in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
shutil.copyfile(dll_filename, target_path)
dll_map.append((dll_filename, dll_name))
if Options.isShowInclusion():
inclusion_logger.info(
"Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
if Utils.getOS() == "Darwin":
# For macOS, the binary and the DLLs needs to be changed to reflect
# the relative DLL location in the ".dist" folder.
for standalone_entry_point in standalone_entry_points:
fixupBinaryDLLPathsMacOS(
binary_filename=standalone_entry_point.dest_path,
dll_map=dll_map,
original_location=standalone_entry_point.source_path,
)
for original_path, dll_filename in dll_map:
fixupBinaryDLLPathsMacOS(
binary_filename=os.path.join(dist_dir, dll_filename),
dll_map=dll_map,
original_location=original_path,
)
# Remove code signature from CPython installed library
candidate = os.path.join(
dist_dir,
"Python",
)
if os.path.exists(candidate):
removeMacOSCodeSignature(candidate)
# Remove rpath settings.
if Utils.getOS() in ("Linux", "Darwin"):
# For Linux, the "rpath" of libraries may be an issue and must be
# removed.
if Utils.getOS() == "Darwin":
start = 0
else:
start = 1
for standalone_entry_point in standalone_entry_points[start:]:
removeSharedLibraryRPATH(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
if Utils.isWin32Windows():
if python_version < 0x300:
# For Win32, we might have to remove SXS paths
for standalone_entry_point in standalone_entry_points[1:]:
removeSxsFromDLL(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSxsFromDLL(os.path.join(dist_dir, dll_filename))
# Indicator if we were called as "nuitka-run" in which case we assume some
# other defaults and work a bit different with parameters.
is_nuitka_run = os.path.basename(sys.argv[0]).lower().endswith("-run")
if not is_nuitka_run:
usage = "usage: %prog [--module] [--run] [options] main_module.py"
else:
usage = "usage: %prog [options] main_module.py"
parser = OptionParser(
usage=usage,
version="\n".join((
getNuitkaVersion(),
"Python: " + sys.version.split("\n")[0],
"Executable: " + sys.executable,
"OS: " + Utils.getOS(),
"Arch: " + Utils.getArchitecture(),
)),
)
parser.add_option(
"--module",
action="store_false",
dest="executable",
default=True,
help="""\
Create an extension module executable instead of a program. Defaults to off.""",
)
parser.add_option(
"--standalone",
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if os.path.isfile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(
0, (binary_filename, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(
source_dir=getSourceDirectoryPath(main_module),
dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points,
)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module), target_filename)
makePath(os.path.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(path=getSourceDirectoryPath(main_module),
ignore_errors=False)
# Modules should not be executable, but Scons creates them like it, fix
# it up here. TODO: Move inside scons file and avoid subprocess call.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, "w") as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
""" % {
"imports":
"\n".join("import %s" % module_name
for module_name in getImportedNames())
})
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment(),
)
technical=technical)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
# Some modules we want to blacklist.
ignore_modules = [
"__main__.py",
"__init__.py",
"antigravity.py",
]
if Utils.getOS() != "Windows":
ignore_modules.append("wintypes.py")
ignore_modules.append("cp65001.py")
def scanStandardLibraryPath(stdlib_dir):
# There is a lot of black-listing here, done in branches, so there
# is many of them, but that's acceptable, pylint: disable=too-many-branches
for root, dirs, filenames in os.walk(stdlib_dir):
import_path = root[len(stdlib_dir):].strip("/\\")
import_path = import_path.replace('\\', '.').replace('/', '.')
if import_path == "":
if "site-packages" in dirs:
dirs.remove("site-packages")
def runScons(main_module, quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
options = {
"name":
os.path.basename(getTreeFilenameWithSuffix(main_module, "")),
"result_name":
getResultBasepath(main_module),
"source_dir":
getSourceDirectoryPath(main_module),
"debug_mode":
_asBoolStr(Options.isDebug()),
"python_debug":
_asBoolStr(Options.isPythonDebug()),
"unstripped_mode":
_asBoolStr(Options.isUnstripped()),
"module_mode":
_asBoolStr(Options.shallMakeModule()),
"full_compat":
_asBoolStr(Options.isFullCompat()),
"experimental":
",".join(Options.getExperimentalIndications()),
"trace_mode":
_asBoolStr(Options.shallTraceExecution()),
"python_version":
python_version_str,
"target_arch":
Utils.getArchitecture(),
"python_prefix":
sys.prefix,
"nuitka_src":
SconsInterface.getSconsDataPath(),
"nuitka_cache":
getCacheDir(),
"module_count":
"%d" % (1 + len(ModuleRegistry.getDoneUserModules()) +
len(ModuleRegistry.getUncompiledNonTechnicalModules())),
}
if not Options.shallMakeModule():
options["result_exe"] = getResultFullpath(main_module)
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = "true"
# For AnaConda default to trying static lib python library, which
# normally is just not available or if it is even unusable.
if "Anaconda" in sys.version:
options["static_libpython"] = "true"
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = "true"
if Options.isStandaloneMode():
options["standalone_mode"] = "true"
if (not Options.isStandaloneMode() and not Options.shallMakeModule()
and isUninstalledPython()):
options["uninstalled_python"] = "true"
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules()))
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw64():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if "." not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if Options.isClang():
options["clang_mode"] = "true"
if Options.getIconPath():
options["icon_path"] = Options.getIconPath()
if Options.isProfile():
options["profile_mode"] = "true"
if "no_warnings" in getPythonFlags():
options["no_python_warnings"] = "true"
if "no_asserts" in getPythonFlags():
options["python_sysflag_optimize"] = "true"
if python_version < 300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = "true"
if python_version < 300 and (sys.flags.division_warning
or sys.flags.py3k_warning):
options["python_sysflag_division_warning"] = "true"
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = "true"
if int(
os.environ.get("NUITKA_SITE_FLAG", "no_site"
in Options.getPythonFlags())):
options["python_sysflag_no_site"] = "true"
if "trace_imports" in Options.getPythonFlags():
options["python_sysflag_verbose"] = "true"
if python_version < 300 and sys.flags.unicode:
options["python_sysflag_unicode"] = "true"
if python_version >= 370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = "true"
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
return SconsInterface.runScons(options, quiet), options
def getCompileTimeFilename(self):
return Utils.abspath(self.getSourceReference().getFilename())
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
# This is terribly complex, because we check the list of used DLLs
# trying to avoid duplicates, and detecting errors with them not
# being binary identical, so we can report them. And then of course
# we also need to handle OS specifics.
# pylint: disable=too-many-branches,too-many-locals
used_dlls = detectUsedDLLs(source_dir, standalone_entry_points)
removed_dlls = set()
# Fist make checks and remove some.
for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != \
os.path.basename(dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
info("""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (
dll_name,
dll_filename1,
dll_filename2,
))
# Check that if a DLL has the same name, if it's identical,
# happens at least for OSC and Fedora 20.
import filecmp
if filecmp.cmp(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
continue
# So we have conflicting DLLs, in which case we do not proceed.
sys.exit("""Error, conflicting DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (dll_name, dll_filename1, "\n ".join(sources1), dll_filename2,
"\n ".join(sources2)))
dll_map = []
for dll_filename, sources in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
shutil.copyfile(dll_filename, target_path)
dll_map.append((dll_filename, dll_name))
if Options.isShowInclusion():
info("Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
if Utils.getOS() == "Darwin":
# For MacOS, the binary and the DLLs needs to be changed to reflect
# the relative DLL location in the ".dist" folder.
for standalone_entry_point in standalone_entry_points:
fixupBinaryDLLPaths(
binary_filename=standalone_entry_point[1],
is_exe=standalone_entry_point is standalone_entry_points[0],
dll_map=dll_map)
for _original_path, dll_filename in dll_map:
fixupBinaryDLLPaths(binary_filename=os.path.join(
dist_dir, dll_filename),
is_exe=False,
dll_map=dll_map)
if Utils.getOS() == "Linux":
# For Linux, the "rpath" of libraries may be an issue and must be
# removed.
for standalone_entry_point in standalone_entry_points[1:]:
removeSharedLibraryRPATH(standalone_entry_point[1])
for _original_path, dll_filename in dll_map:
removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
def main():
# PyLint for Python3 thinks we import from ourselves if we really
# import from package, pylint:disable=I0021,no-name-in-module
# Also high complexity.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
if "NUITKA_BINARY_NAME" in os.environ:
sys.argv[0] = os.environ["NUITKA_BINARY_NAME"]
if "NUITKA_PYTHONPATH" in os.environ:
# Restore the PYTHONPATH gained from the site module, that we chose not
# to have imported. pylint: disable=eval-used
sys.path = eval(os.environ["NUITKA_PYTHONPATH"])
del os.environ["NUITKA_PYTHONPATH"]
else:
# Remove path element added for being called via "__main__.py", this can
# only lead to trouble, having e.g. a "distutils" in sys.path that comes
# from "nuitka.distutils".
sys.path = [
path_element
for path_element in sys.path
if os.path.dirname(os.path.abspath(__file__)) != path_element
]
# For re-execution, we might not have done this.
from nuitka import Options # isort:skip
Options.parseArgs()
from nuitka.utils import Utils, Execution # isort:skip
import logging # isort:skip
logging.basicConfig(format = "Nuitka:%(levelname)s:%(message)s")
# We don't care, and these are triggered by run time calculations of "range" and
# others, while on python2.7 they are disabled by default.
warnings.simplefilter("ignore", DeprecationWarning)
# We will run with the Python configuration as specified by the user, if it does
# not match, we restart ourselves with matching configuration.
needs_reexec = False
current_version = "%d.%d" % (sys.version_info[0], sys.version_info[1])
# We support to execute with a specified version.
intended_version = Options.getIntendedPythonVersion()
if intended_version is None:
intended_version = current_version
# If it's a different version, we find it by guessing it, otherwise we use the
# one previously used.
if current_version != intended_version:
if Utils.getOS() == "Windows":
python_binary = Execution.getPythonExePathWindows(
intended_version,
Options.getIntendedPythonArch()
)
else:
python_binary = Execution.getExecutablePath("python" + intended_version)
if python_binary is None:
sys.exit(
"Error, cannot find Python %s binary in PATH (%s)." % (
intended_version,
os.environ.get("PATH", "")
)
)
needs_reexec = True
else:
python_binary = sys.executable
if sys.flags.no_site == 0:
needs_reexec = True
# The hash randomization totally changes the created source code created,
# changing it every single time Nuitka is run. This kills any attempt at
# caching it, and comparing generated source code. While the created binary
# actually may still use it, during compilation we don't want to. So lets
# disable it.
if os.environ.get("PYTHONHASHSEED", "-1") != '0':
needs_reexec = True
# In case we need to re-execute.
if needs_reexec:
if not Options.isAllowedToReexecute():
sys.exit("Error, not allowed to re-execute, but that would be needed.")
our_filename = sys.modules[__name__].__file__
# Workaround for --python-version which will choke on existing, but
# not matching .pyc files.
if current_version != intended_version:
pyc_filename = our_filename[:-2] + ".pyc"
if os.path.exists(pyc_filename):
try:
os.unlink(pyc_filename)
except OSError:
pass
# Execute with full path as the process name, so it can find itself and its
# libraries.
args = [
python_binary,
python_binary,
"-S",
our_filename,
]
os.environ["NUITKA_BINARY_NAME"] = sys.modules["__main__"].__file__
if Options.is_nuitka_run:
args.append("--run")
# Same arguments as before.
args += sys.argv[1:] + list(Options.getMainArgs())
if current_version == intended_version:
os.environ["NUITKA_PYTHONPATH"] = repr(
sys.path
)
from nuitka.importing.PreloadedPackages import detectPreLoadedPackagePaths, detectPthImportedPackages
os.environ["NUITKA_NAMESPACES"] = repr(
detectPreLoadedPackagePaths()
)
if "site" in sys.modules:
os.environ["NUITKA_SITE_FILENAME"] = sys.modules["site"].__file__
os.environ["NUITKA_PTH_IMPORTED"] = repr(detectPthImportedPackages())
os.environ["NUITKA_SITE_FLAG"] = str(sys.flags.no_site) \
if "no_site" not in Options.getPythonFlags() \
else '1'
os.environ["PYTHONHASHSEED"] = '0'
Execution.callExec(args)
if Options.isShowMemory():
from nuitka.utils import MemoryUsage
MemoryUsage.startMemoryTracing()
# Inform the user about potential issues.
if current_version not in Options.getSupportedPythonVersions():
# Do not disturb run of automatic tests, detected from the presence of
# that environment variable.
if "PYTHON" not in os.environ:
logging.warning(
"The version '%s' is not currently supported. Expect problems.",
current_version
)
if "NUITKA_NAMESPACES" in os.environ:
# Restore the detected name space packages, that were force loaded in
# site.py, and will need a free pass later on. pylint: disable=eval-used
from nuitka.importing.PreloadedPackages import setPreloadedPackagePaths
setPreloadedPackagePaths(eval(os.environ["NUITKA_NAMESPACES"]))
del os.environ["NUITKA_NAMESPACES"]
if "NUITKA_PTH_IMPORTED" in os.environ:
# Restore the packages that the ".pth" files asked to import.
# pylint: disable=eval-used
from nuitka.importing.PreloadedPackages import setPthImportedPackages
setPthImportedPackages(eval(os.environ["NUITKA_PTH_IMPORTED"]))
del os.environ["NUITKA_PTH_IMPORTED"]
# Now the real main program of Nuitka can take over.
from nuitka import MainControl # isort:skip
MainControl.main()
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
def getSourceDirectoryPath(main_module):
assert main_module.isCompiledPythonModule()
return Options.getOutputPath(
path=Utils.basename(getTreeFilenameWithSuffix(main_module, ".build")))
def getScanDirectories(package_name, original_dir):
# Many cases, pylint: disable=too-many-branches
cache_key = package_name, original_dir
if cache_key in _scan_dir_cache:
return _scan_dir_cache[cache_key]
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if (Utils.isWin32Windows() and package_name is not None
and package_name.isBelowNamespace("win32com")):
pywin32_dir = getPyWin32Dir()
if pywin32_dir is not None:
scan_dirs.append(pywin32_dir)
for path_dir in os.environ["PATH"].split(";"):
if not os.path.isdir(path_dir):
continue
if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "System32")):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
continue
scan_dirs.append(path_dir)
result = []
# Remove directories that hold no DLLs.
for scan_dir in scan_dirs:
sys.stdout.flush()
# These are useless, but plenty.
if os.path.basename(scan_dir) == "__pycache__":
continue
scan_dir = getDirectoryRealPath(scan_dir)
# No DLLs, no use.
if not any(entry[1].lower().endswith(".dll")
for entry in listDir(scan_dir)):
continue
result.append(os.path.realpath(scan_dir))
_scan_dir_cache[cache_key] = result
return result
def getOutputFilename(self):
return Utils.dirname(self.getFilename())
def _detectBinaryPathDLLsPosix(dll_filename):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
process = subprocess.Popen(
args=["ldd", dll_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Do not include kernel / glibc specific libraries. This list has been
# assembled by looking what are the most common .so files provided by
# glibc packages from ArchLinux, Debian Stretch and CentOS.
#
# Online sources:
# - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
# - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
# - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
# - https://packages.debian.org/stretch/amd64/libc6/filelist
#
# Note: This list may still be incomplete. Some additional libraries
# might be provided by glibc - it may vary between the package versions
# and between Linux distros. It might or might not be a problem in the
# future, but it should be enough for now.
if os.path.basename(filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(sub_dll_filename))
return sub_result
def scanStandardLibraryPath(stdlib_dir):
# There is a lot of black-listing here, done in branches, so there
# is many of them, but that's acceptable, pylint: disable=too-many-branches
for root, dirs, filenames in os.walk(stdlib_dir):
import_path = root[len(stdlib_dir):].strip("/\\")
import_path = import_path.replace('\\', '.').replace('/', '.')
if import_path == "":
if "site-packages" in dirs:
dirs.remove("site-packages")
if "dist-packages" in dirs:
dirs.remove("dist-packages")
if "test" in dirs:
dirs.remove("test")
if "idlelib" in dirs:
dirs.remove("idlelib")
if "turtledemo" in dirs:
dirs.remove("turtledemo")
if "ensurepip" in filenames:
filenames.remove("ensurepip")
if "ensurepip" in dirs:
dirs.remove("ensurepip")
# Ignore "lib-dynload" and "lib-tk" and alikes.
dirs[:] = [
dirname for dirname in dirs if not dirname.startswith("lib-")
if dirname != "Tools"
]
if import_path in ("tkinter", "importlib", "ctypes", "unittest",
"sqlite3", "distutils", "email", "bsddb"):
if "test" in dirs:
dirs.remove("test")
if import_path in ("lib2to3", "json", "distutils"):
if "tests" in dirs:
dirs.remove("tests")
if python_version >= 340 and Utils.getOS() == "Windows":
if import_path == "multiprocessing":
filenames.remove("popen_fork.py")
filenames.remove("popen_forkserver.py")
filenames.remove("popen_spawn_posix.py")
if Utils.getOS() == "NetBSD":
if import_path == "xml.sax":
filenames.remove("expatreader.py")
for filename in filenames:
if filename.endswith(".py") and filename not in ignore_modules:
module_name = filename[:-3]
if import_path == "":
yield module_name
else:
yield import_path + '.' + module_name
if python_version >= 300:
if "__pycache__" in dirs:
dirs.remove("__pycache__")
for dirname in dirs:
if import_path == "":
yield dirname
else:
yield import_path + '.' + dirname
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=R0912,R0914
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name = module.getFullName(),
module_filename = module.getCompileTimeFilename()
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if uncompiled_module.isUserProvided():
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
warning(
"Didn't recurse to '%s', apparently not used." % \
any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(
source_dir = source_dir,
modules = ModuleRegistry.getDoneModules()
)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
cpp_filename = module_filenames[module]
prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode(
global_context = global_context,
module = module,
module_name = module.getFullName(),
)
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[cpp_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
cpp_filename = module_filenames[module]
template_values, module_context = prepared_modules[cpp_filename]
source_code = CodeGeneration.generateModuleCode(
module_context = module_context,
template_values = template_values
)
writeSourceCode(
filename = cpp_filename,
source_code = source_code
)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split('.')
)
if Utils.getOS() == "Windows":
target_filename += ".pyd"
else:
target_filename += ".so"
target_dir = Utils.dirname(target_filename)
if not Utils.isDir(target_dir):
Utils.makePath(target_dir)
shutil.copy(
module.getFilename(),
target_filename
)
standalone_entry_points.append(
(target_filename, module.getPackage())
)
elif module.isUncompiledPythonModule():
pass
else:
assert False, module
writeSourceCode(
filename = Utils.joinpath(
source_dir,
"__constants.c"
),
source_code = ConstantCodes.getConstantsDefinitionCode(
context = global_context
)
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules()
)
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.h"),
source_code = helper_decl_code
)
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.c"),
source_code = helper_impl_code
)
def runScons(main_module, quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches, pylint: disable=R0912
python_version_str = "%d.%d" % (sys.version_info[0], sys.version_info[1])
if hasattr(sys, "abiflags"):
if Options.isPythonDebug() or \
hasattr(sys, "getobjects"):
if sys.abiflags.startswith('d'):
python_version_str += sys.abiflags
else:
python_version_str += 'd' + sys.abiflags
else:
python_version_str += sys.abiflags
def asBoolStr(value):
return "true" if value else "false"
options = {
"name" : Utils.basename(
getTreeFilenameWithSuffix(main_module, "")
),
"result_name" : getResultBasepath(main_module),
"source_dir" : getSourceDirectoryPath(main_module),
"debug_mode" : asBoolStr(Options.isDebug()),
"python_debug" : asBoolStr(Options.isPythonDebug()),
"unstripped_mode" : asBoolStr(Options.isUnstripped()),
"module_mode" : asBoolStr(Options.shallMakeModule()),
"optimize_mode" : asBoolStr(Options.isOptimize()),
"full_compat" : asBoolStr(Options.isFullCompat()),
"experimental" : asBoolStr(Options.isExperimental()),
"trace_mode" : asBoolStr(Options.shallTraceExecution()),
"python_version" : python_version_str,
"target_arch" : Utils.getArchitecture(),
"python_prefix" : sys.prefix,
"nuitka_src" : SconsInterface.getSconsDataPath(),
"module_count" : "%d" % (
1 + \
len(ModuleRegistry.getDoneUserModules()) + \
len(ModuleRegistry.getUncompiledNonTechnicalModules())
)
}
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = "true"
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = "true"
if Options.isStandaloneMode():
options["standalone_mode"] = "true"
if not Options.isStandaloneMode() and \
not Options.shallMakeModule() and \
isUninstalledPython():
options["uninstalled_python"] = "true"
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules())
)
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if '.' not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if Options.isClang():
options["clang_mode"] = "true"
if Options.getIconPath():
options["icon_path"] = Options.getIconPath()
if Options.isProfile():
options["profile_mode"] = "true"
if "no_warnings" in getPythonFlags():
options["no_python_warnings"] = "true"
if python_version < 300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = "true"
if python_version < 300 and (sys.flags.division_warning or sys.flags.py3k_warning):
options["python_sysflag_division_warning"] = "true"
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = "true"
if int(os.environ.get("NUITKA_SITE_FLAG", "no_site" in Options.getPythonFlags())):
options["python_sysflag_no_site"] = "true"
if "trace_imports" in Options.getPythonFlags():
options["python_sysflag_verbose"] = "true"
if python_version < 300 and sys.flags.unicode:
options["python_sysflag_unicode"] = "true"
return SconsInterface.runScons(options, quiet), options
def getOutputPath(path):
if options.output_dir:
return Utils.normpath(Utils.joinpath(options.output_dir, path))
else:
return path