def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(
path = target_dir,
ignore_errors = False
)
command = [
nuitka,
"--module",
"--recurse-none",
"--plugin-enable=pylint-warnings",
"--output-dir=%s"% tmp_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
diffRecursive(os.path.join(package, target), target_dir)
shutil.rmtree(target_dir)
if os.name == "nt":
target_filename = filename.replace(".py", ".pyd")
else:
target_filename = filename.replace(".py", ".so")
os.unlink(os.path.join(tmp_dir, target_filename))
def _unpackPathElement(path_entry):
if not path_entry:
return "." # empty means current directory
if os.path.isfile(path_entry) and path_entry.lower().endswith(".egg"):
if path_entry not in _egg_files:
with open(path_entry, "rb") as f:
checksum = hashlib.md5(f.read()).hexdigest()
target_dir = os.path.join(getCacheDir(), "egg-content", checksum)
if not os.path.exists(target_dir):
try:
# Not all Python versions allow using with here, pylint: disable=consider-using-with
zip_ref = zipfile.ZipFile(path_entry, "r")
zip_ref.extractall(target_dir)
zip_ref.close()
except BaseException:
removeDirectory(target_dir, ignore_errors=True)
raise
_egg_files[path_entry] = target_dir
return _egg_files[path_entry]
return path_entry
def _runOnefileScons(quiet):
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=True),
"result_exe": OutputDirectories.getResultFullpath(onefile=True),
"source_dir": source_dir,
"debug_mode": asBoolStr(Options.is_debug),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"target_arch": getArchitecture(),
"python_prefix": sys.prefix,
"nuitka_src": SconsInterface.getSconsDataPath(),
"compiled_exe": OutputDirectories.getResultFullpath(onefile=False),
}
SconsInterface.setCommonOptions(options)
onefile_env_values = {}
if Options.isWindowsOnefileTempDirMode() or getOS() != "Windows":
onefile_env_values["ONEFILE_TEMP_SPEC"] = Options.getWindowsOnefileTempDirSpec(
use_default=True
)
else:
# Merge version information if possible, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources["CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources["ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverriden(onefile_env_values):
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="WindowsOnefile.scons"
)
# Exit if compilation failed.
if not result:
scons_logger.sysexit("Error, one file bootstrap build for Windows failed.")
if Options.isRemoveBuildDir():
general.info("Removing onefile build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping onefile build directory %r." % source_dir)
def withVirtualenv(env_name,
base_dir=None,
python=None,
delete=True,
style=None):
""" Create a virtualenv and change into it.
Activating for actual use will be your task.
"""
print("Creating virtualenv for quick test:")
if python is None:
python = sys.executable
if base_dir is not None:
env_dir = os.path.join(base_dir, env_name)
else:
env_dir = env_name
removeDirectory(env_dir, ignore_errors=False)
with withDirectoryChange(base_dir, allow_none=True):
command = [python, "-m", "virtualenv", env_name]
if style is not None:
my_print("Executing: %s" % " ".join(command))
subprocess.check_call(command)
yield Virtualenv(env_dir)
if delete:
removeDirectory(env_dir, ignore_errors=False)
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(
dist_dir,
full_name,
"qt-plugins"
)
shutil.copytree(
plugin_dir,
target_plugin_dir
)
plugin_options = self.getPluginOptions()
if not plugin_options:
plugin_options.append("all")
if plugin_options == ["sensible"]:
plugin_options = ["imageformats", "iconengines", "mediaservice", "printsupport"]
info(
"Copying '%s' Qt plug-ins to '%s'." % (
','.join(plugin_options),
target_plugin_dir
)
)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors = False)
for plugin_candidate in plugin_options:
if not os.path.isdir(os.path.join(target_plugin_dir, plugin_candidate)):
sys.exit("Error, no such Qt plugin family: %s" % plugin_candidate)
return [
(
filename,
os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)),
full_name
)
for filename in
getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)))
]
return ()
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule())
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename,
module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
if Options.isExperimental():
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def _removeCPythonTestSuiteDir():
# Cleanup, some tests apparently forget that.
try:
if os.path.isdir("@test"):
removeDirectory("@test", ignore_errors=False)
elif os.path.isfile("@test"):
os.unlink("@test")
except OSError:
# TODO: Move this into removeDirectory maybe. Doing an external
# call as last resort could be a good idea.
# This seems to work for broken "lnk" files.
if os.name == "nt":
os.system("rmdir /S /Q @test")
if os.path.exists("@test"):
raise
def withVirtualenv(env_name, base_dir=None, python=None):
""" Create a virtualenv and change into it.
Activating it will be your task.
"""
print("Creating virtualenv for quick test:")
if python is None:
python = sys.executable
if base_dir is not None:
env_dir = os.path.join(base_dir, env_name)
else:
env_dir = env_name
removeDirectory(env_dir, ignore_errors=False)
with withDirectoryChange(base_dir, allow_none=True):
subprocess.check_call([python, "-m", "virtualenv", env_name])
yield Virtualenv(env_dir)
removeDirectory(env_dir, ignore_errors=False)
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
full_name = module.getFullName()
if full_name == self.binding_name:
if not self.getQtPluginDirs():
self.sysexit(
"Error, failed to detect %r plugin directories." % self.binding_name
)
target_plugin_dir = os.path.join(dist_dir, full_name.asPath(), "qt-plugins")
self.info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(
sorted(x for x in self.getQtPluginsSelected() if x != "xml")
),
target_plugin_dir,
)
)
# TODO: Change this to filtering copyTree while it's doing it.
for plugin_dir in self.getQtPluginDirs():
copyTree(plugin_dir, target_plugin_dir)
if "all" not in self.getQtPluginsSelected():
for plugin_candidate in getSubDirectories(target_plugin_dir):
if (
os.path.basename(plugin_candidate)
not in self.getQtPluginsSelected()
):
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in self.getQtPluginsSelected():
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
self.sysexit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = self.findDLLs(
full_name=full_name,
target_plugin_dir=target_plugin_dir,
)
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs()),
[],
)
self.info("Copying OpenSSL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if (
"qml" in self.getQtPluginsSelected()
or "all" in self.getQtPluginsSelected()
):
result += self.copyQmlFiles(
full_name=full_name,
target_plugin_dir=target_plugin_dir,
)
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
self.info("Copying OpenGL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
elif full_name == self.binding_name + ".QtNetwork":
if not isWin32Windows():
dll_path = locateDLL("crypto")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
dll_path = locateDLL("ssl")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
elif (
full_name
in (
self.binding_name + ".QtWebEngine",
self.binding_name + ".QtWebEngineCore",
self.binding_name + ".QtWebEngineWidgets",
)
and not self.webengine_done
):
self.webengine_done = True # prevent multiple copies
self.info("Copying QtWebEngine components")
plugin_parent = os.path.dirname(self.getQtPluginDirs()[0])
if isWin32Windows():
bin_dir = os.path.join(plugin_parent, "bin")
else: # TODO verify this for non-Windows!
bin_dir = os.path.join(plugin_parent, "libexec")
target_bin_dir = os.path.join(dist_dir)
for f in os.listdir(bin_dir):
if f.startswith("QtWebEngineProcess"):
shutil.copy(os.path.join(bin_dir, f), target_bin_dir)
resources_dir = os.path.join(plugin_parent, "resources")
target_resources_dir = os.path.join(dist_dir)
for f in os.listdir(resources_dir):
shutil.copy(os.path.join(resources_dir, f), target_resources_dir)
translations_dir = os.path.join(plugin_parent, "translations")
pos = len(translations_dir) + 1
target_translations_dir = os.path.join(
dist_dir,
full_name.getTopLevelPackageName().asPath(),
"Qt",
"translations",
)
for f in getFileList(translations_dir):
tar_f = os.path.join(target_translations_dir, f[pos:])
makePath(os.path.dirname(tar_f))
shutil.copyfile(f, tar_f)
return ()
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace("/", os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(path=target_dir, ignore_errors=False)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--no-pyi-file",
path,
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
my_print("Command: ", " ".join(command))
exit_nuitka = subprocess.call(command)
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
command2 = wrapCommandForDebuggerForSubprocess(*command)
subprocess.call(command2)
if exit_nuitka != 0:
my_print("An error exit %s occurred, aborting." % exit_nuitka)
sys.exit(exit_nuitka)
has_diff = diffRecursive(os.path.join(package, target), target_dir)
if has_diff:
sys.exit("There were differences!")
shutil.rmtree(target_dir)
target_filename = filename.replace(
".py", getSharedLibrarySuffix(preferred=True)
)
os.unlink(os.path.join(tmp_dir, target_filename))
def createNodeTree(filename):
"""Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = OutputDirectories.getStandaloneDirectoryPath()
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=OutputDirectories.getResultFullpath(), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=ModuleName(package_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=ModuleName(module_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(
dirname = None,
filename = filename
)
if active:
my_print("Consider distutils example:", filename)
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"), ignore_errors = False)
removeDirectory(os.path.join(case_dir, "dist"), ignore_errors = False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(
commands = [
'cd "%s"' % case_dir,
"python setup.py bdist_wheel",
]
)
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand(
'pip install "%s"' % (
os.path.join(
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if os.path.isfile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat() and \
Options.getIntendedPythonVersion() is None:
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of e.g. '--python-version=%s' option, if that's not the one expected.
""" % (python_version_str, suggested_python_version_str)
sys.exit(error_message)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(path=getSourceDirectoryPath(main_module),
ignore_errors=False)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (None, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module), target_filename)
makePath(os.path.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment())
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
full_name = module.getFullName()
elements = full_name.split(".")
plugin_dirs = None
if elements[0] in ("PyQt4", "PyQt5"):
qt_version = int(elements[0][-1])
plugin_dirs = self.getPyQtPluginDirs(qt_version)
if full_name in ("PyQt4", "PyQt5"):
if not plugin_dirs:
sys.exit("Error, failed to detect %s plugin directories." % full_name)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = set(self.qt_plugins.split(","))
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dirs, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
self.info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
for plugin_dir in plugin_dirs:
copyTree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for plugin_dir in plugin_dirs
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(
getFileList(qt_bin_dir)
for qt_bin_dir in self._getQtBinDirs(plugin_dirs)
),
[],
)
self.info("Copying OpenSSL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
for plugin_dir in plugin_dirs:
qml_plugin_dir = os.path.normpath(
os.path.join(plugin_dir, "..", "qml")
)
if os.path.exists(qml_plugin_dir):
break
else:
sys.exit("Error, no such Qt plugin family: qml")
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
self.info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
copyTree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
self.info("Copying OpenGL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
elif full_name == "PyQt5.QtNetwork":
if not isWin32Windows():
dll_path = locateDLL("crypto")
if dll_path is None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
dll_path = locateDLL("ssl")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
elif (
full_name.startswith(("PyQt4.QtWebEngine", "PyQt5.QtWebEngine"))
and not self.webengine_done
):
self.webengine_done = True # prevent multiple copies
self.info("Copying QtWebEngine components")
plugin_parent = os.path.dirname(plugin_dirs[0])
if isWin32Windows():
bin_dir = os.path.join(plugin_parent, "bin")
else: # TODO verify this for non-Windows!
bin_dir = os.path.join(plugin_parent, "libexec")
target_bin_dir = os.path.join(dist_dir)
for f in os.listdir(bin_dir):
if f.startswith("QtWebEngineProcess"):
shutil.copy(os.path.join(bin_dir, f), target_bin_dir)
resources_dir = os.path.join(plugin_parent, "resources")
target_resources_dir = os.path.join(dist_dir)
for f in os.listdir(resources_dir):
shutil.copy(os.path.join(resources_dir, f), target_resources_dir)
translations_dir = os.path.join(plugin_parent, "translations")
pos = len(translations_dir) + 1
target_translations_dir = os.path.join(
dist_dir, elements[0], "Qt", "translations"
)
for f in getFileList(translations_dir):
tar_f = os.path.join(target_translations_dir, f[pos:])
makePath(os.path.dirname(tar_f))
shutil.copyfile(f, tar_f)
return ()
def _build(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.__past__ import ( # pylint: disable=I0021,redefined-builtin
Iterable, unicode,
)
from nuitka.importing.Importing import (
findModule,
setMainScriptDirectory,
)
from nuitka.utils.ModuleNames import ModuleName
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory(".")
to_builds = self._find_to_build()
for to_build in to_builds:
package, main_filename, finding = findModule(
importing=None,
module_name=ModuleName(to_build.module_name),
parent_package=None,
level=0,
warn=False,
)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
if package is not None:
output_dir = os.path.join(build_lib, package)
else:
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % output_dir,
"--nofollow-import-to=*.tests",
"--show-modules",
"--remove-output",
]
if type(to_build) is PyPackage:
command += ("--include-package=%s" %
package_name.replace("/", ".")
for package_name in to_build.related_packages)
else: # type(to_build) is PyModule
command += ("--include-module=%s" % module_name
for module_name in to_build.related_modules)
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, value in self.distribution.command_options[
"nuitka"].items():
option = "--" + option.lstrip("-")
if value is None:
command.append(option)
elif isinstance(value, bool):
option = "--" + ("no" if not value else
"") + option.lstrip("-")
command.append(option)
elif isinstance(value, Iterable) and not isinstance(
value, (unicode, bytes, str)):
for val in value:
command.append("%s=%s" % (option, val))
else:
command.append("%s=%s" % (option, value))
command.append(main_filename)
# added for clarity
my_print("Building: %s" % to_build, style="yellow")
check_call(command, cwd=build_lib)
for root, _, filenames in os.walk(build_lib):
for filename in filenames:
fullpath = os.path.join(root, filename)
if fullpath.lower().endswith(
(".py", ".pyw", ".pyc", ".pyo")):
os.unlink(fullpath)
# If the Python module has more than one parent package (e.g.
# 'a.b.mod'), the compiled module will be in 'a.b/mod.so'. Move it
# to 'a/b/mod.so', to make imports work.
if package and "." in package:
compiled_package_path = os.path.join(build_lib, package)
assert os.path.isdir(
compiled_package_path), compiled_package_path
parts = package.split(".")
fixed_package_path = os.path.join(build_lib, *parts)
copyTree(compiled_package_path, fixed_package_path)
removeDirectory(compiled_package_path, ignore_errors=False)
os.chdir(old_dir)
self.build_lib = build_lib
def main():
"""Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches,too-many-statements
# In case we are in a PGO run, we read its information first, so it becomes
# available for later parts.
pgo_filename = getPythonPgoInput()
if pgo_filename is not None:
readPGOInputFile(pgo_filename)
if not Options.shallDumpBuiltTreeXML():
general.info(
"Starting Python compilation with Nuitka %r on Python %r commercial %r."
% (getNuitkaVersion(), python_version_str, getCommercialVersion())
)
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
addIncludedDataFilesFromFileOptions()
# Turn that source code into a node tree structure.
try:
main_module = _createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
addIncludedDataFilesFromPackageOptions()
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree()
# Exit if compilation failed.
if not result:
sys.exit(1)
# Relaunch in case of Python PGO input to be produced.
if Options.shallCreatePgoInput():
# Will not return.
pgo_filename = OutputDirectories.getPgoRunInputFilename()
general.info(
"Restarting compilation using collected information from '%s'."
% pgo_filename
)
reExecuteNuitka(pgo_filename=pgo_filename)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing()
copyDataFiles()
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
setMainEntryPoint(binary_filename)
dist_dir = OutputDirectories.getStandaloneDirectoryPath()
for module in ModuleRegistry.getDoneModules():
addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))
copyDllsUsed(
source_dir=OutputDirectories.getSourceDirectoryPath(),
dist_dir=dist_dir,
standalone_entry_points=getStandaloneEntryPoints(),
)
Plugins.onStandaloneDistributionFinished(dist_dir)
if Options.isOnefileMode():
packDistFolderToOnefile(dist_dir, binary_filename)
if Options.isRemoveBuildDir():
general.info("Removing dist folder %r." % dist_dir)
removeDirectory(path=dist_dir, ignore_errors=False)
else:
general.info(
"Keeping dist folder %r for inspection, no need to use it."
% dist_dir
)
# Remove the source directory (now build directory too) if asked to.
source_dir = OutputDirectories.getSourceDirectoryPath()
if Options.isRemoveBuildDir():
general.info("Removing build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping build directory %r." % source_dir)
final_filename = OutputDirectories.getResultFullpath(
onefile=Options.isOnefileMode()
)
if Options.isStandaloneMode() and isMacOS():
general.info(
"Created binary that runs on macOS %s (%s) or higher."
% (options["macos_min_version"], options["macos_target_arch"])
)
Plugins.onFinalResult(final_filename)
general.info("Successfully created %r." % final_filename)
report_filename = Options.getCompilationReportFilename()
if report_filename:
writeCompilationReport(report_filename)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
run_filename = OutputDirectories.getResultRunFilename(
onefile=Options.isOnefileMode()
)
general.info("Launching %r." % run_filename)
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=run_filename,
clean_path=Options.shallClearPythonPathEnvironment(),
)
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--oneline",
"remove_output",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of standalone mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "System32")
),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if r"azure dev spaces cli" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/"):
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith(
(
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)
):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith(
(
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)
):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith(
(
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)
):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (
loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")
):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith("/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
lib_prefix_dir = "/usr/lib/python%d.%s" % (
python_version[0],
python_version[1],
)
# PySide accesses its directory.
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
continue
# GTK accesses package directories only.
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
continue
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gio"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
test_logger.warning("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
displayError(displayError, filename)
# Run with traces to help debugging, specifically in CI environment.
if sys.platform == "darwin" or sys.platform.startswith("freebsd"):
test_logger.info("dtruss:")
os.system("sudo dtruss %s" % binary_filename)
else:
test_logger.info("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def executePASS1():
my_print("PASS 1: Compiling from compiler running from .py files.")
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
target_dir = package
removeDirectory(
path = target_dir,
ignore_errors = False
)
os.mkdir(target_dir)
for path, filename in listDir(target_dir):
if filename.endswith(".so"):
os.unlink(path)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % target_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
else:
shutil.copyfile(path, os.path.join(target_dir, filename))
my_print("Compiling '%s'." % nuitka_main_path)
shutil.copyfile(nuitka_main_path, "nuitka-runner.py")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--nofollow-imports",
"--plugin-enable=pylint-warnings",
"--output-dir=.",
"--python-flag=-S",
"nuitka-runner.py"
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
shutil.move("nuitka-runner" + exe_suffix, "nuitka" + exe_suffix)
scons_inline_copy_path = os.path.join(
base_dir,
"nuitka",
"build",
"inline_copy"
)
if os.path.exists(scons_inline_copy_path):
shutil.copytree(
scons_inline_copy_path,
os.path.join("nuitka", "build", "inline_copy")
)
shutil.copyfile(
os.path.join(base_dir, "nuitka", "build", "SingleExe.scons"),
os.path.join("nuitka", "build", "SingleExe.scons")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "static_src"),
os.path.join("nuitka", "build", "static_src")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "include"),
os.path.join("nuitka", "build", "include")
)
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace("/", os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(path=target_dir, ignore_errors=False)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--no-pyi-file",
path,
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
my_print("Command: ", " ".join(command))
result = subprocess.call(command)
if result != 0:
sys.exit(result)
has_diff = diffRecursive(os.path.join(package, target),
target_dir)
# TODO: Temporary, until we have something better than marshal which behaves
# differently in compiled Nuitka:
if has_diff and filename not in (
"Contexts.py",
"Whitelisting.py",
"ImplicitImports.py",
):
sys.exit("There were differences!")
shutil.rmtree(target_dir)
if os.name == "nt":
target_filename = filename.replace(".py", ".pyd")
else:
target_filename = filename.replace(".py", ".so")
os.unlink(os.path.join(tmp_dir, target_filename))
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing(getResultFullpath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(0, (binary_filename, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir=getSourceDirectoryPath(main_module),
dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points,
)
data_files = []
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
copyDataFiles(dist_dir=dist_dir, data_files=data_files)
Plugins.onStandaloneDistributionFinished(dist_dir)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path=getSourceDirectoryPath(main_module), ignore_errors=False
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, "w") as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name
for module_name in getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment(),
)
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=package_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=module_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def _runOnefileScons(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=True),
"result_exe": OutputDirectories.getResultFullpath(onefile=True),
"source_dir": source_dir,
"debug_mode": asBoolStr(Options.is_debug),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"target_arch": getArchitecture(),
"python_prefix": sys.prefix,
"nuitka_src": SconsInterface.getSconsDataPath(),
"compiled_exe": OutputDirectories.getResultFullpath(onefile=False),
}
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = "true"
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = "true"
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw64():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if "." not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if getOS() == "Windows":
options["noelf_mode"] = "true"
if Options.isClang():
options["clang_mode"] = "true"
cpp_defines = Plugins.getPreprocessorSymbols()
if cpp_defines:
options["cpp_defines"] = ",".join(
"%s%s%s" % (key, "=" if value else "", value or "")
for key, value in cpp_defines.items())
link_libraries = Plugins.getExtraLinkLibraries()
if link_libraries:
options["link_libraries"] = ",".join(link_libraries)
if Options.shallRunInDebugger():
options["full_names"] = "true"
if Options.assumeYesForDownloads():
options["assume_yes_for_downloads"] = "true"
onefile_env_values = {}
if not Options.isWindowsOnefileTempDirMode():
# Merge version information if necessary, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources[
"CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources[
"ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverriden(onefile_env_values):
result = SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="WindowsOnefile.scons")
# Exit if compilation failed.
if not result:
scons_logger.sysexit(
"Error, one file bootstrap build for Windows failed.")
if Options.isRemoveBuildDir():
general.info("Removing onefile build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping onefile build directory %r." % source_dir)
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
sys.exit(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(
path = target_dir,
ignore_errors = False
)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s"% tmp_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
has_diff = diffRecursive(os.path.join(package, target), target_dir)
if has_diff:
sys.exit("There were differences!")
shutil.rmtree(target_dir)
if os.name == "nt":
target_filename = filename.replace(".py", ".pyd")
else:
target_filename = filename.replace(".py", ".so")
os.unlink(os.path.join(tmp_dir, target_filename))
def main():
# Complex stuff, even more should become common code or project options though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(suite="standalone", needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--standalone",
"remove_output",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
if filename == "Urllib3Using.py" and os.name == "nt":
reportSkip(
"Socket module early import not working on Windows currently",
".",
filename,
)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version < (3, ):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
if getOS() == "Windows":
reportSkip("Can hang on Windows CI.", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
# TODO: Once we have a no-Qt Plugin, we should use that.
if filename == "MatplotlibUsing.py":
# For the plugin warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
if filename == "PandasUsing.py":
extra_flags.append("plugin_enable:numpy")
extra_flags.append("plugin_disable:pylint-warnings")
extra_flags.append("plugin_disable:pyqt5")
extra_flags.append("plugin_disable:pyside2")
extra_flags.append("plugin_disable:pyside6")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "GlfwUsing.py":
# For the warnings.
extra_flags.append("plugin_enable:numpy")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")):
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information
if filename.startswith("PySide2"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PySide6"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PyQt5"):
extra_flags.append("plugin_enable:pyqt5")
elif filename.startswith("PyQt6"):
extra_flags.append("plugin_enable:pyqt6")
test_logger.info(
"Consider output of standalone mode compiled program: %s" %
filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
test_logger.warning(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
with TimerReport("Determining run time loaded files took %.2f",
logger=test_logger):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, command=[binary_filename])
illegal_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd())
if illegal_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning("Should not access these file(s): '%r'." %
illegal_accesses)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
search_mode.finish()
):
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider distutils example:", filename)
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"), ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"), ignore_errors=False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(
commands=['cd "%s"' % case_dir, "python setup.py bdist_wheel"]
)
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand(
'pip install "%s"' % (os.path.join(dist_dir, os.listdir(dist_dir)[0]))
)
runner_binary = os.path.join(
venv.getVirtualenvDir(),
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = [
"expect_success",
"standalone",
"remove_output",
# For enum plugin info
"ignore_infos",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
# catch error
if filename == "Boto3Using.py":
reportSkip("boto3 test not fully working yet", ".", filename)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version.startswith("2"):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
# extra_flags.append("plugin_enable:data-files")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PySideUsing.py":
# TODO: Disabled due to lack of upstream support.
reportSkip("PySide not supported yet", ".", filename)
continue
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide", "PyQt")):
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information.
if getPythonVendor() != "Anaconda":
extra_flags.append("plugin_enable:qt-plugins")
else:
# For the plug-in not used information.
extra_flags.append("ignore_warnings")
my_print("Consider output of recursively compiled program:", filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
my_print(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"System32")),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64")),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith(
"/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/"):
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith((
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith((
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith(
"/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
# PySide accesses its directory.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/PySide"):
continue
# GTK accesses package directories only.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gtk"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/glib"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gio"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(
sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def _createNodeTree(filename):
"""Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# Many cases to deal with, pylint: disable=too-many-branches
# First, build the raw node tree from the source code.
main_module = Building.buildMainModuleTree(
filename=filename,
is_main=not Options.shallMakeModule(),
)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
SconsInterface.cleanSconsDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = OutputDirectories.getStandaloneDirectoryPath(bundle=False)
removeDirectory(path=standalone_dir, ignore_errors=True)
if Options.shallCreateAppBundle():
removeDirectory(
path=changeFilenameExtension(standalone_dir, ".app"), ignore_errors=True
)
# Delete result file, to avoid confusion with previous build and to
# avoid locking issues after the build.
deleteFile(
path=OutputDirectories.getResultFullpath(onefile=False), must_exist=False
)
if Options.isOnefileMode():
deleteFile(
path=OutputDirectories.getResultFullpath(onefile=True), must_exist=False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_name, package_directory, kind = Importing.locateModule(
module_name=ModuleName(package_name),
parent_package=None,
level=0,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate package %r you asked to include."
% package_name.asString()
)
Recursion.checkPluginPath(
plugin_filename=package_directory,
module_package=package_name.getPackageName(),
)
for module_name in Options.getMustIncludeModules():
module_name, module_filename, kind = Importing.locateModule(
module_name=ModuleName(module_name),
parent_package=None,
level=0,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate module '%s' you asked to include."
% module_name.asString()
)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_name.getPackageName()
)
# Allow plugins to add more modules based on the initial set being complete.
Plugins.onModuleInitialSet()
# Then optimize the tree and potentially recursed modules.
optimizeModules(main_module.getOutputFilename())
# Allow plugins to comment on final module set.
Plugins.onModuleCompleteSet()
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = self.getPluginOptions()
plugin_options = set(plugin_options)
if not plugin_options:
plugin_options.add("sensible")
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
)
if self.hasPluginFamily(plugin_dir, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
# Seems platforms is required on Windows.
if os.name == "nt":
plugin_options.add("platforms")
info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
shutil.copytree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if "qml" in plugin_options or "all" in plugin_options:
qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml"))
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
shutil.copytree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
return result
return ()
def removeTempDir():
removeDirectory(path=tmp_dir, ignore_errors=True)
def main():
"""Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches,too-many-statements
if not Options.shallDumpBuiltTreeXML():
general.info("Starting Python compilation.")
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Turn that source code into a node tree structure.
try:
main_module = _createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree()
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing()
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = OutputDirectories.getResultBasepath() + ".pyi"
putTextFileContents(
filename=pyi_filename,
contents="""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name for module_name in getImportedNames()
)
},
)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
setMainEntryPoint(binary_filename)
dist_dir = OutputDirectories.getStandaloneDirectoryPath()
for module in ModuleRegistry.getDoneModules():
addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(
source_dir=OutputDirectories.getSourceDirectoryPath(),
dist_dir=dist_dir,
standalone_entry_points=getStandaloneEntryPoints(),
)
copyDataFiles(dist_dir=dist_dir)
Plugins.onStandaloneDistributionFinished(dist_dir)
if Options.isOnefileMode():
packDistFolderToOnefile(dist_dir, binary_filename)
if Options.isRemoveBuildDir():
general.info("Removing dist folder %r." % dist_dir)
removeDirectory(path=dist_dir, ignore_errors=False)
else:
general.info(
"Keeping dist folder %r for inspection, no need to use it."
% dist_dir
)
# Remove the source directory (now build directory too) if asked to.
source_dir = OutputDirectories.getSourceDirectoryPath()
if Options.isRemoveBuildDir():
general.info("Removing build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping build directory %r." % source_dir)
final_filename = OutputDirectories.getResultFullpath(
onefile=Options.isOnefileMode()
)
Plugins.onFinalResult(final_filename)
general.info("Successfully created %r." % final_filename)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
general.info("Launching %r." % final_filename)
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=final_filename,
clean_path=Options.shallClearPythonPathEnvironment(),
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if os.path.isfile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(
0,
(binary_filename, binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir = getSourceDirectoryPath(main_module),
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for module in ModuleRegistry.getDoneModules():
data_files.extend(
Plugins.considerDataFiles(module)
)
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module),
target_filename
)
makePath(os.path.dirname(target_filename))
shutil.copy2(
source_filename,
target_filename
)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path = getSourceDirectoryPath(main_module),
ignore_errors = False
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here. TODO: Move inside scons file and avoid subprocess call.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, 'w') as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
""" % {
"imports" : '\n'.join(
"import %s" % module_name
for module_name in
getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
# Win API can be assumed.
if loaded_basename.upper().startswith("API-MS-WIN"):
continue
# MSVC run time DLLs, seem to sometimes come from system. TODO:
# clarify if that means we did it wrong.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
# These stopped being loaded by system on Windows 10.
if loaded_basename.upper() in ("MSVCP_WIN.DLL", "WIN32U.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
sys.exit(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
search_mode.finish()
def executePASS1():
my_print("PASS 1: Compiling from compiler running from .py files.")
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
target_dir = package
removeDirectory(
path = target_dir,
ignore_errors = False
)
os.mkdir(target_dir)
for path, filename in listDir(target_dir):
if filename.endswith(".so"):
os.unlink(path)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--module",
"--recurse-none",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % target_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
else:
shutil.copyfile(path, os.path.join(target_dir, filename))
my_print("Compiling '%s'." % nuitka_main_path)
shutil.copyfile(nuitka_main_path, "nuitka-runner.py")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--recurse-none",
"--plugin-enable=pylint-warnings",
"--output-dir=.",
"--python-flag=-S",
"nuitka-runner.py"
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
shutil.move("nuitka-runner.exe", "nuitka.exe")
scons_inline_copy_path = os.path.join(
base_dir,
"nuitka",
"build",
"inline_copy"
)
if os.path.exists(scons_inline_copy_path):
shutil.copytree(
scons_inline_copy_path,
os.path.join("nuitka", "build", "inline_copy")
)
shutil.copy(
os.path.join(base_dir, "nuitka", "build", "SingleExe.scons"),
os.path.join("nuitka", "build", "SingleExe.scons")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "static_src"),
os.path.join("nuitka", "build", "static_src")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "include"),
os.path.join("nuitka", "build", "include")
)
def compareWithCPython(dirname, filename, extra_flags, search_mode,
needs_2to3):
""" Call the comparison tool. For a given directory filename.
The search mode decides if the test case aborts on error or gets extra
flags that are exceptions.
"""
# pylint: disable=too-many-branches
if dirname is None:
path = filename
else:
path = os.path.join(dirname, filename)
# Apply 2to3 conversion if necessary.
if needs_2to3:
path, converted = convertUsing2to3(path)
else:
converted = False
command = [
sys.executable,
os.path.join("..", "..", "bin", "compare_with_cpython"), path, "silent"
]
if extra_flags is not None:
command += extra_flags
command += search_mode.getExtraFlags(dirname, filename)
try:
result = subprocess.call(command)
except KeyboardInterrupt:
result = 2
# Cleanup, some tests apparently forget that.
try:
if os.path.isdir("@test"):
removeDirectory("@test", ignore_errors=False)
elif os.path.isfile("@test"):
os.unlink("@test")
except OSError:
# TODO: Move this into removeDirectory maybe. Doing an external
# call as last resort could be a good idea.
# This seems to work for broken "lnk" files.
if os.name == "nt":
os.system("rmdir /S /Q @test")
if os.path.exists("@test"):
raise
if result != 0 and \
result != 2 and \
search_mode.abortOnFinding(dirname, filename):
my_print("Error exit!", result)
sys.exit(result)
if converted:
os.unlink(path)
if result == 2:
sys.stderr.write("Interrupted, with CTRL-C\n")
sys.exit(2)
def main():
# Complex stuff, pylint: disable=too-many-locals,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(
(".build", ".dist")) or filename.startswith("venv_")):
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider distutils example:", filename)
py3_only_examples = ("example_3", "nested_namespaces")
if python_version < (3, ) and filename in py3_only_examples:
reportSkip("Skipped, only relevant for Python3", ".", filename)
continue
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_wheel"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
# Need to call CPython binary for Windows.
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
my_print("STDOUT CPython:")
my_print(stdout_cpython)
my_print("STDERR CPython:")
my_print(stderr_cpython)
assert exit_cpython == 0, exit_cpython
my_print("EXIT was OK.")
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_nuitka") as venv:
# Install nuitka from source.
venv.runCommand(commands=[
'cd "%s"' % nuitka_dir, "python setup.py install"
])
# Remove that left over from the install command.
removeDirectory(
path=os.path.join(nuitka_dir, "Nuitka.egg-info"),
ignore_errors=False,
)
# Create the wheel with Nuitka compilation.
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_nuitka"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
runner_binary,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
my_print("STDOUT Nuitka:")
my_print(stdout_nuitka)
my_print("STDERR Nuitka:")
my_print(stderr_nuitka)
assert exit_nuitka == 0, exit_nuitka
my_print("EXIT was OK.")
exit_code_stdout = compareOutput(
"stdout",
stdout_cpython,
stdout_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return:
my_print("""\
Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka))
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
sys.exit("Error, outputs differed.")
if search_mode.abortIfExecuted():
break
search_mode.finish()
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = self.getPluginOptions()
plugin_options = set(plugin_options)
if not plugin_options:
plugin_options.add("sensible")
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dir, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
shutil.copytree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if os.name == "nt":
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "bin"))
qt_bin_files = getFileList(qt_bin_dir)
info("Copying OpenSSL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml"))
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
shutil.copytree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if os.name == "nt":
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
info("Copying OpenGL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
return ()