def getResultFullpath(onefile):
"""Get the final output binary result full path."""
result = getResultBasepath(onefile=onefile)
if Options.shallMakeModule():
result += getSharedLibrarySuffix(preferred=True)
else:
output_filename = Options.getOutputFilename()
if Options.isOnefileMode() and output_filename is not None:
if onefile:
result = output_filename
else:
result = os.path.join(
getStandaloneDirectoryPath(),
os.path.basename(output_filename),
)
elif output_filename is not None:
result = output_filename
elif getOS() == "Windows":
result += ".exe"
elif (not Options.isStandaloneMode()
or onefile and not Options.shallCreateAppBundle()):
result += ".bin"
return result
def executePASS5():
my_print("PASS 5: Compiling the compiler 'nuitka' package to single '.so' file.")
path = os.path.join("..", "..", "nuitka")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--include-plugin-dir=%s" % path,
"--nofollow-import-to=nuitka.build.inline_copy",
"--nofollow-import-to=nuitka.build.include",
"--nofollow-import-to=nuitka.build.static_src",
"--module",
path,
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
os.unlink(os.path.join(tmp_dir, "nuitka" + getSharedLibrarySuffix(preferred=True)))
os.unlink(os.path.join(tmp_dir, "nuitka.pyi"))
shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
def action(stage_dir, _root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"),
"--module",
"--output-dir",
stage_dir,
"--remove-output",
"--plugin-enable=pylint-warnings",
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(path)
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
if os.path.basename(path) in nosyntax_errors:
my_print("Syntax error is known unreliable with file file.")
else:
my_print("Falling back to full comparison due to error exit.")
checkCompilesNotWithCPython(dirname=None,
filename=path,
search_mode=search_mode)
else:
my_print("OK")
suffix = getSharedLibrarySuffix(preferred=True)
target_filename = os.path.basename(path)[:-3] + suffix
target_filename = target_filename.replace("(", "").replace(")", "")
os.unlink(os.path.join(stage_dir, target_filename))
def considerExtraDlls(self, dist_dir, module):
module_name = module.getFullName()
if module_name == "zmq.libzmq" and isWin32Windows():
# TODO: Very strange thing for zmq on Windows, needs the .pyd file in wrong dir too. Have
# this done in a dedicated form somewhere.
shutil.copyfile(
os.path.join(dist_dir, "zmq\\libzmq.pyd"),
os.path.join(dist_dir, "libzmq" +
getSharedLibrarySuffix(preferred=True)),
)
return NuitkaPluginBase.considerExtraDlls(self,
dist_dir=dist_dir,
module=module)
def getResultFullpath():
"""Get the final output binary result full path."""
result = getResultBasepath()
if Options.shallMakeModule():
result += getSharedLibrarySuffix(preferred=True)
else:
if Options.getOutputFilename() is not None:
result = Options.getOutputFilename()
elif getOS() == "Windows":
result += ".exe"
elif not Options.isStandaloneMode():
result += ".bin"
return result
def addShlibEntryPoint(module):
target_filename = os.path.join(getStandaloneDirectoryPath(),
*module.getFullName().split("."))
target_filename += getSharedLibrarySuffix(preferred=False)
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append(
makeExtensionModuleEntryPoint(
source_path=module.getFilename(),
dest_path=target_filename,
package_name=module.getFullName().getPackageName(),
))
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=False),
"source_dir": OutputDirectories.getSourceDirectoryPath(),
"nuitka_python": asBoolStr(isNuitkaPython()),
"debug_mode": asBoolStr(Options.is_debug),
"python_debug": asBoolStr(Options.isPythonDebug()),
"module_mode": asBoolStr(Options.shallMakeModule()),
"full_compat": asBoolStr(Options.is_fullcompat),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"python_version": python_version_str,
"target_arch": getArchitecture(),
"python_prefix": getDirectoryRealPath(getSystemPrefixPath()),
"nuitka_src": SconsInterface.getSconsDataPath(),
"module_count": "%d"
% (
1
+ len(ModuleRegistry.getDoneModules())
+ len(ModuleRegistry.getUncompiledNonTechnicalModules())
),
}
if Options.isLowMemory():
options["low_memory"] = asBoolStr(True)
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False)
main_module = ModuleRegistry.getRootTopModule()
assert main_module.isMainModule()
main_module_name = main_module.getFullName()
if main_module_name != "__main__":
options["main_module_name"] = main_module_name
if Options.shallUseStaticLibPython():
options["static_libpython"] = getSystemStaticLibPythonPath()
if isDebianPackagePython():
options["debian_python"] = asBoolStr(True)
if isMSYS2MingwPython():
options["msys2_mingw_python"] = asBoolStr(True)
if isAnacondaPython():
options["anaconda_python"] = asBoolStr(True)
if isApplePython():
options["apple_python"] = asBoolStr(True)
if isPyenvPython():
options["pyenv_python"] = asBoolStr(True)
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.isOnefileMode():
options["onefile_mode"] = asBoolStr(True)
if Options.isOnefileTempDirMode():
options["onefile_temp_mode"] = asBoolStr(True)
if Options.getForcedStdoutPath():
options["forced_stdout_path"] = Options.getForcedStdoutPath()
if Options.getForcedStderrPath():
options["forced_stderr_path"] = Options.getForcedStderrPath()
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules())
)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if hasPythonFlagNoWarnings():
options["no_python_warnings"] = asBoolStr(True)
if hasPythonFlagNoAsserts():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (
sys.flags.division_warning or sys.flags.py3k_warning
):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())):
options["python_sysflag_no_site"] = asBoolStr(True)
if Options.hasPythonFlagTraceImports():
options["python_sysflag_verbose"] = asBoolStr(True)
if Options.hasPythonFlagNoRandomization():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
if hasPythonFlagUnbuffered():
options["python_sysflag_unbuffered"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
SconsInterface.setCommonOptions(options)
if Options.shallCreatePgoInput():
options["pgo_mode"] = "python"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runPythonPgoBinary()
return True, options
# Need to restart compilation from scratch here.
if Options.isPgoMode():
# For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile
# and standalone mode proper support, which might need data files to be
# there, which currently are not yet there, so it won't run.
if Options.isPgoMode():
options["pgo_mode"] = "generate"
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
)
if not result:
return result, options
# Need to make it usable before executing it.
executePostProcessing()
_runCPgoBinary()
options["pgo_mode"] = "use"
result = (
SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
),
options,
)
if options.get("pgo_mode") == "use" and _wasMsvcMode():
_deleteMsvcPGOFiles(pgo_mode="use")
return result
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=False),
"source_dir": OutputDirectories.getSourceDirectoryPath(),
"debug_mode": asBoolStr(Options.is_debug),
"python_debug": asBoolStr(Options.isPythonDebug()),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"module_mode": asBoolStr(Options.shallMakeModule()),
"full_compat": asBoolStr(Options.is_fullcompat),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"python_version": python_version_str,
"target_arch": Utils.getArchitecture(),
"python_prefix": getDirectoryRealPath(sys.prefix),
"nuitka_src": SconsInterface.getSconsDataPath(),
"module_count": "%d"
% (
1
+ len(ModuleRegistry.getDoneModules())
+ len(ModuleRegistry.getUncompiledNonTechnicalModules())
),
}
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False)
if Options.shallUseStaticLibPython():
options["static_libpython"] = asBoolStr(True)
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.isOnefileMode():
options["onefile_mode"] = asBoolStr(True)
if Options.isWindowsOnefileTempDirMode():
options["onefile_temp_mode"] = asBoolStr(True)
if Options.getForcedStdoutPath():
options["forced_stdout_path"] = Options.getForcedStdoutPath()
if Options.getForcedStderrPath():
options["forced_stderr_path"] = Options.getForcedStderrPath()
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules())
)
if Utils.getOS() == "Windows":
options["noelf_mode"] = asBoolStr(True)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if "no_warnings" in getPythonFlags():
options["no_python_warnings"] = asBoolStr(True)
if "no_asserts" in getPythonFlags():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (
sys.flags.division_warning or sys.flags.py3k_warning
):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(os.environ.get("NUITKA_SITE_FLAG", Options.hasPythonFlagNoSite())):
options["python_sysflag_no_site"] = asBoolStr(True)
if "trace_imports" in Options.getPythonFlags():
options["python_sysflag_verbose"] = asBoolStr(True)
if "no_randomization" in Options.getPythonFlags():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
SconsInterface.setCommonOptions(options)
return (
SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="Backend.scons"
),
options,
)
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace("/", os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(path=target_dir, ignore_errors=False)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--no-pyi-file",
path,
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
my_print("Command: ", " ".join(command))
exit_nuitka = subprocess.call(command)
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
command2 = wrapCommandForDebuggerForSubprocess(*command)
subprocess.call(command2)
if exit_nuitka != 0:
my_print("An error exit %s occurred, aborting." % exit_nuitka)
sys.exit(exit_nuitka)
has_diff = diffRecursive(os.path.join(package, target), target_dir)
if has_diff:
sys.exit("There were differences!")
shutil.rmtree(target_dir)
target_filename = filename.replace(
".py", getSharedLibrarySuffix(preferred=True)
)
os.unlink(os.path.join(tmp_dir, target_filename))
def runSconsBackend(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name":
OutputDirectories.getResultBasepath(onefile=False),
"source_dir":
OutputDirectories.getSourceDirectoryPath(),
"debug_mode":
asBoolStr(Options.is_debug),
"python_debug":
asBoolStr(Options.isPythonDebug()),
"unstripped_mode":
asBoolStr(Options.isUnstripped()),
"module_mode":
asBoolStr(Options.shallMakeModule()),
"full_compat":
asBoolStr(Options.is_fullcompat),
"experimental":
",".join(Options.getExperimentalIndications()),
"trace_mode":
asBoolStr(Options.shallTraceExecution()),
"python_version":
python_version_str,
"target_arch":
Utils.getArchitecture(),
"python_prefix":
getDirectoryRealPath(sys.prefix),
"nuitka_src":
SconsInterface.getSconsDataPath(),
"module_count":
"%d" % (1 + len(ModuleRegistry.getDoneModules()) +
len(ModuleRegistry.getUncompiledNonTechnicalModules())),
}
if not Options.shallMakeModule():
options["result_exe"] = OutputDirectories.getResultFullpath()
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = asBoolStr(True)
if Options.shallUseStaticLibPython():
options["static_libpython"] = asBoolStr(True)
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = asBoolStr(True)
if Options.isStandaloneMode():
options["standalone_mode"] = asBoolStr(True)
if Options.shallTreatUninstalledPython():
options["uninstalled_python"] = asBoolStr(True)
if ModuleRegistry.getUncompiledTechnicalModules():
options["frozen_modules"] = str(
len(ModuleRegistry.getUncompiledTechnicalModules()))
if Options.isShowScons():
options["show_scons"] = asBoolStr(True)
if Options.isMingw64():
options["mingw_mode"] = asBoolStr(True)
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if "." not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if Utils.getOS() == "Windows":
options["noelf_mode"] = asBoolStr(True)
if Options.isClang():
options["clang_mode"] = asBoolStr(True)
if Options.isProfile():
options["profile_mode"] = asBoolStr(True)
if "no_warnings" in getPythonFlags():
options["no_python_warnings"] = asBoolStr(True)
if "no_asserts" in getPythonFlags():
options["python_sysflag_optimize"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.py3k_warning:
options["python_sysflag_py3k_warning"] = asBoolStr(True)
if python_version < 0x300 and (sys.flags.division_warning
or sys.flags.py3k_warning):
options["python_sysflag_division_warning"] = asBoolStr(True)
if sys.flags.bytes_warning:
options["python_sysflag_bytes_warning"] = asBoolStr(True)
if int(
os.environ.get("NUITKA_SITE_FLAG", "no_site"
in Options.getPythonFlags())):
options["python_sysflag_no_site"] = asBoolStr(True)
if "trace_imports" in Options.getPythonFlags():
options["python_sysflag_verbose"] = asBoolStr(True)
if "no_randomization" in Options.getPythonFlags():
options["python_sysflag_no_randomization"] = asBoolStr(True)
if python_version < 0x300 and sys.flags.unicode:
options["python_sysflag_unicode"] = asBoolStr(True)
if python_version >= 0x370 and sys.flags.utf8_mode:
options["python_sysflag_utf8"] = asBoolStr(True)
abiflags = getPythonABI()
if abiflags:
options["abiflags"] = abiflags
cpp_defines = Plugins.getPreprocessorSymbols()
if cpp_defines:
options["cpp_defines"] = ",".join(
"%s%s%s" % (key, "=" if value else "", value or "")
for key, value in cpp_defines.items())
link_libraries = Plugins.getExtraLinkLibraries()
if link_libraries:
options["link_libraries"] = ",".join(link_libraries)
if Options.shallMakeModule():
options["module_suffix"] = getSharedLibrarySuffix(preferred=True)
if Options.shallRunInDebugger():
options["full_names"] = asBoolStr(True)
if Options.assumeYesForDownloads():
options["assume_yes_for_downloads"] = asBoolStr(True)
return (
SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="Backend.scons"),
options,
)
def call_analyzer(f, call_list, import_calls, import_files, trace_logic):
""" Analyze the call hierarchy to determine valid called names.
Notes:
Always called with a CALL record type.
Recursive function calling itself for every level change. Each CALL on
each level will be followed by exactly one RESULT (or EXCEPTION),
potentially with interspersed CALL / RESULT pairs at lower levels.
Args:
f: file to read from (created by the script wrapped in hinting logic)
call_list: list representing a CALL record
import_calls: list to receive computed import names
import_files: list to receive imported files
trace_logic: bool to switch on tracing the logic
Returns:
No direct returns, output will be written to call_file.
"""
global line_number
def normalize_file(t):
# step 1: remove any platform tags from shared libraries
folder = os.path.dirname(t) # folder part
datei = os.path.basename(t) # filename
_, ext = os.path.splitext(datei) # extension
if ext in (".pyd", ".so"): # shared library?
datei_arr = datei.split(".") # split
if len(datei_arr) > 2: # platform tag present?
datei = ".".join(datei_arr[:-2]) # yes, omit
else:
datei = ".".join(datei_arr[:-1]) # just omit ext
t = os.path.join(folder, datei) # rebuild filename for step 2
# step 2: turn slashes into '.', remove __init__.py and extensions
t = t.replace("\\", ".").replace("/", ".").replace("$PYTHONPATH.", "")
if t.endswith(".__init__.py"):
t = t[:-12]
return t
if t.endswith(".py"):
t = t[:-3]
return t
if ext not in (".pyd", ".so"):
sys.exit("found unknown Python module type '%s'" % t)
return t
def write_mod(t, f): # write a call entry
import_calls.append((t, f))
if trace_logic:
print(line_number, "call:", t)
return
def write_file(t): # write a file entry
import_files.append(t)
if trace_logic:
print(line_number, "file:", t)
return
level = call_list[0] # nesting level
CALLED = call_list[2] # the imported module
implist = call_list[3] # list accompanying the import statement
text = reader(f) # read the next record
if not bool(text): # EOF should not happen here!
print("unexpected EOF at %s" % str(call_list))
sys.exit("line number %i" % line_number)
if len(text) < 3:
print("unexpected record format", text)
sys.exit("at line number %i" % line_number)
while "CALL" in text: # any CALL records will be recursed into
call_analyzer(f, text, import_calls, import_files, trace_logic)
text = reader(f)
if len(text) < 3:
return
if text[0] != level: # this record should have our level!
matching = False
else:
matching = True
if text[1] == "EXCEPTION": # no output if an exception resulted
return
if text[1] != "RESULT": # this must be a RESULT now
sys.exit("%i: expected RESULT after %s" %
(line_number, str(call_list)))
RESULT = text[2] # resulting module name
if RESULT == "__main__": # skip current script
return
res_file = text[3] # resulting file name
if res_file == "built-in": # skip output for built-in stuff
return
if res_file.endswith(".dll"): # special handling for pythoncom and friends
res_file = RESULT + ".py"
if RESULT.startswith("win32com"): # special handling for win32com
res_file = "$PYTHONPATH\\win32com\\__init__.py"
if trace_logic:
print(line_number, ":", str(call_list))
print(line_number, ":", str(text))
normalized_file = normalize_file(res_file)
write_file(normalized_file)
if not matching:
print("No result matches %i, %s, %s" % (level, CALLED, str(implist)))
write_mod(RESULT, normalized_file) # this is a sure output
# members of shared modules cannot be filtered out, so allow them all
# TODO: This should consider all possible suffixes, should it not.
if (hasFilenameExtension(res_file, getSharedLibrarySuffix(
preferred=True)) # a shared module!
or normalized_file in accept_always):
write_mod(RESULT + ".*", normalized_file)
return
if not CALLED: # case: the CALL name is empty
if not implist: # should not happen, but let's ignore this
return
for item in implist: # return RESULT.item for items in list
write_mod(RESULT + "." + item, normalized_file)
return
if (CALLED.startswith(RESULT) or RESULT.startswith(CALLED)
or RESULT.endswith(CALLED)):
# CALL and RESULT names contain each other in some way
if not implist:
if CALLED != RESULT:
write_mod(CALLED, normalized_file)
return
if CALLED == RESULT:
cmod = CALLED
elif RESULT.endswith(CALLED):
cmod = RESULT
elif RESULT.startswith(CALLED):
cmod = RESULT
else:
cmod = CALLED
for item in implist: # this is a list of items
write_mod(cmod + "." + item, normalized_file)
return
""" Case:
RESULT and CALL names neither contain each other, nor is CALLED empty.
We then assume that the true call name should be RESULT.CALLED in output.
"""
cmod = RESULT + "." + CALLED # equals RESULT.CALLED
write_mod(cmod, normalized_file) # output it
if not implist: # no list there: done
return
for item in implist: # or again a list of items
write_mod(cmod + "." + item, normalized_file)
return
