def subprocess_spawn(args):
sh, _cmd, args, env = args
_stdout, stderr, exit_code = executeProcess(
command=[sh, "-c", " ".join(args)], env=env)
ignore_next = False
for line in stderr.splitlines():
if ignore_next:
ignore_next = False
continue
if isIgnoredError(line):
ignore_next = True
continue
if str is not bytes:
line = decodeData(line)
if exit_code != 0 and "terminated with signal 11" in line:
exit_code = -11
my_print(line, style="yellow", file=sys.stderr)
return exit_code
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
# Many cases to deal with, pylint: disable=too-many-branches
if isWin32Windows():
sys.exit("Error, valgrind is not available on Windows.")
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
_stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess(
command)
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
copyFile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def _executePylint(filenames, pylint_options, extra_options):
# This is kind of a singleton module, pylint: disable=global-statement
global our_exit_code
command = ([os.environ["PYTHON"], "-m", "pylint"] + pylint_options +
extra_options + filenames)
stdout, stderr, exit_code = executeProcess(command)
if exit_code == -11:
sys.exit("Error, segfault from pylint.")
stdout = _cleanupPylintOutput(stdout)
stderr = _cleanupPylintOutput(stderr)
if stderr:
our_exit_code = 1
for line in stderr:
my_print(line)
if stdout:
# If we filtered everything away, remove the leading file name reports.
while stdout and stdout[-1].startswith("******"):
del stdout[-1]
for line in stdout:
my_print(line)
if stdout:
our_exit_code = 1
sys.stdout.flush()
def getCompilerArch(mingw_mode, msvc_mode, the_cc_name, compiler_path):
assert not mingw_mode or not msvc_mode
if compiler_path not in _compiler_arch:
if mingw_mode:
_compiler_arch[compiler_path] = _getBinaryArch(
binary=compiler_path, mingw_mode=mingw_mode)
elif msvc_mode:
cmdline = [compiler_path]
if "-cl" in the_cc_name:
cmdline.append("--version")
# The cl.exe without further args will give error
stdout, stderr, _rv = executeProcess(command=cmdline, )
# The MSVC will output on error, while clang outputs in stdout and they
# use different names for arches.
if b"x64" in stderr or b"x86_64" in stdout:
_compiler_arch[compiler_path] = "pei-x86-64"
elif b"x86" in stderr or b"i686" in stdout:
_compiler_arch[compiler_path] = "pei-i386"
elif b"ARM64" in stderr:
# TODO: The ARM64 output for Clang is not known yet.
_compiler_arch[compiler_path] = "pei-arm64"
else:
assert False, (stdout, stderr)
else:
assert False, compiler_path
return _compiler_arch[compiler_path]
def updateWorkingFile(path, orig_object_hash, new_object_hash):
patch = check_output(
["git", "diff", "--no-color", orig_object_hash, new_object_hash])
git_path = path.replace(os.path.sep, "/").encode("utf8")
def updateLine(line):
if line.startswith(b"diff --git"):
line = b"diff --git a/%s b/%s" % (git_path, git_path)
elif line.startswith(b"--- a/"):
line = b"--- a/" + git_path
elif line.startswith(b"+++ b/"):
line = b"+++ b/" + git_path
return line
# Substitute object hashes in patch header with path to working tree file
patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n"
# Apply the patch.
output, err, exit_code = executeProcess(
["git", "apply", "-"],
stdin=patch,
)
# Windows extra ball, new files have new lines that make the patch fail.
if exit_code != 0 and os.name == "nt":
from .autoformat.Autoformat import cleanupWindowsNewlines
cleanupWindowsNewlines(path)
output, err, exit_code = executeProcess(
["git", "apply", "-"],
stdin=patch,
)
success = exit_code == 0
if not success:
# TODO: In case of failure, do we need to abort, or what do we do.
if output:
my_print(output, style="yellow")
if err:
my_print(err, style="yellow")
return success
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs):
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with withFileLock(
"Finding out dependency walker path and creating DWP file for %s"
% binary_filename
):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
putTextFileContents(
dwp_filename,
contents="""\
%(scan_dirs)s
SxS
"""
% {
"scan_dirs": "\n".join(
"UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs
)
},
)
# Starting the process while locked, so file handles are not duplicated.
# TODO: At least exit code should be checked, output goes to a filename,
# but errors might be interesting potentially.
_stdout, _stderr, _exit_code = executeProcess(
command=(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
external_cwd=True,
)
if not os.path.exists(output_filename):
inclusion_logger.sysexit(
"Error, 'depends.exe' failed to produce expected output."
)
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
result = parseDependsExeOutput(output_filename)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
return result
def run(self):
try:
# execute the command, queue the result
with self.timer_report:
self.data, self.err, self.exit_code = executeProcess(
command=self.cmdline, env=self.env)
except Exception as e: # will rethrow all, pylint: disable=broad-except
self.exception = e
def _getBinaryArch(binary, mingw_mode):
if "linux" in sys.platform or mingw_mode:
assert os.path.exists(binary), binary
command = ["objdump", "-f", binary]
try:
data, _err, rv = executeProcess(command)
except OSError:
return None
if rv != 0:
return None
if str is not bytes:
data = decodeData(data)
for line in data.splitlines():
if " file format " in line:
return line.split(" file format ")[-1]
else:
# TODO: Missing for macOS, FreeBSD, other Linux
return None
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
def hasArgValue(arg_option, default=None):
for arg in args:
if arg.startswith(arg_option + "="):
args.remove(arg)
return arg[len(arg_option) + 1:]
return default
def hasArgValues(arg_option):
result = []
for arg in tuple(args):
if arg.startswith(arg_option + "="):
args.remove(arg)
result.append(arg[len(arg_option) + 1:])
return result
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("--module")
coverage_mode = hasArg("coverage")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (hasArg("trace_command")
or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0")
remove_output = hasArg("remove_output")
remove_binary = not hasArg("--keep-binary")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site") or coverage_mode
report = hasArgValue("--report")
nofollow_imports = hasArg("recurse_none") or hasArg("--nofollow-imports")
follow_imports = hasArg("recurse_all") or hasArg("--follow-imports")
timing = hasArg("timing")
original_file = hasArg("original_file") or hasArg(
"--file-reference-choice=original")
runtime_file = hasArg("runtime_file") or hasArg(
"--file-reference-choice=runtime")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
send_kill = hasArg("--send-ctrl-c")
output_dir = hasArgValue("--output-dir", None)
include_packages = hasArgValues("--include-package")
include_modules = hasArgValues("--include-module")
python_flag_m = hasArg("--python-flag=-m")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:"):])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:"):])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:"):])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:"):])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:"):])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
# We need to split it, so we know when to kill.
if send_kill:
two_step_execution = True
assert not standalone_mode or not module_mode
assert not follow_imports or not nofollow_imports
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(
os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print("""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".
format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
))
else:
my_print("""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".
format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
))
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
module_name = os.path.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s" %
(repr(os.path.dirname(filename)), module_name),
]
if no_warnings:
cpython_cmd[1:1] = [
"-W",
"ignore",
]
else:
cpython_cmd = [os.environ["PYTHON"]]
if no_warnings:
cpython_cmd[1:1] = [
"-W",
"ignore",
]
if python_flag_m:
cpython_cmd += ["-m", os.path.basename(filename)]
os.chdir(os.path.dirname(filename))
else:
cpython_cmd.append(filename)
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if python_flag_m:
extra_options.append("--python-flag=-m")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output([
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
])
if sys.version_info >= (3, ):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if report:
extra_options.append("--report=%s" % report)
if nofollow_imports:
extra_options.append("--nofollow-imports")
if follow_imports:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--enable-plugin=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--disable-plugin=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" %
filename)
if output_dir is not None:
extra_options.append("--output-dir=%s" % output_dir)
else:
# TODO: The run-tests uses NUITKA_EXTRA_OPTIONS still.
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
for include_package in include_packages:
extra_options.append("--include-package=%s" % include_package)
for include_module in include_modules:
extra_options.append("--include-module=%s" % include_module)
# Progress bar is not used.
extra_options.append("--no-progressbar")
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
extra_options.append("--module")
elif onefile_mode:
extra_options.append("--onefile")
elif standalone_mode:
extra_options.append("--standalone")
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (nuitka_call + extra_options +
["--module", os.path.abspath(filename)])
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + [
"--standalone", filename
]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
if module_mode:
module_name = os.path.basename(filename)
if module_name.endswith(".py"):
module_name = module_name[:-3]
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % module_name,
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
if os.name == "nt":
exe_filename += ".exe"
else:
exe_filename += ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=False,
send_kill=send_kill,
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = getTempDir()
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(
stdout_nuitka) and checkNoPermissionError(stderr_nuitka):
break
my_print(
"Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
stdout_nuitka1, stderr_nuitka1, exit_nuitka1 = executeProcess(
nuitka_cmd1)
if exit_nuitka1 != 0:
if (not expect_failure and not comparison_mode
and not os.path.exists(".coverage")):
sys.exit("""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
""" % (os.environ["PYTHON"], stderr_nuitka1))
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
stdout_nuitka2 = b"not run due to compilation error:\n" + stdout_nuitka1
stderr_nuitka2 = stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if os.path.exists(nuitka_cmd2[0][:-4] + ".cmd"):
nuitka_cmd2[0] = nuitka_cmd2[0][:-4] + ".cmd"
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
# Need full manual control, and not all Python versions allow using
# context manager here, pylint: disable=consider-using-with
process = subprocess.Popen(args=nuitka_cmd2,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Lambda is used immediately in same loop, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Nuitka compiled program",
process.pid),
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(
*nuitka_cmd2)
callProcess(nuitka_cmd2, shell=False)
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(
stdout_nuitka) and checkNoPermissionError(stderr_nuitka):
break
my_print(
"Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout",
stdout_cpython,
stdout_nuitka2 if two_step_execution else stdout_nuitka,
ignore_warnings,
syntax_errors,
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka2 if two_step_execution else stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)"""
.format(exit_cpython=exit_cpython,
exit_nuitka=exit_nuitka))
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False)
if not int(os.environ.get("NUITKA_CPYTHON_NO_CACHE_UPDATE", "0")):
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
send_kill=send_kill,
)
if not silent_mode:
if (old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (exit_code_return and exit_nuitka in (-11, -6)
and sys.platform != "nt" and not module_mode
and not two_step_execution):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
callProcess(nuitka_cmd, shell=False)
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]) and remove_binary:
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(
filename) + getSharedLibrarySuffix(preferred=True)
if os.path.exists(module_filename) and remove_binary:
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." %
(cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
# TODO: Actually would be better to pass it as env to the created process instead.
with withEnvironmentPathAdded(
"LD_LIBRARY_PATH",
*_getLdLibraryPath(
package_name=package_name,
python_rpath=_detected_python_rpath,
original_dir=original_dir,
)):
# TODO: Check exit code, should never fail.
stdout, stderr, _exit_code = executeProcess(command=("ldd",
dll_filename))
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
result = set()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Normalize, sometimes the DLLs produce "something/../", this has
# been seen with Qt at least.
filename = os.path.normpath(filename)
# Do not include kernel DLLs on the ignore list.
filename_base = os.path.basename(filename)
if any(filename_base == entry or filename_base.startswith(entry + ".")
for entry in _linux_dll_ignore_list):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(
dll_filename=sub_dll_filename,
package_name=package_name,
original_dir=original_dir,
))
return sub_result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result