def subprocess_spawn(args): sh, _cmd, args, env = args _stdout, stderr, exit_code = executeProcess( command=[sh, "-c", " ".join(args)], env=env) ignore_next = False for line in stderr.splitlines(): if ignore_next: ignore_next = False continue if isIgnoredError(line): ignore_next = True continue if str is not bytes: line = decodeData(line) if exit_code != 0 and "terminated with signal 11" in line: exit_code = -11 my_print(line, style="yellow", file=sys.stderr) return exit_code
def runValgrind(descr, tool, args, include_startup, save_logfilename=None): # Many cases to deal with, pylint: disable=too-many-branches if isWin32Windows(): sys.exit("Error, valgrind is not available on Windows.") if descr: my_print(descr, tool, file=sys.stderr, end="... ") with withTemporaryFile() as log_file: log_filename = log_file.name command = ["valgrind", "-q"] if tool == "callgrind": command += ("--tool=callgrind", "--callgrind-out-file=%s" % log_filename) elif tool == "massif": command += ("--tool=massif", "--massif-out-file=%s" % log_filename) else: sys.exit("Error, no support for tool '%s' yet." % tool) # Do not count things before main module starts its work. if not include_startup: command += ( "--zero-before=init__main__()", "--zero-before=init__main__", "--zero-before=PyInit___main__", "--zero-before=PyInit___main__()", ) command.extend(args) _stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess( command) assert exit_valgrind == 0, stderr_valgrind if descr: my_print("OK", file=sys.stderr) if save_logfilename is not None: copyFile(log_filename, save_logfilename) max_mem = None for line in getFileContentByLine(log_filename): if tool == "callgrind" and line.startswith("summary:"): return int(line.split()[1]) elif tool == "massif" and line.startswith("mem_heap_B="): mem = int(line.split("=")[1]) if max_mem is None: max_mem = 0 max_mem = max(mem, max_mem) if tool == "massif" and max_mem is not None: return max_mem sys.exit("Error, didn't parse Valgrind log file successfully.")
def _executePylint(filenames, pylint_options, extra_options): # This is kind of a singleton module, pylint: disable=global-statement global our_exit_code command = ([os.environ["PYTHON"], "-m", "pylint"] + pylint_options + extra_options + filenames) stdout, stderr, exit_code = executeProcess(command) if exit_code == -11: sys.exit("Error, segfault from pylint.") stdout = _cleanupPylintOutput(stdout) stderr = _cleanupPylintOutput(stderr) if stderr: our_exit_code = 1 for line in stderr: my_print(line) if stdout: # If we filtered everything away, remove the leading file name reports. while stdout and stdout[-1].startswith("******"): del stdout[-1] for line in stdout: my_print(line) if stdout: our_exit_code = 1 sys.stdout.flush()
def getCompilerArch(mingw_mode, msvc_mode, the_cc_name, compiler_path): assert not mingw_mode or not msvc_mode if compiler_path not in _compiler_arch: if mingw_mode: _compiler_arch[compiler_path] = _getBinaryArch( binary=compiler_path, mingw_mode=mingw_mode) elif msvc_mode: cmdline = [compiler_path] if "-cl" in the_cc_name: cmdline.append("--version") # The cl.exe without further args will give error stdout, stderr, _rv = executeProcess(command=cmdline, ) # The MSVC will output on error, while clang outputs in stdout and they # use different names for arches. if b"x64" in stderr or b"x86_64" in stdout: _compiler_arch[compiler_path] = "pei-x86-64" elif b"x86" in stderr or b"i686" in stdout: _compiler_arch[compiler_path] = "pei-i386" elif b"ARM64" in stderr: # TODO: The ARM64 output for Clang is not known yet. _compiler_arch[compiler_path] = "pei-arm64" else: assert False, (stdout, stderr) else: assert False, compiler_path return _compiler_arch[compiler_path]
def updateWorkingFile(path, orig_object_hash, new_object_hash): patch = check_output( ["git", "diff", "--no-color", orig_object_hash, new_object_hash]) git_path = path.replace(os.path.sep, "/").encode("utf8") def updateLine(line): if line.startswith(b"diff --git"): line = b"diff --git a/%s b/%s" % (git_path, git_path) elif line.startswith(b"--- a/"): line = b"--- a/" + git_path elif line.startswith(b"+++ b/"): line = b"+++ b/" + git_path return line # Substitute object hashes in patch header with path to working tree file patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n" # Apply the patch. output, err, exit_code = executeProcess( ["git", "apply", "-"], stdin=patch, ) # Windows extra ball, new files have new lines that make the patch fail. if exit_code != 0 and os.name == "nt": from .autoformat.Autoformat import cleanupWindowsNewlines cleanupWindowsNewlines(path) output, err, exit_code = executeProcess( ["git", "apply", "-"], stdin=patch, ) success = exit_code == 0 if not success: # TODO: In case of failure, do we need to abort, or what do we do. if output: my_print(output, style="yellow") if err: my_print(err, style="yellow") return success
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs): dwp_filename = binary_filename + ".dwp" output_filename = binary_filename + ".depends" # User query should only happen once if at all. with withFileLock( "Finding out dependency walker path and creating DWP file for %s" % binary_filename ): depends_exe = getDependsExePath() # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. putTextFileContents( dwp_filename, contents="""\ %(scan_dirs)s SxS """ % { "scan_dirs": "\n".join( "UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs ) }, ) # Starting the process while locked, so file handles are not duplicated. # TODO: At least exit code should be checked, output goes to a filename, # but errors might be interesting potentially. _stdout, _stderr, _exit_code = executeProcess( command=( depends_exe, "-c", "-ot%s" % output_filename, "-d:%s" % dwp_filename, "-f1", "-pa1", "-ps1", binary_filename, ), external_cwd=True, ) if not os.path.exists(output_filename): inclusion_logger.sysexit( "Error, 'depends.exe' failed to produce expected output." ) # Opening the result under lock, so it is not getting locked by new processes. # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. result = parseDependsExeOutput(output_filename) deleteFile(output_filename, must_exist=True) deleteFile(dwp_filename, must_exist=True) return result
def run(self): try: # execute the command, queue the result with self.timer_report: self.data, self.err, self.exit_code = executeProcess( command=self.cmdline, env=self.env) except Exception as e: # will rethrow all, pylint: disable=broad-except self.exception = e
def _getBinaryArch(binary, mingw_mode): if "linux" in sys.platform or mingw_mode: assert os.path.exists(binary), binary command = ["objdump", "-f", binary] try: data, _err, rv = executeProcess(command) except OSError: return None if rv != 0: return None if str is not bytes: data = decodeData(data) for line in data.splitlines(): if " file format " in line: return line.split(" file format ")[-1] else: # TODO: Missing for macOS, FreeBSD, other Linux return None
def main(): # Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements filename = sys.argv[1] args = sys.argv[2:] def hasArg(arg): if arg in args: args.remove(arg) return True else: return False def hasArgValue(arg_option, default=None): for arg in args: if arg.startswith(arg_option + "="): args.remove(arg) return arg[len(arg_option) + 1:] return default def hasArgValues(arg_option): result = [] for arg in tuple(args): if arg.startswith(arg_option + "="): args.remove(arg) result.append(arg[len(arg_option) + 1:]) return result # For output keep it arguments = list(args) silent_mode = hasArg("silent") ignore_stderr = hasArg("ignore_stderr") ignore_warnings = hasArg("ignore_warnings") expect_success = hasArg("expect_success") expect_failure = hasArg("expect_failure") python_debug = hasArg("python_debug") module_mode = hasArg("--module") coverage_mode = hasArg("coverage") two_step_execution = hasArg("two_step_execution") binary_python_path = hasArg("binary_python_path") keep_python_path = hasArg("keep_python_path") trace_command = (hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0") remove_output = hasArg("remove_output") remove_binary = not hasArg("--keep-binary") standalone_mode = hasArg("--standalone") onefile_mode = hasArg("--onefile") no_site = hasArg("no_site") or coverage_mode report = hasArgValue("--report") nofollow_imports = hasArg("recurse_none") or hasArg("--nofollow-imports") follow_imports = hasArg("recurse_all") or hasArg("--follow-imports") timing = hasArg("timing") original_file = hasArg("original_file") or hasArg( "--file-reference-choice=original") runtime_file = hasArg("runtime_file") or hasArg( "--file-reference-choice=runtime") no_warnings = not hasArg("warnings") full_compat = not hasArg("improved") cpython_cached = hasArg("cpython_cache") syntax_errors = hasArg("syntax_errors") noprefer_source = hasArg("noprefer_source") noverbose_log = hasArg("noverbose_log") noinclusion_log = hasArg("noinclusion_log") send_kill = hasArg("--send-ctrl-c") output_dir = hasArgValue("--output-dir", None) include_packages = hasArgValues("--include-package") include_modules = hasArgValues("--include-module") python_flag_m = hasArg("--python-flag=-m") plugins_enabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_enable:"): plugins_enabled.append(arg[len("plugin_enable:"):]) del args[count] plugins_disabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_disable:"): plugins_disabled.append(arg[len("plugin_disable:"):]) del args[count] user_plugins = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("user_plugin:"): user_plugins.append(arg[len("user_plugin:"):]) del args[count] recurse_not = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_not:"): recurse_not.append(arg[len("recurse_not:"):]) del args[count] recurse_to = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_to:"): recurse_to.append(arg[len("recurse_to:"):]) del args[count] if args: sys.exit("Error, non understood mode(s) '%s'," % ",".join(args)) # In coverage mode, we don't want to execute, and to do this only in one mode, # we enable two step execution, which splits running the binary from the actual # compilation: if coverage_mode: two_step_execution = True # The coverage mode doesn't work with debug mode. if coverage_mode: python_debug = False comparison_mode = not coverage_mode # We need to split it, so we know when to kill. if send_kill: two_step_execution = True assert not standalone_mode or not module_mode assert not follow_imports or not nofollow_imports if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" os.environ["PYTHONWARNINGS"] = "ignore" if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() if "--python-debug" in extra_options or "--python-dbg" in extra_options: python_debug = True if python_debug: if os.path.exists( os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")): os.environ["PYTHON"] += "-dbg" if os.name == "nt": if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"): os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe" if os.environ["PYTHON"].endswith("-dbg"): python_debug = True if os.environ["PYTHON"].lower().endswith("_d.exe"): python_debug = True if comparison_mode: my_print("""\ Comparing output of '{filename}' using '{python}' with flags {args} ...""". format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), )) else: my_print("""\ Taking coverage of '{filename}' using '{python}' with flags {args} ...""". format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), )) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("CPython:") my_print("*" * 80) if two_step_execution: filename = os.path.abspath(filename) if module_mode: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] cpython_cmd = [ os.environ["PYTHON"], "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), module_name), ] if no_warnings: cpython_cmd[1:1] = [ "-W", "ignore", ] else: cpython_cmd = [os.environ["PYTHON"]] if no_warnings: cpython_cmd[1:1] = [ "-W", "ignore", ] if python_flag_m: cpython_cmd += ["-m", os.path.basename(filename)] os.chdir(os.path.dirname(filename)) else: cpython_cmd.append(filename) if no_site: cpython_cmd.insert(1, "-S") if "NUITKA" in os.environ: # Would need to extract which "python" this is going to use. assert not coverage_mode, "Not implemented for binaries." nuitka_call = [os.environ["NUITKA"]] else: if comparison_mode: nuitka_call = [ os.environ["PYTHON"], "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] else: assert coverage_mode nuitka_call = [ os.environ["PYTHON"], "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] if python_debug: extra_options.append("--python-debug") if no_warnings: extra_options.append("--python-flag=no_warnings") if remove_output: extra_options.append("--remove-output") if original_file: extra_options.append("--file-reference-choice=original") if runtime_file: extra_options.append("--file-reference-choice=runtime") if full_compat: extra_options.append("--full-compat") if noprefer_source: extra_options.append("--no-prefer-source") if python_flag_m: extra_options.append("--python-flag=-m") if coverage_mode: # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output([ os.environ["PYTHON"], "-c", "import sys, os; print(os.pathsep.join(sys.path))", ]) if sys.version_info >= (3, ): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() if binary_python_path: addToPythonPath(os.path.dirname(os.path.abspath(filename))) if keep_python_path or binary_python_path: extra_options.append("--execute-with-pythonpath") if report: extra_options.append("--report=%s" % report) if nofollow_imports: extra_options.append("--nofollow-imports") if follow_imports: extra_options.append("--follow-imports") if recurse_not: extra_options.extend("--nofollow-import-to=" + v for v in recurse_not) if coverage_mode: extra_options.append("--must-not-re-execute") extra_options.append("--generate-c-only") for plugin_enabled in plugins_enabled: extra_options.append("--enable-plugin=" + plugin_enabled) for plugin_disabled in plugins_disabled: extra_options.append("--disable-plugin=" + plugin_disabled) for user_plugin in user_plugins: extra_options.append("--user-plugin=" + user_plugin) if not noverbose_log: extra_options.append("--verbose-output=%s.optimization.log" % filename) if not noinclusion_log: extra_options.append("--show-modules-output=%s.inclusion.log" % filename) if output_dir is not None: extra_options.append("--output-dir=%s" % output_dir) else: # TODO: The run-tests uses NUITKA_EXTRA_OPTIONS still. for extra_option in extra_options: dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option) if dir_match: output_dir = dir_match.group(1) break else: # The default. output_dir = "." for include_package in include_packages: extra_options.append("--include-package=%s" % include_package) for include_module in include_modules: extra_options.append("--include-module=%s" % include_module) # Progress bar is not used. extra_options.append("--no-progressbar") # Now build the command to run Nuitka. if not two_step_execution: if module_mode: extra_options.append("--module") elif onefile_mode: extra_options.append("--onefile") elif standalone_mode: extra_options.append("--standalone") nuitka_cmd = nuitka_call + extra_options + ["--run", filename] if no_site: nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S") else: if module_mode: nuitka_cmd1 = (nuitka_call + extra_options + ["--module", os.path.abspath(filename)]) elif standalone_mode: nuitka_cmd1 = nuitka_call + extra_options + [ "--standalone", filename ] else: nuitka_cmd1 = nuitka_call + extra_options + [filename] if no_site: nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S") if module_mode: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] nuitka_cmd2 = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import %s" % module_name, ] else: exe_filename = os.path.basename(filename) if filename.endswith(".py"): exe_filename = exe_filename[:-3] exe_filename = exe_filename.replace(")", "").replace("(", "") if os.name == "nt": exe_filename += ".exe" else: exe_filename += ".bin" nuitka_cmd2 = [os.path.join(output_dir, exe_filename)] pdb_filename = exe_filename[:-4] + ".pdb" if trace_command: my_print("CPython command:", *cpython_cmd) if comparison_mode: cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False, send_kill=send_kill, ) if not silent_mode: displayOutput(stdout_cpython, stderr_cpython) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("Nuitka:") my_print("*" * 80) if two_step_execution: if output_dir: os.chdir(output_dir) else: tmp_dir = getTempDir() os.chdir(tmp_dir) if trace_command: my_print("Going to output directory", os.getcwd()) stop_watch = StopWatch() stop_watch.start() if not two_step_execution: if trace_command: my_print("Nuitka command:", nuitka_cmd) # Try a couple of times for permission denied, on Windows it can # be transient. for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen(args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_nuitka, stderr_nuitka = process.communicate() exit_nuitka = process.returncode if checkNoPermissionError( stdout_nuitka) and checkNoPermissionError(stderr_nuitka): break my_print( "Retrying nuitka exe due to permission problems after delay.") time.sleep(2) else: if trace_command: my_print("Nuitka command 1:", nuitka_cmd1) for _i in range(5): with withPythonPathChange(nuitka_package_dir): stdout_nuitka1, stderr_nuitka1, exit_nuitka1 = executeProcess( nuitka_cmd1) if exit_nuitka1 != 0: if (not expect_failure and not comparison_mode and not os.path.exists(".coverage")): sys.exit("""\ Error, failed to take coverage with '%s'. Stderr was: %s """ % (os.environ["PYTHON"], stderr_nuitka1)) exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 stdout_nuitka2 = b"not run due to compilation error:\n" + stdout_nuitka1 stderr_nuitka2 = stderr_nuitka1 else: # No execution second step for coverage mode. if comparison_mode: if os.path.exists(nuitka_cmd2[0][:-4] + ".cmd"): nuitka_cmd2[0] = nuitka_cmd2[0][:-4] + ".cmd" if trace_command: my_print("Nuitka command 2:", nuitka_cmd2) # Need full manual control, and not all Python versions allow using # context manager here, pylint: disable=consider-using-with process = subprocess.Popen(args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if send_kill: # Lambda is used immediately in same loop, pylint: disable=cell-var-from-loop executeAfterTimePassed( 1.0, lambda: killProcess("Nuitka compiled program", process.pid), ) stdout_nuitka2, stderr_nuitka2 = process.communicate() stdout_nuitka = stdout_nuitka1 + stdout_nuitka2 stderr_nuitka = stderr_nuitka1 + stderr_nuitka2 exit_nuitka = process.returncode # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": nuitka_cmd2 = wrapCommandForDebuggerForSubprocess( *nuitka_cmd2) callProcess(nuitka_cmd2, shell=False) else: exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 if checkNoPermissionError( stdout_nuitka) and checkNoPermissionError(stderr_nuitka): break my_print( "Retrying nuitka exe due to permission problems after delay.") time.sleep(2) stop_watch.stop() nuitka_time = stop_watch.getDelta() if not silent_mode: displayOutput(stdout_nuitka, stderr_nuitka) if coverage_mode: assert not stdout_nuitka assert not stderr_nuitka if comparison_mode: def makeComparisons(trace_result): exit_code_stdout = compareOutput( "stdout", stdout_cpython, stdout_nuitka2 if two_step_execution else stdout_nuitka, ignore_warnings, syntax_errors, ) if ignore_stderr: exit_code_stderr = 0 else: exit_code_stderr = compareOutput( "stderr", stderr_cpython, stderr_nuitka2 if two_step_execution else stderr_nuitka, ignore_warnings, syntax_errors, ) exit_code_return = exit_cpython != exit_nuitka if exit_code_return and trace_result: my_print( """Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""" .format(exit_cpython=exit_cpython, exit_nuitka=exit_nuitka)) return exit_code_stdout, exit_code_stderr, exit_code_return if cpython_cached: exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=False) if not int(os.environ.get("NUITKA_CPYTHON_NO_CACHE_UPDATE", "0")): if exit_code_stdout or exit_code_stderr or exit_code_return: old_stdout_cpython = stdout_cpython old_stderr_cpython = stderr_cpython old_exit_cpython = exit_cpython my_print( "Updating CPython cache by force due to non-matching comparison results.", style="yellow", ) ( cpython_time, stdout_cpython, stderr_cpython, exit_cpython, ) = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=True, send_kill=send_kill, ) if not silent_mode: if (old_stdout_cpython != stdout_cpython or old_stderr_cpython != stderr_cpython or old_exit_cpython != exit_cpython): displayOutput(stdout_cpython, stderr_cpython) exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=True) # In case of segfault, also output the call stack by entering debugger # without stdin forwarded. if (exit_code_return and exit_nuitka in (-11, -6) and sys.platform != "nt" and not module_mode and not two_step_execution): nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger") with withPythonPathChange(nuitka_package_dir): callProcess(nuitka_cmd, shell=False) exit_code = exit_code_stdout or exit_code_stderr or exit_code_return if exit_code: problems = [] if exit_code_stdout: problems.append("stdout") if exit_code_stderr: problems.append("stderr") if exit_code_return: problems.append("exit_code") sys.exit("Error, results differed (%s)." % ",".join(problems)) if expect_success and exit_cpython != 0: if silent_mode: displayOutput(stdout_cpython, stderr_cpython) sys.exit("Unexpected error exit from CPython.") if expect_failure and exit_cpython == 0: sys.exit("Unexpected success exit from CPython.") if remove_output: if not module_mode: if os.path.exists(nuitka_cmd2[0]) and remove_binary: if os.name == "nt": # It appears there is a tiny lock race that we randomly cause, # likely because --run spawns a subprocess that might still # be doing the cleanup work. if os.path.exists(nuitka_cmd2[0] + ".away"): os.unlink(nuitka_cmd2[0] + ".away") for _i in range(10): try: os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away") except OSError: time.sleep(0.1) continue for _i in range(10): try: os.unlink(nuitka_cmd2[0] + ".away") except OSError: time.sleep(2) continue else: break if os.path.exists(pdb_filename): os.unlink(pdb_filename) else: os.unlink(nuitka_cmd2[0]) else: module_filename = os.path.basename( filename) + getSharedLibrarySuffix(preferred=True) if os.path.exists(module_filename) and remove_binary: os.unlink(module_filename) if comparison_mode and timing: my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time)) if comparison_mode and not silent_mode: my_print("OK, same outputs.")
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( "$ORIGIN", os.path.dirname(sys.executable)) # TODO: Actually would be better to pass it as env to the created process instead. with withEnvironmentPathAdded( "LD_LIBRARY_PATH", *_getLdLibraryPath( package_name=package_name, python_rpath=_detected_python_rpath, original_dir=original_dir, )): # TODO: Check exit code, should never fail. stdout, stderr, _exit_code = executeProcess(command=("ldd", dll_filename)) stderr = b"\n".join( line for line in stderr.splitlines() if not line.startswith( b"ldd: warning: you do not have execution permission for")) inclusion_logger.debug("ldd output for %s is:\n%s" % (dll_filename, stdout)) if stderr: inclusion_logger.debug("ldd error for %s is:\n%s" % (dll_filename, stderr)) result = set() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 0x300: filename = filename.decode("utf8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Normalize, sometimes the DLLs produce "something/../", this has # been seen with Qt at least. filename = os.path.normpath(filename) # Do not include kernel DLLs on the ignore list. filename_base = os.path.basename(filename) if any(filename_base == entry or filename_base.startswith(entry + ".") for entry in _linux_dll_ignore_list): continue result.add(filename) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsPosix( dll_filename=sub_dll_filename, package_name=package_name, original_dir=original_dir, )) return sub_result
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command if str is not bytes: command = command.encode("utf8") _stdout, stderr, exit_code = executeProcess( command=( sys.executable, "-s", "-S", "-v", "-c", "import sys;exec(sys.stdin.read())", ), stdin=command, env=dict(os.environ, PYTHONIOENCODING="utf-8"), ) assert type(stderr) is bytes # Don't let errors here go unnoticed. if exit_code != 0: # An error by the user pressing CTRL-C should not lead to the below output. if b"KeyboardInterrupt" in stderr: general.sysexit("Pressed CTRL-C while detecting early imports.") general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"from" and python_version < 0x300: filename = parts[1][len(b"from "):] if str is not bytes: # For consistency, and maybe later reuse filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) else: assert False elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300 and module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "extension", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "extension", filename)) for module_name, _priority, kind, filename in sorted(detections): if isStandardLibraryNoAutoInclusionModule(module_name): continue if kind == "extension": _detectedExtensionModule( filename=filename, module_name=module_name, result=result, technical=technical, ) elif kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) else: assert False, kind return result