def getResultFullpath(onefile): """Get the final output binary result full path.""" result = getResultBasepath(onefile=onefile) if Options.shallMakeModule(): result += getSharedLibrarySuffix(preferred=True) else: output_filename = Options.getOutputFilename() if Options.isOnefileMode() and output_filename is not None: if onefile: result = output_filename else: result = os.path.join( getStandaloneDirectoryPath(), os.path.basename(output_filename), ) elif output_filename is not None: result = output_filename elif getOS() == "Windows": result += ".exe" elif (not Options.isStandaloneMode() or onefile and not Options.shallCreateAppBundle()): result += ".bin" return result
def executePASS5(): my_print("PASS 5: Compiling the compiler 'nuitka' package to single '.so' file.") path = os.path.join("..", "..", "nuitka") command = [ os.environ["PYTHON"], nuitka_main_path, "--plugin-enable=pylint-warnings", "--output-dir=%s" % tmp_dir, "--include-plugin-dir=%s" % path, "--nofollow-import-to=nuitka.build.inline_copy", "--nofollow-import-to=nuitka.build.include", "--nofollow-import-to=nuitka.build.static_src", "--module", path, ] result = subprocess.call(command) if result != 0: sys.exit(result) os.unlink(os.path.join(tmp_dir, "nuitka" + getSharedLibrarySuffix(preferred=True))) os.unlink(os.path.join(tmp_dir, "nuitka.pyi")) shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
def action(stage_dir, _root, path): command = [ sys.executable, os.path.join("..", "..", "bin", "nuitka"), "--module", "--output-dir", stage_dir, "--remove-output", "--plugin-enable=pylint-warnings", ] command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(path) try: subprocess.check_call(command) except subprocess.CalledProcessError: if os.path.basename(path) in nosyntax_errors: my_print("Syntax error is known unreliable with file file.") else: my_print("Falling back to full comparison due to error exit.") checkCompilesNotWithCPython(dirname=None, filename=path, search_mode=search_mode) else: my_print("OK") suffix = getSharedLibrarySuffix(preferred=True) target_filename = os.path.basename(path)[:-3] + suffix target_filename = target_filename.replace("(", "").replace(")", "") os.unlink(os.path.join(stage_dir, target_filename))
def considerExtraDlls(self, dist_dir, module): module_name = module.getFullName() if module_name == "zmq.libzmq" and isWin32Windows(): # TODO: Very strange thing for zmq on Windows, needs the .pyd file in wrong dir too. Have # this done in a dedicated form somewhere. shutil.copyfile( os.path.join(dist_dir, "zmq\\libzmq.pyd"), os.path.join(dist_dir, "libzmq" + getSharedLibrarySuffix(preferred=True)), ) return NuitkaPluginBase.considerExtraDlls(self, dist_dir=dist_dir, module=module)
def getResultFullpath(): """Get the final output binary result full path.""" result = getResultBasepath() if Options.shallMakeModule(): result += getSharedLibrarySuffix(preferred=True) else: if Options.getOutputFilename() is not None: result = Options.getOutputFilename() elif getOS() == "Windows": result += ".exe" elif not Options.isStandaloneMode(): result += ".bin" return result
def addShlibEntryPoint(module): target_filename = os.path.join(getStandaloneDirectoryPath(), *module.getFullName().split(".")) target_filename += getSharedLibrarySuffix(preferred=False) target_dir = os.path.dirname(target_filename) if not os.path.isdir(target_dir): makePath(target_dir) shutil.copyfile(module.getFilename(), target_filename) standalone_entry_points.append( makeExtensionModuleEntryPoint( source_path=module.getFilename(), dest_path=target_filename, package_name=module.getFullName().getPackageName(), ))
def runSconsBackend(quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=False), "source_dir": OutputDirectories.getSourceDirectoryPath(), "nuitka_python": asBoolStr(isNuitkaPython()), "debug_mode": asBoolStr(Options.is_debug), "python_debug": asBoolStr(Options.isPythonDebug()), "module_mode": asBoolStr(Options.shallMakeModule()), "full_compat": asBoolStr(Options.is_fullcompat), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "python_version": python_version_str, "target_arch": getArchitecture(), "python_prefix": getDirectoryRealPath(getSystemPrefixPath()), "nuitka_src": SconsInterface.getSconsDataPath(), "module_count": "%d" % ( 1 + len(ModuleRegistry.getDoneModules()) + len(ModuleRegistry.getUncompiledNonTechnicalModules()) ), } if Options.isLowMemory(): options["low_memory"] = asBoolStr(True) if not Options.shallMakeModule(): options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False) main_module = ModuleRegistry.getRootTopModule() assert main_module.isMainModule() main_module_name = main_module.getFullName() if main_module_name != "__main__": options["main_module_name"] = main_module_name if Options.shallUseStaticLibPython(): options["static_libpython"] = getSystemStaticLibPythonPath() if isDebianPackagePython(): options["debian_python"] = asBoolStr(True) if isMSYS2MingwPython(): options["msys2_mingw_python"] = asBoolStr(True) if isAnacondaPython(): options["anaconda_python"] = asBoolStr(True) if isApplePython(): options["apple_python"] = asBoolStr(True) if isPyenvPython(): options["pyenv_python"] = asBoolStr(True) if Options.isStandaloneMode(): options["standalone_mode"] = asBoolStr(True) if Options.isOnefileMode(): options["onefile_mode"] = asBoolStr(True) if Options.isOnefileTempDirMode(): options["onefile_temp_mode"] = asBoolStr(True) if Options.getForcedStdoutPath(): options["forced_stdout_path"] = Options.getForcedStdoutPath() if Options.getForcedStderrPath(): options["forced_stderr_path"] = Options.getForcedStderrPath() if Options.shallTreatUninstalledPython(): options["uninstalled_python"] = asBoolStr(True) if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules()) ) if Options.isProfile(): options["profile_mode"] = asBoolStr(True) if hasPythonFlagNoWarnings(): options["no_python_warnings"] = asBoolStr(True) if hasPythonFlagNoAsserts(): options["python_sysflag_optimize"] = asBoolStr(True) if python_version < 0x300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = asBoolStr(True) if python_version < 0x300 and ( sys.flags.division_warning or sys.flags.py3k_warning ): options["python_sysflag_division_warning"] = asBoolStr(True) if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = asBoolStr(True) if int(os.environ.get("NUITKA_NOSITE_FLAG", Options.hasPythonFlagNoSite())): options["python_sysflag_no_site"] = asBoolStr(True) if Options.hasPythonFlagTraceImports(): options["python_sysflag_verbose"] = asBoolStr(True) if Options.hasPythonFlagNoRandomization(): options["python_sysflag_no_randomization"] = asBoolStr(True) if python_version < 0x300 and sys.flags.unicode: options["python_sysflag_unicode"] = asBoolStr(True) if python_version >= 0x370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = asBoolStr(True) if hasPythonFlagUnbuffered(): options["python_sysflag_unbuffered"] = asBoolStr(True) abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags if Options.shallMakeModule(): options["module_suffix"] = getSharedLibrarySuffix(preferred=True) SconsInterface.setCommonOptions(options) if Options.shallCreatePgoInput(): options["pgo_mode"] = "python" result = SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ) if not result: return result, options # Need to make it usable before executing it. executePostProcessing() _runPythonPgoBinary() return True, options # Need to restart compilation from scratch here. if Options.isPgoMode(): # For C level PGO, we have a 2 pass system. TODO: Make it more global for onefile # and standalone mode proper support, which might need data files to be # there, which currently are not yet there, so it won't run. if Options.isPgoMode(): options["pgo_mode"] = "generate" result = SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ) if not result: return result, options # Need to make it usable before executing it. executePostProcessing() _runCPgoBinary() options["pgo_mode"] = "use" result = ( SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ), options, ) if options.get("pgo_mode") == "use" and _wasMsvcMode(): _deleteMsvcPGOFiles(pgo_mode="use") return result
def main(): # Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements filename = sys.argv[1] args = sys.argv[2:] def hasArg(arg): if arg in args: args.remove(arg) return True else: return False # For output keep it arguments = list(args) silent_mode = hasArg("silent") ignore_stderr = hasArg("ignore_stderr") ignore_warnings = hasArg("ignore_warnings") expect_success = hasArg("expect_success") expect_failure = hasArg("expect_failure") python_debug = hasArg("python_debug") module_mode = hasArg("module_mode") two_step_execution = hasArg("two_step_execution") binary_python_path = hasArg("binary_python_path") keep_python_path = hasArg("keep_python_path") trace_command = ( hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0" ) remove_output = hasArg("remove_output") standalone_mode = hasArg("--standalone") onefile_mode = hasArg("--onefile") no_site = hasArg("no_site") recurse_none = hasArg("recurse_none") recurse_all = hasArg("recurse_all") timing = hasArg("timing") coverage_mode = hasArg("coverage") original_file = hasArg("original_file") runtime_file = hasArg("runtime_file") no_warnings = not hasArg("warnings") full_compat = not hasArg("improved") cpython_cached = hasArg("cpython_cache") syntax_errors = hasArg("syntax_errors") noprefer_source = hasArg("noprefer_source") noverbose_log = hasArg("noverbose_log") noinclusion_log = hasArg("noinclusion_log") plugins_enabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_enable:"): plugins_enabled.append(arg[len("plugin_enable:") :]) del args[count] plugins_disabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_disable:"): plugins_disabled.append(arg[len("plugin_disable:") :]) del args[count] user_plugins = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("user_plugin:"): user_plugins.append(arg[len("user_plugin:") :]) del args[count] recurse_not = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_not:"): recurse_not.append(arg[len("recurse_not:") :]) del args[count] recurse_to = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_to:"): recurse_to.append(arg[len("recurse_to:") :]) del args[count] if args: sys.exit("Error, non understood mode(s) '%s'," % ",".join(args)) # In coverage mode, we don't want to execute, and to do this only in one mode, # we enable two step execution, which splits running the binary from the actual # compilation: if coverage_mode: two_step_execution = True # The coverage mode doesn't work with debug mode. if coverage_mode: python_debug = False comparison_mode = not coverage_mode assert not standalone_mode or not module_mode assert not recurse_all or not recurse_none if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" os.environ["PYTHONWARNINGS"] = "ignore" if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() if "--python-debug" in extra_options or "--python-dbg" in extra_options: python_debug = True if python_debug: if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")): os.environ["PYTHON"] += "-dbg" if os.name == "nt": if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"): os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe" if os.environ["PYTHON"].endswith("-dbg"): python_debug = True if os.environ["PYTHON"].lower().endswith("_d.exe"): python_debug = True if comparison_mode: my_print( """\ Comparing output of '{filename}' using '{python}' with flags {args} ...""".format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), ) ) else: my_print( """\ Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), ) ) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("CPython:") my_print("*" * 80) if two_step_execution: filename = os.path.abspath(filename) if module_mode: if no_warnings: cpython_cmd = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), os.path.basename(filename)), ] else: cpython_cmd = [ os.environ["PYTHON"], "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), os.path.basename(filename)), ] else: if no_warnings: cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename] else: cpython_cmd = [os.environ["PYTHON"], filename] if no_site: cpython_cmd.insert(1, "-S") if "NUITKA" in os.environ: # Would need to extract which "python" this is going to use. assert not coverage_mode, "Not implemented for binaries." nuitka_call = [os.environ["NUITKA"]] else: if comparison_mode: nuitka_call = [ os.environ["PYTHON"], "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] else: assert coverage_mode nuitka_call = [ os.environ["PYTHON"], "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] if python_debug: extra_options.append("--python-debug") if no_warnings: extra_options.append("--python-flag=no_warnings") if remove_output: extra_options.append("--remove-output") if original_file: extra_options.append("--file-reference-choice=original") if runtime_file: extra_options.append("--file-reference-choice=runtime") if full_compat: extra_options.append("--full-compat") if noprefer_source: extra_options.append("--no-prefer-source") if coverage_mode: # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output( [ os.environ["PYTHON"], "-c", "import sys, os; print(os.pathsep.join(sys.path))", ] ) if sys.version_info >= (3,): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() if binary_python_path: addToPythonPath(os.path.dirname(os.path.abspath(filename))) if keep_python_path or binary_python_path: extra_options.append("--execute-with-pythonpath") if recurse_none: extra_options.append("--nofollow-imports") if recurse_all: extra_options.append("--follow-imports") if recurse_not: extra_options.extend("--nofollow-import-to=" + v for v in recurse_not) if coverage_mode: extra_options.append("--must-not-re-execute") extra_options.append("--generate-c-only") for plugin_enabled in plugins_enabled: extra_options.append("--plugin-enable=" + plugin_enabled) for plugin_disabled in plugins_disabled: extra_options.append("--plugin-disable=" + plugin_disabled) for user_plugin in user_plugins: extra_options.append("--user-plugin=" + user_plugin) if not noverbose_log: extra_options.append("--verbose-output=%s.optimization.log" % filename) if not noinclusion_log: extra_options.append("--show-modules-output=%s.inclusion.log" % filename) # Now build the command to run Nuitka. if not two_step_execution: if module_mode: nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename] elif onefile_mode: nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename] elif standalone_mode: nuitka_cmd = ( nuitka_call + extra_options + ["--run", "--standalone", filename] ) else: nuitka_cmd = nuitka_call + extra_options + ["--run", filename] if no_site: nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S") else: if module_mode: nuitka_cmd1 = ( nuitka_call + extra_options + ["--module", os.path.abspath(filename)] ) elif standalone_mode: nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename] else: nuitka_cmd1 = nuitka_call + extra_options + [filename] if no_site: nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S") for extra_option in extra_options: dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option) if dir_match: output_dir = dir_match.group(1) break else: # The default. output_dir = "." if module_mode: nuitka_cmd2 = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import %s" % os.path.basename(filename), ] else: exe_filename = os.path.basename(filename) if filename.endswith(".py"): exe_filename = exe_filename[:-3] exe_filename = exe_filename.replace(")", "").replace("(", "") exe_filename += ".exe" if os.name == "nt" else ".bin" nuitka_cmd2 = [os.path.join(output_dir, exe_filename)] pdb_filename = exe_filename[:-4] + ".pdb" if trace_command: my_print("CPython command:", *cpython_cmd) if comparison_mode: cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False ) if not silent_mode: displayOutput(stdout_cpython, stderr_cpython) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("Nuitka:") my_print("*" * 80) if two_step_execution: if output_dir: os.chdir(output_dir) else: tmp_dir = tempfile.gettempdir() # Try to avoid RAM disk /tmp and use the disk one instead. if tmp_dir == "/tmp" and os.path.exists("/var/tmp"): tmp_dir = "/var/tmp" os.chdir(tmp_dir) if trace_command: my_print("Going to output directory", os.getcwd()) stop_watch = StopWatch() stop_watch.start() if not two_step_execution: if trace_command: my_print("Nuitka command:", nuitka_cmd) # Try a couple of times for permission denied, on Windows it can # be transient. for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen( args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka, stderr_nuitka = process.communicate() exit_nuitka = process.returncode if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError( stderr_nuitka ): break my_print("Retrying nuitka exe due to permission problems after delay.") time.sleep(2) else: if trace_command: my_print("Nuitka command 1:", nuitka_cmd1) for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen( args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka1, stderr_nuitka1 = process.communicate() exit_nuitka1 = process.returncode if exit_nuitka1 != 0: if ( not expect_failure and not comparison_mode and not os.path.exists(".coverage") ): sys.exit( """\ Error, failed to take coverage with '%s'. Stderr was: %s """ % (os.environ["PYTHON"], stderr_nuitka1) ) exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 else: # No execution second step for coverage mode. if comparison_mode: if trace_command: my_print("Nuitka command 2:", nuitka_cmd2) process = subprocess.Popen( args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka2, stderr_nuitka2 = process.communicate() stdout_nuitka = stdout_nuitka1 + stdout_nuitka2 stderr_nuitka = stderr_nuitka1 + stderr_nuitka2 exit_nuitka = process.returncode # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2) process = subprocess.Popen( args=nuitka_cmd2, stdin=subprocess.PIPE ) process.communicate() else: exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError( stderr_nuitka ): break my_print("Retrying nuitka exe due to permission problems after delay.") time.sleep(2) stop_watch.stop() nuitka_time = stop_watch.getDelta() if not silent_mode: displayOutput(stdout_nuitka, stderr_nuitka) if coverage_mode: assert not stdout_nuitka assert not stderr_nuitka if comparison_mode: def makeComparisons(trace_result): exit_code_stdout = compareOutput( "stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors ) if ignore_stderr: exit_code_stderr = 0 else: exit_code_stderr = compareOutput( "stderr", stderr_cpython, stderr_nuitka, ignore_warnings, syntax_errors, ) exit_code_return = exit_cpython != exit_nuitka if exit_code_return and trace_result: my_print( """Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format( exit_cpython=exit_cpython, exit_nuitka=exit_nuitka ) ) return exit_code_stdout, exit_code_stderr, exit_code_return if cpython_cached: exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=False ) if exit_code_stdout or exit_code_stderr or exit_code_return: old_stdout_cpython = stdout_cpython old_stderr_cpython = stderr_cpython old_exit_cpython = exit_cpython my_print( "Updating CPython cache by force due to non-matching comparison results.", style="yellow", ) ( cpython_time, stdout_cpython, stderr_cpython, exit_cpython, ) = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=True, ) if not silent_mode: if ( old_stdout_cpython != stdout_cpython or old_stderr_cpython != stderr_cpython or old_exit_cpython != exit_cpython ): displayOutput(stdout_cpython, stderr_cpython) exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=True ) # In case of segfault, also output the call stack by entering debugger # without stdin forwarded. if ( exit_code_return and exit_nuitka in (-11, -6) and sys.platform != "nt" and not module_mode and not two_step_execution ): nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger") with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE) process.communicate() exit_code = exit_code_stdout or exit_code_stderr or exit_code_return if exit_code: problems = [] if exit_code_stdout: problems.append("stdout") if exit_code_stderr: problems.append("stderr") if exit_code_return: problems.append("exit_code") sys.exit("Error, results differed (%s)." % ",".join(problems)) if expect_success and exit_cpython != 0: if silent_mode: displayOutput(stdout_cpython, stderr_cpython) sys.exit("Unexpected error exit from CPython.") if expect_failure and exit_cpython == 0: sys.exit("Unexpected success exit from CPython.") if remove_output: if not module_mode: if os.path.exists(nuitka_cmd2[0]): if os.name == "nt": # It appears there is a tiny lock race that we randomly cause, # likely because --run spawns a subprocess that might still # be doing the cleanup work. if os.path.exists(nuitka_cmd2[0] + ".away"): os.unlink(nuitka_cmd2[0] + ".away") for _i in range(10): try: os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away") except OSError: time.sleep(0.1) continue for _i in range(10): try: os.unlink(nuitka_cmd2[0] + ".away") except OSError: time.sleep(2) continue else: break if os.path.exists(pdb_filename): os.unlink(pdb_filename) else: os.unlink(nuitka_cmd2[0]) else: module_filename = os.path.basename(filename) + getSharedLibrarySuffix( preferred=True ) if os.path.exists(module_filename): os.unlink(module_filename) if comparison_mode and timing: my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time)) if comparison_mode and not silent_mode: my_print("OK, same outputs.")
def runSconsBackend(quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=False), "source_dir": OutputDirectories.getSourceDirectoryPath(), "debug_mode": asBoolStr(Options.is_debug), "python_debug": asBoolStr(Options.isPythonDebug()), "unstripped_mode": asBoolStr(Options.isUnstripped()), "module_mode": asBoolStr(Options.shallMakeModule()), "full_compat": asBoolStr(Options.is_fullcompat), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "python_version": python_version_str, "target_arch": Utils.getArchitecture(), "python_prefix": getDirectoryRealPath(sys.prefix), "nuitka_src": SconsInterface.getSconsDataPath(), "module_count": "%d" % ( 1 + len(ModuleRegistry.getDoneModules()) + len(ModuleRegistry.getUncompiledNonTechnicalModules()) ), } if not Options.shallMakeModule(): options["result_exe"] = OutputDirectories.getResultFullpath(onefile=False) if Options.shallUseStaticLibPython(): options["static_libpython"] = asBoolStr(True) if Options.isStandaloneMode(): options["standalone_mode"] = asBoolStr(True) if Options.isOnefileMode(): options["onefile_mode"] = asBoolStr(True) if Options.isWindowsOnefileTempDirMode(): options["onefile_temp_mode"] = asBoolStr(True) if Options.getForcedStdoutPath(): options["forced_stdout_path"] = Options.getForcedStdoutPath() if Options.getForcedStderrPath(): options["forced_stderr_path"] = Options.getForcedStderrPath() if Options.shallTreatUninstalledPython(): options["uninstalled_python"] = asBoolStr(True) if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules()) ) if Utils.getOS() == "Windows": options["noelf_mode"] = asBoolStr(True) if Options.isProfile(): options["profile_mode"] = asBoolStr(True) if "no_warnings" in getPythonFlags(): options["no_python_warnings"] = asBoolStr(True) if "no_asserts" in getPythonFlags(): options["python_sysflag_optimize"] = asBoolStr(True) if python_version < 0x300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = asBoolStr(True) if python_version < 0x300 and ( sys.flags.division_warning or sys.flags.py3k_warning ): options["python_sysflag_division_warning"] = asBoolStr(True) if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = asBoolStr(True) if int(os.environ.get("NUITKA_SITE_FLAG", Options.hasPythonFlagNoSite())): options["python_sysflag_no_site"] = asBoolStr(True) if "trace_imports" in Options.getPythonFlags(): options["python_sysflag_verbose"] = asBoolStr(True) if "no_randomization" in Options.getPythonFlags(): options["python_sysflag_no_randomization"] = asBoolStr(True) if python_version < 0x300 and sys.flags.unicode: options["python_sysflag_unicode"] = asBoolStr(True) if python_version >= 0x370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = asBoolStr(True) abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags if Options.shallMakeModule(): options["module_suffix"] = getSharedLibrarySuffix(preferred=True) SconsInterface.setCommonOptions(options) return ( SconsInterface.runScons( options=options, quiet=quiet, scons_filename="Backend.scons" ), options, )
def compileAndCompareWith(nuitka): if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" base_dir = os.path.join("..", "..") for package in PACKAGE_LIST: package = package.replace("/", os.path.sep) source_dir = os.path.join(base_dir, package) for path, filename in listDir(source_dir): if not filename.endswith(".py"): continue if filename.startswith(".#"): continue path = os.path.join(source_dir, filename) if filename != "__init__.py": my_print("Compiling '%s'." % path) target = filename.replace(".py", ".build") target_dir = os.path.join(tmp_dir, target) removeDirectory(path=target_dir, ignore_errors=False) command = [ nuitka, "--module", "--plugin-enable=pylint-warnings", "--output-dir=%s" % tmp_dir, "--no-pyi-file", path, ] command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() my_print("Command: ", " ".join(command)) exit_nuitka = subprocess.call(command) # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": command2 = wrapCommandForDebuggerForSubprocess(*command) subprocess.call(command2) if exit_nuitka != 0: my_print("An error exit %s occurred, aborting." % exit_nuitka) sys.exit(exit_nuitka) has_diff = diffRecursive(os.path.join(package, target), target_dir) if has_diff: sys.exit("There were differences!") shutil.rmtree(target_dir) target_filename = filename.replace( ".py", getSharedLibrarySuffix(preferred=True) ) os.unlink(os.path.join(tmp_dir, target_filename))
def runSconsBackend(quiet): # Scons gets transported many details, that we express as variables, and # have checks for them, leading to many branches and statements, # pylint: disable=too-many-branches,too-many-statements asBoolStr = SconsInterface.asBoolStr options = { "result_name": OutputDirectories.getResultBasepath(onefile=False), "source_dir": OutputDirectories.getSourceDirectoryPath(), "debug_mode": asBoolStr(Options.is_debug), "python_debug": asBoolStr(Options.isPythonDebug()), "unstripped_mode": asBoolStr(Options.isUnstripped()), "module_mode": asBoolStr(Options.shallMakeModule()), "full_compat": asBoolStr(Options.is_fullcompat), "experimental": ",".join(Options.getExperimentalIndications()), "trace_mode": asBoolStr(Options.shallTraceExecution()), "python_version": python_version_str, "target_arch": Utils.getArchitecture(), "python_prefix": getDirectoryRealPath(sys.prefix), "nuitka_src": SconsInterface.getSconsDataPath(), "module_count": "%d" % (1 + len(ModuleRegistry.getDoneModules()) + len(ModuleRegistry.getUncompiledNonTechnicalModules())), } if not Options.shallMakeModule(): options["result_exe"] = OutputDirectories.getResultFullpath() # Ask Scons to cache on Windows, except where the directory is thrown # away. On non-Windows you can should use ccache instead. if not Options.isRemoveBuildDir() and Utils.getOS() == "Windows": options["cache_mode"] = "true" if Options.isLto(): options["lto_mode"] = asBoolStr(True) if Options.shallUseStaticLibPython(): options["static_libpython"] = asBoolStr(True) if Options.shallDisableConsoleWindow(): options["win_disable_console"] = asBoolStr(True) if Options.isStandaloneMode(): options["standalone_mode"] = asBoolStr(True) if Options.shallTreatUninstalledPython(): options["uninstalled_python"] = asBoolStr(True) if ModuleRegistry.getUncompiledTechnicalModules(): options["frozen_modules"] = str( len(ModuleRegistry.getUncompiledTechnicalModules())) if Options.isShowScons(): options["show_scons"] = asBoolStr(True) if Options.isMingw64(): options["mingw_mode"] = asBoolStr(True) if Options.getMsvcVersion(): msvc_version = Options.getMsvcVersion() msvc_version = msvc_version.replace("exp", "Exp") if "." not in msvc_version: msvc_version += ".0" options["msvc_version"] = msvc_version if Utils.getOS() == "Windows": options["noelf_mode"] = asBoolStr(True) if Options.isClang(): options["clang_mode"] = asBoolStr(True) if Options.isProfile(): options["profile_mode"] = asBoolStr(True) if "no_warnings" in getPythonFlags(): options["no_python_warnings"] = asBoolStr(True) if "no_asserts" in getPythonFlags(): options["python_sysflag_optimize"] = asBoolStr(True) if python_version < 0x300 and sys.flags.py3k_warning: options["python_sysflag_py3k_warning"] = asBoolStr(True) if python_version < 0x300 and (sys.flags.division_warning or sys.flags.py3k_warning): options["python_sysflag_division_warning"] = asBoolStr(True) if sys.flags.bytes_warning: options["python_sysflag_bytes_warning"] = asBoolStr(True) if int( os.environ.get("NUITKA_SITE_FLAG", "no_site" in Options.getPythonFlags())): options["python_sysflag_no_site"] = asBoolStr(True) if "trace_imports" in Options.getPythonFlags(): options["python_sysflag_verbose"] = asBoolStr(True) if "no_randomization" in Options.getPythonFlags(): options["python_sysflag_no_randomization"] = asBoolStr(True) if python_version < 0x300 and sys.flags.unicode: options["python_sysflag_unicode"] = asBoolStr(True) if python_version >= 0x370 and sys.flags.utf8_mode: options["python_sysflag_utf8"] = asBoolStr(True) abiflags = getPythonABI() if abiflags: options["abiflags"] = abiflags cpp_defines = Plugins.getPreprocessorSymbols() if cpp_defines: options["cpp_defines"] = ",".join( "%s%s%s" % (key, "=" if value else "", value or "") for key, value in cpp_defines.items()) link_libraries = Plugins.getExtraLinkLibraries() if link_libraries: options["link_libraries"] = ",".join(link_libraries) if Options.shallMakeModule(): options["module_suffix"] = getSharedLibrarySuffix(preferred=True) if Options.shallRunInDebugger(): options["full_names"] = asBoolStr(True) if Options.assumeYesForDownloads(): options["assume_yes_for_downloads"] = asBoolStr(True) return ( SconsInterface.runScons(options=options, quiet=quiet, scons_filename="Backend.scons"), options, )
def call_analyzer(f, call_list, import_calls, import_files, trace_logic): """ Analyze the call hierarchy to determine valid called names. Notes: Always called with a CALL record type. Recursive function calling itself for every level change. Each CALL on each level will be followed by exactly one RESULT (or EXCEPTION), potentially with interspersed CALL / RESULT pairs at lower levels. Args: f: file to read from (created by the script wrapped in hinting logic) call_list: list representing a CALL record import_calls: list to receive computed import names import_files: list to receive imported files trace_logic: bool to switch on tracing the logic Returns: No direct returns, output will be written to call_file. """ global line_number def normalize_file(t): # step 1: remove any platform tags from shared libraries folder = os.path.dirname(t) # folder part datei = os.path.basename(t) # filename _, ext = os.path.splitext(datei) # extension if ext in (".pyd", ".so"): # shared library? datei_arr = datei.split(".") # split if len(datei_arr) > 2: # platform tag present? datei = ".".join(datei_arr[:-2]) # yes, omit else: datei = ".".join(datei_arr[:-1]) # just omit ext t = os.path.join(folder, datei) # rebuild filename for step 2 # step 2: turn slashes into '.', remove __init__.py and extensions t = t.replace("\\", ".").replace("/", ".").replace("$PYTHONPATH.", "") if t.endswith(".__init__.py"): t = t[:-12] return t if t.endswith(".py"): t = t[:-3] return t if ext not in (".pyd", ".so"): sys.exit("found unknown Python module type '%s'" % t) return t def write_mod(t, f): # write a call entry import_calls.append((t, f)) if trace_logic: print(line_number, "call:", t) return def write_file(t): # write a file entry import_files.append(t) if trace_logic: print(line_number, "file:", t) return level = call_list[0] # nesting level CALLED = call_list[2] # the imported module implist = call_list[3] # list accompanying the import statement text = reader(f) # read the next record if not bool(text): # EOF should not happen here! print("unexpected EOF at %s" % str(call_list)) sys.exit("line number %i" % line_number) if len(text) < 3: print("unexpected record format", text) sys.exit("at line number %i" % line_number) while "CALL" in text: # any CALL records will be recursed into call_analyzer(f, text, import_calls, import_files, trace_logic) text = reader(f) if len(text) < 3: return if text[0] != level: # this record should have our level! matching = False else: matching = True if text[1] == "EXCEPTION": # no output if an exception resulted return if text[1] != "RESULT": # this must be a RESULT now sys.exit("%i: expected RESULT after %s" % (line_number, str(call_list))) RESULT = text[2] # resulting module name if RESULT == "__main__": # skip current script return res_file = text[3] # resulting file name if res_file == "built-in": # skip output for built-in stuff return if res_file.endswith(".dll"): # special handling for pythoncom and friends res_file = RESULT + ".py" if RESULT.startswith("win32com"): # special handling for win32com res_file = "$PYTHONPATH\\win32com\\__init__.py" if trace_logic: print(line_number, ":", str(call_list)) print(line_number, ":", str(text)) normalized_file = normalize_file(res_file) write_file(normalized_file) if not matching: print("No result matches %i, %s, %s" % (level, CALLED, str(implist))) write_mod(RESULT, normalized_file) # this is a sure output # members of shared modules cannot be filtered out, so allow them all # TODO: This should consider all possible suffixes, should it not. if (hasFilenameExtension(res_file, getSharedLibrarySuffix( preferred=True)) # a shared module! or normalized_file in accept_always): write_mod(RESULT + ".*", normalized_file) return if not CALLED: # case: the CALL name is empty if not implist: # should not happen, but let's ignore this return for item in implist: # return RESULT.item for items in list write_mod(RESULT + "." + item, normalized_file) return if (CALLED.startswith(RESULT) or RESULT.startswith(CALLED) or RESULT.endswith(CALLED)): # CALL and RESULT names contain each other in some way if not implist: if CALLED != RESULT: write_mod(CALLED, normalized_file) return if CALLED == RESULT: cmod = CALLED elif RESULT.endswith(CALLED): cmod = RESULT elif RESULT.startswith(CALLED): cmod = RESULT else: cmod = CALLED for item in implist: # this is a list of items write_mod(cmod + "." + item, normalized_file) return """ Case: RESULT and CALL names neither contain each other, nor is CALLED empty. We then assume that the true call name should be RESULT.CALLED in output. """ cmod = RESULT + "." + CALLED # equals RESULT.CALLED write_mod(cmod, normalized_file) # output it if not implist: # no list there: done return for item in implist: # or again a list of items write_mod(cmod + "." + item, normalized_file) return