def main(): _python_version = setup() search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not os.path.isdir(filename) or filename.endswith(".build"): continue extra_flags = [ "expect_success", "remove_output", "module_mode", "two_step_execution", ] # The use of "__main__" in the test package gives a warning. if filename == "sub_package": extra_flags.append("ignore_warnings") active = search_mode.consider(dirname=None, filename=filename) if active: my_print("Consider output of recursively compiled program:", filename) filename_main = None for filename_main in os.listdir(filename): if not os.path.isdir(os.path.join(filename, filename_main)): continue if filename_main not in ("..", "."): break else: search_mode.onErrorDetected("""\ Error, no package in dir '%s' found, incomplete test case.""" % filename) extensions = [ "--include-package=%s" % os.path.basename(filename_main) ] if "--output-dir" not in os.environ.get("NUITKA_EXTRA_OPTIONS", ""): extensions.append("--output-dir=%s" % getTempDir()) with withExtendedExtraOptions(*extensions): compareWithCPython( dirname=filename, filename=filename_main, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, ) if search_mode.abortIfExecuted(): break else: my_print("Skipping", filename) search_mode.finish()
def main(): setup(suite="packages") search_mode = createSearchMode() for filename in sorted(os.listdir(".")): if not os.path.isdir(filename) or filename.endswith(".build"): continue extra_flags = [ "--module", "expect_success", "remove_output", "two_step_execution", ] active = search_mode.consider(dirname=None, filename=filename) if active: my_print("Consider output of compiled package:", filename) filename_main = None filename_main = os.path.join( filename, "".join(part.title() for part in filename.split("_")) + ".py" ) if os.path.exists(filename_main): filename_main = os.path.basename(filename_main) else: filename_main = None if filename_main is None: for filename_main in os.listdir(filename): if not os.path.isdir(os.path.join(filename, filename_main)): continue if filename_main not in ("..", "."): break else: search_mode.onErrorDetected( """\ Error, no package in test directory '%s' found, incomplete test case.""" % filename ) extra_flags.append( "--include-package=%s" % os.path.basename(filename_main) ) extra_flags.append("--output-dir=%s" % getTempDir()) compareWithCPython( dirname=filename, filename=filename_main, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False, ) search_mode.finish()
def main(): parser = OptionParser() parser.add_option( "--checked-module", action="store", dest="checked_module", help="""\ Module with main() function to be checked for reference count stability.""", ) parser.add_option( "--explain", action="store_true", dest="explain", default=False, help="""\ Try to explain the differences by comparing object counts.""", ) options, positional_args = parser.parse_args() if positional_args and options.checked_module is None: options.checked_module = positional_args.pop() if positional_args and options.checked_module: parser.print_help() sys.exit("\nError, no positional argument allowed.") # First with pure Python. checked_module = importFileAsModule(options.checked_module) print("Using", checked_module.main) checkReferenceCount(checked_module.main, explain=options.explain) temp_dir = getTempDir() command = [ sys.executable, "-m", "nuitka", "--module", options.checked_module, "--output-dir", temp_dir, ] if hasattr(sys, "gettotalrefcount"): command.append("--python-debug") # print(command) check_call(command) module_name = os.path.basename(options.checked_module).split(".")[0] sys.path.insert(0, temp_dir) checked_module = __import__(module_name) print("Using", checked_module.main) checkReferenceCount(checked_module.main)
def getExtraFlags(where, name, flags): if options.assume_yes_for_downloads and name in ("onefile", "standalone"): yield "--assume-yes-for-downloads" if os.name == "nt" and options.mingw64: yield "--mingw64" if where is not None: tmp_dir = getTempDir() where = os.path.join(tmp_dir, name, where) if not os.path.exists(where): os.makedirs(where) yield "--output-dir=%s" % where yield flags
def main(): python_version = setup() os.chdir("subject") nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka") tmp_dir = getTempDir() command = [ os.environ["PYTHON"], nuitka_main_path, "--plugin-enable=pylint-warnings", "--output-dir=%s" % tmp_dir, "--follow-imports", "--include-package=package", "--nofollow-import-to=*.tests", "--python-flag=-v", "--debug", "--module", "package", ] result = subprocess.call(command) if result != 0: sys.exit(result) os.makedirs(os.path.join(tmp_dir, "package.ext")) copyTree("package", os.path.join(tmp_dir, "package.ext/package")) os.chdir(tmp_dir) # We compile the package non-closed, so we can smuggle in tests # and user code. This is going to be the example code. with open("package.ext/package/user_provided.py", "w") as output: # TODO: Maybe assert that the type name of a local function and one from # the package are not the same, i.e. we are running inside the compiled # package. output.write(""" from __future__ import print_function import package print("__name__:", package.__name__) print("__package__:", package.__package__) print("__path__:", package.__path__) print("__file__:", package.__file__) # print("__loader__:", package.__loader__) import package.sub_package1 print("__name__:", package.sub_package1.__name__) print("__package__:", package.sub_package1.__package__) print("__path__:", package.sub_package1.__path__) print("__file__:", package.sub_package1.__file__) # print("__loader__:", package.sub_package1.__loader__) import package.sub_package1.tests; print("__name__:", package.sub_package1.tests.__name__) print("__package__:", package.sub_package1.tests.__package__) print("__path__:", package.sub_package1.tests.__path__) print("__file__:", package.sub_package1.tests.__file__) # print("__loader__:", package.sub_package1.tests.__loader__) """) os.makedirs("nose") with open("nose/usage.txt", "w") as output: pass os.system("find | sort") # Inform about the extra path, format is NUITKA_PACKAGE_fullname where # dots become "_" and should point to the directory where external code # to be loaded will live under. Probably should be an absolute path, but # we avoid it here. os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package" # Lets make sure these to not work. These will be used in the compiled # form only. for module_path in ("__init__.py", "sub_package1__init__.py"): with open(os.path.join("./package.ext/package", module_path), "w") as output: output.write("assert False") # Check the compiled package is functional for importing. my_print("Running package as basic test:") command = [os.environ["PYTHON"], "-c", "import package"] result = subprocess.call(command) if result != 0: sys.exit(result) my_print("Running nose tests:") # assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0 my_print("Running py.test tests:") command = [ os.environ["PYTHON"], "-m", "pytest", "-v", "--pyargs", "package.sub_package1.tests", ] result = subprocess.call(command) if result != 0: sys.exit(result)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) my_print("TEST_CASE_HASH='%s'" % hashlib.md5(open(test_case, "rb").read()).hexdigest()) needs_2to3 = python_version.startswith('3') and \ not test_case.endswith("32.py") and \ not test_case.endswith("33.py") if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case))) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) case_1_source, case_2_source = generateConstructCases(open(test_case).read()) with open(test_case_1, 'w') as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, 'w') as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, needs_delete = convertUsing2to3(test_case_1) test_case_2, needs_delete = convertUsing2to3(test_case_2)
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option( "--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", ""), ) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option( "--code-diff", action="store", dest="diff_filename", default="", ) parser.add_option( "--copy-source-to", action="store", dest="target_dir", default="", ) options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit( "Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) python_version = setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % python_version) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) my_print("TEST_CASE_HASH='%s'" % hashlib.md5(open(test_case, "rb").read()).hexdigest()) needs_2to3 = python_version.startswith('3') and \ not test_case.endswith("32.py") and \ not test_case.endswith("33.py") if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case))) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) case_1_source, case_2_source = generateConstructCases( open(test_case).read()) with open(test_case_1, 'w') as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, 'w') as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = '0' if nuitka: nuitka_id = check_output("cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case) ] nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. shutil.copyfile(test_case_1, os.path.basename(test_case)) subprocess.check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build")) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix)) shutil.copyfile(test_case_2, os.path.basename(test_case)) subprocess.check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build")) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix)) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join( test_case_1.replace(".py", ".build"), "module.__main__" + suffix, ) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join( test_case_2.replace(".py", ".build"), "module.__main__" + suffix, ) if os.path.exists(cpp_2): break else: assert False import difflib open(options.diff_filename, 'w').write(difflib.HtmlDiff().make_table( open(cpp_1).readlines(), open(cpp_2).readlines(), "Construct", "Baseline", True)) nuitka_1 = runValgrind("Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix), ), include_startup=True) nuitka_2 = runValgrind("Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix), ), include_startup=True) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % ' '.join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_1 = runValgrind("CPython construct", "callgrind", (os.environ["PYTHON"], "-S", test_case_1), include_startup=True) cpython_2 = runValgrind("CPython baseline", "callgrind", (os.environ["PYTHON"], "-S", test_case_2), include_startup=True) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", ".." ) ) ) from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip python_version = setup() nuitka_main_path = os.path.join("..", "..", "bin", "nuitka") tmp_dir = getTempDir() # Cannot detect this more automatic, either need whitelist or # blacklist not needed stuff. PACKAGE_LIST = ( "nuitka", "nuitka/nodes", "nuitka/specs", "nuitka/nodes/shapes", "nuitka/tree", "nuitka/importing", "nuitka/build", "nuitka/freezer", "nuitka/gui", "nuitka/codegen", "nuitka/codegen/templates",
active = search_mode.consider(dirname=None, filename=filename) if active: my_print("Consider output of recursively compiled program:", filename) for filename_main in os.listdir(filename): if not os.path.isdir(os.path.join(filename, filename_main)): continue if filename_main not in ("..", '.'): break else: sys.exit("""\ Error, no package in dir '%s' found, incomplete test case.""" % filename) extensions = ["--include-package=%s" % os.path.basename(filename_main)] if not "--output-dir" in os.environ.get("NUITKA_EXTRA_OPTIONS", ""): extensions.append("--output-dir=%s" % getTempDir()) with withExtendedExtraOptions(*extensions): compareWithCPython(dirname=filename, filename=filename_main, extra_flags=extra_flags, search_mode=search_mode, needs_2to3=False) else: my_print("Skipping", filename) search_mode.finish()
import shutil from nuitka.tools.testing.Common import getTempDir from nuitka.tools.testing.Valgrind import getBinarySizes, runValgrind input_file = sys.argv[1] nuitka_binary = os.environ.get( "NUITKA_BINARY", os.path.join(os.path.dirname(__file__), "../bin/nuitka") ) nuitka_binary = os.path.normpath(nuitka_binary) basename = os.path.basename(input_file) tempdir = getTempDir() output_binary = os.path.join( tempdir, (basename[:-3] if input_file.endswith(".py") else basename) + ".bin" ) os.environ["PYTHONHASHSEED"] = "0" # To make that python run well despite the "-S" flag for things that need site # to expand sys.path. os.environ["PYTHONPATH"] = os.pathsep.join(sys.path) os.system( "%s %s --python-flag=-S --no-progress --output-dir=%s %s %s %s %s" % ( sys.executable,
os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", ".." ) ) ) from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip python_version = setup() nuitka_main_path = os.path.join("..", "..", "bin", "nuitka") tmp_dir = getTempDir() # Cannot detect this more automatic, either need whitelist or # blacklist not needed stuff. PACKAGE_LIST = ( "nuitka", "nuitka/nodes", "nuitka/specs", "nuitka/nodes/shapes", "nuitka/tree", "nuitka/importing", "nuitka/build", "nuitka/freezer", "nuitka/codegen", "nuitka/codegen/templates", "nuitka/codegen/c_types",
def main(): setup(suite="extension_modules", needs_io_encoding=True) search_mode = createSearchMode() tmp_dir = getTempDir() done = set() def decide(root, filename): if os.path.sep + "Cython" + os.path.sep in root: return False if (root.endswith(os.path.sep + "matplotlib") or os.path.sep + "matplotlib" + os.path.sep in root): return False if filename.endswith("linux-gnu_d.so"): return False if root.endswith(os.path.sep + "msgpack"): return False first_part = filename.split(".")[0] if first_part in done: return False done.add(first_part) return filename.endswith( (".so", ".pyd")) and not filename.startswith("libpython") current_dir = os.path.normpath(os.getcwd()) current_dir = os.path.normcase(current_dir) def action(stage_dir, root, path): command = [ sys.executable, os.path.join("..", "..", "bin", "nuitka"), "--stand", "--run", "--output-dir=%s" % stage_dir, "--remove-output", ] filename = os.path.join(stage_dir, "importer.py") assert path.startswith(root) module_name = path[len(root) + 1:] module_name = module_name.split(".")[0] module_name = module_name.replace(os.path.sep, ".") module_name = ModuleName(module_name) with openTextFile(filename, "w") as output: plugin_names = set(["pylint-warnings", "anti-bloat"]) if module_name.hasNamespace("PySide2"): plugin_names.add("pyside2") elif module_name.hasNamespace("PySide6"): plugin_names.add("pyside2") elif module_name.hasNamespace("PyQt5"): plugin_names.add("pyqt5") else: # TODO: We do not have a noqt plugin yet. plugin_names.add("pyqt5") for plugin_name in plugin_names: output.write("# nuitka-project: --enable-plugin=%s\n" % plugin_name) # Make it an error to find unwanted bloat compiled in. output.write("# nuitka-project: --noinclude-default-mode=error\n") output.write("import " + module_name.asString() + "\n") output.write("print('OK.')") command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(filename) if checkSucceedsWithCPython(filename): try: output = check_output(command).splitlines() except Exception: # only trying to check for no exception, pylint: disable=try-except-raise raise else: assert os.path.exists(filename[:-3] + ".dist") binary_filename = os.path.join( filename[:-3] + ".dist", "importer.exe" if os.name == "nt" else "importer", ) loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename], ) outside_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd()) if outside_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning( "Should not access these file(s): '%r'." % outside_accesses) search_mode.onErrorDetected(1) if output[-1] != b"OK.": my_print(" ".join(command)) my_print(filename) my_print(output) test_logger.sysexit("FAIL.") my_print("OK.") assert not outside_accesses, outside_accesses shutil.rmtree(filename[:-3] + ".dist") else: my_print("SKIP (does not work with CPython)") compileLibraryTest( search_mode=search_mode, stage_dir=os.path.join(tmp_dir, "compile_extensions"), decide=decide, action=action, ) my_print("FINISHED, all extension modules compiled.")
if not os.path.isdir(os.path.join(filename, filename_main)): continue if filename_main not in ("..", '.'): break else: sys.exit( """\ Error, no package in dir '%s' found, incomplete test case.""" % filename ) extensions = [ "--include-package=%s" % os.path.basename(filename_main) ] if not "--output-dir" in os.environ.get("NUITKA_EXTRA_OPTIONS", ""): extensions.append("--output-dir=%s" % getTempDir()) with withExtendedExtraOptions(*extensions): compareWithCPython( dirname = filename, filename = filename_main, extra_flags = extra_flags, search_mode = search_mode, needs_2to3 = False ) else: my_print("Skipping", filename) search_mode.finish()
def main(): # Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements filename = sys.argv[1] args = sys.argv[2:] def hasArg(arg): if arg in args: args.remove(arg) return True else: return False def hasArgValue(arg_option, default=None): for arg in args: if arg.startswith(arg_option + "="): args.remove(arg) return arg[len(arg_option) + 1:] return default def hasArgValues(arg_option): result = [] for arg in tuple(args): if arg.startswith(arg_option + "="): args.remove(arg) result.append(arg[len(arg_option) + 1:]) return result # For output keep it arguments = list(args) silent_mode = hasArg("silent") ignore_stderr = hasArg("ignore_stderr") ignore_warnings = hasArg("ignore_warnings") expect_success = hasArg("expect_success") expect_failure = hasArg("expect_failure") python_debug = hasArg("python_debug") module_mode = hasArg("--module") coverage_mode = hasArg("coverage") two_step_execution = hasArg("two_step_execution") binary_python_path = hasArg("binary_python_path") keep_python_path = hasArg("keep_python_path") trace_command = (hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0") remove_output = hasArg("remove_output") remove_binary = not hasArg("--keep-binary") standalone_mode = hasArg("--standalone") onefile_mode = hasArg("--onefile") no_site = hasArg("no_site") or coverage_mode report = hasArgValue("--report") nofollow_imports = hasArg("recurse_none") or hasArg("--nofollow-imports") follow_imports = hasArg("recurse_all") or hasArg("--follow-imports") timing = hasArg("timing") original_file = hasArg("original_file") or hasArg( "--file-reference-choice=original") runtime_file = hasArg("runtime_file") or hasArg( "--file-reference-choice=runtime") no_warnings = not hasArg("warnings") full_compat = not hasArg("improved") cpython_cached = hasArg("cpython_cache") syntax_errors = hasArg("syntax_errors") noprefer_source = hasArg("noprefer_source") noverbose_log = hasArg("noverbose_log") noinclusion_log = hasArg("noinclusion_log") send_kill = hasArg("--send-ctrl-c") output_dir = hasArgValue("--output-dir", None) include_packages = hasArgValues("--include-package") include_modules = hasArgValues("--include-module") python_flag_m = hasArg("--python-flag=-m") plugins_enabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_enable:"): plugins_enabled.append(arg[len("plugin_enable:"):]) del args[count] plugins_disabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_disable:"): plugins_disabled.append(arg[len("plugin_disable:"):]) del args[count] user_plugins = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("user_plugin:"): user_plugins.append(arg[len("user_plugin:"):]) del args[count] recurse_not = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_not:"): recurse_not.append(arg[len("recurse_not:"):]) del args[count] recurse_to = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_to:"): recurse_to.append(arg[len("recurse_to:"):]) del args[count] if args: sys.exit("Error, non understood mode(s) '%s'," % ",".join(args)) # In coverage mode, we don't want to execute, and to do this only in one mode, # we enable two step execution, which splits running the binary from the actual # compilation: if coverage_mode: two_step_execution = True # The coverage mode doesn't work with debug mode. if coverage_mode: python_debug = False comparison_mode = not coverage_mode # We need to split it, so we know when to kill. if send_kill: two_step_execution = True assert not standalone_mode or not module_mode assert not follow_imports or not nofollow_imports if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" os.environ["PYTHONWARNINGS"] = "ignore" if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() if "--python-debug" in extra_options or "--python-dbg" in extra_options: python_debug = True if python_debug: if os.path.exists( os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")): os.environ["PYTHON"] += "-dbg" if os.name == "nt": if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"): os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe" if os.environ["PYTHON"].endswith("-dbg"): python_debug = True if os.environ["PYTHON"].lower().endswith("_d.exe"): python_debug = True if comparison_mode: my_print("""\ Comparing output of '{filename}' using '{python}' with flags {args} ...""". format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), )) else: my_print("""\ Taking coverage of '{filename}' using '{python}' with flags {args} ...""". format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), )) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("CPython:") my_print("*" * 80) if two_step_execution: filename = os.path.abspath(filename) if module_mode: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] cpython_cmd = [ os.environ["PYTHON"], "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), module_name), ] if no_warnings: cpython_cmd[1:1] = [ "-W", "ignore", ] else: cpython_cmd = [os.environ["PYTHON"]] if no_warnings: cpython_cmd[1:1] = [ "-W", "ignore", ] if python_flag_m: cpython_cmd += ["-m", os.path.basename(filename)] os.chdir(os.path.dirname(filename)) else: cpython_cmd.append(filename) if no_site: cpython_cmd.insert(1, "-S") if "NUITKA" in os.environ: # Would need to extract which "python" this is going to use. assert not coverage_mode, "Not implemented for binaries." nuitka_call = [os.environ["NUITKA"]] else: if comparison_mode: nuitka_call = [ os.environ["PYTHON"], "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] else: assert coverage_mode nuitka_call = [ os.environ["PYTHON"], "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] if python_debug: extra_options.append("--python-debug") if no_warnings: extra_options.append("--python-flag=no_warnings") if remove_output: extra_options.append("--remove-output") if original_file: extra_options.append("--file-reference-choice=original") if runtime_file: extra_options.append("--file-reference-choice=runtime") if full_compat: extra_options.append("--full-compat") if noprefer_source: extra_options.append("--no-prefer-source") if python_flag_m: extra_options.append("--python-flag=-m") if coverage_mode: # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output([ os.environ["PYTHON"], "-c", "import sys, os; print(os.pathsep.join(sys.path))", ]) if sys.version_info >= (3, ): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() if binary_python_path: addToPythonPath(os.path.dirname(os.path.abspath(filename))) if keep_python_path or binary_python_path: extra_options.append("--execute-with-pythonpath") if report: extra_options.append("--report=%s" % report) if nofollow_imports: extra_options.append("--nofollow-imports") if follow_imports: extra_options.append("--follow-imports") if recurse_not: extra_options.extend("--nofollow-import-to=" + v for v in recurse_not) if coverage_mode: extra_options.append("--must-not-re-execute") extra_options.append("--generate-c-only") for plugin_enabled in plugins_enabled: extra_options.append("--enable-plugin=" + plugin_enabled) for plugin_disabled in plugins_disabled: extra_options.append("--disable-plugin=" + plugin_disabled) for user_plugin in user_plugins: extra_options.append("--user-plugin=" + user_plugin) if not noverbose_log: extra_options.append("--verbose-output=%s.optimization.log" % filename) if not noinclusion_log: extra_options.append("--show-modules-output=%s.inclusion.log" % filename) if output_dir is not None: extra_options.append("--output-dir=%s" % output_dir) else: # TODO: The run-tests uses NUITKA_EXTRA_OPTIONS still. for extra_option in extra_options: dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option) if dir_match: output_dir = dir_match.group(1) break else: # The default. output_dir = "." for include_package in include_packages: extra_options.append("--include-package=%s" % include_package) for include_module in include_modules: extra_options.append("--include-module=%s" % include_module) # Progress bar is not used. extra_options.append("--no-progressbar") # Now build the command to run Nuitka. if not two_step_execution: if module_mode: extra_options.append("--module") elif onefile_mode: extra_options.append("--onefile") elif standalone_mode: extra_options.append("--standalone") nuitka_cmd = nuitka_call + extra_options + ["--run", filename] if no_site: nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S") else: if module_mode: nuitka_cmd1 = (nuitka_call + extra_options + ["--module", os.path.abspath(filename)]) elif standalone_mode: nuitka_cmd1 = nuitka_call + extra_options + [ "--standalone", filename ] else: nuitka_cmd1 = nuitka_call + extra_options + [filename] if no_site: nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S") if module_mode: module_name = os.path.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] nuitka_cmd2 = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import %s" % module_name, ] else: exe_filename = os.path.basename(filename) if filename.endswith(".py"): exe_filename = exe_filename[:-3] exe_filename = exe_filename.replace(")", "").replace("(", "") if os.name == "nt": exe_filename += ".exe" else: exe_filename += ".bin" nuitka_cmd2 = [os.path.join(output_dir, exe_filename)] pdb_filename = exe_filename[:-4] + ".pdb" if trace_command: my_print("CPython command:", *cpython_cmd) if comparison_mode: cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False, send_kill=send_kill, ) if not silent_mode: displayOutput(stdout_cpython, stderr_cpython) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("Nuitka:") my_print("*" * 80) if two_step_execution: if output_dir: os.chdir(output_dir) else: tmp_dir = getTempDir() os.chdir(tmp_dir) if trace_command: my_print("Going to output directory", os.getcwd()) stop_watch = StopWatch() stop_watch.start() if not two_step_execution: if trace_command: my_print("Nuitka command:", nuitka_cmd) # Try a couple of times for permission denied, on Windows it can # be transient. for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen(args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_nuitka, stderr_nuitka = process.communicate() exit_nuitka = process.returncode if checkNoPermissionError( stdout_nuitka) and checkNoPermissionError(stderr_nuitka): break my_print( "Retrying nuitka exe due to permission problems after delay.") time.sleep(2) else: if trace_command: my_print("Nuitka command 1:", nuitka_cmd1) for _i in range(5): with withPythonPathChange(nuitka_package_dir): stdout_nuitka1, stderr_nuitka1, exit_nuitka1 = executeProcess( nuitka_cmd1) if exit_nuitka1 != 0: if (not expect_failure and not comparison_mode and not os.path.exists(".coverage")): sys.exit("""\ Error, failed to take coverage with '%s'. Stderr was: %s """ % (os.environ["PYTHON"], stderr_nuitka1)) exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 stdout_nuitka2 = b"not run due to compilation error:\n" + stdout_nuitka1 stderr_nuitka2 = stderr_nuitka1 else: # No execution second step for coverage mode. if comparison_mode: if os.path.exists(nuitka_cmd2[0][:-4] + ".cmd"): nuitka_cmd2[0] = nuitka_cmd2[0][:-4] + ".cmd" if trace_command: my_print("Nuitka command 2:", nuitka_cmd2) # Need full manual control, and not all Python versions allow using # context manager here, pylint: disable=consider-using-with process = subprocess.Popen(args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if send_kill: # Lambda is used immediately in same loop, pylint: disable=cell-var-from-loop executeAfterTimePassed( 1.0, lambda: killProcess("Nuitka compiled program", process.pid), ) stdout_nuitka2, stderr_nuitka2 = process.communicate() stdout_nuitka = stdout_nuitka1 + stdout_nuitka2 stderr_nuitka = stderr_nuitka1 + stderr_nuitka2 exit_nuitka = process.returncode # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": nuitka_cmd2 = wrapCommandForDebuggerForSubprocess( *nuitka_cmd2) callProcess(nuitka_cmd2, shell=False) else: exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 if checkNoPermissionError( stdout_nuitka) and checkNoPermissionError(stderr_nuitka): break my_print( "Retrying nuitka exe due to permission problems after delay.") time.sleep(2) stop_watch.stop() nuitka_time = stop_watch.getDelta() if not silent_mode: displayOutput(stdout_nuitka, stderr_nuitka) if coverage_mode: assert not stdout_nuitka assert not stderr_nuitka if comparison_mode: def makeComparisons(trace_result): exit_code_stdout = compareOutput( "stdout", stdout_cpython, stdout_nuitka2 if two_step_execution else stdout_nuitka, ignore_warnings, syntax_errors, ) if ignore_stderr: exit_code_stderr = 0 else: exit_code_stderr = compareOutput( "stderr", stderr_cpython, stderr_nuitka2 if two_step_execution else stderr_nuitka, ignore_warnings, syntax_errors, ) exit_code_return = exit_cpython != exit_nuitka if exit_code_return and trace_result: my_print( """Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""" .format(exit_cpython=exit_cpython, exit_nuitka=exit_nuitka)) return exit_code_stdout, exit_code_stderr, exit_code_return if cpython_cached: exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=False) if not int(os.environ.get("NUITKA_CPYTHON_NO_CACHE_UPDATE", "0")): if exit_code_stdout or exit_code_stderr or exit_code_return: old_stdout_cpython = stdout_cpython old_stderr_cpython = stderr_cpython old_exit_cpython = exit_cpython my_print( "Updating CPython cache by force due to non-matching comparison results.", style="yellow", ) ( cpython_time, stdout_cpython, stderr_cpython, exit_cpython, ) = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=True, send_kill=send_kill, ) if not silent_mode: if (old_stdout_cpython != stdout_cpython or old_stderr_cpython != stderr_cpython or old_exit_cpython != exit_cpython): displayOutput(stdout_cpython, stderr_cpython) exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=True) # In case of segfault, also output the call stack by entering debugger # without stdin forwarded. if (exit_code_return and exit_nuitka in (-11, -6) and sys.platform != "nt" and not module_mode and not two_step_execution): nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger") with withPythonPathChange(nuitka_package_dir): callProcess(nuitka_cmd, shell=False) exit_code = exit_code_stdout or exit_code_stderr or exit_code_return if exit_code: problems = [] if exit_code_stdout: problems.append("stdout") if exit_code_stderr: problems.append("stderr") if exit_code_return: problems.append("exit_code") sys.exit("Error, results differed (%s)." % ",".join(problems)) if expect_success and exit_cpython != 0: if silent_mode: displayOutput(stdout_cpython, stderr_cpython) sys.exit("Unexpected error exit from CPython.") if expect_failure and exit_cpython == 0: sys.exit("Unexpected success exit from CPython.") if remove_output: if not module_mode: if os.path.exists(nuitka_cmd2[0]) and remove_binary: if os.name == "nt": # It appears there is a tiny lock race that we randomly cause, # likely because --run spawns a subprocess that might still # be doing the cleanup work. if os.path.exists(nuitka_cmd2[0] + ".away"): os.unlink(nuitka_cmd2[0] + ".away") for _i in range(10): try: os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away") except OSError: time.sleep(0.1) continue for _i in range(10): try: os.unlink(nuitka_cmd2[0] + ".away") except OSError: time.sleep(2) continue else: break if os.path.exists(pdb_filename): os.unlink(pdb_filename) else: os.unlink(nuitka_cmd2[0]) else: module_filename = os.path.basename( filename) + getSharedLibrarySuffix(preferred=True) if os.path.exists(module_filename) and remove_binary: os.unlink(module_filename) if comparison_mode and timing: my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time)) if comparison_mode and not silent_mode: my_print("OK, same outputs.")
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option( "--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "") ) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option("--code-diff", action="store", dest="diff_filename", default="") parser.add_option("--copy-source-to", action="store", dest="target_dir", default="") options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit("Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) python_version = setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % python_version) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) with open(test_case, "rb") as f: my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest()) needs_2to3 = ( python_version.startswith("3") and not test_case.endswith("32.py") and not test_case.endswith("33.py") ) if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case)) ) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) with open(test_case) as f: case_1_source, case_2_source = generateConstructCases(f.read()) with open(test_case_1, "w") as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, "w") as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = "0" if nuitka: nuitka_id = check_output( "cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True ) nuitka_id = nuitka_id.strip() if sys.version_info > (3,): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case), ] nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. shutil.copyfile(test_case_1, os.path.basename(test_case)) subprocess.check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix), ) shutil.copyfile(test_case_2, os.path.basename(test_case)) subprocess.check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix), ) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join( test_case_1.replace(".py", ".build"), "module.__main__" + suffix ) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join( test_case_2.replace(".py", ".build"), "module.__main__" + suffix ) if os.path.exists(cpp_2): break else: assert False import difflib with open(options.diff_filename, "w") as f: with open(cpp_1) as cpp1: with open(cpp_2) as cpp2: f.write( difflib.HtmlDiff().make_table( cpp1.readlines(), cpp2.readlines(), "Construct", "Baseline", True, ) ) nuitka_1 = runValgrind( "Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix),), include_startup=True, ) nuitka_2 = runValgrind( "Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix),), include_startup=True, ) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_1 = runValgrind( "CPython construct", "callgrind", (os.environ["PYTHON"], "-S", test_case_1), include_startup=True, ) cpython_2 = runValgrind( "CPython baseline", "callgrind", (os.environ["PYTHON"], "-S", test_case_2), include_startup=True, ) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def main(): python_version = setup() os.chdir("subject") nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka") tmp_dir = getTempDir() command = [ os.environ["PYTHON"], nuitka_main_path, "--plugin-enable=pylint-warnings", "--output-dir=%s" % tmp_dir, "--follow-imports", "--include-package=package", "--nofollow-import-to=*.tests", "--python-flag=-v", "--debug", "--module", "package" ] result = subprocess.call( command ) if result != 0: sys.exit(result) os.makedirs(os.path.join(tmp_dir, "package.ext")) shutil.copytree( "package", os.path.join(tmp_dir, "package.ext/package") ) os.chdir(tmp_dir) # We compile the package non-closed, so we can smuggle in tests # and user code. This is going to be the example code. with open("package.ext/package/user_provided.py", "w") as output: # TODO: Maybe assert that the type name of a local function and one from # the package are not the same, i.e. we are running inside the compiled # package. output.write(""" from __future__ import print_function import package print("__name__:", package.__name__) print("__package__:", package.__package__) print("__path__:", package.__path__) print("__file__:", package.__file__) # print("__loader__:", package.__loader__) import package.sub_package1 print("__name__:", package.sub_package1.__name__) print("__package__:", package.sub_package1.__package__) print("__path__:", package.sub_package1.__path__) print("__file__:", package.sub_package1.__file__) # print("__loader__:", package.sub_package1.__loader__) import package.sub_package1.tests; print("__name__:", package.sub_package1.tests.__name__) print("__package__:", package.sub_package1.tests.__package__) print("__path__:", package.sub_package1.tests.__path__) print("__file__:", package.sub_package1.tests.__file__) # print("__loader__:", package.sub_package1.tests.__loader__) """ ) os.makedirs("nose") with open("nose/usage.txt", "w") as output: pass os.system("find | sort") # Inform about the extra path, format is NUITKA_PACKAGE_fullname where # dots become "_" and should point to the directory where external code # to be loaded will live under. Probably should be an absolute path, but # we avoid it here. os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package" # Lets make sure these to not work. These will be used in the compiled # form only. for module_path in ("__init__.py", "sub_package1__init__.py"): with open(os.path.join("./package.ext/package", module_path), "w") as output: output.write("assert False") # Check the compiled package is functional for importing. my_print("Running package as basic test:") assert os.system(os.environ["PYTHON"] + " -c 'import package'" ) == 0 my_print("Running nose tests:") # assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0 my_print("Running py.test tests:") assert os.system(os.environ["PYTHON"] + " -m pytest --pyargs package.sub_package1.tests" ) == 0