def action(stage_dir, root, path): command = [ sys.executable, os.path.join("..", "..", "bin", "nuitka"), "--stand", "--run", "--output-dir", stage_dir, "--remove-output", "--plugin-enable=pylint-warnings", ] filename = os.path.join(stage_dir, "importer.py") assert path.startswith(root) module_name = path[len(root) + 1:] module_name = module_name.split(".")[0] module_name = module_name.replace(os.path.sep, ".") with open(filename, "w") as output: output.write("import " + module_name + "\n") output.write("print('OK')") command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(filename) if checkSucceedsWithCPython(filename): try: output = check_output(command).splitlines() except Exception: # only trying to check for no exception, pylint: disable=try-except-raise raise else: assert os.path.exists(filename[:-3] + ".dist") loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, path=os.path.join(filename[:-3] + ".dist", "importer.exe"), ) outside_accesses = checkRuntimeLoadedFilesForOutsideAccesses( loaded_filenames, [ filename[:-3] + ".dist", current_dir, os.path.expanduser("~/.config") ], ) if output[-1] != b"OK": sys.exit("FAIL") my_print("OK") assert not outside_accesses, outside_accesses shutil.rmtree(filename[:-3] + ".dist") else: my_print("SKIP (does not work with CPython)")
def action(stage_dir, root, path): command = [ sys.executable, os.path.join( "..", "..", "bin", "nuitka" ), "--stand", "--run", "--output-dir", stage_dir, "--remove-output", "--plugin-enable=pylint-warnings" ] filename = os.path.join(stage_dir, "importer.py") assert path.startswith(root) module_name = path[len(root)+1:] module_name = module_name.split(".")[0] module_name = module_name.replace(os.path.sep, ".") with open(filename, "w") as output: output.write("import " + module_name + "\n") output.write("print('OK')") command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(filename) try: output = check_output(command).splitlines() except Exception: raise else: if output[-1] != b"OK": sys.exit("FAIL") my_print("OK") shutil.rmtree(filename[:-3] + ".dist")
with open(test_case_1, 'w') as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, 'w') as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, needs_delete = convertUsing2to3(test_case_1) test_case_2, needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = '0' if nuitka: nuitka_id = check_output("cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case) ]
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option( "--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", ""), ) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option( "--code-diff", action="store", dest="diff_filename", default="", ) parser.add_option( "--copy-source-to", action="store", dest="target_dir", default="", ) options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit( "Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) python_version = setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % python_version) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) my_print("TEST_CASE_HASH='%s'" % hashlib.md5(open(test_case, "rb").read()).hexdigest()) needs_2to3 = python_version.startswith('3') and \ not test_case.endswith("32.py") and \ not test_case.endswith("33.py") if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case))) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) case_1_source, case_2_source = generateConstructCases( open(test_case).read()) with open(test_case_1, 'w') as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, 'w') as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = '0' if nuitka: nuitka_id = check_output("cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case) ] nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. shutil.copyfile(test_case_1, os.path.basename(test_case)) subprocess.check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build")) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix)) shutil.copyfile(test_case_2, os.path.basename(test_case)) subprocess.check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build")) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix)) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join( test_case_1.replace(".py", ".build"), "module.__main__" + suffix, ) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join( test_case_2.replace(".py", ".build"), "module.__main__" + suffix, ) if os.path.exists(cpp_2): break else: assert False import difflib open(options.diff_filename, 'w').write(difflib.HtmlDiff().make_table( open(cpp_1).readlines(), open(cpp_2).readlines(), "Construct", "Baseline", True)) nuitka_1 = runValgrind("Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix), ), include_startup=True) nuitka_2 = runValgrind("Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix), ), include_startup=True) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % ' '.join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_1 = runValgrind("CPython construct", "callgrind", (os.environ["PYTHON"], "-S", test_case_1), include_startup=True) cpython_2 = runValgrind("CPython baseline", "callgrind", (os.environ["PYTHON"], "-S", test_case_2), include_startup=True) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def main(): # Complex stuff, pylint: disable=too-many-branches for filename in sorted(os.listdir(".")): if not filename.endswith(".py") or filename.startswith("run_"): continue if not decideFilenameVersionSkip(filename): continue active = search_mode.consider(dirname=None, filename=filename) if active: # Apply 2to3 conversion if necessary. if python_version >= (3, ): filename, changed = convertUsing2to3(filename) else: changed = False my_print("Consider", filename, end=" ") command = [ os.environ["PYTHON"], os.path.abspath(os.path.join("..", "..", "bin", "nuitka")), "--xml", "--quiet", "--module", filename, ] if search_mode.isCoverage(): # To avoid re-execution, which is not acceptable to coverage. if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output([ os.environ["PYTHON"], "-c", "import sys, os; print(os.pathsep.join(sys.path))", ]) if sys.version_info >= (3, ): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() command.insert(2, "--must-not-re-execute") command = (command[0:1] + [ "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a" ] + command[1:]) result = check_output(command) # Parse the result into XML and check it. try: root = lxml.etree.fromstring(result) except lxml.etree.XMLSyntaxError: my_print("Problematic XML output:") my_print(result) raise module_body = root[0] module_statements_sequence = module_body[0] assert len(module_statements_sequence) == 1 module_statements = next(iter(module_statements_sequence)) try: checkSequence(filename, module_statements) for function in root.xpath('role[@name="functions"]/node'): (function_body, ) = function.xpath('role[@name="body"]') function_statements_sequence = function_body[0] assert len(function_statements_sequence) == 1 function_statements = next( iter(function_statements_sequence)) checkSequence(filename, function_statements) if changed: os.unlink(filename) except SystemExit: my_print("FAIL.") raise my_print("OK.") if search_mode.abortIfExecuted(): break search_mode.finish()
command = [ os.environ["PYTHON"], os.path.abspath(os.path.join("..", "..", "bin", "nuitka")), "--xml", "--module", filename ] if search_mode.isCoverage(): # To avoid re-execution, which is not acceptable to coverage. if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = '0' # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output([ os.environ["PYTHON"], "-c" "import sys, os; print(os.pathsep.join(sys.path))" ]) if sys.version_info >= (3, ): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() command.insert(2, "--must-not-re-execute") command = command[0:1] + [ "-S", "-m", "coverage", "run", "--rcfile",
def action(stage_dir, root, path): command = [ sys.executable, os.path.join("..", "..", "bin", "nuitka"), "--stand", "--run", "--output-dir=%s" % stage_dir, "--remove-output", ] filename = os.path.join(stage_dir, "importer.py") assert path.startswith(root) module_name = path[len(root) + 1:] module_name = module_name.split(".")[0] module_name = module_name.replace(os.path.sep, ".") module_name = ModuleName(module_name) with openTextFile(filename, "w") as output: plugin_names = set(["pylint-warnings", "anti-bloat"]) if module_name.hasNamespace("PySide2"): plugin_names.add("pyside2") elif module_name.hasNamespace("PySide6"): plugin_names.add("pyside2") elif module_name.hasNamespace("PyQt5"): plugin_names.add("pyqt5") else: # TODO: We do not have a noqt plugin yet. plugin_names.add("pyqt5") for plugin_name in plugin_names: output.write("# nuitka-project: --enable-plugin=%s\n" % plugin_name) # Make it an error to find unwanted bloat compiled in. output.write("# nuitka-project: --noinclude-default-mode=error\n") output.write("import " + module_name.asString() + "\n") output.write("print('OK.')") command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(filename) if checkSucceedsWithCPython(filename): try: output = check_output(command).splitlines() except Exception: # only trying to check for no exception, pylint: disable=try-except-raise raise else: assert os.path.exists(filename[:-3] + ".dist") binary_filename = os.path.join( filename[:-3] + ".dist", "importer.exe" if os.name == "nt" else "importer", ) loaded_filenames = getRuntimeTraceOfLoadedFiles( logger=test_logger, command=[binary_filename], ) outside_accesses = checkLoadedFileAccesses( loaded_filenames=loaded_filenames, current_dir=os.getcwd()) if outside_accesses: displayError(None, filename) displayRuntimeTraces(test_logger, binary_filename) test_logger.warning( "Should not access these file(s): '%r'." % outside_accesses) search_mode.onErrorDetected(1) if output[-1] != b"OK.": my_print(" ".join(command)) my_print(filename) my_print(output) test_logger.sysexit("FAIL.") my_print("OK.") assert not outside_accesses, outside_accesses shutil.rmtree(filename[:-3] + ".dist") else: my_print("SKIP (does not work with CPython)")
os.environ["PYTHONPATH"] = python_path.strip() command.insert(2, "--must-not-re-execute") command = command[0:1] + [ "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", ] + command[1:] result = check_output( command ) # Parse the result into XML and check it. root = lxml.etree.fromstring(result) module_body = root[0] module_statements_sequence = module_body[0] assert len(module_statements_sequence) == 1 module_statements = next(iter(module_statements_sequence)) checkSequence(module_statements) for function in root.xpath('role[@name="functions"]/node'): function_body, = function.xpath('role[@name="body"]') function_statements_sequence = function_body[0]
def action(stage_dir, root, path): command = [ sys.executable, os.path.join( "..", "..", "bin", "nuitka" ), "--stand", "--run", "--output-dir", stage_dir, "--remove-output", "--plugin-enable=pylint-warnings" ] filename = os.path.join(stage_dir, "importer.py") assert path.startswith(root) module_name = path[len(root)+1:] module_name = module_name.split(".")[0] module_name = module_name.replace(os.path.sep, ".") with open(filename, "w") as output: output.write("import " + module_name + "\n") output.write("print('OK')") command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() command.append(filename) if checkSucceedsWithCPython(filename): try: output = check_output(command).splitlines() except Exception: raise else: assert os.path.exists(filename[:-3] + ".dist") loaded_filenames = getRuntimeTraceOfLoadedFiles( path = os.path.join( filename[:-3] + ".dist", "importer.exe" ) ) outside_accesses = checkRuntimeLoadedFilesForOutsideAccesses( loaded_filenames, [ filename[:-3] + ".dist", current_dir, os.path.expanduser("~/.config") ] ) if output[-1] != b"OK": sys.exit("FAIL") my_print("OK") assert not outside_accesses, outside_accesses shutil.rmtree(filename[:-3] + ".dist") else: my_print("SKIP (does not work with CPython)")
"--module", filename ] if search_mode.isCoverage(): # To avoid re-execution, which is not acceptable to coverage. if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = '0' # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output( [ os.environ["PYTHON"], "-c" "import sys, os; print(os.pathsep.join(sys.path))" ] ) if sys.version_info >= (3,): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() command.insert(2, "--must-not-re-execute") command = command[0:1] + [ "-S", "-m",
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option( "--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "") ) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option("--code-diff", action="store", dest="diff_filename", default="") parser.add_option("--copy-source-to", action="store", dest="target_dir", default="") options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit("Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) python_version = setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % python_version) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) with open(test_case, "rb") as f: my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest()) needs_2to3 = ( python_version.startswith("3") and not test_case.endswith("32.py") and not test_case.endswith("33.py") ) if options.target_dir: shutil.copyfile( test_case, os.path.join(options.target_dir, os.path.basename(test_case)) ) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) with open(test_case) as f: case_1_source, case_2_source = generateConstructCases(f.read()) with open(test_case_1, "w") as case_1_file: case_1_file.write(case_1_source) with open(test_case_2, "w") as case_2_file: case_2_file.write(case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = "0" if nuitka: nuitka_id = check_output( "cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True ) nuitka_id = nuitka_id.strip() if sys.version_info > (3,): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--python-flag=-S", os.path.basename(test_case), ] nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. shutil.copyfile(test_case_1, os.path.basename(test_case)) subprocess.check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix), ) shutil.copyfile(test_case_2, os.path.basename(test_case)) subprocess.check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix), ) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join( test_case_1.replace(".py", ".build"), "module.__main__" + suffix ) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join( test_case_2.replace(".py", ".build"), "module.__main__" + suffix ) if os.path.exists(cpp_2): break else: assert False import difflib with open(options.diff_filename, "w") as f: with open(cpp_1) as cpp1: with open(cpp_2) as cpp2: f.write( difflib.HtmlDiff().make_table( cpp1.readlines(), cpp2.readlines(), "Construct", "Baseline", True, ) ) nuitka_1 = runValgrind( "Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix),), include_startup=True, ) nuitka_2 = runValgrind( "Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix),), include_startup=True, ) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_1 = runValgrind( "CPython construct", "callgrind", (os.environ["PYTHON"], "-S", test_case_1), include_startup=True, ) cpython_2 = runValgrind( "CPython baseline", "callgrind", (os.environ["PYTHON"], "-S", test_case_2), include_startup=True, ) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))