def getBranchName(): branch_name = check_output("git symbolic-ref --short HEAD".split()).strip() if str is not bytes: branch_name = branch_name.decode() return branch_name
def isStaticallyLinkedPython(): try: import sysconfig except ImportError: # Cannot detect this properly for Python 2.6, but we don't care much # about that anyway. return False result = sysconfig.get_config_var("Py_ENABLE_SHARED") == 0 if result: from nuitka.utils.Execution import check_output with open(os.devnull, "w") as devnull: output = check_output( (os.path.realpath(sys.executable) + "-config", "--ldflags"), stderr=devnull, ) if str is not bytes: output = output.decode("utf8") import shlex output = shlex.split(output) python_abi_version = python_version_str + getPythonABI() result = ("-lpython" + python_abi_version) not in output return result
def publishCoverageData(): def copyToGlobalCoverageData(source, target): coverage_dir = os.environ.get("COVERAGE_DIR", None) if coverage_dir is None: return subprocess.check_call(("scp", source, os.path.join(coverage_dir, target))) if os.name == "nt": suffix = "win" else: import platform suffix = platform.uname()[0] + "." + platform.uname()[4] with open("data.coverage", "w") as data_file: source_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) with withDirectoryChange(source_dir): nuitka_id = check_output("git rev-parse HEAD".split()) nuitka_id = nuitka_id.strip() if sys.version_info > (3,): nuitka_id = nuitka_id.decode() data_file.write("NUITKA_SOURCE_DIR=%r\n" % source_dir) data_file.write("NUITKA_COMMIT=%r\n" % nuitka_id) copyToGlobalCoverageData("data.coverage", "meta.coverage." + suffix) def makeCoverageRelative(filename): """ Normalize coverage data. """ with open(filename) as input_file: data = input_file.read() data = data.replace( (os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), "" ) if os.path.sep != "/": data.replace(os.path.sep, "/") with open(filename, "w") as output_file: output_file.write(data) coverage_file = os.environ.get("COVERAGE_FILE", ".coverage") makeCoverageRelative(coverage_file) copyToGlobalCoverageData(coverage_file, "data.coverage." + suffix)
def publishCoverageData(): def copyToGlobalCoverageData(source, target): coverage_dir = os.environ.get("COVERAGE_DIR") if coverage_dir is None: return check_call(("scp", source, os.path.join(coverage_dir, target))) if os.name == "nt": suffix = "win" else: import platform suffix = platform.uname()[0] + "." + platform.uname()[4] with open("data.coverage", "w") as data_file: source_dir = os.path.abspath(os.path.dirname( os.path.dirname(__file__))) with withDirectoryChange(source_dir): nuitka_id = check_output("git rev-parse HEAD".split()) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() data_file.write("NUITKA_SOURCE_DIR=%r\n" % source_dir) data_file.write("NUITKA_COMMIT=%r\n" % nuitka_id) copyToGlobalCoverageData("data.coverage", "meta.coverage." + suffix) def makeCoverageRelative(filename): """Normalize coverage data.""" with open(filename) as input_file: data = input_file.read() data = data.replace( (os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), "") if os.path.sep != "/": data.replace(os.path.sep, "/") with open(filename, "w") as output_file: output_file.write(data) coverage_file = os.environ.get("COVERAGE_FILE", ".coverage") makeCoverageRelative(coverage_file) copyToGlobalCoverageData(coverage_file, "data.coverage." + suffix)
def queryRuntimeInformationMultiple(self, info_name, setup_codes, values): if info_name in self._runtime_information_cache: return self._runtime_information_cache[info_name] keys = [] query_codes = [] for key, value_expression in values: keys.append(key) query_codes.append("print(repr(%s))" % value_expression) query_codes.append('print("-" * 27)') if type(setup_codes) is str: setup_codes = [setup_codes] cmd = r"""\ from __future__ import print_function from __future__ import absolute_import %(setup_codes)s %(query_codes)s """ % { "setup_codes": "\n".join(setup_codes), "query_codes": "\n".join(query_codes), } feedback = check_output([sys.executable, "-c", cmd]) if str is not bytes: # We want to work with strings, that's hopefully OK. feedback = feedback.decode("utf8") # Ignore Windows newlines difference. feedback = [line.strip() for line in feedback.splitlines()] if feedback.count("-" * 27) != len(keys): self.sysexit( "Error, mismatch in output retrieving %r information." % info_name ) feedback = [line for line in feedback if line != "-" * 27] NamedTupleResult = namedtuple(info_name, keys) # We are being lazy here, the code is trusted, pylint: disable=eval-used self._runtime_information_cache[info_name] = NamedTupleResult( *(eval(value) for value in feedback) ) return self._runtime_information_cache[info_name]
def _checkRequiredVersion(tool, tool_call): required_version = _getRequiredVersion(tool) for line in _getRequirementsContentsByLine(): if line.startswith(tool + " =="): required_version = line.split()[2] break else: sys.exit("Error, cannot find %r in requirements-devel.txt" % tool) if tool == "rstfmt": if "-m" in tool_call: return False, "rstfmt doesn't support that yet." else: return True, "unchecked" tool_call = list(tool_call) + ["--version"] try: version_output = check_output(tool_call) except NuitkaCalledProcessError: return False, "failed to execute" if str is not bytes: version_output = version_output.decode("utf8") for line in version_output.splitlines(): line = line.strip() if line.startswith(("black, version", "__main__.py, version ")): actual_version = line.split()[-1] break if line.startswith("VERSION "): actual_version = line.split()[-1] break else: sys.exit( "Error, couldn't determine version output of %r (%r)" % (tool, " ".join(tool_call)) ) message = "Version of %r via %r is required to be %r and not %r." % ( tool, " ".join(tool_call), required_version, actual_version, ) return required_version == actual_version, message
def _checkRequiredVersion(tool, tool_call): required_version = _getRequiredVersion(tool) for line in _getRequirementsContentsByLine(): if line.startswith(tool + " =="): required_version = line.split()[2] break else: sys.exit("Error, cannot find %r in requirements-devel.txt" % tool) tool_call = list(tool_call) + ["--version"] try: version_output = check_output(tool_call) except NuitkaCalledProcessError as e: return False, "failed to execute: %s" % e.stderr if str is not bytes: version_output = version_output.decode("utf8") for line in version_output.splitlines(): line = line.strip() if line.startswith(("black, ", "python -m black,", "__main__.py, ")): if "(" in line: line = line[:line.rfind("(")].strip() actual_version = line.split()[-1] break if line.startswith("VERSION "): actual_version = line.split()[-1] break if line.startswith("rstfmt "): actual_version = line.split()[-1] break else: sys.exit("Error, couldn't determine version output of %r (%r)" % (tool, " ".join(tool_call))) message = "Version of %r via %r is required to be %r and not %r." % ( tool, " ".join(tool_call), required_version, actual_version, ) return required_version == actual_version, message
def updateWorkingFile(path, orig_object_hash, new_object_hash): patch = check_output( ["git", "diff", "--no-color", orig_object_hash, new_object_hash]) git_path = path.replace(os.path.sep, "/").encode("utf8") def updateLine(line): if line.startswith(b"diff --git"): line = b"diff --git a/%s b/%s" % (git_path, git_path) elif line.startswith(b"--- a/"): line = b"--- a/" + git_path elif line.startswith(b"+++ b/"): line = b"+++ b/" + git_path return line # Substitute object hashes in patch header with path to working tree file patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n" # Apply the patch. output, err, exit_code = executeProcess( ["git", "apply", "-"], stdin=patch, ) # Windows extra ball, new files have new lines that make the patch fail. if exit_code != 0 and os.name == "nt": from .autoformat.Autoformat import cleanupWindowsNewlines cleanupWindowsNewlines(path) output, err, exit_code = executeProcess( ["git", "apply", "-"], stdin=patch, ) success = exit_code == 0 if not success: # TODO: In case of failure, do we need to abort, or what do we do. if output: my_print(output, style="yellow") if err: my_print(err, style="yellow") return success
def getUnpushedPaths(): result = set() output = check_output( ["git", "diff", "--stat", "--name-only", "@{upstream}"]) for line in output.splitlines(): if str is not bytes: line = line.decode("utf8") # Removed files appear too, but are useless to talk about. if not os.path.exists(line): continue result.add(line) return tuple(sorted(result))
def updateWorkingFile(path, orig_object_hash, new_object_hash): patch = check_output(["git", "diff", orig_object_hash, new_object_hash]) # Substitute object hashes in patch header with path to working tree file patch_b = patch.replace(orig_object_hash.encode(), path.encode()).replace(new_object_hash.encode(), path.encode()) apply_patch = subprocess.Popen( ["git", "apply", "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) _output, _err = apply_patch.communicate(input=patch_b) # TODO: In case of failure, do we need to abort? return apply_patch.returncode != 0
def checkVersion(): # pylint: disable=global-statement global pylint_version if not hasModule("pylint"): sys.exit( "Error, pylint is not installed for this interpreter %r version." % os.environ["PYTHON"]) if pylint_version is None: pylint_version = check_output( [os.environ["PYTHON"], "-m", "pylint", "--version"], stderr=getNullOutput()) if str is not bytes: pylint_version = pylint_version.decode("utf8") pylint_version = pylint_version.split("\n")[0].split()[-1].strip(",") my_print("Using PyLint version:", pylint_version)
def updateWorkingFile(path, orig_object_hash, new_object_hash): patch = check_output( ["git", "diff", "--no-color", orig_object_hash, new_object_hash]) git_path = path.replace(os.path.sep, "/").encode("utf8") def updateLine(line): if line.startswith(b"diff --git"): line = b"diff --git a/%s b/%s" % (git_path, git_path) elif line.startswith(b"--- a/"): line = b"--- a/" + git_path elif line.startswith(b"+++ b/"): line = b"+++ b/" + git_path return line # Substitute object hashes in patch header with path to working tree file patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n" # Cannot use executeProcess, because we supply input, and we cannot # use context managers, because not all Python versions have it that # way, xpylintx: disable=consider-using-with apply_patch = subprocess.Popen( ["git", "apply", "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) output, err = apply_patch.communicate(input=patch) success = apply_patch.returncode == 0 if not success: # TODO: In case of failure, do we need to abort, or what do we do. if output: my_print(output, style="yellow") if err: my_print(err, style="yellow") return success
def _checkRequiredVersion(tool, tool_call): required_version = _getRequiredVersion(tool) for line in _getRequirementsContentsByLine(): if line.startswith(tool + " =="): required_version = line.split()[2] break else: sys.exit("Error, cannot find %r in requirements-devel.txt" % tool) tool_call = list(tool_call) + ["--version"] version_output = check_output(tool_call) if str is not bytes: version_output = version_output.decode("utf8") for line in version_output.splitlines(): line = line.strip() if line.startswith(("black, version", "__main__.py, version ")): actual_version = line.split()[-1] break if line.startswith("VERSION "): actual_version = line.split()[-1] break else: sys.exit("Error, couldn't determine version output of %r (%r)" % (tool, " ".join(tool_call))) message = "Version of %r via %r is required to be %r and not %r." % ( tool, " ".join(tool_call), required_version, actual_version, ) return required_version == actual_version, message
0, os.path.normpath( os.path.join( os.path.dirname(__file__), "..", ) ) ) from nuitka.tools.release.Release import checkAtHome, checkBranchName from nuitka.utils.Execution import check_output checkAtHome() nuitka_version = check_output( "./bin/nuitka --version", shell = True ).strip() branch_name = checkBranchName() if branch_name == "factory": for remote in "origin", "github": assert 0 == os.system("git push --recurse-submodules=no --force-with-lease %s factory" % remote) sys.exit(0) assert 0 == os.system("rsync -rvlpt --exclude=deb_dist dist/ [email protected]:/var/www/releases/") for filename in ("README.pdf", "Changelog.pdf", "Developer_Manual.pdf"): assert 0 == os.system("rsync %s [email protected]:/var/www/doc/" % filename)
def setup(suite="", needs_io_encoding=False, silent=False, go_main=True): if go_main: goMainDir() if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable # Allow test code to use this to make caching specific. os.environ["NUITKA_TEST_SUITE"] = suite # Allow providing 33, 27, and expand that to python2.7 if ( len(os.environ["PYTHON"]) == 2 and os.environ["PYTHON"].isdigit() and os.name != "nt" ): os.environ["PYTHON"] = "python%s.%s" % ( os.environ["PYTHON"][0], os.environ["PYTHON"][1], ) if needs_io_encoding and "PYTHONIOENCODING" not in os.environ: os.environ["PYTHONIOENCODING"] = "utf-8" version_output = check_output( ( os.environ["PYTHON"], "-c", """\ import sys, os;\ print(".".join(str(s) for s in list(sys.version_info)[:3]));\ print(("x86_64" if "AMD64" in sys.version else "x86") if os.name == "nt" else os.uname()[4]);\ print(sys.executable);\ print("Anaconda" if os.path.exists(os.path.join(sys.prefix, 'conda-meta')) else "Unknown")\ """, ), stderr=subprocess.STDOUT, ) global _python_version_str, _python_version, _python_arch, _python_executable, _python_vendor # singleton, pylint: disable=global-statement _python_version_str = version_output.split(b"\n")[0].strip() _python_arch = version_output.split(b"\n")[1].strip() _python_executable = version_output.split(b"\n")[2].strip() _python_vendor = version_output.split(b"\n")[3].strip() if str is not bytes: _python_version_str = _python_version_str.decode("utf-8") _python_arch = _python_arch.decode("utf-8") _python_executable = _python_executable.decode("utf-8") _python_vendor = _python_vendor.decode("utf-8") assert type(_python_version_str) is str, repr(_python_version_str) assert type(_python_arch) is str, repr(_python_arch) assert type(_python_executable) is str, repr(_python_executable) if not silent: my_print("Using concrete python", _python_version_str, "on", _python_arch) if "COVERAGE_FILE" not in os.environ: os.environ["COVERAGE_FILE"] = os.path.join( os.path.dirname(__file__), "..", "..", "..", ".coverage" ) _python_version = tuple(int(d) for d in _python_version_str.split(".")) return _python_version
import os import sys # Unchanged, running from checkout, use the parent directory, the nuitka # package ought be there. sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) # isort:start from nuitka.tools.release.Release import checkAtHome, checkBranchName from nuitka.utils.Execution import check_output checkAtHome() nuitka_version = check_output( "%s ./bin/nuitka --version" % sys.executable, shell=True ).strip() branch_name = checkBranchName() if branch_name == "factory": for remote in "origin", "github": assert ( os.system( "git push --recurse-submodules=no --force-with-lease %s factory" % remote ) == 0 ) sys.exit(0)
def main(): print("Querying openSUSE build service status of Nuitka packages.") osc_cmd = ["osc", "pr", "-c", "home:kayhayen"] stdout_osc = check_output(args=osc_cmd) # Response is really a CSV file, so use that for parsing. csvfile = StringIO(stdout_osc) osc_reader = csv.reader(csvfile, delimiter=";") osc_reader = iter(osc_reader) bad = ("failed", "unresolvable", "broken", "blocked") titles = osc_reader.next()[1:] # Nuitka (follow git master branch) row1 = osc_reader.next() # Nuitka-Unstable (follow git develop branch) row2 = osc_reader.next() problems = [] def decideConsideration(title): # Ignore other arch builds, they might to not even boot at times. if "ppc" in title or "aarch" in title or "arm" in title: return False # This fails for other reasons often, and is not critical to Nuitka. if "openSUSE_Tumbleweed" in title: return False return True for count, title in enumerate(titles): if not decideConsideration(title): continue status = row1[count + 1] if status in bad: problems.append((row1[0], title, status)) for count, title in enumerate(titles): if not decideConsideration(title): continue status = row2[count + 1] if status in bad: problems.append((row2[0], title, status)) if problems: print("There are problems with:") print("\n".join("%s: %s (%s)" % problem for problem in problems)) sys.exit(1) else: print("Looks good.") sys.exit(0)
def main(): # Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements filename = sys.argv[1] args = sys.argv[2:] def hasArg(arg): if arg in args: args.remove(arg) return True else: return False # For output keep it arguments = list(args) silent_mode = hasArg("silent") ignore_stderr = hasArg("ignore_stderr") ignore_warnings = hasArg("ignore_warnings") expect_success = hasArg("expect_success") expect_failure = hasArg("expect_failure") python_debug = hasArg("python_debug") module_mode = hasArg("module_mode") two_step_execution = hasArg("two_step_execution") binary_python_path = hasArg("binary_python_path") keep_python_path = hasArg("keep_python_path") trace_command = ( hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0" ) remove_output = hasArg("remove_output") standalone_mode = hasArg("--standalone") onefile_mode = hasArg("--onefile") no_site = hasArg("no_site") recurse_none = hasArg("recurse_none") recurse_all = hasArg("recurse_all") timing = hasArg("timing") coverage_mode = hasArg("coverage") original_file = hasArg("original_file") runtime_file = hasArg("runtime_file") no_warnings = not hasArg("warnings") full_compat = not hasArg("improved") cpython_cached = hasArg("cpython_cache") syntax_errors = hasArg("syntax_errors") noprefer_source = hasArg("noprefer_source") noverbose_log = hasArg("noverbose_log") noinclusion_log = hasArg("noinclusion_log") plugins_enabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_enable:"): plugins_enabled.append(arg[len("plugin_enable:") :]) del args[count] plugins_disabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_disable:"): plugins_disabled.append(arg[len("plugin_disable:") :]) del args[count] user_plugins = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("user_plugin:"): user_plugins.append(arg[len("user_plugin:") :]) del args[count] recurse_not = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_not:"): recurse_not.append(arg[len("recurse_not:") :]) del args[count] recurse_to = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_to:"): recurse_to.append(arg[len("recurse_to:") :]) del args[count] if args: sys.exit("Error, non understood mode(s) '%s'," % ",".join(args)) # In coverage mode, we don't want to execute, and to do this only in one mode, # we enable two step execution, which splits running the binary from the actual # compilation: if coverage_mode: two_step_execution = True # The coverage mode doesn't work with debug mode. if coverage_mode: python_debug = False comparison_mode = not coverage_mode assert not standalone_mode or not module_mode assert not recurse_all or not recurse_none if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" os.environ["PYTHONWARNINGS"] = "ignore" if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() if "--python-debug" in extra_options or "--python-dbg" in extra_options: python_debug = True if python_debug: if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")): os.environ["PYTHON"] += "-dbg" if os.name == "nt": if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"): os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe" if os.environ["PYTHON"].endswith("-dbg"): python_debug = True if os.environ["PYTHON"].lower().endswith("_d.exe"): python_debug = True if comparison_mode: my_print( """\ Comparing output of '{filename}' using '{python}' with flags {args} ...""".format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), ) ) else: my_print( """\ Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format( filename=filename, python=os.environ["PYTHON"], args=", ".join(arguments), ) ) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("CPython:") my_print("*" * 80) if two_step_execution: filename = os.path.abspath(filename) if module_mode: if no_warnings: cpython_cmd = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), os.path.basename(filename)), ] else: cpython_cmd = [ os.environ["PYTHON"], "-c", "import sys; sys.path.append(%s); import %s" % (repr(os.path.dirname(filename)), os.path.basename(filename)), ] else: if no_warnings: cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename] else: cpython_cmd = [os.environ["PYTHON"], filename] if no_site: cpython_cmd.insert(1, "-S") if "NUITKA" in os.environ: # Would need to extract which "python" this is going to use. assert not coverage_mode, "Not implemented for binaries." nuitka_call = [os.environ["NUITKA"]] else: if comparison_mode: nuitka_call = [ os.environ["PYTHON"], "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] else: assert coverage_mode nuitka_call = [ os.environ["PYTHON"], "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", "-m", "nuitka.__main__", # Note: Needed for Python2.6 ] if python_debug: extra_options.append("--python-debug") if no_warnings: extra_options.append("--python-flag=no_warnings") if remove_output: extra_options.append("--remove-output") if original_file: extra_options.append("--file-reference-choice=original") if runtime_file: extra_options.append("--file-reference-choice=runtime") if full_compat: extra_options.append("--full-compat") if noprefer_source: extra_options.append("--no-prefer-source") if coverage_mode: # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output( [ os.environ["PYTHON"], "-c", "import sys, os; print(os.pathsep.join(sys.path))", ] ) if sys.version_info >= (3,): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() if binary_python_path: addToPythonPath(os.path.dirname(os.path.abspath(filename))) if keep_python_path or binary_python_path: extra_options.append("--execute-with-pythonpath") if recurse_none: extra_options.append("--nofollow-imports") if recurse_all: extra_options.append("--follow-imports") if recurse_not: extra_options.extend("--nofollow-import-to=" + v for v in recurse_not) if coverage_mode: extra_options.append("--must-not-re-execute") extra_options.append("--generate-c-only") for plugin_enabled in plugins_enabled: extra_options.append("--plugin-enable=" + plugin_enabled) for plugin_disabled in plugins_disabled: extra_options.append("--plugin-disable=" + plugin_disabled) for user_plugin in user_plugins: extra_options.append("--user-plugin=" + user_plugin) if not noverbose_log: extra_options.append("--verbose-output=%s.optimization.log" % filename) if not noinclusion_log: extra_options.append("--show-modules-output=%s.inclusion.log" % filename) # Now build the command to run Nuitka. if not two_step_execution: if module_mode: nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename] elif onefile_mode: nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename] elif standalone_mode: nuitka_cmd = ( nuitka_call + extra_options + ["--run", "--standalone", filename] ) else: nuitka_cmd = nuitka_call + extra_options + ["--run", filename] if no_site: nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S") else: if module_mode: nuitka_cmd1 = ( nuitka_call + extra_options + ["--module", os.path.abspath(filename)] ) elif standalone_mode: nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename] else: nuitka_cmd1 = nuitka_call + extra_options + [filename] if no_site: nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S") for extra_option in extra_options: dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option) if dir_match: output_dir = dir_match.group(1) break else: # The default. output_dir = "." if module_mode: nuitka_cmd2 = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import %s" % os.path.basename(filename), ] else: exe_filename = os.path.basename(filename) if filename.endswith(".py"): exe_filename = exe_filename[:-3] exe_filename = exe_filename.replace(")", "").replace("(", "") exe_filename += ".exe" if os.name == "nt" else ".bin" nuitka_cmd2 = [os.path.join(output_dir, exe_filename)] pdb_filename = exe_filename[:-4] + ".pdb" if trace_command: my_print("CPython command:", *cpython_cmd) if comparison_mode: cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False ) if not silent_mode: displayOutput(stdout_cpython, stderr_cpython) if comparison_mode and not silent_mode: my_print("*" * 80) my_print("Nuitka:") my_print("*" * 80) if two_step_execution: if output_dir: os.chdir(output_dir) else: tmp_dir = tempfile.gettempdir() # Try to avoid RAM disk /tmp and use the disk one instead. if tmp_dir == "/tmp" and os.path.exists("/var/tmp"): tmp_dir = "/var/tmp" os.chdir(tmp_dir) if trace_command: my_print("Going to output directory", os.getcwd()) stop_watch = StopWatch() stop_watch.start() if not two_step_execution: if trace_command: my_print("Nuitka command:", nuitka_cmd) # Try a couple of times for permission denied, on Windows it can # be transient. for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen( args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka, stderr_nuitka = process.communicate() exit_nuitka = process.returncode if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError( stderr_nuitka ): break my_print("Retrying nuitka exe due to permission problems after delay.") time.sleep(2) else: if trace_command: my_print("Nuitka command 1:", nuitka_cmd1) for _i in range(5): with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen( args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka1, stderr_nuitka1 = process.communicate() exit_nuitka1 = process.returncode if exit_nuitka1 != 0: if ( not expect_failure and not comparison_mode and not os.path.exists(".coverage") ): sys.exit( """\ Error, failed to take coverage with '%s'. Stderr was: %s """ % (os.environ["PYTHON"], stderr_nuitka1) ) exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 else: # No execution second step for coverage mode. if comparison_mode: if trace_command: my_print("Nuitka command 2:", nuitka_cmd2) process = subprocess.Popen( args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout_nuitka2, stderr_nuitka2 = process.communicate() stdout_nuitka = stdout_nuitka1 + stdout_nuitka2 stderr_nuitka = stderr_nuitka1 + stderr_nuitka2 exit_nuitka = process.returncode # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2) process = subprocess.Popen( args=nuitka_cmd2, stdin=subprocess.PIPE ) process.communicate() else: exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError( stderr_nuitka ): break my_print("Retrying nuitka exe due to permission problems after delay.") time.sleep(2) stop_watch.stop() nuitka_time = stop_watch.getDelta() if not silent_mode: displayOutput(stdout_nuitka, stderr_nuitka) if coverage_mode: assert not stdout_nuitka assert not stderr_nuitka if comparison_mode: def makeComparisons(trace_result): exit_code_stdout = compareOutput( "stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors ) if ignore_stderr: exit_code_stderr = 0 else: exit_code_stderr = compareOutput( "stderr", stderr_cpython, stderr_nuitka, ignore_warnings, syntax_errors, ) exit_code_return = exit_cpython != exit_nuitka if exit_code_return and trace_result: my_print( """Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format( exit_cpython=exit_cpython, exit_nuitka=exit_nuitka ) ) return exit_code_stdout, exit_code_stderr, exit_code_return if cpython_cached: exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=False ) if exit_code_stdout or exit_code_stderr or exit_code_return: old_stdout_cpython = stdout_cpython old_stderr_cpython = stderr_cpython old_exit_cpython = exit_cpython my_print( "Updating CPython cache by force due to non-matching comparison results.", style="yellow", ) ( cpython_time, stdout_cpython, stderr_cpython, exit_cpython, ) = getCPythonResults( cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=True, ) if not silent_mode: if ( old_stdout_cpython != stdout_cpython or old_stderr_cpython != stderr_cpython or old_exit_cpython != exit_cpython ): displayOutput(stdout_cpython, stderr_cpython) exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons( trace_result=True ) # In case of segfault, also output the call stack by entering debugger # without stdin forwarded. if ( exit_code_return and exit_nuitka in (-11, -6) and sys.platform != "nt" and not module_mode and not two_step_execution ): nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger") with withPythonPathChange(nuitka_package_dir): process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE) process.communicate() exit_code = exit_code_stdout or exit_code_stderr or exit_code_return if exit_code: problems = [] if exit_code_stdout: problems.append("stdout") if exit_code_stderr: problems.append("stderr") if exit_code_return: problems.append("exit_code") sys.exit("Error, results differed (%s)." % ",".join(problems)) if expect_success and exit_cpython != 0: if silent_mode: displayOutput(stdout_cpython, stderr_cpython) sys.exit("Unexpected error exit from CPython.") if expect_failure and exit_cpython == 0: sys.exit("Unexpected success exit from CPython.") if remove_output: if not module_mode: if os.path.exists(nuitka_cmd2[0]): if os.name == "nt": # It appears there is a tiny lock race that we randomly cause, # likely because --run spawns a subprocess that might still # be doing the cleanup work. if os.path.exists(nuitka_cmd2[0] + ".away"): os.unlink(nuitka_cmd2[0] + ".away") for _i in range(10): try: os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away") except OSError: time.sleep(0.1) continue for _i in range(10): try: os.unlink(nuitka_cmd2[0] + ".away") except OSError: time.sleep(2) continue else: break if os.path.exists(pdb_filename): os.unlink(pdb_filename) else: os.unlink(nuitka_cmd2[0]) else: module_filename = os.path.basename(filename) + getSharedLibrarySuffix( preferred=True ) if os.path.exists(module_filename): os.unlink(module_filename) if comparison_mode and timing: my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time)) if comparison_mode and not silent_mode: my_print("OK, same outputs.")
# Unchanged, running from checkout, use the parent directory, the nuitka # package ought be there. sys.path.insert( 0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) # isort:start import shutil import subprocess from nuitka.tools.release.Release import checkAtHome, checkBranchName from nuitka.utils.Execution import check_output checkAtHome() nuitka_version = check_output("./bin/nuitka --version", shell=True).strip() branch_name = checkBranchName() if branch_name == "factory": for remote in "origin", "github": assert 0 == os.system( "git push --recurse-submodules=no --force-with-lease %s factory" % remote) sys.exit(0) assert 0 == os.system( "rsync -rvlpt --exclude=deb_dist dist/ [email protected]:/var/www/releases/") for filename in ("README.pdf", "Changelog.pdf", "Developer_Manual.pdf"):
def getFileHashContent(object_hash): return check_output(["git", "cat-file", "-p", object_hash])
def main(): my_print("Querying openSUSE build service status of Nuitka packages.") osc_cmd = ["osc", "pr", "-c", "home:kayhayen"] stdout_osc = check_output(args=osc_cmd) if str is not bytes: stdout_osc = stdout_osc.decode("utf8") # Response is really a CSV file, so use that for parsing. csvfile = StringIO(stdout_osc) osc_reader = csv.reader(csvfile, delimiter=";") osc_reader = iter(osc_reader) bad = ("failed", "unresolvable", "broken", "blocked") titles = next(osc_reader)[1:] # Nuitka (follow git main branch) row1 = next(osc_reader) # Nuitka-Unstable (follow git develop branch) row2 = next(osc_reader) # Nuitka-Experimental (follow git factory branch) row3 = next(osc_reader) problems = [] def decideConsideration(title, status): # Ignore other arch builds, they might to not even boot at times. if "ppc" in title or "aarch" in title or "arm" in title: return False # This fails for other reasons often, and is not critical to Nuitka. if "openSUSE_Tumbleweed" in title: return False # Ignore old Fedora and RHEL6 32 bit being blocked. if status == "blocked": if ("Fedora_2" in title or "RedHat_RHEL-6/i586" in title or "CentOS_CentOS-6/i586" in title): return False # It makes building visible now, that's not an error of course. if status == "building": return False return True for count, title in enumerate(titles): status = row1[count + 1] if not decideConsideration(title, status): continue if status in bad: problems.append((row1[0], title, status)) for count, title in enumerate(titles): status = row2[count + 1] if not decideConsideration(title, status): continue if status in bad: problems.append((row2[0], title, status)) for count, title in enumerate(titles): status = row3[count + 1] if not decideConsideration(title, status): continue if status in bad: problems.append((row3[0], title, status)) if problems: my_print("There are problems with:", style="yellow") my_print("\n".join("%s: %s (%s)" % problem for problem in problems), style="yellow") sys.exit(1) else: my_print("Looks good.", style="blue") sys.exit(0)
def main(): # There are freaking many options to honor, pylint: disable=too-many-branches # Lets honor this Debian option here. if "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split(): print("Skipped all tests as per DEB_BUILD_OPTIONS environment.") sys.exit(0) goHome() parser = OptionParser() parser.add_option("--skip-basic-tests", action="store_false", dest="basic_tests", default=True, help="""\ The basic tests, execute these to check if Nuitka is healthy. Default is %default.""") parser.add_option("--skip-syntax-tests", action="store_false", dest="syntax_tests", default=True, help="""\ The syntax tests, execute these to check if Nuitka handles Syntax errors fine. Default is %default.""") parser.add_option("--skip-program-tests", action="store_false", dest="program_tests", default=True, help="""\ The programs tests, execute these to check if Nuitka handles programs, e.g. import recursions, etc. fine. Default is %default.""") parser.add_option("--skip-package-tests", action="store_false", dest="package_tests", default=True, help="""\ The packages tests, execute these to check if Nuitka handles packages, e.g. import recursions, etc. fine. Default is %default.""") parser.add_option("--skip-optimizations-tests", action="store_false", dest="optimization_tests", default=True, help="""\ The optimization tests, execute these to check if Nuitka does optimize certain constructs fully away. Default is %default.""") parser.add_option( "--skip-standalone-tests", action="store_false", dest="standalone_tests", default=os.name != "posix" or os.uname()[0] != "NetBSD", # @UndefinedVariable help="""\ The standalone tests, execute these to check if Nuitka standalone mode, e.g. not referring to outside, important 3rd library packages like PyQt fine. Default is %default.""") parser.add_option("--skip-reflection-test", action="store_false", dest="reflection_test", default=True, help="""\ The reflection test compiles Nuitka with Nuitka, and then Nuitka with the compile Nuitka and compares the outputs. Default is %default.""") parser.add_option("--skip-cpython26-tests", action="store_false", dest="cpython26", default=True, help="""\ The standard CPython2.6 test suite. Execute this for all corner cases to be covered. With Python 2.7 this covers exception behavior quite well. Default is %default.""") parser.add_option("--skip-cpython27-tests", action="store_false", dest="cpython27", default=True, help="""\ The standard CPython2.7 test suite. Execute this for all corner cases to be covered. With Python 2.6 these are not run. Default is %default.""") parser.add_option("--skip-cpython32-tests", action="store_false", dest="cpython32", default=True, help="""\ The standard CPython3.2 test suite. Execute this for all corner cases to be covered. With Python 2.6 these are not run. Default is %default.""") parser.add_option("--skip-cpython33-tests", action="store_false", dest="cpython33", default=True, help="""\ The standard CPython3.3 test suite. Execute this for all corner cases to be covered. With Python 2.x these are not run. Default is %default.""") parser.add_option("--skip-cpython34-tests", action="store_false", dest="cpython34", default=True, help="""\ The standard CPython3.4 test suite. Execute this for all corner cases to be covered. With Python 2.x these are not run. Default is %default.""") parser.add_option("--skip-cpython35-tests", action="store_false", dest="cpython35", default=True, help="""\ The standard CPython3.5 test suite. Execute this for all corner cases to be covered. With Python 2.x these are not run. Default is %default.""") parser.add_option("--skip-cpython36-tests", action="store_false", dest="cpython36", default=True, help="""\ The standard CPython3.6 test suite. Execute this for all corner cases to be covered. With Python 2.x these are not run. Default is %default.""") parser.add_option("--skip-other-cpython-tests", action="store_true", dest="cpython_no_other", default=False, help="""\ Do not execute any CPython test suite other than the one matching the running Python. Default is %default.""") parser.add_option("--skip-all-cpython-tests", action="store_true", dest="cpython_none", default=False, help="""\ Do not execute any CPython test suite other than the one matching the running Python. Default is %default.""") parser.add_option("--no-other-python", action="store_true", dest="no_other", default=False, help="""\ Do not use any other Python than the one running, even if available on the system. Default is %default.""") parser.add_option("--no-python2.6", action="store_true", dest="no26", default=False, help="""\ Do not use Python2.6 even if available on the system. Default is %default.""") parser.add_option("--no-python2.7", action="store_true", dest="no27", default=False, help="""\ Do not use Python2.7 even if available on the system. Default is %default.""") parser.add_option("--no-python3.2", action="store_true", dest="no32", default=False, help="""\ Do not use Python3.2 even if available on the system. Default is %default.""") parser.add_option("--no-python3.3", action="store_true", dest="no33", default=False, help="""\ Do not use Python3.3 even if available on the system. Default is %default.""") parser.add_option("--no-python3.4", action="store_true", dest="no34", default=False, help="""\ Do not use Python3.4 even if available on the system. Default is %default.""") parser.add_option("--no-python3.5", action="store_true", dest="no35", default=False, help="""\ Do not use Python3.5 even if available on the system. Default is %default.""") parser.add_option("--no-python3.6", action="store_true", dest="no36", default=False, help="""\ Do not use Python3.6 even if available on the system. Default is %default.""") parser.add_option("--coverage", action="store_true", dest="coverage", default=False, help="""\ Make a coverage analysis, that does not really check. Default is %default.""") options, positional_args = parser.parse_args() if positional_args: parser.print_help() sys.exit("\nError, no positional argument allowed.") if options.no_other: if sys.version_info[0:2] != (2, 6): options.no26 = True if sys.version_info[0:2] != (2, 7): options.no27 = True if sys.version_info[0:2] != (3, 2): options.no32 = True if sys.version_info[0:2] != (3, 3): options.no33 = True if sys.version_info[0:2] != (3, 4): options.no34 = True if sys.version_info[0:2] != (3, 5): options.no35 = True if sys.version_info[0:2] != (3, 6): options.no36 = True if options.cpython_no_other: if sys.version_info[0:2] != (2, 6): options.cpython26 = False if sys.version_info[0:2] != (2, 7): options.cpython27 = False if sys.version_info[0:2] != (3, 2): options.cpython32 = False if sys.version_info[0:2] != (3, 3): options.cpython33 = False if sys.version_info[0:2] != (3, 4): options.cpython34 = False if sys.version_info[0:2] != (3, 5): options.cpython35 = False if sys.version_info[0:2] != (3, 6): options.cpython36 = False if options.cpython_none: options.cpython26 = False options.cpython27 = False options.cpython32 = False options.cpython33 = False options.cpython34 = False options.cpython35 = False options.cpython36 = False if options.coverage and os.path.exists(".coverage"): os.unlink(".coverage") # Add the local bin directory to search path start. os.environ["PATH"] = \ os.path.join( os.getcwd(), "bin" ) + \ os.pathsep + \ os.environ["PATH"] def checkExecutableCommand(command): """ Check if a command is executable. """ # Many cases, pylint: disable=too-many-branches,too-many-return-statements # Do respect given options to disable specific Python versions if command == "python2.6" and options.no26: return False if command == "python2.7" and options.no27: return False if command == "python3.2" and options.no32: return False if command == "python3.3" and options.no33: return False if command == "python3.4" and options.no34: return False if command == "python3.5" and options.no35: return False if command == "python3.6" and options.no36: return False # Shortcuts for python versions, also needed for Windows as it won't have # the version number in the Python binaries at all. if command == "python2.6" and sys.version_info[0:2] == (2, 6): return True if command == "python2.7" and sys.version_info[0:2] == (2, 7): return True if command == "python3.2" and sys.version_info[0:2] == (3, 2): return True if command == "python3.3" and sys.version_info[0:2] == (3, 3): return True if command == "python3.4" and sys.version_info[0:2] == (3, 4): return True if command == "python3.5" and sys.version_info[0:2] == (3, 5): return True if command == "python3.6" and sys.version_info[0:2] == (3, 6): return True path = os.environ["PATH"] suffixes = (".exe", ) if os.name == "nt" else ("", ) for part in path.split(os.pathsep): if not part: continue for suffix in suffixes: if os.path.exists(os.path.join(part, command + suffix)): return True if os.name == "nt": if command.startswith("python"): remainder = command[6:] if len(remainder) == 3 and remainder[1] == '.': command = getPythonExePathWindows(search=remainder, arch=None) return True return False def setExtraFlags(where, name, flags): if where is not None: tmp_dir = tempfile.gettempdir() # Try to avoid RAM disk /tmp and use the disk one instead. if tmp_dir == "/tmp" and os.path.exists("/var/tmp"): tmp_dir = "/var/tmp" where = os.path.join(tmp_dir, name, where) if not os.path.exists(where): os.makedirs(where) os.environ[ "NUITKA_EXTRA_OPTIONS"] = flags + " --output-dir=" + where else: os.environ["NUITKA_EXTRA_OPTIONS"] = flags def executeSubTest(command, hide_output=False): if options.coverage and "search" in command: command = command.replace("search", "coverage") parts = command.split() parts[0] = parts[0].replace('/', os.path.sep) # The running Python will be good enough, on some platforms there is no # "python", and we need to pass this alone then. parts.insert(0, sys.executable) print("Run '%s' in '%s'." % (' '.join(parts), os.getcwd())) sys.stdout.flush() if hide_output: result = subprocess.call(parts, stdout=open(os.devnull, 'w')) else: result = subprocess.call(parts) if result != 0: sys.exit(result) def execute_tests(where, use_python, flags): # Many cases, pylint: disable=too-many-branches,too-many-statements print( "Executing test case called %s with CPython %s and extra flags '%s'." % (where, use_python, flags)) intended_version = use_python[6:] if sys.version.startswith(intended_version): os.environ["PYTHON"] = sys.executable else: if os.name == "nt": os.environ["PYTHON"] = getPythonExePathWindows( search=intended_version, arch=None) else: os.environ["PYTHON"] = getExecutablePath(use_python) if options.basic_tests: print("Running the basic tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "basics", flags) executeSubTest("./tests/basics/run_all.py search") if options.syntax_tests: print("Running the syntax tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "syntax", flags) executeSubTest("./tests/syntax/run_all.py search") if options.program_tests: print("Running the program tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "programs", flags) executeSubTest("./tests/programs/run_all.py search") if options.package_tests: print("Running the package tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "packages", flags) executeSubTest("./tests/packages/run_all.py search") # At least one Debian Jessie, these versions won't have lxml installed, so # don't run them there. Also these won't be very version dependent in their # results. if use_python != "python2.6" and use_python != "python3.2": if options.optimization_tests: print( "Running the optimizations tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "optimizations", flags) executeSubTest("./tests/optimizations/run_all.py search") if options.standalone_tests and not options.coverage: print("Running the standalone tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(None, "standalone", flags) executeSubTest("./tests/standalone/run_all.py search") if options.reflection_test and not options.coverage: print("Running the reflection test with options '%s' with %s:" % (flags, use_python)) setExtraFlags(None, "reflected", flags) executeSubTest("./tests/reflected/compile_itself.py search") if not use_python.startswith("python3"): if os.path.exists("./tests/CPython26/run_all.py"): if options.cpython26: print( "Running the CPython 2.6 tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "26tests", flags) executeSubTest("./tests/CPython26/run_all.py search") else: print("The CPython2.6 tests are not present, not run.") # Running the Python 2.7 test suite with CPython 2.6 gives little # insight, because "importlib" will not be there and that's it. if use_python != "python2.6": if os.path.exists("./tests/CPython27/run_all.py"): if options.cpython27: print( "Running the CPython 2.7 tests with options '%s' with %s:" % (flags, use_python)) setExtraFlags(where, "27tests", flags) executeSubTest("./tests/CPython27/run_all.py search") else: print("The CPython2.7 tests are not present, not run.") if "--debug" not in flags: # Not running the Python 3.2 test suite with CPython2.6, as that's about # the same as CPython2.7 and won't have any new insights. if use_python != "python2.6" and \ use_python != "python2.7" or not options.coverage: if os.path.exists("./tests/CPython32/run_all.py"): if options.cpython32: setExtraFlags(where, "32tests", flags) executeSubTest("./tests/CPython32/run_all.py search") else: print("The CPython3.2 tests are not present, not run.") # Running the Python 3.3 test suite only with CPython3.x. if not use_python.startswith("python2"): if os.path.exists("./tests/CPython33/run_all.py"): if options.cpython33: setExtraFlags(where, "33tests", flags) executeSubTest("./tests/CPython33/run_all.py search") else: print("The CPython3.3 tests are not present, not run.") # Running the Python 3.4 test suite only with CPython3.x. if not use_python.startswith("python2"): if os.path.exists("./tests/CPython34/run_all.py"): if options.cpython34: setExtraFlags(where, "34tests", flags) executeSubTest("./tests/CPython34/run_all.py search") else: print("The CPython3.4 tests are not present, not run.") # Running the Python 3.4 test suite only with CPython3.x. if not use_python.startswith("python2"): if os.path.exists("./tests/CPython35/run_all.py"): if options.cpython35: setExtraFlags(where, "35tests", flags) executeSubTest("./tests/CPython35/run_all.py search") else: print("The CPython3.5 tests are not present, not run.") # Running the Python 3.4 test suite only with CPython3.x. if not use_python.startswith("python2"): if os.path.exists("./tests/CPython36/run_all.py"): if options.cpython36: setExtraFlags(where, "36tests", flags) executeSubTest("./tests/CPython36/run_all.py search") else: print("The CPython3.6 tests are not present, not run.") if "NUITKA_EXTRA_OPTIONS" in os.environ: del os.environ["NUITKA_EXTRA_OPTIONS"] assert checkExecutableCommand("python2.6") or \ checkExecutableCommand("python2.7") or \ checkExecutableCommand("python3.2") or \ checkExecutableCommand("python3.3") or \ checkExecutableCommand("python3.4") or \ checkExecutableCommand("python3.5") or \ checkExecutableCommand("python3.6") # Just the quick syntax test, full tests are run later. if checkExecutableCommand("python3.2"): executeSubTest("./bin/nuitka --python-version=3.2 --version", hide_output=True) if checkExecutableCommand("python2.6"): execute_tests("python2.6-debug", "python2.6", "--debug") else: print( "Cannot execute tests with Python 2.6, disabled or not installed.") if checkExecutableCommand("python2.7"): execute_tests("python2.7-debug", "python2.7", "--debug") else: print( "Cannot execute tests with Python 2.7, disabled or not installed.") if checkExecutableCommand("python3.2"): execute_tests("python3.2-debug", "python3.2", "--debug") else: print( "Cannot execute tests with Python 3.2, disabled or not installed.") if checkExecutableCommand("python2.6"): execute_tests("python2.6-nodebug", "python2.6", "") else: print( "Cannot execute tests with Python 2.6, disabled or not installed.") if checkExecutableCommand("python2.7"): execute_tests("python2.7-nodebug", "python2.7", "") else: print( "Cannot execute tests with Python 2.7, disabled or not installed.") if checkExecutableCommand("python3.2"): execute_tests("python3.2-nodebug", "python3.2", "") else: print( "Cannot execute tests with Python 3.2, disabled or not installed.") if checkExecutableCommand("python3.3"): execute_tests("python3.3-nodebug", "python3.3", "") else: print( "Cannot execute tests with Python 3.3, disabled or not installed.") if checkExecutableCommand("python3.4"): execute_tests("python3.4-nodebug", "python3.4", "") else: print( "Cannot execute tests with Python 3.4, disabled or not installed.") if checkExecutableCommand("python3.5"): execute_tests("python3.5-nodebug", "python3.5", "") else: print( "Cannot execute tests with Python 3.5, disabled or not installed.") if checkExecutableCommand("python3.6"): execute_tests("python3.6-nodebug", "python3.6", "") else: print( "Cannot execute tests with Python 3.6, disabled or not installed.") if options.coverage: def copyToGlobalCoverageData(source, target): coverage_dir = os.environ.get("COVERAGE_DIR", None) if coverage_dir is None: return assert subprocess.call( ("C:\\MinGW\\msys\\1.0\\bin\\scp.exe" if os.name == "nt" else "scp", source, os.path.join(coverage_dir, target))) == 0 if os.name == "nt": suffix = "win" else: import platform suffix = platform.uname()[0] + '.' + platform.uname()[4] with open("data.coverage", 'w') as data_file: source_dir = os.path.abspath( os.path.dirname(os.path.dirname(__file__))) nuitka_id = check_output("cd '%s'; git rev-parse HEAD" % source_dir, shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() data_file.write("NUITKA_COMMIT='%s'\n" % nuitka_id) copyToGlobalCoverageData("data.coverage", "data.coverage." + suffix) def makeCoverageRelative(filename): """ Normalize coverage data. """ with open(filename) as input_file: data = input_file.read() data = data.replace(os.path.abspath('.') + os.path.sep, "") if os.path.sep != '/': data.replace(os.path.sep, '/') with open(filename, 'w') as output_file: output_file.write(data) coverage_file = os.environ.get("COVERAGE_FILE", ".coverage") makeCoverageRelative(coverage_file) copyToGlobalCoverageData(coverage_file, ".coverage." + suffix) print("OK.")
def getBinarySizes(filename): command = ["size", filename] sizes = check_output(command).strip() sizes = sizes.split(b'\n')[-1].replace(b'\t', b"").split() return int(sizes[0]), int(sizes[1])
def getBinarySizes(filename): command = ["size", filename] sizes = check_output(command).strip() sizes = sizes.split(b"\n")[-1].replace(b"\t", b"").split() return int(sizes[0]), int(sizes[1])
def main(): # Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements from nuitka.utils.Execution import check_output filename = sys.argv[1] args = sys.argv[2:] def hasArg(arg): if arg in args: args.remove(arg) return True else: return False # For output keep it arguments = list(args) silent_mode = hasArg("silent") ignore_stderr = hasArg("ignore_stderr") ignore_warnings = hasArg("ignore_warnings") ignore_infos = hasArg("ignore_infos") expect_success = hasArg("expect_success") expect_failure = hasArg("expect_failure") python_debug = hasArg("python_debug") module_mode = hasArg("module_mode") two_step_execution = hasArg("two_step_execution") binary_python_path = hasArg("binary_python_path") keep_python_path = hasArg("keep_python_path") trace_command = hasArg("trace_command") remove_output = hasArg("remove_output") standalone_mode = hasArg("standalone") no_site = hasArg("no_site") recurse_none = hasArg("recurse_none") recurse_all = hasArg("recurse_all") timing = hasArg("timing") coverage_mode = hasArg("coverage") original_file = hasArg("original_file") no_warnings = not hasArg("warnings") full_compat = not hasArg("improved") syntax_errors = hasArg("syntax_errors") plugins_enabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_enable:"): plugins_enabled.append(arg[len("plugin_enable:"):]) del args[count] plugins_disabled = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("plugin_disable:"): plugins_disabled.append(arg[len("plugin_disable:"):]) del args[count] recurse_not = [] for count, arg in reversed(tuple(enumerate(args))): if arg.startswith("recurse_not:"): recurse_not.append(arg[len("recurse_not:"):]) del args[count] if args: sys.exit("Error, non understood mode(s) '%s'," % ','.join(args)) # In coverage mode, we don't want to execute, and to do this only in one mode, # we enable two step execution, which splits running the binary from the actual # compilation: if coverage_mode: two_step_execution = True # The coverage mode doesn't work with debug mode. if coverage_mode: python_debug = False comparison_mode = not coverage_mode assert not standalone_mode or not module_mode assert not recurse_all or not recurse_none if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = '0' os.environ["PYTHONWARNINGS"] = "ignore" if "PYTHON" not in os.environ: os.environ["PYTHON"] = sys.executable extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() if "--python-debug" in extra_options or "--python-dbg" in extra_options: python_debug = True if python_debug: if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")): os.environ["PYTHON"] += "-dbg" if os.name == "nt": if os.path.exists(os.environ["PYTHON"][:-4]+"_d.exe"): os.environ["PYTHON"] = os.environ["PYTHON"][:-4]+"_d.exe" if os.environ["PYTHON"].endswith("-dbg"): python_debug = True if os.environ["PYTHON"].lower().endswith("_d.exe"): python_debug = True if comparison_mode: my_print( """\ Comparing output of '{filename}' using '{python}' with flags {args} ...""". format( filename = filename, python = os.environ["PYTHON"], args = ", ".join(arguments) ) ) else: my_print( """\ Taking coverage of '{filename}' using '{python}' with flags {args} ...""". format( filename = filename, python = os.environ["PYTHON"], args = ", ".join(arguments) ) ) if comparison_mode and not silent_mode: my_print('*' * 80) my_print("CPython:") my_print('*' * 80) if two_step_execution: filename = os.path.abspath(filename) if module_mode: if no_warnings: cpython_cmd = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import sys; sys.path.append(%s); import %s" % ( repr(os.path.dirname(filename)), os.path.basename(filename) ) ] else: cpython_cmd = [ os.environ["PYTHON"], "-c", "import sys; sys.path.append(%s); import %s" % ( repr(os.path.dirname(filename)), os.path.basename(filename) ) ] else: if no_warnings: cpython_cmd = [ os.environ["PYTHON"], "-W", "ignore", filename ] else: cpython_cmd = [ os.environ["PYTHON"], filename ] if no_site: cpython_cmd.insert(1, "-S") if "NUITKA" in os.environ: # Would need to extract which "python" this is going to use. assert not coverage_mode, "Not implemented for binaries." nuitka_call = [os.environ["NUITKA"]] else: if comparison_mode: nuitka_call = [ os.environ["PYTHON"], os.path.abspath( os.path.join( os.path.dirname(__file__), "..", "..", "..", "..", "bin", "nuitka" ) ) ] else: assert coverage_mode nuitka_call = [ os.environ["PYTHON"], "-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a", os.path.abspath( os.path.join( os.path.dirname(__file__), "..", "..", "..", "..", "bin", "nuitka" ) ) ] if python_debug: extra_options.append("--python-debug") if no_warnings: extra_options.append("--python-flag=no_warnings") if remove_output: extra_options.append("--remove-output") if original_file: extra_options.append("--file-reference-choice=original") if full_compat: extra_options.append("--full-compat") if coverage_mode: # Coverage modules hates Nuitka to re-execute, and so we must avoid # that. python_path = check_output( [ os.environ["PYTHON"], "-c" "import sys, os; print(os.pathsep.join(sys.path))" ] ) if sys.version_info >= (3,): python_path = python_path.decode("utf8") os.environ["PYTHONPATH"] = python_path.strip() if binary_python_path: python_path = os.environ.get("PYTHONPATH", "") os.environ["PYTHONPATH"] = os.pathsep.join( python_path.split(os.pathsep) + \ [os.path.dirname(os.path.abspath(filename))] ) if keep_python_path or binary_python_path: extra_options.append("--keep-pythonpath") if recurse_none: extra_options.append("--recurse-none") if recurse_all: extra_options.append("--recurse-all") if recurse_not: extra_options.extend("--recurse-not-to=" + v for v in recurse_not) if coverage_mode: extra_options.append("--must-not-re-execute") extra_options.append("--generate-c-only") for plugin_enabled in plugins_enabled: extra_options.append("--plugin-enable=" + plugin_enabled) for plugin_disabled in plugins_disabled: extra_options.append("--plugin-disable=" + plugin_disabled) # Now build the command to run Nuitka. if not two_step_execution: if module_mode: nuitka_cmd = nuitka_call + extra_options + \ ["--execute", "--module", filename] elif standalone_mode: nuitka_cmd = nuitka_call + extra_options + \ ["--execute", "--standalone", filename] else: nuitka_cmd = nuitka_call + extra_options + \ ["--execute", filename] if no_site: nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S") else: if module_mode: nuitka_cmd1 = nuitka_call + extra_options + \ ["--module", os.path.abspath(filename)] elif standalone_mode: nuitka_cmd1 = nuitka_call + extra_options + \ ["--standalone", filename] else: nuitka_cmd1 = nuitka_call + extra_options + \ [filename] if no_site: nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S") for extra_option in extra_options: dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option) if dir_match: output_dir = dir_match.group(1) break else: # The default. output_dir = '.' if module_mode: nuitka_cmd2 = [ os.environ["PYTHON"], "-W", "ignore", "-c", "import %s" % os.path.basename(filename) ] else: exe_filename = os.path.basename(filename) if filename.endswith(".py"): exe_filename = exe_filename[:-3] exe_filename = exe_filename.replace(')', "").replace('(', "") exe_filename += ".exe" nuitka_cmd2 = [ os.path.join(output_dir, exe_filename) ] pdb_filename = exe_filename[:-4] + ".pdb" if trace_command: my_print("CPython command:", *cpython_cmd) if comparison_mode: start_time = time.time() process = subprocess.Popen( args = cpython_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout_cpython, stderr_cpython = process.communicate() exit_cpython = process.returncode cpython_time = time.time() - start_time if comparison_mode and not silent_mode: displayOutput(stdout_cpython, stderr_cpython) if comparison_mode and not silent_mode: my_print('*' * 80) my_print("Nuitka:") my_print('*' * 80) if two_step_execution: if output_dir: os.chdir(output_dir) else: tmp_dir = tempfile.gettempdir() # Try to avoid RAM disk /tmp and use the disk one instead. if tmp_dir == "/tmp" and os.path.exists("/var/tmp"): tmp_dir = "/var/tmp" os.chdir(tmp_dir) if trace_command: my_print("Going to output directory", os.getcwd()) start_time = time.time() if not two_step_execution: if trace_command: my_print("Nuitka command:", nuitka_cmd) process = subprocess.Popen( args = nuitka_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout_nuitka, stderr_nuitka = process.communicate() exit_nuitka = process.returncode else: if trace_command: my_print("Nuitka command 1:", nuitka_cmd1) process = subprocess.Popen( args = nuitka_cmd1, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout_nuitka1, stderr_nuitka1 = process.communicate() exit_nuitka1 = process.returncode if exit_nuitka1 != 0: if not expect_failure and \ not comparison_mode and \ not os.path.exists(".coverage"): sys.exit( """\ Error, failed to take coverage with '%s'. Stderr was: %s """ % ( os.environ["PYTHON"], stderr_nuitka1 ) ) exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 else: # No execution second step for coverage mode. if comparison_mode: if trace_command: my_print("Nuitka command 2:", nuitka_cmd2) process = subprocess.Popen( args = nuitka_cmd2, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout_nuitka2, stderr_nuitka2 = process.communicate() stdout_nuitka = stdout_nuitka1 + stdout_nuitka2 stderr_nuitka = stderr_nuitka1 + stderr_nuitka2 exit_nuitka = process.returncode else: exit_nuitka = exit_nuitka1 stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1 nuitka_time = time.time() - start_time if not silent_mode: displayOutput(stdout_nuitka, stderr_nuitka) if coverage_mode: assert not stdout_nuitka assert not stderr_nuitka if comparison_mode: exit_code_stdout = compareOutput( "stdout", stdout_cpython, stdout_nuitka, ignore_warnings, ignore_infos, syntax_errors ) if ignore_stderr: exit_code_stderr = 0 else: exit_code_stderr = compareOutput( "stderr", stderr_cpython, stderr_nuitka, ignore_warnings, ignore_infos, syntax_errors ) exit_code_return = exit_cpython != exit_nuitka if exit_code_return: my_print( """\ Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format( exit_cpython = exit_cpython, exit_nuitka = exit_nuitka ) ) # In case of segfault, also output the call stack by entering debugger # without stdin forwarded. if exit_code_return and exit_nuitka == -11 and sys.platform != "nt": nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger") process = subprocess.Popen( args = nuitka_cmd, stdin = subprocess.PIPE ) process.communicate() exit_code = exit_code_stdout or exit_code_stderr or exit_code_return if exit_code: sys.exit("Error, outputs differed.") if expect_success and exit_cpython != 0: if silent_mode: displayOutput(stdout_cpython, stderr_cpython) sys.exit("Unexpected error exit from CPython.") if expect_failure and exit_cpython == 0: sys.exit("Unexpected success exit from CPython.") if remove_output: if not module_mode: if os.path.exists(nuitka_cmd2[0]): if os.name == "nt": # It appears there is a tiny lock race that we randomly cause, # likely because --run spawns a subprocess that might still # be doing the cleanup work. for _i in range(10): try: os.rename(nuitka_cmd2[0], nuitka_cmd2[0]+".away") except OSError: time.sleep(0.1) continue for _i in range(10): try: os.unlink(nuitka_cmd2[0]+".away") except OSError: time.sleep(2) continue else: break assert not os.path.exists(nuitka_cmd2[0]+".away") if os.path.exists(pdb_filename): os.unlink(pdb_filename) else: os.unlink(nuitka_cmd2[0]) else: if os.name == "nt": module_filename = os.path.basename(filename) + ".pyd" else: module_filename = os.path.basename(filename) + ".so" if os.path.exists(module_filename): os.unlink(module_filename) if comparison_mode and timing: my_print( "CPython took %.2fs vs %0.2fs Nuitka." % ( cpython_time, nuitka_time ) ) if comparison_mode and not silent_mode: my_print("OK, same outputs.")