def getBranchName():
branch_name = check_output("git symbolic-ref --short HEAD".split()).strip()
if str is not bytes:
branch_name = branch_name.decode()
return branch_name
def isStaticallyLinkedPython():
try:
import sysconfig
except ImportError:
# Cannot detect this properly for Python 2.6, but we don't care much
# about that anyway.
return False
result = sysconfig.get_config_var("Py_ENABLE_SHARED") == 0
if result:
from nuitka.utils.Execution import check_output
with open(os.devnull, "w") as devnull:
output = check_output(
(os.path.realpath(sys.executable) + "-config", "--ldflags"),
stderr=devnull,
)
if str is not bytes:
output = output.decode("utf8")
import shlex
output = shlex.split(output)
python_abi_version = python_version_str + getPythonABI()
result = ("-lpython" + python_abi_version) not in output
return result
def publishCoverageData():
def copyToGlobalCoverageData(source, target):
coverage_dir = os.environ.get("COVERAGE_DIR", None)
if coverage_dir is None:
return
subprocess.check_call(("scp", source, os.path.join(coverage_dir, target)))
if os.name == "nt":
suffix = "win"
else:
import platform
suffix = platform.uname()[0] + "." + platform.uname()[4]
with open("data.coverage", "w") as data_file:
source_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
with withDirectoryChange(source_dir):
nuitka_id = check_output("git rev-parse HEAD".split())
nuitka_id = nuitka_id.strip()
if sys.version_info > (3,):
nuitka_id = nuitka_id.decode()
data_file.write("NUITKA_SOURCE_DIR=%r\n" % source_dir)
data_file.write("NUITKA_COMMIT=%r\n" % nuitka_id)
copyToGlobalCoverageData("data.coverage", "meta.coverage." + suffix)
def makeCoverageRelative(filename):
""" Normalize coverage data.
"""
with open(filename) as input_file:
data = input_file.read()
data = data.replace(
(os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), ""
)
if os.path.sep != "/":
data.replace(os.path.sep, "/")
with open(filename, "w") as output_file:
output_file.write(data)
coverage_file = os.environ.get("COVERAGE_FILE", ".coverage")
makeCoverageRelative(coverage_file)
copyToGlobalCoverageData(coverage_file, "data.coverage." + suffix)
def publishCoverageData():
def copyToGlobalCoverageData(source, target):
coverage_dir = os.environ.get("COVERAGE_DIR")
if coverage_dir is None:
return
check_call(("scp", source, os.path.join(coverage_dir, target)))
if os.name == "nt":
suffix = "win"
else:
import platform
suffix = platform.uname()[0] + "." + platform.uname()[4]
with open("data.coverage", "w") as data_file:
source_dir = os.path.abspath(os.path.dirname(
os.path.dirname(__file__)))
with withDirectoryChange(source_dir):
nuitka_id = check_output("git rev-parse HEAD".split())
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
data_file.write("NUITKA_SOURCE_DIR=%r\n" % source_dir)
data_file.write("NUITKA_COMMIT=%r\n" % nuitka_id)
copyToGlobalCoverageData("data.coverage", "meta.coverage." + suffix)
def makeCoverageRelative(filename):
"""Normalize coverage data."""
with open(filename) as input_file:
data = input_file.read()
data = data.replace(
(os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), "")
if os.path.sep != "/":
data.replace(os.path.sep, "/")
with open(filename, "w") as output_file:
output_file.write(data)
coverage_file = os.environ.get("COVERAGE_FILE", ".coverage")
makeCoverageRelative(coverage_file)
copyToGlobalCoverageData(coverage_file, "data.coverage." + suffix)
def queryRuntimeInformationMultiple(self, info_name, setup_codes, values):
if info_name in self._runtime_information_cache:
return self._runtime_information_cache[info_name]
keys = []
query_codes = []
for key, value_expression in values:
keys.append(key)
query_codes.append("print(repr(%s))" % value_expression)
query_codes.append('print("-" * 27)')
if type(setup_codes) is str:
setup_codes = [setup_codes]
cmd = r"""\
from __future__ import print_function
from __future__ import absolute_import
%(setup_codes)s
%(query_codes)s
""" % {
"setup_codes": "\n".join(setup_codes),
"query_codes": "\n".join(query_codes),
}
feedback = check_output([sys.executable, "-c", cmd])
if str is not bytes: # We want to work with strings, that's hopefully OK.
feedback = feedback.decode("utf8")
# Ignore Windows newlines difference.
feedback = [line.strip() for line in feedback.splitlines()]
if feedback.count("-" * 27) != len(keys):
self.sysexit(
"Error, mismatch in output retrieving %r information." % info_name
)
feedback = [line for line in feedback if line != "-" * 27]
NamedTupleResult = namedtuple(info_name, keys)
# We are being lazy here, the code is trusted, pylint: disable=eval-used
self._runtime_information_cache[info_name] = NamedTupleResult(
*(eval(value) for value in feedback)
)
return self._runtime_information_cache[info_name]
def _checkRequiredVersion(tool, tool_call):
required_version = _getRequiredVersion(tool)
for line in _getRequirementsContentsByLine():
if line.startswith(tool + " =="):
required_version = line.split()[2]
break
else:
sys.exit("Error, cannot find %r in requirements-devel.txt" % tool)
if tool == "rstfmt":
if "-m" in tool_call:
return False, "rstfmt doesn't support that yet."
else:
return True, "unchecked"
tool_call = list(tool_call) + ["--version"]
try:
version_output = check_output(tool_call)
except NuitkaCalledProcessError:
return False, "failed to execute"
if str is not bytes:
version_output = version_output.decode("utf8")
for line in version_output.splitlines():
line = line.strip()
if line.startswith(("black, version", "__main__.py, version ")):
actual_version = line.split()[-1]
break
if line.startswith("VERSION "):
actual_version = line.split()[-1]
break
else:
sys.exit(
"Error, couldn't determine version output of %r (%r)"
% (tool, " ".join(tool_call))
)
message = "Version of %r via %r is required to be %r and not %r." % (
tool,
" ".join(tool_call),
required_version,
actual_version,
)
return required_version == actual_version, message
def _checkRequiredVersion(tool, tool_call):
required_version = _getRequiredVersion(tool)
for line in _getRequirementsContentsByLine():
if line.startswith(tool + " =="):
required_version = line.split()[2]
break
else:
sys.exit("Error, cannot find %r in requirements-devel.txt" % tool)
tool_call = list(tool_call) + ["--version"]
try:
version_output = check_output(tool_call)
except NuitkaCalledProcessError as e:
return False, "failed to execute: %s" % e.stderr
if str is not bytes:
version_output = version_output.decode("utf8")
for line in version_output.splitlines():
line = line.strip()
if line.startswith(("black, ", "python -m black,", "__main__.py, ")):
if "(" in line:
line = line[:line.rfind("(")].strip()
actual_version = line.split()[-1]
break
if line.startswith("VERSION "):
actual_version = line.split()[-1]
break
if line.startswith("rstfmt "):
actual_version = line.split()[-1]
break
else:
sys.exit("Error, couldn't determine version output of %r (%r)" %
(tool, " ".join(tool_call)))
message = "Version of %r via %r is required to be %r and not %r." % (
tool,
" ".join(tool_call),
required_version,
actual_version,
)
return required_version == actual_version, message
def updateWorkingFile(path, orig_object_hash, new_object_hash):
patch = check_output(
["git", "diff", "--no-color", orig_object_hash, new_object_hash])
git_path = path.replace(os.path.sep, "/").encode("utf8")
def updateLine(line):
if line.startswith(b"diff --git"):
line = b"diff --git a/%s b/%s" % (git_path, git_path)
elif line.startswith(b"--- a/"):
line = b"--- a/" + git_path
elif line.startswith(b"+++ b/"):
line = b"+++ b/" + git_path
return line
# Substitute object hashes in patch header with path to working tree file
patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n"
# Apply the patch.
output, err, exit_code = executeProcess(
["git", "apply", "-"],
stdin=patch,
)
# Windows extra ball, new files have new lines that make the patch fail.
if exit_code != 0 and os.name == "nt":
from .autoformat.Autoformat import cleanupWindowsNewlines
cleanupWindowsNewlines(path)
output, err, exit_code = executeProcess(
["git", "apply", "-"],
stdin=patch,
)
success = exit_code == 0
if not success:
# TODO: In case of failure, do we need to abort, or what do we do.
if output:
my_print(output, style="yellow")
if err:
my_print(err, style="yellow")
return success
def getUnpushedPaths():
result = set()
output = check_output(
["git", "diff", "--stat", "--name-only", "@{upstream}"])
for line in output.splitlines():
if str is not bytes:
line = line.decode("utf8")
# Removed files appear too, but are useless to talk about.
if not os.path.exists(line):
continue
result.add(line)
return tuple(sorted(result))
def updateWorkingFile(path, orig_object_hash, new_object_hash):
patch = check_output(["git", "diff", orig_object_hash, new_object_hash])
# Substitute object hashes in patch header with path to working tree file
patch_b = patch.replace(orig_object_hash.encode(),
path.encode()).replace(new_object_hash.encode(),
path.encode())
apply_patch = subprocess.Popen(
["git", "apply", "-"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_output, _err = apply_patch.communicate(input=patch_b)
# TODO: In case of failure, do we need to abort?
return apply_patch.returncode != 0
def checkVersion():
# pylint: disable=global-statement
global pylint_version
if not hasModule("pylint"):
sys.exit(
"Error, pylint is not installed for this interpreter %r version." %
os.environ["PYTHON"])
if pylint_version is None:
pylint_version = check_output(
[os.environ["PYTHON"], "-m", "pylint", "--version"],
stderr=getNullOutput())
if str is not bytes:
pylint_version = pylint_version.decode("utf8")
pylint_version = pylint_version.split("\n")[0].split()[-1].strip(",")
my_print("Using PyLint version:", pylint_version)
def updateWorkingFile(path, orig_object_hash, new_object_hash):
patch = check_output(
["git", "diff", "--no-color", orig_object_hash, new_object_hash])
git_path = path.replace(os.path.sep, "/").encode("utf8")
def updateLine(line):
if line.startswith(b"diff --git"):
line = b"diff --git a/%s b/%s" % (git_path, git_path)
elif line.startswith(b"--- a/"):
line = b"--- a/" + git_path
elif line.startswith(b"+++ b/"):
line = b"+++ b/" + git_path
return line
# Substitute object hashes in patch header with path to working tree file
patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n"
# Cannot use executeProcess, because we supply input, and we cannot
# use context managers, because not all Python versions have it that
# way, xpylintx: disable=consider-using-with
apply_patch = subprocess.Popen(
["git", "apply", "-"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, err = apply_patch.communicate(input=patch)
success = apply_patch.returncode == 0
if not success:
# TODO: In case of failure, do we need to abort, or what do we do.
if output:
my_print(output, style="yellow")
if err:
my_print(err, style="yellow")
return success
def _checkRequiredVersion(tool, tool_call):
required_version = _getRequiredVersion(tool)
for line in _getRequirementsContentsByLine():
if line.startswith(tool + " =="):
required_version = line.split()[2]
break
else:
sys.exit("Error, cannot find %r in requirements-devel.txt" % tool)
tool_call = list(tool_call) + ["--version"]
version_output = check_output(tool_call)
if str is not bytes:
version_output = version_output.decode("utf8")
for line in version_output.splitlines():
line = line.strip()
if line.startswith(("black, version", "__main__.py, version ")):
actual_version = line.split()[-1]
break
if line.startswith("VERSION "):
actual_version = line.split()[-1]
break
else:
sys.exit("Error, couldn't determine version output of %r (%r)" %
(tool, " ".join(tool_call)))
message = "Version of %r via %r is required to be %r and not %r." % (
tool,
" ".join(tool_call),
required_version,
actual_version,
)
return required_version == actual_version, message
0,
os.path.normpath(
os.path.join(
os.path.dirname(__file__),
"..",
)
)
)
from nuitka.tools.release.Release import checkAtHome, checkBranchName
from nuitka.utils.Execution import check_output
checkAtHome()
nuitka_version = check_output(
"./bin/nuitka --version", shell = True
).strip()
branch_name = checkBranchName()
if branch_name == "factory":
for remote in "origin", "github":
assert 0 == os.system("git push --recurse-submodules=no --force-with-lease %s factory" % remote)
sys.exit(0)
assert 0 == os.system("rsync -rvlpt --exclude=deb_dist dist/ [email protected]:/var/www/releases/")
for filename in ("README.pdf", "Changelog.pdf", "Developer_Manual.pdf"):
assert 0 == os.system("rsync %s [email protected]:/var/www/doc/" % filename)
def setup(suite="", needs_io_encoding=False, silent=False, go_main=True):
if go_main:
goMainDir()
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
# Allow test code to use this to make caching specific.
os.environ["NUITKA_TEST_SUITE"] = suite
# Allow providing 33, 27, and expand that to python2.7
if (
len(os.environ["PYTHON"]) == 2
and os.environ["PYTHON"].isdigit()
and os.name != "nt"
):
os.environ["PYTHON"] = "python%s.%s" % (
os.environ["PYTHON"][0],
os.environ["PYTHON"][1],
)
if needs_io_encoding and "PYTHONIOENCODING" not in os.environ:
os.environ["PYTHONIOENCODING"] = "utf-8"
version_output = check_output(
(
os.environ["PYTHON"],
"-c",
"""\
import sys, os;\
print(".".join(str(s) for s in list(sys.version_info)[:3]));\
print(("x86_64" if "AMD64" in sys.version else "x86") if os.name == "nt" else os.uname()[4]);\
print(sys.executable);\
print("Anaconda" if os.path.exists(os.path.join(sys.prefix, 'conda-meta')) else "Unknown")\
""",
),
stderr=subprocess.STDOUT,
)
global _python_version_str, _python_version, _python_arch, _python_executable, _python_vendor # singleton, pylint: disable=global-statement
_python_version_str = version_output.split(b"\n")[0].strip()
_python_arch = version_output.split(b"\n")[1].strip()
_python_executable = version_output.split(b"\n")[2].strip()
_python_vendor = version_output.split(b"\n")[3].strip()
if str is not bytes:
_python_version_str = _python_version_str.decode("utf-8")
_python_arch = _python_arch.decode("utf-8")
_python_executable = _python_executable.decode("utf-8")
_python_vendor = _python_vendor.decode("utf-8")
assert type(_python_version_str) is str, repr(_python_version_str)
assert type(_python_arch) is str, repr(_python_arch)
assert type(_python_executable) is str, repr(_python_executable)
if not silent:
my_print("Using concrete python", _python_version_str, "on", _python_arch)
if "COVERAGE_FILE" not in os.environ:
os.environ["COVERAGE_FILE"] = os.path.join(
os.path.dirname(__file__), "..", "..", "..", ".coverage"
)
_python_version = tuple(int(d) for d in _python_version_str.split("."))
return _python_version
import os
import sys
# Unchanged, running from checkout, use the parent directory, the nuitka
# package ought be there.
sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
# isort:start
from nuitka.tools.release.Release import checkAtHome, checkBranchName
from nuitka.utils.Execution import check_output
checkAtHome()
nuitka_version = check_output(
"%s ./bin/nuitka --version" % sys.executable, shell=True
).strip()
branch_name = checkBranchName()
if branch_name == "factory":
for remote in "origin", "github":
assert (
os.system(
"git push --recurse-submodules=no --force-with-lease %s factory"
% remote
)
== 0
)
sys.exit(0)
def main():
print("Querying openSUSE build service status of Nuitka packages.")
osc_cmd = ["osc", "pr", "-c", "home:kayhayen"]
stdout_osc = check_output(args=osc_cmd)
# Response is really a CSV file, so use that for parsing.
csvfile = StringIO(stdout_osc)
osc_reader = csv.reader(csvfile, delimiter=";")
osc_reader = iter(osc_reader)
bad = ("failed", "unresolvable", "broken", "blocked")
titles = osc_reader.next()[1:]
# Nuitka (follow git master branch)
row1 = osc_reader.next()
# Nuitka-Unstable (follow git develop branch)
row2 = osc_reader.next()
problems = []
def decideConsideration(title):
# Ignore other arch builds, they might to not even boot at times.
if "ppc" in title or "aarch" in title or "arm" in title:
return False
# This fails for other reasons often, and is not critical to Nuitka.
if "openSUSE_Tumbleweed" in title:
return False
return True
for count, title in enumerate(titles):
if not decideConsideration(title):
continue
status = row1[count + 1]
if status in bad:
problems.append((row1[0], title, status))
for count, title in enumerate(titles):
if not decideConsideration(title):
continue
status = row2[count + 1]
if status in bad:
problems.append((row2[0], title, status))
if problems:
print("There are problems with:")
print("\n".join("%s: %s (%s)" % problem for problem in problems))
sys.exit(1)
else:
print("Looks good.")
sys.exit(0)
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
# Unchanged, running from checkout, use the parent directory, the nuitka
# package ought be there.
sys.path.insert(
0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
# isort:start
import shutil
import subprocess
from nuitka.tools.release.Release import checkAtHome, checkBranchName
from nuitka.utils.Execution import check_output
checkAtHome()
nuitka_version = check_output("./bin/nuitka --version", shell=True).strip()
branch_name = checkBranchName()
if branch_name == "factory":
for remote in "origin", "github":
assert 0 == os.system(
"git push --recurse-submodules=no --force-with-lease %s factory" %
remote)
sys.exit(0)
assert 0 == os.system(
"rsync -rvlpt --exclude=deb_dist dist/ [email protected]:/var/www/releases/")
for filename in ("README.pdf", "Changelog.pdf", "Developer_Manual.pdf"):
def getFileHashContent(object_hash):
return check_output(["git", "cat-file", "-p", object_hash])
def main():
my_print("Querying openSUSE build service status of Nuitka packages.")
osc_cmd = ["osc", "pr", "-c", "home:kayhayen"]
stdout_osc = check_output(args=osc_cmd)
if str is not bytes:
stdout_osc = stdout_osc.decode("utf8")
# Response is really a CSV file, so use that for parsing.
csvfile = StringIO(stdout_osc)
osc_reader = csv.reader(csvfile, delimiter=";")
osc_reader = iter(osc_reader)
bad = ("failed", "unresolvable", "broken", "blocked")
titles = next(osc_reader)[1:]
# Nuitka (follow git main branch)
row1 = next(osc_reader)
# Nuitka-Unstable (follow git develop branch)
row2 = next(osc_reader)
# Nuitka-Experimental (follow git factory branch)
row3 = next(osc_reader)
problems = []
def decideConsideration(title, status):
# Ignore other arch builds, they might to not even boot at times.
if "ppc" in title or "aarch" in title or "arm" in title:
return False
# This fails for other reasons often, and is not critical to Nuitka.
if "openSUSE_Tumbleweed" in title:
return False
# Ignore old Fedora and RHEL6 32 bit being blocked.
if status == "blocked":
if ("Fedora_2" in title or "RedHat_RHEL-6/i586" in title
or "CentOS_CentOS-6/i586" in title):
return False
# It makes building visible now, that's not an error of course.
if status == "building":
return False
return True
for count, title in enumerate(titles):
status = row1[count + 1]
if not decideConsideration(title, status):
continue
if status in bad:
problems.append((row1[0], title, status))
for count, title in enumerate(titles):
status = row2[count + 1]
if not decideConsideration(title, status):
continue
if status in bad:
problems.append((row2[0], title, status))
for count, title in enumerate(titles):
status = row3[count + 1]
if not decideConsideration(title, status):
continue
if status in bad:
problems.append((row3[0], title, status))
if problems:
my_print("There are problems with:", style="yellow")
my_print("\n".join("%s: %s (%s)" % problem for problem in problems),
style="yellow")
sys.exit(1)
else:
my_print("Looks good.", style="blue")
sys.exit(0)
def main():
# There are freaking many options to honor, pylint: disable=too-many-branches
# Lets honor this Debian option here.
if "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split():
print("Skipped all tests as per DEB_BUILD_OPTIONS environment.")
sys.exit(0)
goHome()
parser = OptionParser()
parser.add_option("--skip-basic-tests",
action="store_false",
dest="basic_tests",
default=True,
help="""\
The basic tests, execute these to check if Nuitka is healthy.
Default is %default.""")
parser.add_option("--skip-syntax-tests",
action="store_false",
dest="syntax_tests",
default=True,
help="""\
The syntax tests, execute these to check if Nuitka handles Syntax errors fine.
Default is %default.""")
parser.add_option("--skip-program-tests",
action="store_false",
dest="program_tests",
default=True,
help="""\
The programs tests, execute these to check if Nuitka handles programs, e.g.
import recursions, etc. fine. Default is %default.""")
parser.add_option("--skip-package-tests",
action="store_false",
dest="package_tests",
default=True,
help="""\
The packages tests, execute these to check if Nuitka handles packages, e.g.
import recursions, etc. fine. Default is %default.""")
parser.add_option("--skip-optimizations-tests",
action="store_false",
dest="optimization_tests",
default=True,
help="""\
The optimization tests, execute these to check if Nuitka does optimize certain
constructs fully away. Default is %default.""")
parser.add_option(
"--skip-standalone-tests",
action="store_false",
dest="standalone_tests",
default=os.name != "posix"
or os.uname()[0] != "NetBSD", # @UndefinedVariable
help="""\
The standalone tests, execute these to check if Nuitka standalone mode, e.g.
not referring to outside, important 3rd library packages like PyQt fine.
Default is %default.""")
parser.add_option("--skip-reflection-test",
action="store_false",
dest="reflection_test",
default=True,
help="""\
The reflection test compiles Nuitka with Nuitka, and then Nuitka with the
compile Nuitka and compares the outputs. Default is %default.""")
parser.add_option("--skip-cpython26-tests",
action="store_false",
dest="cpython26",
default=True,
help="""\
The standard CPython2.6 test suite. Execute this for all corner cases to be
covered. With Python 2.7 this covers exception behavior quite well. Default
is %default.""")
parser.add_option("--skip-cpython27-tests",
action="store_false",
dest="cpython27",
default=True,
help="""\
The standard CPython2.7 test suite. Execute this for all corner cases to be
covered. With Python 2.6 these are not run. Default is %default.""")
parser.add_option("--skip-cpython32-tests",
action="store_false",
dest="cpython32",
default=True,
help="""\
The standard CPython3.2 test suite. Execute this for all corner cases to be
covered. With Python 2.6 these are not run. Default is %default.""")
parser.add_option("--skip-cpython33-tests",
action="store_false",
dest="cpython33",
default=True,
help="""\
The standard CPython3.3 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython34-tests",
action="store_false",
dest="cpython34",
default=True,
help="""\
The standard CPython3.4 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython35-tests",
action="store_false",
dest="cpython35",
default=True,
help="""\
The standard CPython3.5 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython36-tests",
action="store_false",
dest="cpython36",
default=True,
help="""\
The standard CPython3.6 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-other-cpython-tests",
action="store_true",
dest="cpython_no_other",
default=False,
help="""\
Do not execute any CPython test suite other than the one matching the running
Python. Default is %default.""")
parser.add_option("--skip-all-cpython-tests",
action="store_true",
dest="cpython_none",
default=False,
help="""\
Do not execute any CPython test suite other than the one matching the running
Python. Default is %default.""")
parser.add_option("--no-other-python",
action="store_true",
dest="no_other",
default=False,
help="""\
Do not use any other Python than the one running, even if available on
the system. Default is %default.""")
parser.add_option("--no-python2.6",
action="store_true",
dest="no26",
default=False,
help="""\
Do not use Python2.6 even if available on the system. Default is %default.""")
parser.add_option("--no-python2.7",
action="store_true",
dest="no27",
default=False,
help="""\
Do not use Python2.7 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.2",
action="store_true",
dest="no32",
default=False,
help="""\
Do not use Python3.2 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.3",
action="store_true",
dest="no33",
default=False,
help="""\
Do not use Python3.3 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.4",
action="store_true",
dest="no34",
default=False,
help="""\
Do not use Python3.4 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.5",
action="store_true",
dest="no35",
default=False,
help="""\
Do not use Python3.5 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.6",
action="store_true",
dest="no36",
default=False,
help="""\
Do not use Python3.6 even if available on the system. Default is %default.""")
parser.add_option("--coverage",
action="store_true",
dest="coverage",
default=False,
help="""\
Make a coverage analysis, that does not really check. Default is %default.""")
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
if options.no_other:
if sys.version_info[0:2] != (2, 6):
options.no26 = True
if sys.version_info[0:2] != (2, 7):
options.no27 = True
if sys.version_info[0:2] != (3, 2):
options.no32 = True
if sys.version_info[0:2] != (3, 3):
options.no33 = True
if sys.version_info[0:2] != (3, 4):
options.no34 = True
if sys.version_info[0:2] != (3, 5):
options.no35 = True
if sys.version_info[0:2] != (3, 6):
options.no36 = True
if options.cpython_no_other:
if sys.version_info[0:2] != (2, 6):
options.cpython26 = False
if sys.version_info[0:2] != (2, 7):
options.cpython27 = False
if sys.version_info[0:2] != (3, 2):
options.cpython32 = False
if sys.version_info[0:2] != (3, 3):
options.cpython33 = False
if sys.version_info[0:2] != (3, 4):
options.cpython34 = False
if sys.version_info[0:2] != (3, 5):
options.cpython35 = False
if sys.version_info[0:2] != (3, 6):
options.cpython36 = False
if options.cpython_none:
options.cpython26 = False
options.cpython27 = False
options.cpython32 = False
options.cpython33 = False
options.cpython34 = False
options.cpython35 = False
options.cpython36 = False
if options.coverage and os.path.exists(".coverage"):
os.unlink(".coverage")
# Add the local bin directory to search path start.
os.environ["PATH"] = \
os.path.join(
os.getcwd(),
"bin" ) + \
os.pathsep + \
os.environ["PATH"]
def checkExecutableCommand(command):
""" Check if a command is executable. """
# Many cases, pylint: disable=too-many-branches,too-many-return-statements
# Do respect given options to disable specific Python versions
if command == "python2.6" and options.no26:
return False
if command == "python2.7" and options.no27:
return False
if command == "python3.2" and options.no32:
return False
if command == "python3.3" and options.no33:
return False
if command == "python3.4" and options.no34:
return False
if command == "python3.5" and options.no35:
return False
if command == "python3.6" and options.no36:
return False
# Shortcuts for python versions, also needed for Windows as it won't have
# the version number in the Python binaries at all.
if command == "python2.6" and sys.version_info[0:2] == (2, 6):
return True
if command == "python2.7" and sys.version_info[0:2] == (2, 7):
return True
if command == "python3.2" and sys.version_info[0:2] == (3, 2):
return True
if command == "python3.3" and sys.version_info[0:2] == (3, 3):
return True
if command == "python3.4" and sys.version_info[0:2] == (3, 4):
return True
if command == "python3.5" and sys.version_info[0:2] == (3, 5):
return True
if command == "python3.6" and sys.version_info[0:2] == (3, 6):
return True
path = os.environ["PATH"]
suffixes = (".exe", ) if os.name == "nt" else ("", )
for part in path.split(os.pathsep):
if not part:
continue
for suffix in suffixes:
if os.path.exists(os.path.join(part, command + suffix)):
return True
if os.name == "nt":
if command.startswith("python"):
remainder = command[6:]
if len(remainder) == 3 and remainder[1] == '.':
command = getPythonExePathWindows(search=remainder,
arch=None)
return True
return False
def setExtraFlags(where, name, flags):
if where is not None:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
where = os.path.join(tmp_dir, name, where)
if not os.path.exists(where):
os.makedirs(where)
os.environ[
"NUITKA_EXTRA_OPTIONS"] = flags + " --output-dir=" + where
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = flags
def executeSubTest(command, hide_output=False):
if options.coverage and "search" in command:
command = command.replace("search", "coverage")
parts = command.split()
parts[0] = parts[0].replace('/', os.path.sep)
# The running Python will be good enough, on some platforms there is no
# "python", and we need to pass this alone then.
parts.insert(0, sys.executable)
print("Run '%s' in '%s'." % (' '.join(parts), os.getcwd()))
sys.stdout.flush()
if hide_output:
result = subprocess.call(parts, stdout=open(os.devnull, 'w'))
else:
result = subprocess.call(parts)
if result != 0:
sys.exit(result)
def execute_tests(where, use_python, flags):
# Many cases, pylint: disable=too-many-branches,too-many-statements
print(
"Executing test case called %s with CPython %s and extra flags '%s'."
% (where, use_python, flags))
intended_version = use_python[6:]
if sys.version.startswith(intended_version):
os.environ["PYTHON"] = sys.executable
else:
if os.name == "nt":
os.environ["PYTHON"] = getPythonExePathWindows(
search=intended_version, arch=None)
else:
os.environ["PYTHON"] = getExecutablePath(use_python)
if options.basic_tests:
print("Running the basic tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "basics", flags)
executeSubTest("./tests/basics/run_all.py search")
if options.syntax_tests:
print("Running the syntax tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "syntax", flags)
executeSubTest("./tests/syntax/run_all.py search")
if options.program_tests:
print("Running the program tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "programs", flags)
executeSubTest("./tests/programs/run_all.py search")
if options.package_tests:
print("Running the package tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "packages", flags)
executeSubTest("./tests/packages/run_all.py search")
# At least one Debian Jessie, these versions won't have lxml installed, so
# don't run them there. Also these won't be very version dependent in their
# results.
if use_python != "python2.6" and use_python != "python3.2":
if options.optimization_tests:
print(
"Running the optimizations tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "optimizations", flags)
executeSubTest("./tests/optimizations/run_all.py search")
if options.standalone_tests and not options.coverage:
print("Running the standalone tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(None, "standalone", flags)
executeSubTest("./tests/standalone/run_all.py search")
if options.reflection_test and not options.coverage:
print("Running the reflection test with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(None, "reflected", flags)
executeSubTest("./tests/reflected/compile_itself.py search")
if not use_python.startswith("python3"):
if os.path.exists("./tests/CPython26/run_all.py"):
if options.cpython26:
print(
"Running the CPython 2.6 tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "26tests", flags)
executeSubTest("./tests/CPython26/run_all.py search")
else:
print("The CPython2.6 tests are not present, not run.")
# Running the Python 2.7 test suite with CPython 2.6 gives little
# insight, because "importlib" will not be there and that's it.
if use_python != "python2.6":
if os.path.exists("./tests/CPython27/run_all.py"):
if options.cpython27:
print(
"Running the CPython 2.7 tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "27tests", flags)
executeSubTest("./tests/CPython27/run_all.py search")
else:
print("The CPython2.7 tests are not present, not run.")
if "--debug" not in flags:
# Not running the Python 3.2 test suite with CPython2.6, as that's about
# the same as CPython2.7 and won't have any new insights.
if use_python != "python2.6" and \
use_python != "python2.7" or not options.coverage:
if os.path.exists("./tests/CPython32/run_all.py"):
if options.cpython32:
setExtraFlags(where, "32tests", flags)
executeSubTest("./tests/CPython32/run_all.py search")
else:
print("The CPython3.2 tests are not present, not run.")
# Running the Python 3.3 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython33/run_all.py"):
if options.cpython33:
setExtraFlags(where, "33tests", flags)
executeSubTest("./tests/CPython33/run_all.py search")
else:
print("The CPython3.3 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython34/run_all.py"):
if options.cpython34:
setExtraFlags(where, "34tests", flags)
executeSubTest("./tests/CPython34/run_all.py search")
else:
print("The CPython3.4 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython35/run_all.py"):
if options.cpython35:
setExtraFlags(where, "35tests", flags)
executeSubTest("./tests/CPython35/run_all.py search")
else:
print("The CPython3.5 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "36tests", flags)
executeSubTest("./tests/CPython36/run_all.py search")
else:
print("The CPython3.6 tests are not present, not run.")
if "NUITKA_EXTRA_OPTIONS" in os.environ:
del os.environ["NUITKA_EXTRA_OPTIONS"]
assert checkExecutableCommand("python2.6") or \
checkExecutableCommand("python2.7") or \
checkExecutableCommand("python3.2") or \
checkExecutableCommand("python3.3") or \
checkExecutableCommand("python3.4") or \
checkExecutableCommand("python3.5") or \
checkExecutableCommand("python3.6")
# Just the quick syntax test, full tests are run later.
if checkExecutableCommand("python3.2"):
executeSubTest("./bin/nuitka --python-version=3.2 --version",
hide_output=True)
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-debug", "python2.6", "--debug")
else:
print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-debug", "python2.7", "--debug")
else:
print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.2"):
execute_tests("python3.2-debug", "python3.2", "--debug")
else:
print(
"Cannot execute tests with Python 3.2, disabled or not installed.")
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-nodebug", "python2.6", "")
else:
print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-nodebug", "python2.7", "")
else:
print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.2"):
execute_tests("python3.2-nodebug", "python3.2", "")
else:
print(
"Cannot execute tests with Python 3.2, disabled or not installed.")
if checkExecutableCommand("python3.3"):
execute_tests("python3.3-nodebug", "python3.3", "")
else:
print(
"Cannot execute tests with Python 3.3, disabled or not installed.")
if checkExecutableCommand("python3.4"):
execute_tests("python3.4-nodebug", "python3.4", "")
else:
print(
"Cannot execute tests with Python 3.4, disabled or not installed.")
if checkExecutableCommand("python3.5"):
execute_tests("python3.5-nodebug", "python3.5", "")
else:
print(
"Cannot execute tests with Python 3.5, disabled or not installed.")
if checkExecutableCommand("python3.6"):
execute_tests("python3.6-nodebug", "python3.6", "")
else:
print(
"Cannot execute tests with Python 3.6, disabled or not installed.")
if options.coverage:
def copyToGlobalCoverageData(source, target):
coverage_dir = os.environ.get("COVERAGE_DIR", None)
if coverage_dir is None:
return
assert subprocess.call(
("C:\\MinGW\\msys\\1.0\\bin\\scp.exe" if os.name == "nt" else
"scp", source, os.path.join(coverage_dir, target))) == 0
if os.name == "nt":
suffix = "win"
else:
import platform
suffix = platform.uname()[0] + '.' + platform.uname()[4]
with open("data.coverage", 'w') as data_file:
source_dir = os.path.abspath(
os.path.dirname(os.path.dirname(__file__)))
nuitka_id = check_output("cd '%s'; git rev-parse HEAD" %
source_dir,
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
data_file.write("NUITKA_COMMIT='%s'\n" % nuitka_id)
copyToGlobalCoverageData("data.coverage", "data.coverage." + suffix)
def makeCoverageRelative(filename):
""" Normalize coverage data.
"""
with open(filename) as input_file:
data = input_file.read()
data = data.replace(os.path.abspath('.') + os.path.sep, "")
if os.path.sep != '/':
data.replace(os.path.sep, '/')
with open(filename, 'w') as output_file:
output_file.write(data)
coverage_file = os.environ.get("COVERAGE_FILE", ".coverage")
makeCoverageRelative(coverage_file)
copyToGlobalCoverageData(coverage_file, ".coverage." + suffix)
print("OK.")
def getBinarySizes(filename):
command = ["size", filename]
sizes = check_output(command).strip()
sizes = sizes.split(b'\n')[-1].replace(b'\t', b"").split()
return int(sizes[0]), int(sizes[1])
def getBinarySizes(filename):
command = ["size", filename]
sizes = check_output(command).strip()
sizes = sizes.split(b"\n")[-1].replace(b"\t", b"").split()
return int(sizes[0]), int(sizes[1])
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
from nuitka.utils.Execution import check_output
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
ignore_infos = hasArg("ignore_infos")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = hasArg("trace_command")
remove_output = hasArg("remove_output")
standalone_mode = hasArg("standalone")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
syntax_errors = hasArg("syntax_errors")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:"):])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:"):])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:"):])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ','.join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4]+"_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4]+"_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".
format(
filename = filename,
python = os.environ["PYTHON"],
args = ", ".join(arguments)
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".
format(
filename = filename,
python = os.environ["PYTHON"],
args = ", ".join(arguments)
)
)
if comparison_mode and not silent_mode:
my_print('*' * 80)
my_print("CPython:")
my_print('*' * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W", "ignore",
"-c", "import sys; sys.path.append(%s); import %s" % (
repr(os.path.dirname(filename)),
os.path.basename(filename)
)
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c", "import sys; sys.path.append(%s); import %s" % (
repr(os.path.dirname(filename)),
os.path.basename(filename)
)
]
else:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W", "ignore",
filename
]
else:
cpython_cmd = [
os.environ["PYTHON"],
filename
]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"bin",
"nuitka"
)
)
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"bin",
"nuitka"
)
)
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if full_compat:
extra_options.append("--full-compat")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c"
"import sys, os; print(os.pathsep.join(sys.path))"
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
python_path = os.environ.get("PYTHONPATH", "")
os.environ["PYTHONPATH"] = os.pathsep.join(
python_path.split(os.pathsep) + \
[os.path.dirname(os.path.abspath(filename))]
)
if keep_python_path or binary_python_path:
extra_options.append("--keep-pythonpath")
if recurse_none:
extra_options.append("--recurse-none")
if recurse_all:
extra_options.append("--recurse-all")
if recurse_not:
extra_options.extend("--recurse-not-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + \
["--execute", "--module", filename]
elif standalone_mode:
nuitka_cmd = nuitka_call + extra_options + \
["--execute", "--standalone", filename]
else:
nuitka_cmd = nuitka_call + extra_options + \
["--execute", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = nuitka_call + extra_options + \
["--module", os.path.abspath(filename)]
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + \
["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + \
[filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = '.'
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W", "ignore",
"-c", "import %s" % os.path.basename(filename)
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(')', "").replace('(', "")
exe_filename += ".exe"
nuitka_cmd2 = [
os.path.join(output_dir, exe_filename)
]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
start_time = time.time()
process = subprocess.Popen(
args = cpython_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
cpython_time = time.time() - start_time
if comparison_mode and not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print('*' * 80)
my_print("Nuitka:")
my_print('*' * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
start_time = time.time()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
process = subprocess.Popen(
args = nuitka_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
process = subprocess.Popen(
args = nuitka_cmd1,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if not expect_failure and \
not comparison_mode and \
not os.path.exists(".coverage"):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
""" % (
os.environ["PYTHON"],
stderr_nuitka1
)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args = nuitka_cmd2,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
nuitka_time = time.time() - start_time
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
exit_code_stdout = compareOutput(
"stdout",
stdout_cpython,
stdout_nuitka,
ignore_warnings,
ignore_infos,
syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
ignore_infos,
syntax_errors
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return:
my_print(
"""\
Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython = exit_cpython,
exit_nuitka = exit_nuitka
)
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if exit_code_return and exit_nuitka == -11 and sys.platform != "nt":
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
process = subprocess.Popen(
args = nuitka_cmd,
stdin = subprocess.PIPE
)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
sys.exit("Error, outputs differed.")
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0]+".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0]+".away")
except OSError:
time.sleep(2)
continue
else:
break
assert not os.path.exists(nuitka_cmd2[0]+".away")
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
if os.name == "nt":
module_filename = os.path.basename(filename) + ".pyd"
else:
module_filename = os.path.basename(filename) + ".so"
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print(
"CPython took %.2fs vs %0.2fs Nuitka." % (
cpython_time,
nuitka_time
)
)
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
