def _getCPythonResults(cpython_cmd, send_kill):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(args=cpython_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if send_kill:
# Doing it per loop iteration hopefully, pylint: disable=cell-var-from-loop
executeAfterTimePassed(
1.0,
lambda: killProcess("Uncompiled Python program", process.pid))
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def _getCPythonResults(cpython_cmd):
stop_watch = StopWatch()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
stop_watch.start()
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args=cpython_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
stop_watch.stop()
if checkNoPermissionError(stdout_cpython) and checkNoPermissionError(
stderr_cpython
):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
cpython_time = stop_watch.getDelta()
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def getCPythonResults(cpython_cmd, cpython_cached):
cached = False
if cpython_cached:
# TODO: Hashing stuff and creating cache filename is duplicate code
# and should be shared.
hash_input = " -- ".join(cpython_cmd)
if str is not bytes:
hash_input = hash_input.encode("utf8")
command_hash = hashlib.md5(hash_input)
cache_filename = os.path.join(
getTestingCPythonOutputsCacheDir(),
command_hash.hexdigest()
)
if os.path.exists(cache_filename):
with open(cache_filename, "rb") as cache_file:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = \
pickle.load(cache_file)
cached = True
if not cached:
start_time = time.time()
# Try a coupile of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(os.getcwd()):
process = subprocess.Popen(
args = cpython_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
if checkNoPermissionError(stdout_cpython) and \
checkNoPermissionError(stderr_cpython):
break
my_print("Retrying CPython due to permission problems after delay.")
time.sleep(2)
start_time = time.time()
cpython_time = time.time() - start_time
if cpython_cached:
with open(cache_filename, "wb") as cache_file:
pickle.dump(
(cpython_time, stdout_cpython, stderr_cpython, exit_cpython),
cache_file
)
return cpython_time, stdout_cpython, stderr_cpython, exit_cpython
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(".build")
or filename.endswith(".dist")):
continue
filename = os.path.relpath(filename)
# For these, we expect that they will fail.
expected_errors = [
"module_exits",
"main_raises",
"main_raises2",
"package_contains_main",
]
# After Python3 those have been made to work.
if python_version < "3.5":
expected_errors.append("cyclic_imports")
# Allowed with Python3, packages need no more "__init__.py"
if python_version < "3":
expected_errors.append("package_missing_init")
# Allowed with Python3 only:
if python_version < "3.5":
expected_errors.append("package_init_issue")
if filename not in expected_errors:
extra_flags = ["expect_success"]
else:
extra_flags = ["expect_failure"]
if filename in (
"reimport_main_static",
"package_missing_init",
"dash_import",
"package_contains_main",
"case_imports3",
"import_variants",
"package_init_import",
"pkgutil_itermodules",
):
extra_flags.append("ignore_warnings")
extra_flags.append("remove_output")
extra_flags.append("recurse_all")
# Use the original __file__ value, at least one case warns about things
# with filename included, but for pkgutil iteration, make sure we do not
# see original Python dirs.
if filename != "pkgutil_itermodules":
extra_flags.append("original_file")
else:
extra_flags.append("runtime_file")
# Cannot include the files with syntax errors, these would then become
# ImportError, but that's not the test. In all other cases, use two
# step execution, which will not add the program original source to
# PYTHONPATH.
if filename != "syntax_errors":
extra_flags.append("two_step_execution")
else:
extra_flags.append("binary_python_path")
if filename == "plugin_import":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-package=some_package")
elif filename == "reimport_main_dynamic":
if python_version < "3":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-plugin-directory=%s" %
(os.path.abspath(filename)))
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-plugin-files=%s/*.py" %
(os.path.abspath(filename)))
extra_flags.append("ignore_warnings")
elif filename == "multiprocessing_using":
if os.name == "nt":
extra_flags += [
"plugin_enable:multiprocessing", "ignore_infos"
]
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
filename_main = getMainProgramFilename(filename)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
if filename_main.endswith("Main"):
break
else:
sys.exit(
"""\
Error, no file ends with 'Main.py' or 'Main' in %s, incomplete test case.""" % (
filename
)
)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in
os.listdir(filename)
if entry.startswith("path")
]
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname = filename,
filename = filename_main,
extra_flags = extra_flags,
search_mode = search_mode,
needs_2to3 = False
)
else:
my_print("Skipping", filename)
search_mode.finish()
if active:
my_print("Consider output of recursively compiled program:", filename)
for filename_main in os.listdir(filename):
if filename_main.endswith("Main.py"):
break
if filename_main.endswith("Main"):
break
else:
sys.exit("""\
Error, no file ends with 'Main.py' or 'Main' in %s, incomplete test case.""" %
(filename))
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
with withPythonPathChange(extra_python_path):
compareWithCPython(dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False)
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
# Of course many cases to deal with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
filename = sys.argv[1]
args = sys.argv[2:]
def hasArg(arg):
if arg in args:
args.remove(arg)
return True
else:
return False
# For output keep it
arguments = list(args)
silent_mode = hasArg("silent")
ignore_stderr = hasArg("ignore_stderr")
ignore_warnings = hasArg("ignore_warnings")
expect_success = hasArg("expect_success")
expect_failure = hasArg("expect_failure")
python_debug = hasArg("python_debug")
module_mode = hasArg("module_mode")
two_step_execution = hasArg("two_step_execution")
binary_python_path = hasArg("binary_python_path")
keep_python_path = hasArg("keep_python_path")
trace_command = (
hasArg("trace_command") or os.environ.get("NUITKA_TRACE_COMMANDS", "0") != "0"
)
remove_output = hasArg("remove_output")
standalone_mode = hasArg("--standalone")
onefile_mode = hasArg("--onefile")
no_site = hasArg("no_site")
recurse_none = hasArg("recurse_none")
recurse_all = hasArg("recurse_all")
timing = hasArg("timing")
coverage_mode = hasArg("coverage")
original_file = hasArg("original_file")
runtime_file = hasArg("runtime_file")
no_warnings = not hasArg("warnings")
full_compat = not hasArg("improved")
cpython_cached = hasArg("cpython_cache")
syntax_errors = hasArg("syntax_errors")
noprefer_source = hasArg("noprefer_source")
noverbose_log = hasArg("noverbose_log")
noinclusion_log = hasArg("noinclusion_log")
plugins_enabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_enable:"):
plugins_enabled.append(arg[len("plugin_enable:") :])
del args[count]
plugins_disabled = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("plugin_disable:"):
plugins_disabled.append(arg[len("plugin_disable:") :])
del args[count]
user_plugins = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("user_plugin:"):
user_plugins.append(arg[len("user_plugin:") :])
del args[count]
recurse_not = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_not:"):
recurse_not.append(arg[len("recurse_not:") :])
del args[count]
recurse_to = []
for count, arg in reversed(tuple(enumerate(args))):
if arg.startswith("recurse_to:"):
recurse_to.append(arg[len("recurse_to:") :])
del args[count]
if args:
sys.exit("Error, non understood mode(s) '%s'," % ",".join(args))
# In coverage mode, we don't want to execute, and to do this only in one mode,
# we enable two step execution, which splits running the binary from the actual
# compilation:
if coverage_mode:
two_step_execution = True
# The coverage mode doesn't work with debug mode.
if coverage_mode:
python_debug = False
comparison_mode = not coverage_mode
assert not standalone_mode or not module_mode
assert not recurse_all or not recurse_none
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
os.environ["PYTHONWARNINGS"] = "ignore"
if "PYTHON" not in os.environ:
os.environ["PYTHON"] = sys.executable
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
if "--python-debug" in extra_options or "--python-dbg" in extra_options:
python_debug = True
if python_debug:
if os.path.exists(os.path.join("/usr/bin/", os.environ["PYTHON"] + "-dbg")):
os.environ["PYTHON"] += "-dbg"
if os.name == "nt":
if os.path.exists(os.environ["PYTHON"][:-4] + "_d.exe"):
os.environ["PYTHON"] = os.environ["PYTHON"][:-4] + "_d.exe"
if os.environ["PYTHON"].endswith("-dbg"):
python_debug = True
if os.environ["PYTHON"].lower().endswith("_d.exe"):
python_debug = True
if comparison_mode:
my_print(
"""\
Comparing output of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
else:
my_print(
"""\
Taking coverage of '{filename}' using '{python}' with flags {args} ...""".format(
filename=filename,
python=os.environ["PYTHON"],
args=", ".join(arguments),
)
)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("CPython:")
my_print("*" * 80)
if two_step_execution:
filename = os.path.abspath(filename)
if module_mode:
if no_warnings:
cpython_cmd = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
cpython_cmd = [
os.environ["PYTHON"],
"-c",
"import sys; sys.path.append(%s); import %s"
% (repr(os.path.dirname(filename)), os.path.basename(filename)),
]
else:
if no_warnings:
cpython_cmd = [os.environ["PYTHON"], "-W", "ignore", filename]
else:
cpython_cmd = [os.environ["PYTHON"], filename]
if no_site:
cpython_cmd.insert(1, "-S")
if "NUITKA" in os.environ:
# Would need to extract which "python" this is going to use.
assert not coverage_mode, "Not implemented for binaries."
nuitka_call = [os.environ["NUITKA"]]
else:
if comparison_mode:
nuitka_call = [
os.environ["PYTHON"],
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
else:
assert coverage_mode
nuitka_call = [
os.environ["PYTHON"],
"-S",
"-m",
"coverage",
"run",
"--rcfile",
os.devnull,
"-a",
"-m",
"nuitka.__main__", # Note: Needed for Python2.6
]
if python_debug:
extra_options.append("--python-debug")
if no_warnings:
extra_options.append("--python-flag=no_warnings")
if remove_output:
extra_options.append("--remove-output")
if original_file:
extra_options.append("--file-reference-choice=original")
if runtime_file:
extra_options.append("--file-reference-choice=runtime")
if full_compat:
extra_options.append("--full-compat")
if noprefer_source:
extra_options.append("--no-prefer-source")
if coverage_mode:
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output(
[
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
]
)
if sys.version_info >= (3,):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
if binary_python_path:
addToPythonPath(os.path.dirname(os.path.abspath(filename)))
if keep_python_path or binary_python_path:
extra_options.append("--execute-with-pythonpath")
if recurse_none:
extra_options.append("--nofollow-imports")
if recurse_all:
extra_options.append("--follow-imports")
if recurse_not:
extra_options.extend("--nofollow-import-to=" + v for v in recurse_not)
if coverage_mode:
extra_options.append("--must-not-re-execute")
extra_options.append("--generate-c-only")
for plugin_enabled in plugins_enabled:
extra_options.append("--plugin-enable=" + plugin_enabled)
for plugin_disabled in plugins_disabled:
extra_options.append("--plugin-disable=" + plugin_disabled)
for user_plugin in user_plugins:
extra_options.append("--user-plugin=" + user_plugin)
if not noverbose_log:
extra_options.append("--verbose-output=%s.optimization.log" % filename)
if not noinclusion_log:
extra_options.append("--show-modules-output=%s.inclusion.log" % filename)
# Now build the command to run Nuitka.
if not two_step_execution:
if module_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--module", filename]
elif onefile_mode:
nuitka_cmd = nuitka_call + extra_options + ["--run", "--onefile", filename]
elif standalone_mode:
nuitka_cmd = (
nuitka_call + extra_options + ["--run", "--standalone", filename]
)
else:
nuitka_cmd = nuitka_call + extra_options + ["--run", filename]
if no_site:
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--python-flag=-S")
else:
if module_mode:
nuitka_cmd1 = (
nuitka_call + extra_options + ["--module", os.path.abspath(filename)]
)
elif standalone_mode:
nuitka_cmd1 = nuitka_call + extra_options + ["--standalone", filename]
else:
nuitka_cmd1 = nuitka_call + extra_options + [filename]
if no_site:
nuitka_cmd1.insert(len(nuitka_cmd1) - 1, "--python-flag=-S")
for extra_option in extra_options:
dir_match = re.search(r"--output-dir=(.*?)(\s|$)", extra_option)
if dir_match:
output_dir = dir_match.group(1)
break
else:
# The default.
output_dir = "."
if module_mode:
nuitka_cmd2 = [
os.environ["PYTHON"],
"-W",
"ignore",
"-c",
"import %s" % os.path.basename(filename),
]
else:
exe_filename = os.path.basename(filename)
if filename.endswith(".py"):
exe_filename = exe_filename[:-3]
exe_filename = exe_filename.replace(")", "").replace("(", "")
exe_filename += ".exe" if os.name == "nt" else ".bin"
nuitka_cmd2 = [os.path.join(output_dir, exe_filename)]
pdb_filename = exe_filename[:-4] + ".pdb"
if trace_command:
my_print("CPython command:", *cpython_cmd)
if comparison_mode:
cpython_time, stdout_cpython, stderr_cpython, exit_cpython = getCPythonResults(
cpython_cmd=cpython_cmd, cpython_cached=cpython_cached, force_update=False
)
if not silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
if comparison_mode and not silent_mode:
my_print("*" * 80)
my_print("Nuitka:")
my_print("*" * 80)
if two_step_execution:
if output_dir:
os.chdir(output_dir)
else:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
os.chdir(tmp_dir)
if trace_command:
my_print("Going to output directory", os.getcwd())
stop_watch = StopWatch()
stop_watch.start()
if not two_step_execution:
if trace_command:
my_print("Nuitka command:", nuitka_cmd)
# Try a couple of times for permission denied, on Windows it can
# be transient.
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
else:
if trace_command:
my_print("Nuitka command 1:", nuitka_cmd1)
for _i in range(5):
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(
args=nuitka_cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka1, stderr_nuitka1 = process.communicate()
exit_nuitka1 = process.returncode
if exit_nuitka1 != 0:
if (
not expect_failure
and not comparison_mode
and not os.path.exists(".coverage")
):
sys.exit(
"""\
Error, failed to take coverage with '%s'.
Stderr was:
%s
"""
% (os.environ["PYTHON"], stderr_nuitka1)
)
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
else:
# No execution second step for coverage mode.
if comparison_mode:
if trace_command:
my_print("Nuitka command 2:", nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout_nuitka2, stderr_nuitka2 = process.communicate()
stdout_nuitka = stdout_nuitka1 + stdout_nuitka2
stderr_nuitka = stderr_nuitka1 + stderr_nuitka2
exit_nuitka = process.returncode
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
nuitka_cmd2 = wrapCommandForDebuggerForSubprocess(*nuitka_cmd2)
process = subprocess.Popen(
args=nuitka_cmd2, stdin=subprocess.PIPE
)
process.communicate()
else:
exit_nuitka = exit_nuitka1
stdout_nuitka, stderr_nuitka = stdout_nuitka1, stderr_nuitka1
if checkNoPermissionError(stdout_nuitka) and checkNoPermissionError(
stderr_nuitka
):
break
my_print("Retrying nuitka exe due to permission problems after delay.")
time.sleep(2)
stop_watch.stop()
nuitka_time = stop_watch.getDelta()
if not silent_mode:
displayOutput(stdout_nuitka, stderr_nuitka)
if coverage_mode:
assert not stdout_nuitka
assert not stderr_nuitka
if comparison_mode:
def makeComparisons(trace_result):
exit_code_stdout = compareOutput(
"stdout", stdout_cpython, stdout_nuitka, ignore_warnings, syntax_errors
)
if ignore_stderr:
exit_code_stderr = 0
else:
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings,
syntax_errors,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return and trace_result:
my_print(
"""Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka
)
)
return exit_code_stdout, exit_code_stderr, exit_code_return
if cpython_cached:
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=False
)
if exit_code_stdout or exit_code_stderr or exit_code_return:
old_stdout_cpython = stdout_cpython
old_stderr_cpython = stderr_cpython
old_exit_cpython = exit_cpython
my_print(
"Updating CPython cache by force due to non-matching comparison results.",
style="yellow",
)
(
cpython_time,
stdout_cpython,
stderr_cpython,
exit_cpython,
) = getCPythonResults(
cpython_cmd=cpython_cmd,
cpython_cached=cpython_cached,
force_update=True,
)
if not silent_mode:
if (
old_stdout_cpython != stdout_cpython
or old_stderr_cpython != stderr_cpython
or old_exit_cpython != exit_cpython
):
displayOutput(stdout_cpython, stderr_cpython)
exit_code_stdout, exit_code_stderr, exit_code_return = makeComparisons(
trace_result=True
)
# In case of segfault, also output the call stack by entering debugger
# without stdin forwarded.
if (
exit_code_return
and exit_nuitka in (-11, -6)
and sys.platform != "nt"
and not module_mode
and not two_step_execution
):
nuitka_cmd.insert(len(nuitka_cmd) - 1, "--debugger")
with withPythonPathChange(nuitka_package_dir):
process = subprocess.Popen(args=nuitka_cmd, stdin=subprocess.PIPE)
process.communicate()
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
problems = []
if exit_code_stdout:
problems.append("stdout")
if exit_code_stderr:
problems.append("stderr")
if exit_code_return:
problems.append("exit_code")
sys.exit("Error, results differed (%s)." % ",".join(problems))
if expect_success and exit_cpython != 0:
if silent_mode:
displayOutput(stdout_cpython, stderr_cpython)
sys.exit("Unexpected error exit from CPython.")
if expect_failure and exit_cpython == 0:
sys.exit("Unexpected success exit from CPython.")
if remove_output:
if not module_mode:
if os.path.exists(nuitka_cmd2[0]):
if os.name == "nt":
# It appears there is a tiny lock race that we randomly cause,
# likely because --run spawns a subprocess that might still
# be doing the cleanup work.
if os.path.exists(nuitka_cmd2[0] + ".away"):
os.unlink(nuitka_cmd2[0] + ".away")
for _i in range(10):
try:
os.rename(nuitka_cmd2[0], nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(0.1)
continue
for _i in range(10):
try:
os.unlink(nuitka_cmd2[0] + ".away")
except OSError:
time.sleep(2)
continue
else:
break
if os.path.exists(pdb_filename):
os.unlink(pdb_filename)
else:
os.unlink(nuitka_cmd2[0])
else:
module_filename = os.path.basename(filename) + getSharedLibrarySuffix(
preferred=True
)
if os.path.exists(module_filename):
os.unlink(module_filename)
if comparison_mode and timing:
my_print("CPython took %.2fs vs %0.2fs Nuitka." % (cpython_time, nuitka_time))
if comparison_mode and not silent_mode:
my_print("OK, same outputs.")
if filename == "YieldFrom33.py":
extra_flags.append("ignore_stderr")
# For Python2 there is a "builtins" package that gives warnings. TODO: We
# ought to NOT import that package and detect statically that __builtins__
# import won't raise ImportError.
if filename == "BuiltinOverload.py":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
if filename.startswith("Referencing") and not hasDebugPython():
my_print("Skipped (no debug Python)")
continue
needs_2to3 = python_version.startswith('3') and \
not filename.endswith("32.py") and \
not filename.endswith("33.py")
with withPythonPathChange(".."):
compareWithCPython(dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=needs_2to3)
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(suite="programs", needs_io_encoding=True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")
for filename, filename_main in scanDirectoryForTestCaseFolders("."):
# For these, we expect that they will fail.
expected_errors = [
"module_exits",
"main_raises",
"main_raises2",
"package_contains_main",
]
# After Python3 those have been made to work.
if python_version < (3, 5):
expected_errors.append("cyclic_imports")
# Allowed with Python3, packages need no more "__init__.py"
if python_version < (3, ):
expected_errors.append("package_missing_init")
# Allowed with Python3.5 only:
if python_version < (3, 5):
expected_errors.append("package_init_issue")
# Allowed with Python3, name imports can be module imports
if python_version < (3, ):
expected_errors.append("named_imports")
extra_variant = []
if filename not in expected_errors:
extra_flags = ["expect_success"]
else:
extra_flags = ["expect_failure"]
# We annotate some tests, use that to lower warnings.
extra_flags.append("plugin_enable:pylint-warnings")
if filename in (
"reimport_main_static",
"package_missing_init",
"dash_import",
"package_contains_main",
"case_imports3",
"import_variants",
"package_init_import",
"pkgutil_itermodules",
):
extra_flags.append("ignore_warnings")
extra_flags.append("remove_output")
extra_flags.append("--follow-imports")
# Use the original "__file__" value normally, at least one case warns
# about things with filename included, but for pkgutil iteration, make
# sure we do not see original Python dirs.
if filename != "pkgutil_itermodules":
extra_flags.append("--file-reference-choice=original")
else:
extra_flags.append("--file-reference-choice=runtime")
# Run it as a package and also as directory.
if filename == "package_program":
# Not really supported for 2.6
if python_version >= (2, 7):
extra_variant.append("--python-flag=-m")
# Cannot include the files with syntax errors, these would then become
# ImportError, but that's not the test. In all other cases, use two
# step execution, which will not add the program original source to
# PYTHONPATH.
if filename != "syntax_errors":
extra_flags.append("two_step_execution")
else:
extra_flags.append("binary_python_path")
if filename == "plugin_import":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-package=some_package")
elif filename == "reimport_main_dynamic":
if python_version < (3, ):
os.environ[
"NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-directory=%s" % (
os.path.abspath(filename))
else:
os.environ[
"NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-files=%s/*.py" % (
os.path.abspath(filename))
extra_flags.append("ignore_warnings")
elif filename == "multiprocessing_using":
# TODO: Still true?
if sys.platform == "darwin" and python_version >= (3, 8):
reportSkip("Hangs for unknown reasons", ".", filename)
continue
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
if extra_python_path:
my_print("Applying extra PYTHONPATH %r." % extra_python_path)
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if extra_variant:
my_print("Extra variation %r." % extra_variant)
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags + extra_variant,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
_python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")
# TODO: Add a directory test case scanner instead of duplicating this kind of code.
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(".build")
or filename.endswith(".dist")):
continue
filename = os.path.relpath(filename)
extra_flags = ["expect_success", "ignore_infos"]
# We annotate some tests, use that to lower warnings.
extra_flags.append("plugin_enable:pylint-warnings")
extra_flags.append("remove_output")
extra_flags.append("recurse_all")
plugin_files = [
p for p in os.listdir(filename) if p.endswith("-plugin.py")
]
assert plugin_files
extra_flags.extend("user_plugin:" +
os.path.abspath(os.path.join(filename, p))
for p in plugin_files)
if filename == "parameters":
os.environ[
"NUITKA_EXTRA_OPTIONS"] = extra_options + " --trace-my-plugin"
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
filename_main = getMainProgramFilename(filename)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
search_mode.finish()
