def main():
setup(suite="syntax", needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = [
"expect_failure",
"remove_output",
"syntax_errors",
"--nofollow-imports",
]
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
if filename == "TryFinallyContinue.py" and python_version >= "3.8":
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = ["expect_failure", "remove_output", "syntax_errors"]
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
filenames = _createTests(python_version)
# Now run all the tests in this directory.
for filename in filenames:
assert filename.endswith(".py")
if not decideFilenameVersionSkip(filename):
continue
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# We annotate some tests, use that to lower warnings.
"plugin_enable:pylint-warnings",
]
# This test should be run with the debug Python, and makes outputs to
active = search_mode.consider(dirname=None, filename=filename)
if active:
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=decideNeeds2to3(filename),
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
# Complex stuff, pylint: disable=too-many-locals,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(
(".build", ".dist")) or filename.startswith("venv_")):
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider distutils example:", filename)
py3_only_examples = ("example_3", "nested_namespaces")
if python_version < (3, ) and filename in py3_only_examples:
reportSkip("Skipped, only relevant for Python3", ".", filename)
continue
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_wheel"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
# Need to call CPython binary for Windows.
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
my_print("STDOUT CPython:")
my_print(stdout_cpython)
my_print("STDERR CPython:")
my_print(stderr_cpython)
assert exit_cpython == 0, exit_cpython
my_print("EXIT was OK.")
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_nuitka") as venv:
# Install nuitka from source.
venv.runCommand(commands=[
'cd "%s"' % nuitka_dir, "python setup.py install"
])
# Remove that left over from the install command.
removeDirectory(
path=os.path.join(nuitka_dir, "Nuitka.egg-info"),
ignore_errors=False,
)
# Create the wheel with Nuitka compilation.
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_nuitka"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
runner_binary,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
my_print("STDOUT Nuitka:")
my_print(stdout_nuitka)
my_print("STDERR Nuitka:")
my_print(stderr_nuitka)
assert exit_nuitka == 0, exit_nuitka
my_print("EXIT was OK.")
exit_code_stdout = compareOutput(
"stdout",
stdout_cpython,
stdout_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return:
my_print("""\
Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka))
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
sys.exit("Error, outputs differed.")
if search_mode.abortIfExecuted():
break
search_mode.finish()
def main():
# Complex stuff, pylint: disable=too-many-branches
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
# Create large constants test on the fly, if it's not there, not going to
# add it to release archives for no good reason.
if not os.path.exists("BigConstants.py"):
with open("BigConstants.py", "w") as output:
output.write(
"# Automatically generated test, not part of releases or git.\n\n"
)
output.write("print('%s')\n" % ("1234" * 17000))
# Now run all the tests in this directory.
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
]
# This test should be run with the debug Python, and makes outputs to
# standard error that might be ignored.
if filename.startswith("Referencing"):
extra_flags.append("python_debug")
extra_flags.append("recurse_not:nuitka")
# This tests warns about __import__() used.
if filename == "OrderChecks.py":
extra_flags.append("ignore_warnings")
# This tests warns about an package relative import despite
# being in no package.
if filename == "Importing.py":
extra_flags.append("ignore_warnings")
# TODO: Nuitka does not give output for ignored exception in dtor, this is
# not fully compatible and potentially an error.
if filename == "YieldFrom33.py":
extra_flags.append("ignore_stderr")
# For Python2 there is a "builtins" package that gives warnings. TODO: We
# ought to NOT import that package and detect statically that __builtins__
# import won't raise ImportError.
if filename == "BuiltinOverload.py":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
if filename.startswith("Referencing") and not hasDebugPython():
my_print("Skipped (no debug Python)")
continue
needs_2to3 = (python_version.startswith("3")
and not filename.endswith("32.py")
and not filename.endswith("33.py")
and not filename.endswith("35.py")
and not filename.endswith("36.py"))
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=needs_2to3,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
search_mode = createSearchMode()
nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))
for filename in sorted(os.listdir(".")):
if (
not os.path.isdir(filename)
or filename.endswith(".build")
or filename.endswith(".dist")
):
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider distutils example:", filename)
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"), ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"), ignore_errors=False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(
commands=['cd "%s"' % case_dir, "python setup.py bdist_wheel"]
def main():
# Complex stuff, pylint: disable=too-many-branches
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py") or filename.startswith("run_"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
# Apply 2to3 conversion if necessary.
if python_version >= (3, ):
filename, changed = convertUsing2to3(filename)
else:
changed = False
my_print("Consider", filename, end=" ")
command = [
os.environ["PYTHON"],
os.path.abspath(os.path.join("..", "..", "bin", "nuitka")),
"--xml",
"--quiet",
"--module",
filename,
]
if search_mode.isCoverage():
# To avoid re-execution, which is not acceptable to coverage.
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
# Coverage modules hates Nuitka to re-execute, and so we must avoid
# that.
python_path = check_output([
os.environ["PYTHON"],
"-c",
"import sys, os; print(os.pathsep.join(sys.path))",
])
if sys.version_info >= (3, ):
python_path = python_path.decode("utf8")
os.environ["PYTHONPATH"] = python_path.strip()
command.insert(2, "--must-not-re-execute")
command = (command[0:1] + [
"-S", "-m", "coverage", "run", "--rcfile", os.devnull, "-a"
] + command[1:])
result = check_output(command)
# Parse the result into XML and check it.
try:
root = lxml.etree.fromstring(result)
except lxml.etree.XMLSyntaxError:
my_print("Problematic XML output:")
my_print(result)
raise
module_body = root[0]
module_statements_sequence = module_body[0]
assert len(module_statements_sequence) == 1
module_statements = next(iter(module_statements_sequence))
try:
checkSequence(filename, module_statements)
for function in root.xpath('role[@name="functions"]/node'):
(function_body, ) = function.xpath('role[@name="body"]')
function_statements_sequence = function_body[0]
assert len(function_statements_sequence) == 1
function_statements = next(
iter(function_statements_sequence))
checkSequence(filename, function_statements)
if changed:
os.unlink(filename)
except SystemExit:
my_print("FAIL.")
raise
my_print("OK.")
if search_mode.abortIfExecuted():
break
search_mode.finish()
"TOKENBINDING.DLL",
"ICUUC.DLL",
"DRVSETUP.DLL",
"HTTPAPI.DLL",
"WDSCORE.DLL",
"ICUIN.DLL",
"WFDSCONMGR.DLL",
"MFC140U.DLL",
"MFCM140U.DLL",
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
path = os.path.relpath(filename)
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = ["expect_success", "standalone", "remove_output"]
if filename == "PySideUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
def main():
python_version = setup(suite="onefile", needs_io_encoding=True)
search_mode = createSearchMode()
if getOS() == "Linux":
addExtendedExtraOptions(
"--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"remove_output",
# Keep the binary, normally "remove_output" includes that.
"--keep-binary",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
# The onefile can warn about zstandard not being installed.
"ignore_warnings",
]
if filename == "KeyboardInterruptTest.py":
if getOS() == "Darwin":
reportSkip(
"Exit code from KeyboardInterrupt on macOS is not yet good.",
".",
filename,
)
continue
if python_version < (3,):
reportSkip(
"Python2 reports KeyboardInterrupt, but too late",
".",
filename,
)
continue
if os.name == "nt":
reportSkip(
"Testing cannot send KeyboardInterrupt on Windows yet",
".",
filename,
)
continue
extra_flags.append("--send-ctrl-c")
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of onefile mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
if filename == "KeyboardInterruptTest.py":
continue
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, command=[binary_filename]
)
illegal_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd()
)
if illegal_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning(
"Should not access these file(s): '%s'." % ",".join(illegal_accesses)
)
search_mode.onErrorDetected(1)
os.unlink(binary_filename)
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = [
"expect_success",
"standalone",
"remove_output",
# For enum plugin info
"ignore_infos",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
# catch error
if filename == "Boto3Using.py":
reportSkip("boto3 test not fully working yet", ".", filename)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version.startswith("2"):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
# extra_flags.append("plugin_enable:data-files")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PySideUsing.py":
# TODO: Disabled due to lack of upstream support.
reportSkip("PySide not supported yet", ".", filename)
continue
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide", "PyQt")):
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information.
if getPythonVendor() != "Anaconda":
extra_flags.append("plugin_enable:qt-plugins")
else:
# For the plug-in not used information.
extra_flags.append("ignore_warnings")
my_print("Consider output of recursively compiled program:", filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
my_print(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"System32")),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64")),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith(
"/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/"):
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith((
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith((
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith(
"/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
# PySide accesses its directory.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/PySide"):
continue
# GTK accesses package directories only.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gtk"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/glib"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gio"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(
sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def main():
# Complex stuff, even more should become common code or project options though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(suite="standalone", needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--standalone",
"remove_output",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
if filename == "Urllib3Using.py" and os.name == "nt":
reportSkip(
"Socket module early import not working on Windows currently",
".",
filename,
)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version < (3, ):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
if getOS() == "Windows":
reportSkip("Can hang on Windows CI.", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
# TODO: Once we have a no-Qt Plugin, we should use that.
if filename == "MatplotlibUsing.py":
# For the plugin warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
if filename == "PandasUsing.py":
extra_flags.append("plugin_enable:numpy")
extra_flags.append("plugin_disable:pylint-warnings")
extra_flags.append("plugin_disable:pyqt5")
extra_flags.append("plugin_disable:pyside2")
extra_flags.append("plugin_disable:pyside6")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "GlfwUsing.py":
# For the warnings.
extra_flags.append("plugin_enable:numpy")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")):
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information
if filename.startswith("PySide2"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PySide6"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PyQt5"):
extra_flags.append("plugin_enable:pyqt5")
elif filename.startswith("PyQt6"):
extra_flags.append("plugin_enable:pyqt6")
test_logger.info(
"Consider output of standalone mode compiled program: %s" %
filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
test_logger.warning(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
with TimerReport("Determining run time loaded files took %.2f",
logger=test_logger):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, command=[binary_filename])
illegal_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd())
if illegal_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning("Should not access these file(s): '%r'." %
illegal_accesses)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
if getOS() == "Linux":
addExtendedExtraOptions(
"--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--onefile",
"remove_output",
# Keep the binary, normally "remove_output" includes that.
"--keep-binary",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
if filename == "KeyboardInteruptTest.py":
if python_version < (3,):
reportSkip(
"Python2 reports KeyboardInterrupt, but too late",
".",
filename,
)
continue
if os.name == "nt":
reportSkip(
".",
filename,
"Testing cannot send KeyboardInterrupt on Windows yet",
)
continue
extra_flags.append("--send-ctrl-c")
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of onefile mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
if filename == "KeyboardInteruptTest.py":
continue
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, path=binary_filename
)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "System32")
),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if r"azure dev spaces cli" in loaded_filename:
continue
if r"tortoisesvn" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp":
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith(
(
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)
):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith(
(
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)
):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith(
(
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)
):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (
loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")
):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith("/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
lib_prefix_dir = "/usr/lib/python%d.%s" % (
python_version[0],
python_version[1],
)
# PySide accesses its directory.
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
continue
# GTK accesses package directories only.
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
continue
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gio"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# Linux onefile uses this
if loaded_basename.startswith("libfuse.so."):
continue
# MSVC run time DLLs, due to SxS come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
test_logger.warning("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
search_mode.onErrorDetected(1)
os.unlink(binary_filename)
search_mode.finish()
