def main():
setup(suite="packages")
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not os.path.isdir(filename) or filename.endswith(".build"):
continue
extra_flags = [
"--module",
"expect_success",
"remove_output",
"two_step_execution",
]
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of compiled package:", filename)
filename_main = None
filename_main = os.path.join(
filename, "".join(part.title() for part in filename.split("_")) + ".py"
)
if os.path.exists(filename_main):
filename_main = os.path.basename(filename_main)
else:
filename_main = None
if filename_main is None:
for filename_main in os.listdir(filename):
if not os.path.isdir(os.path.join(filename, filename_main)):
continue
if filename_main not in ("..", "."):
break
else:
search_mode.onErrorDetected(
"""\
Error, no package in test directory '%s' found, incomplete test case."""
% filename
)
extra_flags.append(
"--include-package=%s" % os.path.basename(filename_main)
)
extra_flags.append("--output-dir=%s" % getTempDir())
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
setup(suite="syntax", needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = [
"expect_failure",
"remove_output",
"syntax_errors",
"--nofollow-imports",
]
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
setup()
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not os.path.isdir(filename) or filename.endswith(".build"):
continue
extra_flags = [
"expect_success",
"remove_output",
"module_mode",
"two_step_execution",
]
# The use of "__main__" in the test package gives a warning.
if filename == "sub_package":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled package:",
filename)
filename_main = None
for filename_main in os.listdir(filename):
if not os.path.isdir(os.path.join(filename, filename_main)):
continue
if filename_main not in ("..", "."):
break
else:
search_mode.onErrorDetected("""\
Error, no package in dir '%s' found, incomplete test case.""" % filename)
extensions = [
"--include-package=%s" % os.path.basename(filename_main)
]
if "--output-dir" not in os.environ.get("NUITKA_EXTRA_OPTIONS",
""):
extensions.append("--output-dir=%s" % getTempDir())
with withExtendedExtraOptions(*extensions):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def main():
setup(needs_io_encoding=True)
executePASS1()
executePASS2()
executePASS3()
executePASS4()
shutil.rmtree("nuitka")
executePASS5()
def main():
if os.name != "nt":
sys.exit("Error, this is only for use on Windows where SxS exists.")
setup(needs_io_encoding=True)
search_mode = createSearchMode()
tmp_dir = tempfile.gettempdir()
compileLibraryTest(search_mode=search_mode,
stage_dir=os.path.join(tmp_dir, "find_sxs_modules"),
decide=decide,
action=action)
my_print("FINISHED, all extension modules checked.")
def main():
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
if filename == "TryFinallyContinue.py" and python_version >= "3.8":
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = ["expect_failure", "remove_output", "syntax_errors"]
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
if os.name != "nt":
sys.exit("Error, this is only for use on Windows where SxS exists.")
setup(needs_io_encoding=True)
search_mode = createSearchMode()
tmp_dir = tempfile.gettempdir()
compileLibraryTest(
search_mode=search_mode,
stage_dir=os.path.join(tmp_dir, "find_sxs_modules"),
decide=decide,
action=action,
)
my_print("FINISHED, all extension modules checked.")
def main():
setup(suite="pgo", needs_io_encoding=True)
search_mode = createSearchMode()
# Now run all the tests in this directory.
for filename, filename_main in scanDirectoryForTestCaseFolders("."):
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, PGO will then have to deal with unused ones.
"--follow-imports",
# The output during compilation from PGO capture is harmful, so
# split compilation and execution of final result.
"two_step_execution",
# Inclusion report is used by the testing of expected things included
# or not.
"--report=%s.xml" % filename,
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
]
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
_python_version = setup(needs_io_encoding=True)
executePASS1()
executePASS2()
executePASS3()
executePASS4()
shutil.rmtree("nuitka")
executePASS5()
os.unlink(os.path.join(tmp_dir, "nuitka" + exe_suffix))
os.rmdir(tmp_dir)
def main():
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
filenames = _createTests(python_version)
# Now run all the tests in this directory.
for filename in filenames:
assert filename.endswith(".py")
if not decideFilenameVersionSkip(filename):
continue
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# We annotate some tests, use that to lower warnings.
"plugin_enable:pylint-warnings",
]
# This test should be run with the debug Python, and makes outputs to
active = search_mode.consider(dirname=None, filename=filename)
if active:
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=decideNeeds2to3(filename),
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
# Singleton, pylint: disable=global-statement
global operations
global candidates
if python_version >= (3, 5):
operations += (("MatMult", "@"),)
if python_version < (3,):
candidates += (("long", "17L", "-9L"),)
candidates += (("unicode", "u'lala'", "u'lele'"),)
else:
candidates += (("bytes", "b'lala'", "b'lele'"),)
template_context = {
"operations": operations,
"ioperations": tuple(
operation
for operation in operations
if operation[0] not in ("Divmod", "Subscript")
),
"candidates": candidates,
"makeOperatorUsage": makeOperatorUsage,
}
# Now run all the tests in this directory.
for filename in scanDirectoryForTestCases(".", template_context=template_context):
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# We annotate some tests, use that to lower warnings.
"plugin_enable:pylint-warnings",
]
# This test should be run with the debug Python, and makes outputs to
active = search_mode.consider(dirname=None, filename=filename)
if active:
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=decideNeeds2to3(filename),
)
search_mode.finish()
# isort:start
import subprocess
from nuitka.tools.testing.Common import (
createSearchMode,
decideFilenameVersionSkip,
my_print,
setup,
)
from nuitka.tools.testing.OutputComparison import compareOutput
from nuitka.tools.testing.Virtualenv import withVirtualenv
from nuitka.utils.FileOperations import removeDirectory
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))
for filename in sorted(os.listdir(".")):
if (
not os.path.isdir(filename)
or filename.endswith(".build")
or filename.endswith(".dist")
):
continue
filename = os.path.relpath(filename)
# Find nuitka package relative to us.
sys.path.insert(
0,
os.path.normpath(
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
".."
)
)
)
from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip
from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip
python_version = setup()
nuitka_main_path = os.path.join("..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
# Cannot detect this more automatic, either need whitelist or
# blacklist not needed stuff.
PACKAGE_LIST = (
"nuitka",
"nuitka/nodes",
"nuitka/specs",
"nuitka/nodes/shapes",
"nuitka/tree",
"nuitka/importing",
"nuitka/build",
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option(
"--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "")
)
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option("--code-diff", action="store", dest="diff_filename", default="")
parser.add_option("--copy-source-to", action="store", dest="target_dir", default="")
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit("Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
python_version = setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
with open(test_case, "rb") as f:
my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest())
needs_2to3 = (
python_version.startswith("3")
and not test_case.endswith("32.py")
and not test_case.endswith("33.py")
)
if options.target_dir:
shutil.copyfile(
test_case, os.path.join(options.target_dir, os.path.basename(test_case))
)
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case))
with open(test_case) as f:
case_1_source, case_2_source = generateConstructCases(f.read())
with open(test_case_1, "w") as case_1_file:
case_1_file.write(case_1_source)
with open(test_case_2, "w") as case_2_file:
case_2_file.write(case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = "0"
if nuitka:
nuitka_id = check_output(
"cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True
)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3,):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
if nuitka:
nuitka_call = [
os.environ["PYTHON"],
nuitka,
"--python-flag=-S",
os.path.basename(test_case),
]
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
shutil.copyfile(test_case_1, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix),
)
shutil.copyfile(test_case_2, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix),
)
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(
test_case_1.replace(".py", ".build"), "module.__main__" + suffix
)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(
test_case_2.replace(".py", ".build"), "module.__main__" + suffix
)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
with open(options.diff_filename, "w") as f:
with open(cpp_1) as cpp1:
with open(cpp_2) as cpp2:
f.write(
difflib.HtmlDiff().make_table(
cpp1.readlines(),
cpp2.readlines(),
"Construct",
"Baseline",
True,
)
)
nuitka_1 = runValgrind(
"Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix),),
include_startup=True,
)
nuitka_2 = runValgrind(
"Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix),),
include_startup=True,
)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_1 = runValgrind(
"CPython construct",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_1),
include_startup=True,
)
cpython_2 = runValgrind(
"CPython baseline",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_2),
include_startup=True,
)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def main():
setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
# Now run all the tests in this directory.
for filename in scanDirectoryForTestCases("."):
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
# We annotate some tests, use that to lower warnings.
"plugin_enable:pylint-warnings",
]
# This test should be run with the debug Python, and makes outputs to
# standard error that might be ignored.
if filename.startswith("Referencing"):
extra_flags.append("recurse_not:nuitka")
if hasDebugPython():
extra_flags.append("python_debug")
# This tests warns about __import__() used.
if filename == "OrderChecks.py":
extra_flags.append("ignore_warnings")
# This tests warns about an package relative import despite
# being in no package.
if filename == "Importing.py":
extra_flags.append("ignore_warnings")
# TODO: Nuitka does not give output for ignored exception in dtor, this is
# not fully compatible and potentially an error.
if filename == "YieldFrom33.py":
extra_flags.append("ignore_stderr")
# For Python2 there is a "builtins" package that gives warnings. TODO: We
# ought to NOT import that package and detect statically that __builtins__
# import won't raise ImportError.
if filename == "BuiltinOverload.py":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=decideNeeds2to3(filename),
)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(suite="programs", needs_io_encoding=True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")
for filename, filename_main in scanDirectoryForTestCaseFolders("."):
# For these, we expect that they will fail.
expected_errors = [
"module_exits",
"main_raises",
"main_raises2",
"package_contains_main",
]
# After Python3 those have been made to work.
if python_version < (3, 5):
expected_errors.append("cyclic_imports")
# Allowed with Python3, packages need no more "__init__.py"
if python_version < (3, ):
expected_errors.append("package_missing_init")
# Allowed with Python3.5 only:
if python_version < (3, 5):
expected_errors.append("package_init_issue")
# Allowed with Python3, name imports can be module imports
if python_version < (3, ):
expected_errors.append("named_imports")
extra_variant = []
if filename not in expected_errors:
extra_flags = ["expect_success"]
else:
extra_flags = ["expect_failure"]
# We annotate some tests, use that to lower warnings.
extra_flags.append("plugin_enable:pylint-warnings")
if filename in (
"reimport_main_static",
"package_missing_init",
"dash_import",
"package_contains_main",
"case_imports3",
"import_variants",
"package_init_import",
"pkgutil_itermodules",
):
extra_flags.append("ignore_warnings")
extra_flags.append("remove_output")
extra_flags.append("--follow-imports")
# Use the original "__file__" value normally, at least one case warns
# about things with filename included, but for pkgutil iteration, make
# sure we do not see original Python dirs.
if filename != "pkgutil_itermodules":
extra_flags.append("--file-reference-choice=original")
else:
extra_flags.append("--file-reference-choice=runtime")
# Run it as a package and also as directory.
if filename == "package_program":
# Not really supported for 2.6
if python_version >= (2, 7):
extra_variant.append("--python-flag=-m")
# Cannot include the files with syntax errors, these would then become
# ImportError, but that's not the test. In all other cases, use two
# step execution, which will not add the program original source to
# PYTHONPATH.
if filename != "syntax_errors":
extra_flags.append("two_step_execution")
else:
extra_flags.append("binary_python_path")
if filename == "plugin_import":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-package=some_package")
elif filename == "reimport_main_dynamic":
if python_version < (3, ):
os.environ[
"NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-directory=%s" % (
os.path.abspath(filename))
else:
os.environ[
"NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-files=%s/*.py" % (
os.path.abspath(filename))
extra_flags.append("ignore_warnings")
elif filename == "multiprocessing_using":
# TODO: Still true?
if sys.platform == "darwin" and python_version >= (3, 8):
reportSkip("Hangs for unknown reasons", ".", filename)
continue
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
if extra_python_path:
my_print("Applying extra PYTHONPATH %r." % extra_python_path)
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if extra_variant:
my_print("Extra variation %r." % extra_variant)
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags + extra_variant,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(".build")
or filename.endswith(".dist")):
continue
filename = os.path.relpath(filename)
# For these, we expect that they will fail.
expected_errors = [
"module_exits",
"main_raises",
"main_raises2",
"package_contains_main",
]
# After Python3 those have been made to work.
if python_version < "3.5":
expected_errors.append("cyclic_imports")
# Allowed with Python3, packages need no more "__init__.py"
if python_version < "3":
expected_errors.append("package_missing_init")
# Allowed with Python3 only:
if python_version < "3.5":
expected_errors.append("package_init_issue")
if filename not in expected_errors:
extra_flags = ["expect_success"]
else:
extra_flags = ["expect_failure"]
if filename in (
"reimport_main_static",
"package_missing_init",
"dash_import",
"package_contains_main",
"case_imports3",
"import_variants",
"package_init_import",
"pkgutil_itermodules",
):
extra_flags.append("ignore_warnings")
extra_flags.append("remove_output")
extra_flags.append("recurse_all")
# Use the original __file__ value, at least one case warns about things
# with filename included, but for pkgutil iteration, make sure we do not
# see original Python dirs.
if filename != "pkgutil_itermodules":
extra_flags.append("original_file")
else:
extra_flags.append("runtime_file")
# Cannot include the files with syntax errors, these would then become
# ImportError, but that's not the test. In all other cases, use two
# step execution, which will not add the program original source to
# PYTHONPATH.
if filename != "syntax_errors":
extra_flags.append("two_step_execution")
else:
extra_flags.append("binary_python_path")
if filename == "plugin_import":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-package=some_package")
elif filename == "reimport_main_dynamic":
if python_version < "3":
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-plugin-directory=%s" %
(os.path.abspath(filename)))
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = (
extra_options + " --include-plugin-files=%s/*.py" %
(os.path.abspath(filename)))
extra_flags.append("ignore_warnings")
elif filename == "multiprocessing_using":
if os.name == "nt":
extra_flags += [
"plugin_enable:multiprocessing", "ignore_infos"
]
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
filename_main = getMainProgramFilename(filename)
extra_python_path = [
os.path.abspath(os.path.join(filename, entry))
for entry in os.listdir(filename) if entry.startswith("path")
]
with withPythonPathChange(extra_python_path):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
python_version = setup(silent=True)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
my_print("TEST_CASE_HASH='%s'" %
hashlib.md5(open(test_case, "rb").read()).hexdigest())
needs_2to3 = python_version.startswith('3') and \
not test_case.endswith("32.py") and \
not test_case.endswith("33.py")
if options.target_dir:
shutil.copyfile(
def main():
# Complex stuff, pylint: disable=too-many-locals,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))
for filename in sorted(os.listdir(".")):
if (not os.path.isdir(filename) or filename.endswith(
(".build", ".dist")) or filename.startswith("venv_")):
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider distutils example:", filename)
py3_only_examples = ("example_3", "nested_namespaces")
if python_version < (3, ) and filename in py3_only_examples:
reportSkip("Skipped, only relevant for Python3", ".", filename)
continue
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_wheel"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
# Need to call CPython binary for Windows.
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_cpython, stderr_cpython = process.communicate()
exit_cpython = process.returncode
my_print("STDOUT CPython:")
my_print(stdout_cpython)
my_print("STDERR CPython:")
my_print(stderr_cpython)
assert exit_cpython == 0, exit_cpython
my_print("EXIT was OK.")
removeDirectory(os.path.join(case_dir, "build"),
ignore_errors=False)
removeDirectory(os.path.join(case_dir, "dist"),
ignore_errors=False)
with withVirtualenv("venv_nuitka") as venv:
# Install nuitka from source.
venv.runCommand(commands=[
'cd "%s"' % nuitka_dir, "python setup.py install"
])
# Remove that left over from the install command.
removeDirectory(
path=os.path.join(nuitka_dir, "Nuitka.egg-info"),
ignore_errors=False,
)
# Create the wheel with Nuitka compilation.
venv.runCommand(commands=[
'cd "%s"' % case_dir, "python setup.py bdist_nuitka"
])
dist_dir = os.path.join(case_dir, "dist")
venv.runCommand('pip install "%s"' %
(os.path.join(dist_dir,
os.listdir(dist_dir)[0])))
runner_binary = os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"runner",
)
if os.path.exists(runner_binary):
process = subprocess.Popen(
args=[
os.path.join(
venv.getVirtualenvDir(),
"bin" if os.name != "nt" else "scripts",
"python",
),
runner_binary,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
else:
assert os.path.exists(runner_binary + ".exe")
process = subprocess.Popen(
args=[runner_binary + ".exe"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_nuitka, stderr_nuitka = process.communicate()
exit_nuitka = process.returncode
my_print("STDOUT Nuitka:")
my_print(stdout_nuitka)
my_print("STDERR Nuitka:")
my_print(stderr_nuitka)
assert exit_nuitka == 0, exit_nuitka
my_print("EXIT was OK.")
exit_code_stdout = compareOutput(
"stdout",
stdout_cpython,
stdout_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_stderr = compareOutput(
"stderr",
stderr_cpython,
stderr_nuitka,
ignore_warnings=True,
syntax_errors=True,
)
exit_code_return = exit_cpython != exit_nuitka
if exit_code_return:
my_print("""\
Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
exit_cpython=exit_cpython, exit_nuitka=exit_nuitka))
exit_code = exit_code_stdout or exit_code_stderr or exit_code_return
if exit_code:
sys.exit("Error, outputs differed.")
if search_mode.abortIfExecuted():
break
search_mode.finish()
import shutil
import tempfile
from nuitka.tools.testing.Common import (
check_output,
checkRuntimeLoadedFilesForOutsideAccesses,
checkSucceedsWithCPython,
compileLibraryTest,
createSearchMode,
getRuntimeTraceOfLoadedFiles,
my_print,
setup,
)
setup(needs_io_encoding=True)
search_mode = createSearchMode()
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
blacklist = (
"__phello__.foo.py", # Triggers error for "." in module name
"idnadata", # Avoid too complex code for main program.
)
done = set()
# Find nuitka package relative to us.
sys.path.insert(
0,
os.path.normpath(
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
".."
)
)
)
from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip
from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip
python_version = setup()
nuitka_main_path = os.path.join("..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
# Cannot detect this more automatic, either need whitelist or
# blacklist not needed stuff.
PACKAGE_LIST = (
"nuitka",
"nuitka/nodes",
"nuitka/specs",
"nuitka/nodes/shapes",
"nuitka/tree",
"nuitka/importing",
"nuitka/build",
def main():
setup(suite="extension_modules", needs_io_encoding=True)
search_mode = createSearchMode()
tmp_dir = getTempDir()
done = set()
def decide(root, filename):
if os.path.sep + "Cython" + os.path.sep in root:
return False
if (root.endswith(os.path.sep + "matplotlib")
or os.path.sep + "matplotlib" + os.path.sep in root):
return False
if filename.endswith("linux-gnu_d.so"):
return False
if root.endswith(os.path.sep + "msgpack"):
return False
first_part = filename.split(".")[0]
if first_part in done:
return False
done.add(first_part)
return filename.endswith(
(".so", ".pyd")) and not filename.startswith("libpython")
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
def action(stage_dir, root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"),
"--stand",
"--run",
"--output-dir=%s" % stage_dir,
"--remove-output",
]
filename = os.path.join(stage_dir, "importer.py")
assert path.startswith(root)
module_name = path[len(root) + 1:]
module_name = module_name.split(".")[0]
module_name = module_name.replace(os.path.sep, ".")
module_name = ModuleName(module_name)
with openTextFile(filename, "w") as output:
plugin_names = set(["pylint-warnings", "anti-bloat"])
if module_name.hasNamespace("PySide2"):
plugin_names.add("pyside2")
elif module_name.hasNamespace("PySide6"):
plugin_names.add("pyside2")
elif module_name.hasNamespace("PyQt5"):
plugin_names.add("pyqt5")
else:
# TODO: We do not have a noqt plugin yet.
plugin_names.add("pyqt5")
for plugin_name in plugin_names:
output.write("# nuitka-project: --enable-plugin=%s\n" %
plugin_name)
# Make it an error to find unwanted bloat compiled in.
output.write("# nuitka-project: --noinclude-default-mode=error\n")
output.write("import " + module_name.asString() + "\n")
output.write("print('OK.')")
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(filename)
if checkSucceedsWithCPython(filename):
try:
output = check_output(command).splitlines()
except Exception: # only trying to check for no exception, pylint: disable=try-except-raise
raise
else:
assert os.path.exists(filename[:-3] + ".dist")
binary_filename = os.path.join(
filename[:-3] + ".dist",
"importer.exe" if os.name == "nt" else "importer",
)
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger,
command=[binary_filename],
)
outside_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd())
if outside_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning(
"Should not access these file(s): '%r'." %
outside_accesses)
search_mode.onErrorDetected(1)
if output[-1] != b"OK.":
my_print(" ".join(command))
my_print(filename)
my_print(output)
test_logger.sysexit("FAIL.")
my_print("OK.")
assert not outside_accesses, outside_accesses
shutil.rmtree(filename[:-3] + ".dist")
else:
my_print("SKIP (does not work with CPython)")
compileLibraryTest(
search_mode=search_mode,
stage_dir=os.path.join(tmp_dir, "compile_extensions"),
decide=decide,
action=action,
)
my_print("FINISHED, all extension modules compiled.")
os.path.normpath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")),
)
# isort:start
from nuitka.tools.testing.Common import (
check_output,
convertUsing2to3,
createSearchMode,
decideFilenameVersionSkip,
my_print,
setup,
)
from nuitka.TreeXML import toString
python_version = setup(suite="optimiations")
search_mode = createSearchMode()
def getKind(node):
result = node.attrib["kind"]
result = result.replace("Statements", "")
result = result.replace("Statement", "")
result = result.replace("Expression", "")
return result
def getRole(node, role):
os.path.dirname(os.path.abspath(__file__)),
"..",
".."
)
)
)
from nuitka.tools.testing.Common import (
my_print,
setup,
createSearchMode,
compareWithCPython,
reportSkip,
withPythonPathChange
)
python_version = setup(needs_io_encoding = True)
search_mode = createSearchMode()
extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS","")
for filename in sorted(os.listdir('.')):
if not os.path.isdir(filename) or \
filename.endswith(".build") or \
filename.endswith(".dist"):
continue
filename = os.path.relpath(filename)
# For these, we expect that they will fail.
expected_errors = [
def main():
setup()
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be version in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if not positional_args:
positional_args = ["bin", "nuitka"]
print("Working on:", positional_args)
blacklist = ["oset.py", "odict.py", "SyntaxHighlighting.py"]
# Avoid checking the Python2 runner with Python3, it has name collisions.
if python_version >= 300:
blacklist.append("nuitka")
filenames = list(scanTargets(positional_args, (".py",), blacklist))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
setup(go_main=False)
# So PyLint finds nuitka package.
addPYTHONPATH(getHomePath())
setupPATH()
parser = OptionParser()
parser.add_option(
"--diff",
action="store_true",
dest="diff",
default=False,
help="""\
Analyse the changed files in git. Default is %default.""",
)
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be verbose in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if positional_args:
if options.diff:
sys.exit("Error, no filenames argument allowed in git diff mode.")
else:
goHome()
if options.diff:
positional_args = [
filename for filename in getModifiedPaths()
if isPythonFile(filename)
]
else:
positional_args = [
"bin", "nuitka", "setup.py", "tests/*/run_all.py"
]
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
if not positional_args:
sys.exit("No files found.")
print("Working on:", positional_args)
ignore_list = []
# Avoid checking the Python2 runner along with the one for Python3, it has name collisions.
if python_version >= 0x300:
ignore_list.append("nuitka")
filenames = list(
scanTargets(positional_args,
suffixes=(".py", ".scons"),
ignore_list=ignore_list))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
)
)
)
from nuitka.tools.testing.Common import (
setup,
my_print,
createSearchMode,
compileLibraryTest,
check_output,
checkSucceedsWithCPython,
getRuntimeTraceOfLoadedFiles,
checkRuntimeLoadedFilesForOutsideAccesses
)
setup(needs_io_encoding = True)
search_mode = createSearchMode()
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
blacklist = (
"__phello__.foo.py", # Triggers error for "." in module name
"idnadata" # Avoid too complex code for main program.
)
done = set()
def main():
setup()
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be verbose in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if not positional_args:
positional_args = ["bin", "nuitka", "tests/*/run_all.py"]
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
print("Working on:", positional_args)
blacklist = ["oset.py", "odict.py"]
# Avoid checking the Python2 runner with Python3, it has name collisions.
if python_version >= 300:
blacklist.append("nuitka")
filenames = list(scanTargets(positional_args, (".py", ), blacklist))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
if getOS() == "Linux":
addExtendedExtraOptions(
"--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--onefile",
"remove_output",
# Keep the binary, normally "remove_output" includes that.
"--keep-binary",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
if filename == "KeyboardInteruptTest.py":
if python_version < (3,):
reportSkip(
"Python2 reports KeyboardInterrupt, but too late",
".",
filename,
)
continue
if os.name == "nt":
reportSkip(
".",
filename,
"Testing cannot send KeyboardInterrupt on Windows yet",
)
continue
extra_flags.append("--send-ctrl-c")
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of onefile mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
if filename == "KeyboardInteruptTest.py":
continue
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, path=binary_filename
)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "System32")
),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if r"azure dev spaces cli" in loaded_filename:
continue
if r"tortoisesvn" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp":
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith(
(
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)
):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith(
(
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)
):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith(
(
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)
):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (
loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")
):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith("/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
lib_prefix_dir = "/usr/lib/python%d.%s" % (
python_version[0],
python_version[1],
)
# PySide accesses its directory.
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
continue
# GTK accesses package directories only.
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
continue
if loaded_filename == os.path.join(
lib_prefix_dir, "dist-packages/gtk-2.0/gio"
):
continue
if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# Linux onefile uses this
if loaded_basename.startswith("libfuse.so."):
continue
# MSVC run time DLLs, due to SxS come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
test_logger.warning("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
search_mode.onErrorDetected(1)
os.unlink(binary_filename)
search_mode.finish()
def main():
# Complex stuff, pylint: disable=too-many-branches
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
# Create large constants test on the fly, if it's not there, not going to
# add it to release archives for no good reason.
if not os.path.exists("BigConstants.py"):
with open("BigConstants.py", "w") as output:
output.write(
"# Automatically generated test, not part of releases or git.\n\n"
)
output.write("print('%s')\n" % ("1234" * 17000))
# Now run all the tests in this directory.
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
]
# This test should be run with the debug Python, and makes outputs to
# standard error that might be ignored.
if filename.startswith("Referencing"):
extra_flags.append("python_debug")
extra_flags.append("recurse_not:nuitka")
# This tests warns about __import__() used.
if filename == "OrderChecks.py":
extra_flags.append("ignore_warnings")
# This tests warns about an package relative import despite
# being in no package.
if filename == "Importing.py":
extra_flags.append("ignore_warnings")
# TODO: Nuitka does not give output for ignored exception in dtor, this is
# not fully compatible and potentially an error.
if filename == "YieldFrom33.py":
extra_flags.append("ignore_stderr")
# For Python2 there is a "builtins" package that gives warnings. TODO: We
# ought to NOT import that package and detect statically that __builtins__
# import won't raise ImportError.
if filename == "BuiltinOverload.py":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
if filename.startswith("Referencing") and not hasDebugPython():
my_print("Skipped (no debug Python)")
continue
needs_2to3 = (python_version.startswith("3")
and not filename.endswith("32.py")
and not filename.endswith("33.py")
and not filename.endswith("35.py")
and not filename.endswith("36.py"))
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=needs_2to3,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def main():
# Complex stuff, even more should become common code or project options though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(suite="standalone", needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"--standalone",
"remove_output",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
if filename == "Urllib3Using.py" and os.name == "nt":
reportSkip(
"Socket module early import not working on Windows currently",
".",
filename,
)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version < (3, ):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
if getOS() == "Windows":
reportSkip("Can hang on Windows CI.", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
# TODO: Once we have a no-Qt Plugin, we should use that.
if filename == "MatplotlibUsing.py":
# For the plugin warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
if filename == "PandasUsing.py":
extra_flags.append("plugin_enable:numpy")
extra_flags.append("plugin_disable:pylint-warnings")
extra_flags.append("plugin_disable:pyqt5")
extra_flags.append("plugin_disable:pyside2")
extra_flags.append("plugin_disable:pyside6")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "GlfwUsing.py":
# For the warnings.
extra_flags.append("plugin_enable:numpy")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")):
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information
if filename.startswith("PySide2"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PySide6"):
extra_flags.append("plugin_enable:pyside6")
elif filename.startswith("PyQt5"):
extra_flags.append("plugin_enable:pyqt5")
elif filename.startswith("PyQt6"):
extra_flags.append("plugin_enable:pyqt6")
test_logger.info(
"Consider output of standalone mode compiled program: %s" %
filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
test_logger.warning(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
with TimerReport("Determining run time loaded files took %.2f",
logger=test_logger):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, command=[binary_filename])
illegal_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd())
if illegal_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning("Should not access these file(s): '%r'." %
illegal_accesses)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
search_mode.finish()
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option(
"--nuitka",
action="store",
dest="nuitka",
default=os.environ.get("NUITKA", ""),
)
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option(
"--code-diff",
action="store",
dest="diff_filename",
default="",
)
parser.add_option(
"--copy-source-to",
action="store",
dest="target_dir",
default="",
)
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit(
"Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
python_version = setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
my_print("TEST_CASE_HASH='%s'" %
hashlib.md5(open(test_case, "rb").read()).hexdigest())
needs_2to3 = python_version.startswith('3') and \
not test_case.endswith("32.py") and \
not test_case.endswith("33.py")
if options.target_dir:
shutil.copyfile(
test_case,
os.path.join(options.target_dir, os.path.basename(test_case)))
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir,
"Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir,
"Variant2_" + os.path.basename(test_case))
case_1_source, case_2_source = generateConstructCases(
open(test_case).read())
with open(test_case_1, 'w') as case_1_file:
case_1_file.write(case_1_source)
with open(test_case_2, 'w') as case_2_file:
case_2_file.write(case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = '0'
if nuitka:
nuitka_id = check_output("cd %s; git rev-parse HEAD" %
os.path.dirname(nuitka),
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
if nuitka:
nuitka_call = [
os.environ["PYTHON"], nuitka, "--python-flag=-S",
os.path.basename(test_case)
]
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
shutil.copyfile(test_case_1, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"))
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix))
shutil.copyfile(test_case_2, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"))
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix))
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(
test_case_1.replace(".py", ".build"),
"module.__main__" + suffix,
)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(
test_case_2.replace(".py", ".build"),
"module.__main__" + suffix,
)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
open(options.diff_filename,
'w').write(difflib.HtmlDiff().make_table(
open(cpp_1).readlines(),
open(cpp_2).readlines(), "Construct", "Baseline", True))
nuitka_1 = runValgrind("Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix), ),
include_startup=True)
nuitka_2 = runValgrind("Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix), ),
include_startup=True)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % ' '.join(nuitka_call),
file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_1 = runValgrind("CPython construct",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_1),
include_startup=True)
cpython_2 = runValgrind("CPython baseline",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_2),
include_startup=True)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def main():
# Complex stuff, even more should become common code though.
# pylint: disable=too-many-branches,too-many-statements
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = [
"expect_success",
"standalone",
"remove_output",
# For enum plugin info
"ignore_infos",
]
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
# catch error
if filename == "Boto3Using.py":
reportSkip("boto3 test not fully working yet", ".", filename)
continue
if "Idna" in filename:
# For the warnings of Python2.
if python_version.startswith("2"):
extra_flags.append("ignore_stderr")
if filename == "CtypesUsing.py":
extra_flags.append("plugin_disable:pylint-warnings")
if filename == "GtkUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith("Win"):
if os.name != "nt":
reportSkip("Windows only test", ".", filename)
continue
if filename == "TkInterUsing.py":
if getOS() == "Darwin":
reportSkip("Not working macOS yet", ".", filename)
continue
# For the plug-in information.
extra_flags.append("plugin_enable:tk-inter")
if filename == "FlaskUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "NumpyUsing.py":
# TODO: Disabled for now.
reportSkip("numpy.test not fully working yet", ".", filename)
continue
# extra_flags.append("plugin_enable:data-files")
if filename == "PmwUsing.py":
extra_flags.append("plugin_enable:pmw-freezer")
if filename == "OpenGLUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PasslibUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename == "PySideUsing.py":
# TODO: Disabled due to lack of upstream support.
reportSkip("PySide not supported yet", ".", filename)
continue
if filename == "Win32ComUsing.py":
# For the warnings.
extra_flags.append("ignore_warnings")
if filename.startswith(("PySide", "PyQt")):
if python_version.startswith("2.6"):
reportSkip("irrelevant Python version", ".", filename)
continue
# For the plug-in information.
if getPythonVendor() != "Anaconda":
extra_flags.append("plugin_enable:qt-plugins")
else:
# For the plug-in not used information.
extra_flags.append("ignore_warnings")
my_print("Consider output of recursively compiled program:", filename)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
# Second check if glibc libraries haven't been accidentally
# shipped with the standalone executable
found_glibc_libs = []
for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
if os.path.basename(dist_filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
found_glibc_libs.append(dist_filename)
if found_glibc_libs:
my_print(
"Should not ship glibc libraries with the standalone executable (found %s)"
% found_glibc_libs)
sys.exit(1)
binary_filename = os.path.join(
filename[:-3] + ".dist",
filename[:-3] + (".exe" if os.name == "nt" else ""))
# Then use "strace" on the result.
loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)
current_dir = os.path.normpath(os.getcwd())
current_dir = os.path.normcase(current_dir)
illegal_access = False
for loaded_filename in loaded_filenames:
loaded_filename = os.path.normpath(loaded_filename)
loaded_filename = os.path.normcase(loaded_filename)
loaded_basename = os.path.basename(loaded_filename)
if os.name == "nt":
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"System32")),
):
continue
if areSamePaths(
os.path.dirname(loaded_filename),
os.path.normpath(
os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64")),
):
continue
if r"windows\winsxs" in loaded_filename:
continue
# Github actions have these in PATH overriding SYSTEMROOT
if r"windows performance toolkit" in loaded_filename:
continue
if r"powershell" in loaded_filename:
continue
if loaded_filename.startswith(current_dir):
continue
if loaded_filename.startswith(os.path.abspath(current_dir)):
continue
if loaded_filename.startswith("/etc/"):
continue
if loaded_filename.startswith(
"/proc/") or loaded_filename == "/proc":
continue
if loaded_filename.startswith("/dev/"):
continue
if loaded_filename.startswith("/tmp/"):
continue
if loaded_filename.startswith("/run/"):
continue
if loaded_filename.startswith("/usr/lib/locale/"):
continue
if loaded_filename.startswith("/usr/share/locale/"):
continue
if loaded_filename.startswith("/usr/share/X11/locale/"):
continue
# Themes may of course be loaded.
if loaded_filename.startswith("/usr/share/themes"):
continue
if "gtk" in loaded_filename and "/engines/" in loaded_filename:
continue
if loaded_filename in (
"/usr",
"/usr/local",
"/usr/local/lib",
"/usr/share",
"/usr/local/share",
"/usr/lib64",
):
continue
# TCL/tk for tkinter for non-Windows is OK.
if loaded_filename.startswith((
"/usr/lib/tcltk/",
"/usr/share/tcltk/",
"/usr/lib/tcl/",
"/usr/lib64/tcl/",
)):
continue
if loaded_filename in (
"/usr/lib/tcltk",
"/usr/share/tcltk",
"/usr/lib/tcl",
"/usr/lib64/tcl",
):
continue
if loaded_filename in (
"/lib",
"/lib64",
"/lib/sse2",
"/lib/tls",
"/lib64/tls",
"/usr/lib/sse2",
"/usr/lib/tls",
"/usr/lib64/tls",
):
continue
if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
continue
if loaded_filename in (
"/usr/share/tcl8.6/init.tcl",
"/usr/share/tcl8.5/init.tcl",
):
continue
if loaded_filename in (
"/usr/share/tcl8.6/encoding",
"/usr/share/tcl8.5/encoding",
):
continue
# System SSL config on Linux. TODO: Should this not be included and
# read from dist folder.
if loaded_basename == "openssl.cnf":
continue
# Taking these from system is harmless and desirable
if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
continue
# System C libraries are to be expected.
if loaded_basename.startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
# Loaded by C library potentially for DNS lookups.
if loaded_basename.startswith((
"libnss_",
"libnsl",
# Some systems load a lot more, this is CentOS 7 on OBS
"libattr.so.",
"libbz2.so.",
"libcap.so.",
"libdw.so.",
"libelf.so.",
"liblzma.so.",
# Some systems load a lot more, this is Fedora 26 on OBS
"libselinux.so.",
"libpcre.so.",
# And this is Fedora 29 on OBS
"libblkid.so.",
"libmount.so.",
"libpcre2-8.so.",
# CentOS 8 on OBS
"libuuid.so.",
)):
continue
# Loaded by dtruss on macOS X.
if loaded_filename.startswith("/usr/lib/dtrace/"):
continue
# Loaded by cowbuilder and pbuilder on Debian
if loaded_basename == ".ilist":
continue
if "cowdancer" in loaded_filename:
continue
if "eatmydata" in loaded_filename:
continue
# Loading from home directories is OK too.
if (loaded_filename.startswith("/home/")
or loaded_filename.startswith("/data/")
or loaded_filename.startswith("/root/")
or loaded_filename in ("/home", "/data", "/root")):
continue
# For Debian builders, /build is OK too.
if loaded_filename.startswith(
"/build/") or loaded_filename == "/build":
continue
# TODO: Unclear, loading gconv from filesystem of installed system
# may be OK or not. I think it should be.
if loaded_basename == "gconv-modules.cache":
continue
if "/gconv/" in loaded_filename:
continue
if loaded_basename.startswith("libicu"):
continue
if loaded_filename.startswith("/usr/share/icu/"):
continue
# Loading from caches is OK.
if loaded_filename.startswith("/var/cache/"):
continue
# PySide accesses its directory.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/PySide"):
continue
# GTK accesses package directories only.
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gtk"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/glib"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gtk-2.0/gio"):
continue
if (loaded_filename == "/usr/lib/python" + python_version[:3] +
"/dist-packages/gobject"):
continue
# PyQt5 seems to do this, but won't use contents then.
if loaded_filename in (
"/usr/lib/qt5/plugins",
"/usr/lib/qt5",
"/usr/lib64/qt5/plugins",
"/usr/lib64/qt5",
"/usr/lib/x86_64-linux-gnu/qt5/plugins",
"/usr/lib/x86_64-linux-gnu/qt5",
"/usr/lib/x86_64-linux-gnu",
"/usr/lib",
):
continue
# Can look at the interpreter of the system.
if loaded_basename == "python3":
continue
# Current Python executable can actually be a symlink and
# the real executable which it points to will be on the
# loaded_filenames list. This is all fine, let's ignore it.
# Also, because the loaded_filename can be yet another symlink
# (this is weird, but it's true), let's better resolve its real
# path too.
if os.path.realpath(loaded_filename) == os.path.realpath(
sys.executable):
continue
# Accessing SE-Linux is OK.
if loaded_filename in ("/sys/fs/selinux", "/selinux"):
continue
# Allow reading time zone info of local system.
if loaded_filename.startswith("/usr/share/zoneinfo/"):
continue
# The access to .pth files has no effect.
if loaded_filename.endswith(".pth"):
continue
# Looking at site-package dir alone is alone.
if loaded_filename.endswith(("site-packages", "dist-packages")):
continue
# QtNetwork insist on doing this it seems.
if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
continue
# macOS uses these:
if loaded_basename in (
"libcrypto.1.0.0.dylib",
"libssl.1.0.0.dylib",
"libcrypto.1.1.dylib",
):
continue
# MSVC run time DLLs, seem to sometimes come from system.
if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
continue
my_print("Should not access '%s'." % loaded_filename)
illegal_access = True
if illegal_access:
if os.name != "nt":
my_print("Listing of dist folder:")
os.system("ls -Rla %s" % filename[:-3] + ".dist")
my_print("strace:")
os.system("strace -s4096 -e file %s" % binary_filename)
search_mode.onErrorDetected(1)
removeDirectory(filename[:-3] + ".dist", ignore_errors=True)
if search_mode.abortIfExecuted():
break
search_mode.finish()
def main():
# pylint: disable=broad-except,too-many-branches,too-many-locals,too-many-statements
setup()
# cache_dir is where the git clones are cached
cache_dir = os.path.join(getCacheDir(), "pypi-git-clones")
base_dir = os.getcwd()
if not os.path.isdir(cache_dir):
os.mkdir(cache_dir)
search_mode = createSearchMode()
results = []
# load json
with open("packages.json", "r") as f:
packages = json.load(f)
for package_name, details in sorted(packages.items()):
active = search_mode.consider(dirname=None, filename=package_name)
if not active:
continue
if str is not bytes:
# running on python3
if package_name in ("futures", "future"):
reportSkip("Does not run on Python3", ".", package_name)
if search_mode.abortIfExecuted():
break
continue
if os.name == "nt":
if package_name in ("cryptography",):
reportSkip("Not working on Windows", ".", package_name)
if search_mode.abortIfExecuted():
break
continue
if package_name == "pyyaml":
reportSkip("Not yet supported, see Issue #476", ".", package_name)
if search_mode.abortIfExecuted():
break
continue
if package_name in ("pycparser", "numpy"):
reportSkip("Not yet supported, see Issue #477", ".", package_name)
if search_mode.abortIfExecuted():
break
continue
if package_name in (
"google-auth", # bdist_nuitka fails AttributeError: single_version_externally_managed
"jinja2", # ModuleNotFoundError: No module named 'jinja2.tests'
"pandas", # ModuleNotFoundError: No module named 'Cython'
"pytz", # need to 'make build'
"rsa", # Now uses Poetry (no setup.py)
):
if search_mode.abortIfExecuted():
break
continue
package_dir = os.path.join(cache_dir, package_name)
try:
gitClone(package_name, details["url"], cache_dir)
os.chdir(base_dir)
with withVirtualenv(
"venv_%s" % package_name, delete=False, style="blue"
) as venv:
dist_dir = os.path.join(package_dir, "dist")
# delete ignored tests if any
if details["ignored_tests"]:
for test in details["ignored_tests"]:
venv.runCommand("rm -rf %s" % os.path.join(package_dir, test))
# setup for pytest
cmds = [
"python -m pip install pytest",
"cd %s" % os.path.join(os.path.dirname(nuitka.__file__), ".."),
"python setup.py develop",
"cd %s" % package_dir,
]
if details["requirements_file"]:
cmds.append(
"python -m pip install -r %s" % details["requirements_file"]
)
if details.get("extra_commands"):
cmds += details["extra_commands"]
# build uncompiled .whl
cmds.append("python setup.py bdist_wheel")
venv.runCommand(commands=cmds)
# install and print out if the active .whl is compiled or not
venv.runCommand(
commands=[
"python -m pip install -U %s"
% os.path.join(dist_dir, os.listdir(dist_dir)[0]),
# use triple quotes for linux
"""python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """
% details.get("package_name", package_name),
]
)
# get uncompiled pytest results
uncompiled_stdout, uncompiled_stderr = venv.runCommandWithOutput(
commands=[
"cd %s" % package_dir,
"python -m pytest --disable-warnings",
],
style="blue",
)
# clean up before building compiled .whl
cmds = ["cd %s" % package_dir, "git clean -dfx"]
if details.get("extra_commands"):
cmds += details["extra_commands"]
# build nuitka compiled .whl
cmds.append("python setup.py bdist_nuitka")
venv.runCommand(commands=cmds)
# install and print out if the active .whl is compiled or not
venv.runCommand(
commands=[
"python -m pip install -U %s"
% os.path.join(dist_dir, os.listdir(dist_dir)[0]),
# use triple quotes for linux
"""python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """
% details.get("package_name", package_name),
]
)
# get compiled pytest results
compiled_stdout, compiled_stderr = venv.runCommandWithOutput(
commands=[
"cd %s" % package_dir,
"python -m pytest --disable-warnings",
],
style="blue",
)
venv.runCommand(commands=["cd %s" % package_dir, "git clean -q -dfx"])
except Exception as e:
my_print(
"Package",
package_name,
"ran into an exception during execution, traceback: ",
)
my_print(e)
results.append((package_name, "ERROR", "ERROR"))
if search_mode.abortIfExecuted():
break
continue
# compare outputs
stdout_diff = compareOutput(
"stdout",
uncompiled_stdout,
compiled_stdout,
ignore_warnings=True,
syntax_errors=True,
)
stderr_diff = compareOutput(
"stderr",
uncompiled_stderr,
compiled_stderr,
ignore_warnings=True,
syntax_errors=True,
)
results.append((package_name, stdout_diff, stderr_diff))
exit_code = stdout_diff or stderr_diff
my_print(
"\n=================================================================================",
"\n--- %s ---" % package_name,
"exit_stdout:",
stdout_diff,
"exit_stderr:",
stderr_diff,
"\nError, outputs differed for package %s." % package_name
if exit_code
else "\nNo differences found for package %s." % package_name,
"\n=================================================================================\n",
style="red" if exit_code else "green",
)
if exit_code != 0 and search_mode.abortOnFinding(
dirname=None, filename=package_name
):
break
if search_mode.abortIfExecuted():
break
search_mode.finish()
# give a summary of all packages
my_print(
"\n\n=====================================SUMMARY=====================================",
style="yellow",
)
for package_name, stdout_diff, stderr_diff in results:
my_print(
package_name,
"-",
end=" ",
style="red" if (stdout_diff or stderr_diff) else "green",
)
my_print(
"stdout:", stdout_diff, end=" ", style="red" if stdout_diff else "green"
)
my_print(
"stderr:", stderr_diff, end="", style="red" if stderr_diff else "green"
)
my_print(
"\n---------------------------------------------------------------------------------"
)
my_print("TOTAL NUMBER OF PACKAGES TESTED: %s" % len(results), style="yellow")
num_failed = 0
num_errors = 0
# tally the number of errors and failed
for _, y, z in results:
if type(y) is str:
# this means the package ran into an exception
num_errors += 1
elif y or z:
num_failed += 1
my_print(
"TOTAL PASSED: %s" % (len(results) - num_failed - num_errors), style="green"
)
my_print("TOTAL FAILED (differences): %s" % num_failed, style="red")
my_print("TOTAL ERRORS (exceptions): %s" % num_errors, style="red")
def main():
python_version = setup(suite="onefile", needs_io_encoding=True)
search_mode = createSearchMode()
if getOS() == "Linux":
addExtendedExtraOptions(
"--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(dirname=None, filename=filename)
if not active:
test_logger.info("Skipping %s" % filename)
continue
extra_flags = [
"expect_success",
"remove_output",
# Keep the binary, normally "remove_output" includes that.
"--keep-binary",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# To understand what is slow.
"timing",
# The onefile can warn about zstandard not being installed.
"ignore_warnings",
]
if filename == "KeyboardInterruptTest.py":
if getOS() == "Darwin":
reportSkip(
"Exit code from KeyboardInterrupt on macOS is not yet good.",
".",
filename,
)
continue
if python_version < (3,):
reportSkip(
"Python2 reports KeyboardInterrupt, but too late",
".",
filename,
)
continue
if os.name == "nt":
reportSkip(
"Testing cannot send KeyboardInterrupt on Windows yet",
".",
filename,
)
continue
extra_flags.append("--send-ctrl-c")
# skip each test if their respective requirements are not met
requirements_met, error_message = checkRequirements(filename)
if not requirements_met:
reportSkip(error_message, ".", filename)
continue
test_logger.info(
"Consider output of onefile mode compiled program: %s" % filename
)
# First compare so we know the program behaves identical.
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
on_error=displayError,
)
binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")
if filename == "KeyboardInterruptTest.py":
continue
# Then use "strace" on the result.
with TimerReport(
"Determining run time loaded files took %.2f", logger=test_logger
):
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger, command=[binary_filename]
)
illegal_accesses = checkLoadedFileAccesses(
loaded_filenames=loaded_filenames, current_dir=os.getcwd()
)
if illegal_accesses:
displayError(None, filename)
displayRuntimeTraces(test_logger, binary_filename)
test_logger.warning(
"Should not access these file(s): '%s'." % ",".join(illegal_accesses)
)
search_mode.onErrorDetected(1)
os.unlink(binary_filename)
search_mode.finish()
def main():
python_version = setup()
os.chdir("subject")
nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--follow-imports",
"--include-package=package",
"--nofollow-import-to=*.tests",
"--python-flag=-v",
"--debug",
"--module",
"package",
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
os.makedirs(os.path.join(tmp_dir, "package.ext"))
copyTree("package", os.path.join(tmp_dir, "package.ext/package"))
os.chdir(tmp_dir)
# We compile the package non-closed, so we can smuggle in tests
# and user code. This is going to be the example code.
with open("package.ext/package/user_provided.py", "w") as output:
# TODO: Maybe assert that the type name of a local function and one from
# the package are not the same, i.e. we are running inside the compiled
# package.
output.write("""
from __future__ import print_function
import package
print("__name__:", package.__name__)
print("__package__:", package.__package__)
print("__path__:", package.__path__)
print("__file__:", package.__file__)
# print("__loader__:", package.__loader__)
import package.sub_package1
print("__name__:", package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:", package.sub_package1.__path__)
print("__file__:", package.sub_package1.__file__)
# print("__loader__:", package.sub_package1.__loader__)
import package.sub_package1.tests;
print("__name__:", package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:", package.sub_package1.tests.__path__)
print("__file__:", package.sub_package1.tests.__file__)
# print("__loader__:", package.sub_package1.tests.__loader__)
""")
os.makedirs("nose")
with open("nose/usage.txt", "w") as output:
pass
os.system("find | sort")
# Inform about the extra path, format is NUITKA_PACKAGE_fullname where
# dots become "_" and should point to the directory where external code
# to be loaded will live under. Probably should be an absolute path, but
# we avoid it here.
os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"
# Lets make sure these to not work. These will be used in the compiled
# form only.
for module_path in ("__init__.py", "sub_package1__init__.py"):
with open(os.path.join("./package.ext/package", module_path),
"w") as output:
output.write("assert False")
# Check the compiled package is functional for importing.
my_print("Running package as basic test:")
command = [os.environ["PYTHON"], "-c", "import package"]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
my_print("Running nose tests:")
# assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0
my_print("Running py.test tests:")
command = [
os.environ["PYTHON"],
"-m",
"pytest",
"-v",
"--pyargs",
"package.sub_package1.tests",
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
def main():
python_version = setup()
os.chdir("subject")
nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--follow-imports",
"--include-package=package",
"--nofollow-import-to=*.tests",
"--python-flag=-v",
"--debug",
"--module",
"package"
]
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
os.makedirs(os.path.join(tmp_dir, "package.ext"))
shutil.copytree(
"package",
os.path.join(tmp_dir, "package.ext/package")
)
os.chdir(tmp_dir)
# We compile the package non-closed, so we can smuggle in tests
# and user code. This is going to be the example code.
with open("package.ext/package/user_provided.py", "w") as output:
# TODO: Maybe assert that the type name of a local function and one from
# the package are not the same, i.e. we are running inside the compiled
# package.
output.write("""
from __future__ import print_function
import package
print("__name__:", package.__name__)
print("__package__:", package.__package__)
print("__path__:", package.__path__)
print("__file__:", package.__file__)
# print("__loader__:", package.__loader__)
import package.sub_package1
print("__name__:", package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:", package.sub_package1.__path__)
print("__file__:", package.sub_package1.__file__)
# print("__loader__:", package.sub_package1.__loader__)
import package.sub_package1.tests;
print("__name__:", package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:", package.sub_package1.tests.__path__)
print("__file__:", package.sub_package1.tests.__file__)
# print("__loader__:", package.sub_package1.tests.__loader__)
"""
)
os.makedirs("nose")
with open("nose/usage.txt", "w") as output:
pass
os.system("find | sort")
# Inform about the extra path, format is NUITKA_PACKAGE_fullname where
# dots become "_" and should point to the directory where external code
# to be loaded will live under. Probably should be an absolute path, but
# we avoid it here.
os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"
# Lets make sure these to not work. These will be used in the compiled
# form only.
for module_path in ("__init__.py", "sub_package1__init__.py"):
with open(os.path.join("./package.ext/package", module_path), "w") as output:
output.write("assert False")
# Check the compiled package is functional for importing.
my_print("Running package as basic test:")
assert os.system(os.environ["PYTHON"] + " -c 'import package'" ) == 0
my_print("Running nose tests:")
# assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0
my_print("Running py.test tests:")
assert os.system(os.environ["PYTHON"] + " -m pytest --pyargs package.sub_package1.tests" ) == 0