Example #1
0
def main():
    setup(suite="packages")

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not os.path.isdir(filename) or filename.endswith(".build"):
            continue

        extra_flags = [
            "--module",
            "expect_success",
            "remove_output",
            "two_step_execution",
        ]

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            my_print("Consider output of compiled package:", filename)

            filename_main = None

            filename_main = os.path.join(
                filename, "".join(part.title() for part in filename.split("_")) + ".py"
            )
            if os.path.exists(filename_main):
                filename_main = os.path.basename(filename_main)
            else:
                filename_main = None

            if filename_main is None:
                for filename_main in os.listdir(filename):
                    if not os.path.isdir(os.path.join(filename, filename_main)):
                        continue

                    if filename_main not in ("..", "."):
                        break
                else:
                    search_mode.onErrorDetected(
                        """\
Error, no package in test directory '%s' found, incomplete test case."""
                        % filename
                    )

                extra_flags.append(
                    "--include-package=%s" % os.path.basename(filename_main)
                )

            extra_flags.append("--output-dir=%s" % getTempDir())

            compareWithCPython(
                dirname=filename,
                filename=filename_main,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=False,
            )

    search_mode.finish()
Example #2
0
def main():
    setup(suite="syntax", needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            extra_flags = [
                "expect_failure",
                "remove_output",
                "syntax_errors",
                "--nofollow-imports",
            ]

            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=False,
            )

    search_mode.finish()
Example #3
0
def main():
    setup()

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not os.path.isdir(filename) or filename.endswith(".build"):
            continue

        extra_flags = [
            "expect_success",
            "remove_output",
            "module_mode",
            "two_step_execution",
        ]

        # The use of "__main__" in the test package gives a warning.
        if filename == "sub_package":
            extra_flags.append("ignore_warnings")

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            my_print("Consider output of recursively compiled package:",
                     filename)

            filename_main = None
            for filename_main in os.listdir(filename):
                if not os.path.isdir(os.path.join(filename, filename_main)):
                    continue

                if filename_main not in ("..", "."):
                    break
            else:
                search_mode.onErrorDetected("""\
Error, no package in dir '%s' found, incomplete test case.""" % filename)

            extensions = [
                "--include-package=%s" % os.path.basename(filename_main)
            ]

            if "--output-dir" not in os.environ.get("NUITKA_EXTRA_OPTIONS",
                                                    ""):
                extensions.append("--output-dir=%s" % getTempDir())

            with withExtendedExtraOptions(*extensions):
                compareWithCPython(
                    dirname=filename,
                    filename=filename_main,
                    extra_flags=extra_flags,
                    search_mode=search_mode,
                    needs_2to3=False,
                )

            if search_mode.abortIfExecuted():
                break

    search_mode.finish()
Example #4
0
def main():
    setup(needs_io_encoding=True)

    executePASS1()
    executePASS2()
    executePASS3()
    executePASS4()

    shutil.rmtree("nuitka")

    executePASS5()
Example #5
0
def main():
    if os.name != "nt":
        sys.exit("Error, this is only for use on Windows where SxS exists.")

    setup(needs_io_encoding=True)
    search_mode = createSearchMode()

    tmp_dir = tempfile.gettempdir()

    compileLibraryTest(search_mode=search_mode,
                       stage_dir=os.path.join(tmp_dir, "find_sxs_modules"),
                       decide=decide,
                       action=action)

    my_print("FINISHED, all extension modules checked.")
Example #6
0
def main():
    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        if filename == "TryFinallyContinue.py" and python_version >= "3.8":
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            extra_flags = ["expect_failure", "remove_output", "syntax_errors"]

            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=False,
            )

            if search_mode.abortIfExecuted():
                break
        else:
            my_print("Skipping", filename)

    search_mode.finish()
Example #7
0
def main():
    if os.name != "nt":
        sys.exit("Error, this is only for use on Windows where SxS exists.")

    setup(needs_io_encoding=True)
    search_mode = createSearchMode()

    tmp_dir = tempfile.gettempdir()

    compileLibraryTest(
        search_mode=search_mode,
        stage_dir=os.path.join(tmp_dir, "find_sxs_modules"),
        decide=decide,
        action=action,
    )

    my_print("FINISHED, all extension modules checked.")
Example #8
0
def main():
    setup(suite="pgo", needs_io_encoding=True)

    search_mode = createSearchMode()

    # Now run all the tests in this directory.
    for filename, filename_main in scanDirectoryForTestCaseFolders("."):

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            extra_flags = [
                # No error exits normally, unless we break tests, and that we would
                # like to know.
                "expect_success",
                # Keep no temporary files.
                "remove_output",
                # Include imported files, PGO will then have to deal with unused ones.
                "--follow-imports",
                # The output during compilation from PGO capture is harmful, so
                # split compilation and execution of final result.
                "two_step_execution",
                # Inclusion report is used by the testing of expected things included
                # or not.
                "--report=%s.xml" % filename,
                # Cache the CPython results for re-use, they will normally not change.
                "cpython_cache",
            ]

            compareWithCPython(
                dirname=filename,
                filename=filename_main,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=False,
            )

    search_mode.finish()
Example #9
0
def main():
    _python_version = setup(needs_io_encoding=True)

    executePASS1()
    executePASS2()
    executePASS3()
    executePASS4()

    shutil.rmtree("nuitka")

    executePASS5()

    os.unlink(os.path.join(tmp_dir, "nuitka" + exe_suffix))
    os.rmdir(tmp_dir)
Example #10
0
def main():
    python_version = setup(suite="basics", needs_io_encoding=True)

    search_mode = createSearchMode()

    filenames = _createTests(python_version)

    # Now run all the tests in this directory.
    for filename in filenames:
        assert filename.endswith(".py")

        if not decideFilenameVersionSkip(filename):
            continue

        extra_flags = [
            # No error exits normally, unless we break tests, and that we would
            # like to know.
            "expect_success",
            # Keep no temporary files.
            "remove_output",
            # Include imported files, mostly nothing though.
            "recurse_all",
            # Use the original __file__ value, at least one case warns about things
            # with filename included.
            "original_file",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # We annotate some tests, use that to lower warnings.
            "plugin_enable:pylint-warnings",
        ]

        # This test should be run with the debug Python, and makes outputs to

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=decideNeeds2to3(filename),
            )

            if search_mode.abortIfExecuted():
                break
        else:
            my_print("Skipping", filename)

    search_mode.finish()
Example #11
0
def main():
    python_version = setup(suite="basics", needs_io_encoding=True)

    search_mode = createSearchMode()

    # Singleton, pylint: disable=global-statement
    global operations
    global candidates

    if python_version >= (3, 5):
        operations += (("MatMult", "@"),)

    if python_version < (3,):
        candidates += (("long", "17L", "-9L"),)
        candidates += (("unicode", "u'lala'", "u'lele'"),)
    else:
        candidates += (("bytes", "b'lala'", "b'lele'"),)

    template_context = {
        "operations": operations,
        "ioperations": tuple(
            operation
            for operation in operations
            if operation[0] not in ("Divmod", "Subscript")
        ),
        "candidates": candidates,
        "makeOperatorUsage": makeOperatorUsage,
    }

    # Now run all the tests in this directory.
    for filename in scanDirectoryForTestCases(".", template_context=template_context):
        extra_flags = [
            # No error exits normally, unless we break tests, and that we would
            # like to know.
            "expect_success",
            # Keep no temporary files.
            "remove_output",
            # Include imported files, mostly nothing though.
            "recurse_all",
            # Use the original __file__ value, at least one case warns about things
            # with filename included.
            "original_file",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # We annotate some tests, use that to lower warnings.
            "plugin_enable:pylint-warnings",
        ]

        # This test should be run with the debug Python, and makes outputs to

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=decideNeeds2to3(filename),
            )

    search_mode.finish()
Example #12
0
# isort:start

import subprocess

from nuitka.tools.testing.Common import (
    createSearchMode,
    decideFilenameVersionSkip,
    my_print,
    setup,
)
from nuitka.tools.testing.OutputComparison import compareOutput
from nuitka.tools.testing.Virtualenv import withVirtualenv
from nuitka.utils.FileOperations import removeDirectory

python_version = setup(needs_io_encoding=True)

search_mode = createSearchMode()

nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))

for filename in sorted(os.listdir(".")):
    if (
        not os.path.isdir(filename)
        or filename.endswith(".build")
        or filename.endswith(".dist")
    ):
        continue

    filename = os.path.relpath(filename)
Example #13
0
# Find nuitka package relative to us.
sys.path.insert(
    0,
    os.path.normpath(
        os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "..",
            ".."
        )
    )
)

from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip
from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip

python_version = setup()

nuitka_main_path = os.path.join("..", "..", "bin", "nuitka")

tmp_dir = getTempDir()

# Cannot detect this more automatic, either need whitelist or
# blacklist not needed stuff.
PACKAGE_LIST = (
    "nuitka",
    "nuitka/nodes",
    "nuitka/specs",
    "nuitka/nodes/shapes",
    "nuitka/tree",
    "nuitka/importing",
    "nuitka/build",
Example #14
0
def main():
    # Complex stuff, not broken down yet
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    parser = OptionParser()

    parser.add_option(
        "--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "")
    )

    parser.add_option(
        "--cpython",
        action="store",
        dest="cpython",
        default=os.environ.get("PYTHON", sys.executable),
    )

    parser.add_option("--code-diff", action="store", dest="diff_filename", default="")

    parser.add_option("--copy-source-to", action="store", dest="target_dir", default="")

    options, positional_args = parser.parse_args()

    if len(positional_args) != 1:
        sys.exit("Error, need to give test case file name as positional argument.")

    test_case = positional_args[0]

    if os.path.exists(test_case):
        test_case = os.path.abspath(test_case)

    if options.cpython == "no":
        options.cpython = ""

    nuitka = options.nuitka

    if os.path.exists(nuitka):
        nuitka = os.path.abspath(nuitka)
    elif nuitka:
        sys.exit("Error, nuitka binary '%s' not found." % nuitka)

    python_version = setup(silent=True, go_main=False)

    assert os.path.exists(test_case), (test_case, os.getcwd())

    my_print("PYTHON='%s'" % python_version)
    my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
    with open(test_case, "rb") as f:
        my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest())

    needs_2to3 = (
        python_version.startswith("3")
        and not test_case.endswith("32.py")
        and not test_case.endswith("33.py")
    )

    if options.target_dir:
        shutil.copyfile(
            test_case, os.path.join(options.target_dir, os.path.basename(test_case))
        )

    # First produce two variants.
    temp_dir = getTempDir()

    test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case))
    test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case))

    with open(test_case) as f:
        case_1_source, case_2_source = generateConstructCases(f.read())

    with open(test_case_1, "w") as case_1_file:
        case_1_file.write(case_1_source)

    with open(test_case_2, "w") as case_2_file:
        case_2_file.write(case_2_source)

    if needs_2to3:
        test_case_1, _needs_delete = convertUsing2to3(test_case_1)
        test_case_2, _needs_delete = convertUsing2to3(test_case_2)

    os.environ["PYTHONHASHSEED"] = "0"

    if nuitka:
        nuitka_id = check_output(
            "cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True
        )
        nuitka_id = nuitka_id.strip()

        if sys.version_info > (3,):
            nuitka_id = nuitka_id.decode()

        my_print("NUITKA_COMMIT='%s'" % nuitka_id)

    os.chdir(getTempDir())

    if nuitka:
        nuitka_call = [
            os.environ["PYTHON"],
            nuitka,
            "--python-flag=-S",
            os.path.basename(test_case),
        ]
        nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())

        # We want to compile under the same filename to minimize differences, and
        # then copy the resulting files afterwards.
        shutil.copyfile(test_case_1, os.path.basename(test_case))

        subprocess.check_call(nuitka_call)

        if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
            exe_suffix = ".exe"
        else:
            exe_suffix = ".bin"

        os.rename(
            os.path.basename(test_case).replace(".py", ".build"),
            os.path.basename(test_case_1).replace(".py", ".build"),
        )
        os.rename(
            os.path.basename(test_case).replace(".py", exe_suffix),
            os.path.basename(test_case_1).replace(".py", exe_suffix),
        )

        shutil.copyfile(test_case_2, os.path.basename(test_case))

        subprocess.check_call(nuitka_call)

        os.rename(
            os.path.basename(test_case).replace(".py", ".build"),
            os.path.basename(test_case_2).replace(".py", ".build"),
        )
        os.rename(
            os.path.basename(test_case).replace(".py", exe_suffix),
            os.path.basename(test_case_2).replace(".py", exe_suffix),
        )

        if options.diff_filename:
            suffixes = [".c", ".cpp"]

            for suffix in suffixes:
                cpp_1 = os.path.join(
                    test_case_1.replace(".py", ".build"), "module.__main__" + suffix
                )

                if os.path.exists(cpp_1):
                    break
            else:
                assert False

            for suffix in suffixes:
                cpp_2 = os.path.join(
                    test_case_2.replace(".py", ".build"), "module.__main__" + suffix
                )
                if os.path.exists(cpp_2):
                    break
            else:
                assert False

            import difflib

            with open(options.diff_filename, "w") as f:
                with open(cpp_1) as cpp1:
                    with open(cpp_2) as cpp2:
                        f.write(
                            difflib.HtmlDiff().make_table(
                                cpp1.readlines(),
                                cpp2.readlines(),
                                "Construct",
                                "Baseline",
                                True,
                            )
                        )

        nuitka_1 = runValgrind(
            "Nuitka construct",
            "callgrind",
            (test_case_1.replace(".py", exe_suffix),),
            include_startup=True,
        )

        nuitka_2 = runValgrind(
            "Nuitka baseline",
            "callgrind",
            (test_case_2.replace(".py", exe_suffix),),
            include_startup=True,
        )

        nuitka_diff = nuitka_1 - nuitka_2

        my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr)
        my_print("NUITKA_RAW=%s" % nuitka_1)
        my_print("NUITKA_BASE=%s" % nuitka_2)
        my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)

    if options.cpython:
        cpython_1 = runValgrind(
            "CPython construct",
            "callgrind",
            (os.environ["PYTHON"], "-S", test_case_1),
            include_startup=True,
        )
        cpython_2 = runValgrind(
            "CPython baseline",
            "callgrind",
            (os.environ["PYTHON"], "-S", test_case_2),
            include_startup=True,
        )

        cpython_diff = cpython_1 - cpython_2

        my_print("CPYTHON_RAW=%d" % cpython_1)
        my_print("CPYTHON_BASE=%d" % cpython_2)
        my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)

    if options.cpython and options.nuitka:
        if nuitka_diff == 0:
            nuitka_gain = float("inf")
        else:
            nuitka_gain = float(100 * cpython_diff) / nuitka_diff

        my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
        my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
        my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def main():
    setup(suite="basics", needs_io_encoding=True)

    search_mode = createSearchMode()

    # Now run all the tests in this directory.
    for filename in scanDirectoryForTestCases("."):
        extra_flags = [
            # No error exits normally, unless we break tests, and that we would
            # like to know.
            "expect_success",
            # Keep no temporary files.
            "remove_output",
            # Include imported files, mostly nothing though.
            "recurse_all",
            # Use the original __file__ value, at least one case warns about things
            # with filename included.
            "original_file",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
            # We annotate some tests, use that to lower warnings.
            "plugin_enable:pylint-warnings",
        ]

        # This test should be run with the debug Python, and makes outputs to
        # standard error that might be ignored.
        if filename.startswith("Referencing"):
            extra_flags.append("recurse_not:nuitka")

            if hasDebugPython():
                extra_flags.append("python_debug")

        # This tests warns about __import__() used.
        if filename == "OrderChecks.py":
            extra_flags.append("ignore_warnings")

        # This tests warns about an package relative import despite
        # being in no package.
        if filename == "Importing.py":
            extra_flags.append("ignore_warnings")

        # TODO: Nuitka does not give output for ignored exception in dtor, this is
        # not fully compatible and potentially an error.
        if filename == "YieldFrom33.py":
            extra_flags.append("ignore_stderr")

        # For Python2 there is a "builtins" package that gives warnings. TODO: We
        # ought to NOT import that package and detect statically that __builtins__
        # import won't raise ImportError.
        if filename == "BuiltinOverload.py":
            extra_flags.append("ignore_warnings")

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=decideNeeds2to3(filename),
            )

            if search_mode.abortIfExecuted():
                break

    search_mode.finish()
Example #16
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(suite="programs", needs_io_encoding=True)

    search_mode = createSearchMode()

    extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")

    for filename, filename_main in scanDirectoryForTestCaseFolders("."):

        # For these, we expect that they will fail.
        expected_errors = [
            "module_exits",
            "main_raises",
            "main_raises2",
            "package_contains_main",
        ]

        # After Python3 those have been made to work.
        if python_version < (3, 5):
            expected_errors.append("cyclic_imports")

        # Allowed with Python3, packages need no more "__init__.py"
        if python_version < (3, ):
            expected_errors.append("package_missing_init")

        # Allowed with Python3.5 only:
        if python_version < (3, 5):
            expected_errors.append("package_init_issue")

        # Allowed with Python3, name imports can be module imports
        if python_version < (3, ):
            expected_errors.append("named_imports")

        extra_variant = []

        if filename not in expected_errors:
            extra_flags = ["expect_success"]
        else:
            extra_flags = ["expect_failure"]

        # We annotate some tests, use that to lower warnings.
        extra_flags.append("plugin_enable:pylint-warnings")

        if filename in (
                "reimport_main_static",
                "package_missing_init",
                "dash_import",
                "package_contains_main",
                "case_imports3",
                "import_variants",
                "package_init_import",
                "pkgutil_itermodules",
        ):
            extra_flags.append("ignore_warnings")

        extra_flags.append("remove_output")

        extra_flags.append("--follow-imports")

        # Use the original "__file__" value normally, at least one case warns
        # about things with filename included, but for pkgutil iteration, make
        # sure we do not see original Python dirs.
        if filename != "pkgutil_itermodules":
            extra_flags.append("--file-reference-choice=original")
        else:
            extra_flags.append("--file-reference-choice=runtime")

        # Run it as a package and also as directory.
        if filename == "package_program":
            # Not really supported for 2.6
            if python_version >= (2, 7):
                extra_variant.append("--python-flag=-m")

        # Cannot include the files with syntax errors, these would then become
        # ImportError, but that's not the test. In all other cases, use two
        # step execution, which will not add the program original source to
        # PYTHONPATH.
        if filename != "syntax_errors":
            extra_flags.append("two_step_execution")
        else:
            extra_flags.append("binary_python_path")

        if filename == "plugin_import":
            os.environ["NUITKA_EXTRA_OPTIONS"] = (
                extra_options + " --include-package=some_package")
        elif filename == "reimport_main_dynamic":
            if python_version < (3, ):
                os.environ[
                    "NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-directory=%s" % (
                        os.path.abspath(filename))
            else:
                os.environ[
                    "NUITKA_EXTRA_OPTIONS"] = extra_options + " --include-plugin-files=%s/*.py" % (
                        os.path.abspath(filename))

            extra_flags.append("ignore_warnings")
        elif filename == "multiprocessing_using":
            # TODO: Still true?
            if sys.platform == "darwin" and python_version >= (3, 8):
                reportSkip("Hangs for unknown reasons", ".", filename)
                continue
        else:
            os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            my_print("Consider output of recursively compiled program:",
                     filename)

            extra_python_path = [
                os.path.abspath(os.path.join(filename, entry))
                for entry in os.listdir(filename) if entry.startswith("path")
            ]

            if extra_python_path:
                my_print("Applying extra PYTHONPATH %r." % extra_python_path)

            with withPythonPathChange(extra_python_path):
                compareWithCPython(
                    dirname=filename,
                    filename=filename_main,
                    extra_flags=extra_flags,
                    search_mode=search_mode,
                    needs_2to3=False,
                )

                if extra_variant:
                    my_print("Extra variation %r." % extra_variant)
                    compareWithCPython(
                        dirname=filename,
                        filename=filename_main,
                        extra_flags=extra_flags + extra_variant,
                        search_mode=search_mode,
                        needs_2to3=False,
                    )

    search_mode.finish()
Example #17
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS", "")

    for filename in sorted(os.listdir(".")):
        if (not os.path.isdir(filename) or filename.endswith(".build")
                or filename.endswith(".dist")):
            continue

        filename = os.path.relpath(filename)

        # For these, we expect that they will fail.
        expected_errors = [
            "module_exits",
            "main_raises",
            "main_raises2",
            "package_contains_main",
        ]

        # After Python3 those have been made to work.
        if python_version < "3.5":
            expected_errors.append("cyclic_imports")

        # Allowed with Python3, packages need no more "__init__.py"
        if python_version < "3":
            expected_errors.append("package_missing_init")

        # Allowed with Python3 only:
        if python_version < "3.5":
            expected_errors.append("package_init_issue")

        if filename not in expected_errors:
            extra_flags = ["expect_success"]
        else:
            extra_flags = ["expect_failure"]

        if filename in (
                "reimport_main_static",
                "package_missing_init",
                "dash_import",
                "package_contains_main",
                "case_imports3",
                "import_variants",
                "package_init_import",
                "pkgutil_itermodules",
        ):
            extra_flags.append("ignore_warnings")

        extra_flags.append("remove_output")

        extra_flags.append("recurse_all")

        # Use the original __file__ value, at least one case warns about things
        # with filename included, but for pkgutil iteration, make sure we do not
        # see original Python dirs.
        if filename != "pkgutil_itermodules":
            extra_flags.append("original_file")
        else:
            extra_flags.append("runtime_file")

        # Cannot include the files with syntax errors, these would then become
        # ImportError, but that's not the test. In all other cases, use two
        # step execution, which will not add the program original source to
        # PYTHONPATH.
        if filename != "syntax_errors":
            extra_flags.append("two_step_execution")
        else:
            extra_flags.append("binary_python_path")

        if filename == "plugin_import":
            os.environ["NUITKA_EXTRA_OPTIONS"] = (
                extra_options + " --include-package=some_package")
        elif filename == "reimport_main_dynamic":
            if python_version < "3":
                os.environ["NUITKA_EXTRA_OPTIONS"] = (
                    extra_options + " --include-plugin-directory=%s" %
                    (os.path.abspath(filename)))
            else:
                os.environ["NUITKA_EXTRA_OPTIONS"] = (
                    extra_options + " --include-plugin-files=%s/*.py" %
                    (os.path.abspath(filename)))

            extra_flags.append("ignore_warnings")
        elif filename == "multiprocessing_using":
            if os.name == "nt":
                extra_flags += [
                    "plugin_enable:multiprocessing", "ignore_infos"
                ]
        else:
            os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            my_print("Consider output of recursively compiled program:",
                     filename)

            filename_main = getMainProgramFilename(filename)

            extra_python_path = [
                os.path.abspath(os.path.join(filename, entry))
                for entry in os.listdir(filename) if entry.startswith("path")
            ]

            with withPythonPathChange(extra_python_path):
                compareWithCPython(
                    dirname=filename,
                    filename=filename_main,
                    extra_flags=extra_flags,
                    search_mode=search_mode,
                    needs_2to3=False,
                )

            if search_mode.abortIfExecuted():
                break
        else:
            my_print("Skipping", filename)

    search_mode.finish()
Example #18
0
test_case = positional_args[0]

if os.path.exists(test_case):
    test_case = os.path.abspath(test_case)

if options.cpython == "no":
    options.cpython = ""

nuitka = options.nuitka

if os.path.exists(nuitka):
    nuitka = os.path.abspath(nuitka)
elif nuitka:
    sys.exit("Error, nuitka binary '%s' not found." % nuitka)

python_version = setup(silent=True)

assert os.path.exists(test_case), (test_case, os.getcwd())

my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
my_print("TEST_CASE_HASH='%s'" %
         hashlib.md5(open(test_case, "rb").read()).hexdigest())


needs_2to3 = python_version.startswith('3') and \
             not test_case.endswith("32.py") and \
             not test_case.endswith("33.py")

if options.target_dir:
    shutil.copyfile(
def main():
    # Complex stuff, pylint: disable=too-many-locals,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    nuitka_dir = os.path.abspath(os.path.join(os.getcwd(), "..", ".."))

    for filename in sorted(os.listdir(".")):
        if (not os.path.isdir(filename) or filename.endswith(
            (".build", ".dist")) or filename.startswith("venv_")):
            continue

        filename = os.path.relpath(filename)

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            my_print("Consider distutils example:", filename)

            py3_only_examples = ("example_3", "nested_namespaces")
            if python_version < (3, ) and filename in py3_only_examples:
                reportSkip("Skipped, only relevant for Python3", ".", filename)
                continue

            case_dir = os.path.join(os.getcwd(), filename)

            removeDirectory(os.path.join(case_dir, "build"),
                            ignore_errors=False)
            removeDirectory(os.path.join(case_dir, "dist"),
                            ignore_errors=False)

            with withVirtualenv("venv_cpython") as venv:
                venv.runCommand(commands=[
                    'cd "%s"' % case_dir, "python setup.py bdist_wheel"
                ])

                dist_dir = os.path.join(case_dir, "dist")

                venv.runCommand('pip install "%s"' %
                                (os.path.join(dist_dir,
                                              os.listdir(dist_dir)[0])))

                runner_binary = os.path.join(
                    venv.getVirtualenvDir(),
                    "bin" if os.name != "nt" else "scripts",
                    "runner",
                )

                if os.path.exists(runner_binary):
                    # Need to call CPython binary for Windows.
                    process = subprocess.Popen(
                        args=[
                            os.path.join(
                                venv.getVirtualenvDir(),
                                "bin" if os.name != "nt" else "scripts",
                                "python",
                            ),
                            os.path.join(
                                venv.getVirtualenvDir(),
                                "bin" if os.name != "nt" else "scripts",
                                "runner",
                            ),
                        ],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE,
                    )
                else:
                    assert os.path.exists(runner_binary + ".exe")

                    process = subprocess.Popen(
                        args=[runner_binary + ".exe"],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE,
                    )

                stdout_cpython, stderr_cpython = process.communicate()
                exit_cpython = process.returncode

                my_print("STDOUT CPython:")
                my_print(stdout_cpython)
                my_print("STDERR CPython:")
                my_print(stderr_cpython)

                assert exit_cpython == 0, exit_cpython
                my_print("EXIT was OK.")

            removeDirectory(os.path.join(case_dir, "build"),
                            ignore_errors=False)
            removeDirectory(os.path.join(case_dir, "dist"),
                            ignore_errors=False)

            with withVirtualenv("venv_nuitka") as venv:
                # Install nuitka from source.
                venv.runCommand(commands=[
                    'cd "%s"' % nuitka_dir, "python setup.py install"
                ])

                # Remove that left over from the install command.
                removeDirectory(
                    path=os.path.join(nuitka_dir, "Nuitka.egg-info"),
                    ignore_errors=False,
                )

                # Create the wheel with Nuitka compilation.
                venv.runCommand(commands=[
                    'cd "%s"' % case_dir, "python setup.py bdist_nuitka"
                ])

                dist_dir = os.path.join(case_dir, "dist")
                venv.runCommand('pip install "%s"' %
                                (os.path.join(dist_dir,
                                              os.listdir(dist_dir)[0])))

                runner_binary = os.path.join(
                    venv.getVirtualenvDir(),
                    "bin" if os.name != "nt" else "scripts",
                    "runner",
                )

                if os.path.exists(runner_binary):
                    process = subprocess.Popen(
                        args=[
                            os.path.join(
                                venv.getVirtualenvDir(),
                                "bin" if os.name != "nt" else "scripts",
                                "python",
                            ),
                            runner_binary,
                        ],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE,
                    )
                else:
                    assert os.path.exists(runner_binary + ".exe")

                    process = subprocess.Popen(
                        args=[runner_binary + ".exe"],
                        stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE,
                    )

                stdout_nuitka, stderr_nuitka = process.communicate()
                exit_nuitka = process.returncode

                my_print("STDOUT Nuitka:")
                my_print(stdout_nuitka)
                my_print("STDERR Nuitka:")
                my_print(stderr_nuitka)

                assert exit_nuitka == 0, exit_nuitka
                my_print("EXIT was OK.")

            exit_code_stdout = compareOutput(
                "stdout",
                stdout_cpython,
                stdout_nuitka,
                ignore_warnings=True,
                syntax_errors=True,
            )

            exit_code_stderr = compareOutput(
                "stderr",
                stderr_cpython,
                stderr_nuitka,
                ignore_warnings=True,
                syntax_errors=True,
            )

            exit_code_return = exit_cpython != exit_nuitka

            if exit_code_return:
                my_print("""\
Exit codes {exit_cpython:d} (CPython) != {exit_nuitka:d} (Nuitka)""".format(
                    exit_cpython=exit_cpython, exit_nuitka=exit_nuitka))

            exit_code = exit_code_stdout or exit_code_stderr or exit_code_return

            if exit_code:
                sys.exit("Error, outputs differed.")

        if search_mode.abortIfExecuted():
            break

    search_mode.finish()
Example #20
0
import shutil
import tempfile

from nuitka.tools.testing.Common import (
    check_output,
    checkRuntimeLoadedFilesForOutsideAccesses,
    checkSucceedsWithCPython,
    compileLibraryTest,
    createSearchMode,
    getRuntimeTraceOfLoadedFiles,
    my_print,
    setup,
)

setup(needs_io_encoding=True)
search_mode = createSearchMode()

tmp_dir = tempfile.gettempdir()

# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
    tmp_dir = "/var/tmp"

blacklist = (
    "__phello__.foo.py",  # Triggers error for "." in module name
    "idnadata",  # Avoid too complex code for main program.
)

done = set()
Example #21
0
# Find nuitka package relative to us.
sys.path.insert(
    0,
    os.path.normpath(
        os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "..",
            ".."
        )
    )
)

from nuitka.tools.testing.Common import getTempDir, my_print, setup # isort:skip
from nuitka.utils.FileOperations import removeDirectory, listDir # isort:skip

python_version = setup()

nuitka_main_path = os.path.join("..", "..", "bin", "nuitka")

tmp_dir = getTempDir()

# Cannot detect this more automatic, either need whitelist or
# blacklist not needed stuff.
PACKAGE_LIST = (
    "nuitka",
    "nuitka/nodes",
    "nuitka/specs",
    "nuitka/nodes/shapes",
    "nuitka/tree",
    "nuitka/importing",
    "nuitka/build",
Example #22
0
def main():
    setup(suite="extension_modules", needs_io_encoding=True)
    search_mode = createSearchMode()

    tmp_dir = getTempDir()

    done = set()

    def decide(root, filename):
        if os.path.sep + "Cython" + os.path.sep in root:
            return False

        if (root.endswith(os.path.sep + "matplotlib")
                or os.path.sep + "matplotlib" + os.path.sep in root):
            return False

        if filename.endswith("linux-gnu_d.so"):
            return False

        if root.endswith(os.path.sep + "msgpack"):
            return False

        first_part = filename.split(".")[0]
        if first_part in done:
            return False
        done.add(first_part)

        return filename.endswith(
            (".so", ".pyd")) and not filename.startswith("libpython")

    current_dir = os.path.normpath(os.getcwd())
    current_dir = os.path.normcase(current_dir)

    def action(stage_dir, root, path):
        command = [
            sys.executable,
            os.path.join("..", "..", "bin", "nuitka"),
            "--stand",
            "--run",
            "--output-dir=%s" % stage_dir,
            "--remove-output",
        ]

        filename = os.path.join(stage_dir, "importer.py")

        assert path.startswith(root)

        module_name = path[len(root) + 1:]
        module_name = module_name.split(".")[0]
        module_name = module_name.replace(os.path.sep, ".")

        module_name = ModuleName(module_name)

        with openTextFile(filename, "w") as output:
            plugin_names = set(["pylint-warnings", "anti-bloat"])
            if module_name.hasNamespace("PySide2"):
                plugin_names.add("pyside2")
            elif module_name.hasNamespace("PySide6"):
                plugin_names.add("pyside2")
            elif module_name.hasNamespace("PyQt5"):
                plugin_names.add("pyqt5")
            else:
                # TODO: We do not have a noqt plugin yet.
                plugin_names.add("pyqt5")

            for plugin_name in plugin_names:
                output.write("# nuitka-project: --enable-plugin=%s\n" %
                             plugin_name)

            # Make it an error to find unwanted bloat compiled in.
            output.write("# nuitka-project: --noinclude-default-mode=error\n")

            output.write("import " + module_name.asString() + "\n")
            output.write("print('OK.')")

        command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()

        command.append(filename)

        if checkSucceedsWithCPython(filename):
            try:
                output = check_output(command).splitlines()
            except Exception:  # only trying to check for no exception, pylint: disable=try-except-raise
                raise
            else:
                assert os.path.exists(filename[:-3] + ".dist")

                binary_filename = os.path.join(
                    filename[:-3] + ".dist",
                    "importer.exe" if os.name == "nt" else "importer",
                )
                loaded_filenames = getRuntimeTraceOfLoadedFiles(
                    logger=test_logger,
                    command=[binary_filename],
                )

                outside_accesses = checkLoadedFileAccesses(
                    loaded_filenames=loaded_filenames, current_dir=os.getcwd())

                if outside_accesses:
                    displayError(None, filename)
                    displayRuntimeTraces(test_logger, binary_filename)

                    test_logger.warning(
                        "Should not access these file(s): '%r'." %
                        outside_accesses)

                    search_mode.onErrorDetected(1)

                if output[-1] != b"OK.":
                    my_print(" ".join(command))
                    my_print(filename)
                    my_print(output)
                    test_logger.sysexit("FAIL.")

                my_print("OK.")

                assert not outside_accesses, outside_accesses

                shutil.rmtree(filename[:-3] + ".dist")
        else:
            my_print("SKIP (does not work with CPython)")

    compileLibraryTest(
        search_mode=search_mode,
        stage_dir=os.path.join(tmp_dir, "compile_extensions"),
        decide=decide,
        action=action,
    )

    my_print("FINISHED, all extension modules compiled.")
Example #23
0
    os.path.normpath(
        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")),
)

# isort:start
from nuitka.tools.testing.Common import (
    check_output,
    convertUsing2to3,
    createSearchMode,
    decideFilenameVersionSkip,
    my_print,
    setup,
)
from nuitka.TreeXML import toString

python_version = setup(suite="optimiations")

search_mode = createSearchMode()


def getKind(node):
    result = node.attrib["kind"]

    result = result.replace("Statements", "")
    result = result.replace("Statement", "")
    result = result.replace("Expression", "")

    return result


def getRole(node, role):
Example #24
0
            os.path.dirname(os.path.abspath(__file__)),
            "..",
            ".."
        )
    )
)
from nuitka.tools.testing.Common import (
    my_print,
    setup,
    createSearchMode,
    compareWithCPython,
    reportSkip,
    withPythonPathChange
)

python_version = setup(needs_io_encoding = True)

search_mode = createSearchMode()

extra_options = os.environ.get("NUITKA_EXTRA_OPTIONS","")

for filename in sorted(os.listdir('.')):
    if not os.path.isdir(filename) or \
       filename.endswith(".build") or \
       filename.endswith(".dist"):
        continue

    filename = os.path.relpath(filename)

    # For these, we expect that they will fail.
    expected_errors = [
Example #25
0
def main():
    setup()
    goHome()

    # So PyLint finds nuitka package.
    addPYTHONPATH(os.getcwd())
    setupPATH()

    parser = OptionParser()

    parser.add_option(
        "--show-todos",
        "--todos",
        action="store_true",
        dest="todos",
        default=False,
        help="""\
Show TODO items. Default is %default.""",
    )

    parser.add_option(
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help="""\
Be version in output. Default is %default.""",
    )

    parser.add_option(
        "--one-by-one",
        action="store_true",
        dest="one_by_one",
        default=False,
        help="""\
Check files one by one. Default is %default.""",
    )

    parser.add_option(
        "--not-installed-is-no-error",
        action="store_true",
        dest="not_installed_is_no_error",
        default=False,
        help="""\
Insist on PyLint to be installed. Default is %default.""",
    )

    options, positional_args = parser.parse_args()

    if options.not_installed_is_no_error and not hasModule("pylint"):
        print("PyLint is not installed for this interpreter version: SKIPPED")
        sys.exit(0)

    if not positional_args:
        positional_args = ["bin", "nuitka"]

    print("Working on:", positional_args)

    blacklist = ["oset.py", "odict.py", "SyntaxHighlighting.py"]

    # Avoid checking the Python2 runner with Python3, it has name collisions.
    if python_version >= 300:
        blacklist.append("nuitka")

    filenames = list(scanTargets(positional_args, (".py",), blacklist))
    PyLint.executePyLint(
        filenames=filenames,
        show_todos=options.todos,
        verbose=options.verbose,
        one_by_one=options.one_by_one,
    )

    if not filenames:
        sys.exit("No files found.")

    sys.exit(PyLint.our_exit_code)
Example #26
0
def main():
    setup(go_main=False)

    # So PyLint finds nuitka package.
    addPYTHONPATH(getHomePath())
    setupPATH()

    parser = OptionParser()

    parser.add_option(
        "--diff",
        action="store_true",
        dest="diff",
        default=False,
        help="""\
Analyse the changed files in git. Default is %default.""",
    )

    parser.add_option(
        "--show-todos",
        "--todos",
        action="store_true",
        dest="todos",
        default=False,
        help="""\
Show TODO items. Default is %default.""",
    )

    parser.add_option(
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help="""\
Be verbose in output. Default is %default.""",
    )

    parser.add_option(
        "--one-by-one",
        action="store_true",
        dest="one_by_one",
        default=False,
        help="""\
Check files one by one. Default is %default.""",
    )

    parser.add_option(
        "--not-installed-is-no-error",
        action="store_true",
        dest="not_installed_is_no_error",
        default=False,
        help="""\
Insist on PyLint to be installed. Default is %default.""",
    )

    options, positional_args = parser.parse_args()

    if options.not_installed_is_no_error and not hasModule("pylint"):
        print("PyLint is not installed for this interpreter version: SKIPPED")
        sys.exit(0)

    if positional_args:
        if options.diff:
            sys.exit("Error, no filenames argument allowed in git diff mode.")
    else:
        goHome()

        if options.diff:
            positional_args = [
                filename for filename in getModifiedPaths()
                if isPythonFile(filename)
            ]
        else:
            positional_args = [
                "bin", "nuitka", "setup.py", "tests/*/run_all.py"
            ]

    positional_args = sum(
        (resolveShellPatternToFilenames(positional_arg)
         for positional_arg in positional_args),
        [],
    )

    if not positional_args:
        sys.exit("No files found.")

    print("Working on:", positional_args)

    ignore_list = []

    # Avoid checking the Python2 runner along with the one for Python3, it has name collisions.
    if python_version >= 0x300:
        ignore_list.append("nuitka")

    filenames = list(
        scanTargets(positional_args,
                    suffixes=(".py", ".scons"),
                    ignore_list=ignore_list))
    PyLint.executePyLint(
        filenames=filenames,
        show_todos=options.todos,
        verbose=options.verbose,
        one_by_one=options.one_by_one,
    )

    if not filenames:
        sys.exit("No files found.")

    sys.exit(PyLint.our_exit_code)
        )
    )
)

from nuitka.tools.testing.Common import (
    setup,
    my_print,
    createSearchMode,
    compileLibraryTest,
    check_output,
    checkSucceedsWithCPython,
    getRuntimeTraceOfLoadedFiles,
    checkRuntimeLoadedFilesForOutsideAccesses
)

setup(needs_io_encoding = True)
search_mode = createSearchMode()

tmp_dir = tempfile.gettempdir()

# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
    tmp_dir = "/var/tmp"

blacklist = (
    "__phello__.foo.py", # Triggers error for "." in module name
    "idnadata"           # Avoid too complex code for main program.
)

done = set()
Example #28
0
def main():
    setup()
    goHome()

    # So PyLint finds nuitka package.
    addPYTHONPATH(os.getcwd())
    setupPATH()

    parser = OptionParser()

    parser.add_option(
        "--show-todos",
        "--todos",
        action="store_true",
        dest="todos",
        default=False,
        help="""\
Show TODO items. Default is %default.""",
    )

    parser.add_option(
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help="""\
Be verbose in output. Default is %default.""",
    )

    parser.add_option(
        "--one-by-one",
        action="store_true",
        dest="one_by_one",
        default=False,
        help="""\
Check files one by one. Default is %default.""",
    )

    parser.add_option(
        "--not-installed-is-no-error",
        action="store_true",
        dest="not_installed_is_no_error",
        default=False,
        help="""\
Insist on PyLint to be installed. Default is %default.""",
    )

    options, positional_args = parser.parse_args()

    if options.not_installed_is_no_error and not hasModule("pylint"):
        print("PyLint is not installed for this interpreter version: SKIPPED")
        sys.exit(0)

    if not positional_args:
        positional_args = ["bin", "nuitka", "tests/*/run_all.py"]

    positional_args = sum(
        (resolveShellPatternToFilenames(positional_arg)
         for positional_arg in positional_args),
        [],
    )

    print("Working on:", positional_args)

    blacklist = ["oset.py", "odict.py"]

    # Avoid checking the Python2 runner with Python3, it has name collisions.
    if python_version >= 300:
        blacklist.append("nuitka")

    filenames = list(scanTargets(positional_args, (".py", ), blacklist))
    PyLint.executePyLint(
        filenames=filenames,
        show_todos=options.todos,
        verbose=options.verbose,
        one_by_one=options.one_by_one,
    )

    if not filenames:
        sys.exit("No files found.")

    sys.exit(PyLint.our_exit_code)
Example #29
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    if getOS() == "Linux":
        addExtendedExtraOptions(
            "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
        )

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "--onefile",
            "remove_output",
            # Keep the binary, normally "remove_output" includes that.
            "--keep-binary",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
        ]

        if filename == "KeyboardInteruptTest.py":
            if python_version < (3,):
                reportSkip(
                    "Python2 reports KeyboardInterrupt, but too late",
                    ".",
                    filename,
                )
                continue

            if os.name == "nt":
                reportSkip(
                    ".",
                    filename,
                    "Testing cannot send KeyboardInterrupt on Windows yet",
                )
                continue

            extra_flags.append("--send-ctrl-c")

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        test_logger.info(
            "Consider output of onefile mode compiled program: %s" % filename
        )

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")

        if filename == "KeyboardInteruptTest.py":
            continue

        # Then use "strace" on the result.
        with TimerReport(
            "Determining run time loaded files took %.2f", logger=test_logger
        ):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, path=binary_filename
            )

        current_dir = os.path.normpath(os.getcwd())
        current_dir = os.path.normcase(current_dir)

        illegal_access = False

        for loaded_filename in loaded_filenames:
            loaded_filename = os.path.normpath(loaded_filename)
            loaded_filename = os.path.normcase(loaded_filename)
            loaded_basename = os.path.basename(loaded_filename)

            if os.name == "nt":
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "System32")
                    ),
                ):
                    continue
                if areSamePaths(
                    os.path.dirname(loaded_filename),
                    os.path.normpath(
                        os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")
                    ),
                ):
                    continue

                if r"windows\winsxs" in loaded_filename:
                    continue

                # Github actions have these in PATH overriding SYSTEMROOT
                if r"windows performance toolkit" in loaded_filename:
                    continue
                if r"powershell" in loaded_filename:
                    continue
                if r"azure dev spaces cli" in loaded_filename:
                    continue
                if r"tortoisesvn" in loaded_filename:
                    continue

            if loaded_filename.startswith(current_dir):
                continue

            if loaded_filename.startswith(os.path.abspath(current_dir)):
                continue

            if loaded_filename.startswith("/etc/"):
                continue

            if loaded_filename.startswith("/proc/") or loaded_filename == "/proc":
                continue

            if loaded_filename.startswith("/dev/"):
                continue

            if loaded_filename.startswith("/tmp/") or loaded_filename == "/tmp":
                continue

            if loaded_filename.startswith("/run/"):
                continue

            if loaded_filename.startswith("/usr/lib/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/X11/locale/"):
                continue

            # Themes may of course be loaded.
            if loaded_filename.startswith("/usr/share/themes"):
                continue
            if "gtk" in loaded_filename and "/engines/" in loaded_filename:
                continue

            if loaded_filename in (
                "/usr",
                "/usr/local",
                "/usr/local/lib",
                "/usr/share",
                "/usr/local/share",
                "/usr/lib64",
            ):
                continue

            # TCL/tk for tkinter for non-Windows is OK.
            if loaded_filename.startswith(
                (
                    "/usr/lib/tcltk/",
                    "/usr/share/tcltk/",
                    "/usr/lib/tcl/",
                    "/usr/lib64/tcl/",
                )
            ):
                continue
            if loaded_filename in (
                "/usr/lib/tcltk",
                "/usr/share/tcltk",
                "/usr/lib/tcl",
                "/usr/lib64/tcl",
            ):
                continue

            if loaded_filename in (
                "/lib",
                "/lib64",
                "/lib/sse2",
                "/lib/tls",
                "/lib64/tls",
                "/usr/lib/sse2",
                "/usr/lib/tls",
                "/usr/lib64/tls",
            ):
                continue

            if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/init.tcl",
                "/usr/share/tcl8.5/init.tcl",
            ):
                continue
            if loaded_filename in (
                "/usr/share/tcl8.6/encoding",
                "/usr/share/tcl8.5/encoding",
            ):
                continue

            # System SSL config on Linux. TODO: Should this not be included and
            # read from dist folder.
            if loaded_basename == "openssl.cnf":
                continue

            # Taking these from system is harmless and desirable
            if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
                continue

            # System C libraries are to be expected.
            if loaded_basename.startswith(
                (
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
                )
            ):
                continue

            # Loaded by C library potentially for DNS lookups.
            if loaded_basename.startswith(
                (
                    "libnss_",
                    "libnsl",
                    # Some systems load a lot more, this is CentOS 7 on OBS
                    "libattr.so.",
                    "libbz2.so.",
                    "libcap.so.",
                    "libdw.so.",
                    "libelf.so.",
                    "liblzma.so.",
                    # Some systems load a lot more, this is Fedora 26 on OBS
                    "libselinux.so.",
                    "libpcre.so.",
                    # And this is Fedora 29 on OBS
                    "libblkid.so.",
                    "libmount.so.",
                    "libpcre2-8.so.",
                    # CentOS 8 on OBS
                    "libuuid.so.",
                )
            ):
                continue

            # Loaded by dtruss on macOS X.
            if loaded_filename.startswith("/usr/lib/dtrace/"):
                continue

            # Loaded by cowbuilder and pbuilder on Debian
            if loaded_basename == ".ilist":
                continue
            if "cowdancer" in loaded_filename:
                continue
            if "eatmydata" in loaded_filename:
                continue

            # Loading from home directories is OK too.
            if (
                loaded_filename.startswith("/home/")
                or loaded_filename.startswith("/data/")
                or loaded_filename.startswith("/root/")
                or loaded_filename in ("/home", "/data", "/root")
            ):
                continue

            # For Debian builders, /build is OK too.
            if loaded_filename.startswith("/build/") or loaded_filename == "/build":
                continue

            # TODO: Unclear, loading gconv from filesystem of installed system
            # may be OK or not. I think it should be.
            if loaded_basename == "gconv-modules.cache":
                continue
            if "/gconv/" in loaded_filename:
                continue
            if loaded_basename.startswith("libicu"):
                continue
            if loaded_filename.startswith("/usr/share/icu/"):
                continue

            # Loading from caches is OK.
            if loaded_filename.startswith("/var/cache/"):
                continue

            lib_prefix_dir = "/usr/lib/python%d.%s" % (
                python_version[0],
                python_version[1],
            )

            # PySide accesses its directory.
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/PySide"):
                continue

            # GTK accesses package directories only.
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gtk"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/glib"):
                continue
            if loaded_filename == os.path.join(
                lib_prefix_dir, "dist-packages/gtk-2.0/gio"
            ):
                continue
            if loaded_filename == os.path.join(lib_prefix_dir, "dist-packages/gobject"):
                continue

            # PyQt5 seems to do this, but won't use contents then.
            if loaded_filename in (
                "/usr/lib/qt5/plugins",
                "/usr/lib/qt5",
                "/usr/lib64/qt5/plugins",
                "/usr/lib64/qt5",
                "/usr/lib/x86_64-linux-gnu/qt5/plugins",
                "/usr/lib/x86_64-linux-gnu/qt5",
                "/usr/lib/x86_64-linux-gnu",
                "/usr/lib",
            ):
                continue

            # Can look at the interpreter of the system.
            if loaded_basename == "python3":
                continue

            # Current Python executable can actually be a symlink and
            # the real executable which it points to will be on the
            # loaded_filenames list. This is all fine, let's ignore it.
            # Also, because the loaded_filename can be yet another symlink
            # (this is weird, but it's true), let's better resolve its real
            # path too.
            if os.path.realpath(loaded_filename) == os.path.realpath(sys.executable):
                continue

            # Accessing SE-Linux is OK.
            if loaded_filename in ("/sys/fs/selinux", "/selinux"):
                continue

            # Allow reading time zone info of local system.
            if loaded_filename.startswith("/usr/share/zoneinfo/"):
                continue

            # The access to .pth files has no effect.
            if loaded_filename.endswith(".pth"):
                continue

            # Looking at site-package dir alone is alone.
            if loaded_filename.endswith(("site-packages", "dist-packages")):
                continue

            # QtNetwork insist on doing this it seems.
            if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
                continue

            # macOS uses these:
            if loaded_basename in (
                "libcrypto.1.0.0.dylib",
                "libssl.1.0.0.dylib",
                "libcrypto.1.1.dylib",
            ):
                continue

            # Linux onefile uses this
            if loaded_basename.startswith("libfuse.so."):
                continue

            # MSVC run time DLLs, due to SxS come from system.
            if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
                continue

            test_logger.warning("Should not access '%s'." % loaded_filename)
            illegal_access = True

        if illegal_access:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            search_mode.onErrorDetected(1)

        os.unlink(binary_filename)

    search_mode.finish()
Example #30
0
def main():
    # Complex stuff, pylint: disable=too-many-branches

    python_version = setup(suite="basics", needs_io_encoding=True)

    search_mode = createSearchMode()

    # Create large constants test on the fly, if it's not there, not going to
    # add it to release archives for no good reason.
    if not os.path.exists("BigConstants.py"):
        with open("BigConstants.py", "w") as output:
            output.write(
                "# Automatically generated test, not part of releases or git.\n\n"
            )
            output.write("print('%s')\n" % ("1234" * 17000))

    # Now run all the tests in this directory.
    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        extra_flags = [
            # No error exits normally, unless we break tests, and that we would
            # like to know.
            "expect_success",
            # Keep no temporary files.
            "remove_output",
            # Include imported files, mostly nothing though.
            "recurse_all",
            # Use the original __file__ value, at least one case warns about things
            # with filename included.
            "original_file",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
        ]

        # This test should be run with the debug Python, and makes outputs to
        # standard error that might be ignored.
        if filename.startswith("Referencing"):
            extra_flags.append("python_debug")

            extra_flags.append("recurse_not:nuitka")

        # This tests warns about __import__() used.
        if filename == "OrderChecks.py":
            extra_flags.append("ignore_warnings")

        # This tests warns about an package relative import despite
        # being in no package.
        if filename == "Importing.py":
            extra_flags.append("ignore_warnings")

        # TODO: Nuitka does not give output for ignored exception in dtor, this is
        # not fully compatible and potentially an error.
        if filename == "YieldFrom33.py":
            extra_flags.append("ignore_stderr")

        # For Python2 there is a "builtins" package that gives warnings. TODO: We
        # ought to NOT import that package and detect statically that __builtins__
        # import won't raise ImportError.
        if filename == "BuiltinOverload.py":
            extra_flags.append("ignore_warnings")

        active = search_mode.consider(dirname=None, filename=filename)

        if active:
            if filename.startswith("Referencing") and not hasDebugPython():
                my_print("Skipped (no debug Python)")
                continue

            needs_2to3 = (python_version.startswith("3")
                          and not filename.endswith("32.py")
                          and not filename.endswith("33.py")
                          and not filename.endswith("35.py")
                          and not filename.endswith("36.py"))

            compareWithCPython(
                dirname=None,
                filename=filename,
                extra_flags=extra_flags,
                search_mode=search_mode,
                needs_2to3=needs_2to3,
            )

            if search_mode.abortIfExecuted():
                break
        else:
            my_print("Skipping", filename)

    search_mode.finish()
Example #31
0
def main():
    # Complex stuff, even more should become common code or project options though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(suite="standalone", needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "--standalone",
            "remove_output",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
        ]

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        if filename == "Urllib3Using.py" and os.name == "nt":
            reportSkip(
                "Socket module early import not working on Windows currently",
                ".",
                filename,
            )
            continue

        if "Idna" in filename:
            # For the warnings of Python2.
            if python_version < (3, ):
                extra_flags.append("ignore_stderr")

        if filename == "CtypesUsing.py":
            extra_flags.append("plugin_disable:pylint-warnings")

        if filename == "GtkUsing.py":
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith("Win"):
            if os.name != "nt":
                reportSkip("Windows only test", ".", filename)
                continue

        if filename == "TkInterUsing.py":
            if getOS() == "Darwin":
                reportSkip("Not working macOS yet", ".", filename)
                continue

            if getOS() == "Windows":
                reportSkip("Can hang on Windows CI.", ".", filename)
                continue

            # For the plug-in information.
            extra_flags.append("plugin_enable:tk-inter")

        if filename == "FlaskUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        # TODO: Once we have a no-Qt Plugin, we should use that.
        if filename == "MatplotlibUsing.py":
            # For the plugin warnings.
            extra_flags.append("ignore_warnings")

        if filename == "NumpyUsing.py":
            # TODO: Disabled for now.
            reportSkip("numpy.test not fully working yet", ".", filename)
            continue

        if filename == "PandasUsing.py":
            extra_flags.append("plugin_enable:numpy")
            extra_flags.append("plugin_disable:pylint-warnings")
            extra_flags.append("plugin_disable:pyqt5")
            extra_flags.append("plugin_disable:pyside2")
            extra_flags.append("plugin_disable:pyside6")

        if filename == "PmwUsing.py":
            extra_flags.append("plugin_enable:pmw-freezer")

        if filename == "OpenGLUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "GlfwUsing.py":
            # For the warnings.
            extra_flags.append("plugin_enable:numpy")

        if filename == "PasslibUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "Win32ComUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith(("PySide2", "PySide6", "PyQt5", "PyQt6")):
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version < (2, 7) or ((3, ) <= python_version < (3, 7)):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the plug-in information
            if filename.startswith("PySide2"):
                extra_flags.append("plugin_enable:pyside6")
            elif filename.startswith("PySide6"):
                extra_flags.append("plugin_enable:pyside6")
            elif filename.startswith("PyQt5"):
                extra_flags.append("plugin_enable:pyqt5")
            elif filename.startswith("PyQt6"):
                extra_flags.append("plugin_enable:pyqt6")

        test_logger.info(
            "Consider output of standalone mode compiled program: %s" %
            filename)

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        # Second check if glibc libraries haven't been accidentally
        # shipped with the standalone executable
        found_glibc_libs = []
        for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
            if os.path.basename(dist_filename).startswith((
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
            )):
                found_glibc_libs.append(dist_filename)

        if found_glibc_libs:
            test_logger.warning(
                "Should not ship glibc libraries with the standalone executable (found %s)"
                % found_glibc_libs)
            sys.exit(1)

        binary_filename = os.path.join(
            filename[:-3] + ".dist",
            filename[:-3] + (".exe" if os.name == "nt" else ""))

        # Then use "strace" on the result.
        with TimerReport("Determining run time loaded files took %.2f",
                         logger=test_logger):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, command=[binary_filename])

        illegal_accesses = checkLoadedFileAccesses(
            loaded_filenames=loaded_filenames, current_dir=os.getcwd())

        if illegal_accesses:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            test_logger.warning("Should not access these file(s): '%r'." %
                                illegal_accesses)

            search_mode.onErrorDetected(1)

        removeDirectory(filename[:-3] + ".dist", ignore_errors=True)

    search_mode.finish()
Example #32
0
def main():
    # Complex stuff, not broken down yet
    # pylint: disable=too-many-branches,too-many-locals,too-many-statements

    parser = OptionParser()

    parser.add_option(
        "--nuitka",
        action="store",
        dest="nuitka",
        default=os.environ.get("NUITKA", ""),
    )

    parser.add_option(
        "--cpython",
        action="store",
        dest="cpython",
        default=os.environ.get("PYTHON", sys.executable),
    )

    parser.add_option(
        "--code-diff",
        action="store",
        dest="diff_filename",
        default="",
    )

    parser.add_option(
        "--copy-source-to",
        action="store",
        dest="target_dir",
        default="",
    )

    options, positional_args = parser.parse_args()

    if len(positional_args) != 1:
        sys.exit(
            "Error, need to give test case file name as positional argument.")

    test_case = positional_args[0]

    if os.path.exists(test_case):
        test_case = os.path.abspath(test_case)

    if options.cpython == "no":
        options.cpython = ""

    nuitka = options.nuitka

    if os.path.exists(nuitka):
        nuitka = os.path.abspath(nuitka)
    elif nuitka:
        sys.exit("Error, nuitka binary '%s' not found." % nuitka)

    python_version = setup(silent=True, go_main=False)

    assert os.path.exists(test_case), (test_case, os.getcwd())

    my_print("PYTHON='%s'" % python_version)
    my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
    my_print("TEST_CASE_HASH='%s'" %
             hashlib.md5(open(test_case, "rb").read()).hexdigest())


    needs_2to3 = python_version.startswith('3') and \
                 not test_case.endswith("32.py") and \
                 not test_case.endswith("33.py")

    if options.target_dir:
        shutil.copyfile(
            test_case,
            os.path.join(options.target_dir, os.path.basename(test_case)))

    # First produce two variants.
    temp_dir = getTempDir()

    test_case_1 = os.path.join(temp_dir,
                               "Variant1_" + os.path.basename(test_case))
    test_case_2 = os.path.join(temp_dir,
                               "Variant2_" + os.path.basename(test_case))

    case_1_source, case_2_source = generateConstructCases(
        open(test_case).read())

    with open(test_case_1, 'w') as case_1_file:
        case_1_file.write(case_1_source)

    with open(test_case_2, 'w') as case_2_file:
        case_2_file.write(case_2_source)

    if needs_2to3:
        test_case_1, _needs_delete = convertUsing2to3(test_case_1)
        test_case_2, _needs_delete = convertUsing2to3(test_case_2)

    os.environ["PYTHONHASHSEED"] = '0'

    if nuitka:
        nuitka_id = check_output("cd %s; git rev-parse HEAD" %
                                 os.path.dirname(nuitka),
                                 shell=True)
        nuitka_id = nuitka_id.strip()

        if sys.version_info > (3, ):
            nuitka_id = nuitka_id.decode()

        my_print("NUITKA_COMMIT='%s'" % nuitka_id)

    os.chdir(getTempDir())

    if nuitka:
        nuitka_call = [
            os.environ["PYTHON"], nuitka, "--python-flag=-S",
            os.path.basename(test_case)
        ]
        nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())

        # We want to compile under the same filename to minimize differences, and
        # then copy the resulting files afterwards.
        shutil.copyfile(test_case_1, os.path.basename(test_case))

        subprocess.check_call(nuitka_call)

        if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
            exe_suffix = ".exe"
        else:
            exe_suffix = ".bin"

        os.rename(
            os.path.basename(test_case).replace(".py", ".build"),
            os.path.basename(test_case_1).replace(".py", ".build"))
        os.rename(
            os.path.basename(test_case).replace(".py", exe_suffix),
            os.path.basename(test_case_1).replace(".py", exe_suffix))

        shutil.copyfile(test_case_2, os.path.basename(test_case))

        subprocess.check_call(nuitka_call)

        os.rename(
            os.path.basename(test_case).replace(".py", ".build"),
            os.path.basename(test_case_2).replace(".py", ".build"))
        os.rename(
            os.path.basename(test_case).replace(".py", exe_suffix),
            os.path.basename(test_case_2).replace(".py", exe_suffix))

        if options.diff_filename:
            suffixes = [".c", ".cpp"]

            for suffix in suffixes:
                cpp_1 = os.path.join(
                    test_case_1.replace(".py", ".build"),
                    "module.__main__" + suffix,
                )

                if os.path.exists(cpp_1):
                    break
            else:
                assert False

            for suffix in suffixes:
                cpp_2 = os.path.join(
                    test_case_2.replace(".py", ".build"),
                    "module.__main__" + suffix,
                )
                if os.path.exists(cpp_2):
                    break
            else:
                assert False

            import difflib
            open(options.diff_filename,
                 'w').write(difflib.HtmlDiff().make_table(
                     open(cpp_1).readlines(),
                     open(cpp_2).readlines(), "Construct", "Baseline", True))

        nuitka_1 = runValgrind("Nuitka construct",
                               "callgrind",
                               (test_case_1.replace(".py", exe_suffix), ),
                               include_startup=True)

        nuitka_2 = runValgrind("Nuitka baseline",
                               "callgrind",
                               (test_case_2.replace(".py", exe_suffix), ),
                               include_startup=True)

        nuitka_diff = nuitka_1 - nuitka_2

        my_print("NUITKA_COMMAND='%s'" % ' '.join(nuitka_call),
                 file=sys.stderr)
        my_print("NUITKA_RAW=%s" % nuitka_1)
        my_print("NUITKA_BASE=%s" % nuitka_2)
        my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)

    if options.cpython:
        cpython_1 = runValgrind("CPython construct",
                                "callgrind",
                                (os.environ["PYTHON"], "-S", test_case_1),
                                include_startup=True)
        cpython_2 = runValgrind("CPython baseline",
                                "callgrind",
                                (os.environ["PYTHON"], "-S", test_case_2),
                                include_startup=True)

        cpython_diff = cpython_1 - cpython_2

        my_print("CPYTHON_RAW=%d" % cpython_1)
        my_print("CPYTHON_BASE=%d" % cpython_2)
        my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)

    if options.cpython and options.nuitka:
        if nuitka_diff == 0:
            nuitka_gain = float("inf")
        else:
            nuitka_gain = float(100 * cpython_diff) / nuitka_diff

        my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
        my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
        my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
Example #33
0
def main():
    # Complex stuff, even more should become common code though.
    # pylint: disable=too-many-branches,too-many-statements

    python_version = setup(needs_io_encoding=True)

    search_mode = createSearchMode()

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            my_print("Skipping", filename)
            continue

        extra_flags = [
            "expect_success",
            "standalone",
            "remove_output",
            # For enum plugin info
            "ignore_infos",
        ]

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        # catch error
        if filename == "Boto3Using.py":
            reportSkip("boto3 test not fully working yet", ".", filename)
            continue

        if "Idna" in filename:
            # For the warnings of Python2.
            if python_version.startswith("2"):
                extra_flags.append("ignore_stderr")

        if filename == "CtypesUsing.py":
            extra_flags.append("plugin_disable:pylint-warnings")

        if filename == "GtkUsing.py":
            # Don't test on platforms not supported by current Debian testing, and
            # which should be considered irrelevant by now.
            if python_version.startswith("2.6"):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith("Win"):
            if os.name != "nt":
                reportSkip("Windows only test", ".", filename)
                continue

        if filename == "TkInterUsing.py":
            if getOS() == "Darwin":
                reportSkip("Not working macOS yet", ".", filename)
                continue

            # For the plug-in information.
            extra_flags.append("plugin_enable:tk-inter")

        if filename == "FlaskUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "NumpyUsing.py":
            # TODO: Disabled for now.
            reportSkip("numpy.test not fully working yet", ".", filename)
            continue

            # extra_flags.append("plugin_enable:data-files")
        if filename == "PmwUsing.py":
            extra_flags.append("plugin_enable:pmw-freezer")

        if filename == "OpenGLUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "PasslibUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename == "PySideUsing.py":
            # TODO: Disabled due to lack of upstream support.
            reportSkip("PySide not supported yet", ".", filename)
            continue

        if filename == "Win32ComUsing.py":
            # For the warnings.
            extra_flags.append("ignore_warnings")

        if filename.startswith(("PySide", "PyQt")):
            if python_version.startswith("2.6"):
                reportSkip("irrelevant Python version", ".", filename)
                continue

            # For the plug-in information.
            if getPythonVendor() != "Anaconda":
                extra_flags.append("plugin_enable:qt-plugins")
            else:
                # For the plug-in not used information.
                extra_flags.append("ignore_warnings")

        my_print("Consider output of recursively compiled program:", filename)

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        # Second check if glibc libraries haven't been accidentally
        # shipped with the standalone executable
        found_glibc_libs = []
        for dist_filename in os.listdir(os.path.join(filename[:-3] + ".dist")):
            if os.path.basename(dist_filename).startswith((
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
            )):
                found_glibc_libs.append(dist_filename)

        if found_glibc_libs:
            my_print(
                "Should not ship glibc libraries with the standalone executable (found %s)"
                % found_glibc_libs)
            sys.exit(1)

        binary_filename = os.path.join(
            filename[:-3] + ".dist",
            filename[:-3] + (".exe" if os.name == "nt" else ""))

        # Then use "strace" on the result.
        loaded_filenames = getRuntimeTraceOfLoadedFiles(binary_filename)

        current_dir = os.path.normpath(os.getcwd())
        current_dir = os.path.normcase(current_dir)

        illegal_access = False

        for loaded_filename in loaded_filenames:
            loaded_filename = os.path.normpath(loaded_filename)
            loaded_filename = os.path.normcase(loaded_filename)
            loaded_basename = os.path.basename(loaded_filename)

            if os.name == "nt":
                if areSamePaths(
                        os.path.dirname(loaded_filename),
                        os.path.normpath(
                            os.path.join(os.environ["SYSTEMROOT"],
                                         "System32")),
                ):
                    continue
                if areSamePaths(
                        os.path.dirname(loaded_filename),
                        os.path.normpath(
                            os.path.join(os.environ["SYSTEMROOT"],
                                         "SysWOW64")),
                ):
                    continue

                if r"windows\winsxs" in loaded_filename:
                    continue

                # Github actions have these in PATH overriding SYSTEMROOT
                if r"windows performance toolkit" in loaded_filename:
                    continue
                if r"powershell" in loaded_filename:
                    continue

            if loaded_filename.startswith(current_dir):
                continue

            if loaded_filename.startswith(os.path.abspath(current_dir)):
                continue

            if loaded_filename.startswith("/etc/"):
                continue

            if loaded_filename.startswith(
                    "/proc/") or loaded_filename == "/proc":
                continue

            if loaded_filename.startswith("/dev/"):
                continue

            if loaded_filename.startswith("/tmp/"):
                continue

            if loaded_filename.startswith("/run/"):
                continue

            if loaded_filename.startswith("/usr/lib/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/locale/"):
                continue

            if loaded_filename.startswith("/usr/share/X11/locale/"):
                continue

            # Themes may of course be loaded.
            if loaded_filename.startswith("/usr/share/themes"):
                continue
            if "gtk" in loaded_filename and "/engines/" in loaded_filename:
                continue

            if loaded_filename in (
                    "/usr",
                    "/usr/local",
                    "/usr/local/lib",
                    "/usr/share",
                    "/usr/local/share",
                    "/usr/lib64",
            ):
                continue

            # TCL/tk for tkinter for non-Windows is OK.
            if loaded_filename.startswith((
                    "/usr/lib/tcltk/",
                    "/usr/share/tcltk/",
                    "/usr/lib/tcl/",
                    "/usr/lib64/tcl/",
            )):
                continue
            if loaded_filename in (
                    "/usr/lib/tcltk",
                    "/usr/share/tcltk",
                    "/usr/lib/tcl",
                    "/usr/lib64/tcl",
            ):
                continue

            if loaded_filename in (
                    "/lib",
                    "/lib64",
                    "/lib/sse2",
                    "/lib/tls",
                    "/lib64/tls",
                    "/usr/lib/sse2",
                    "/usr/lib/tls",
                    "/usr/lib64/tls",
            ):
                continue

            if loaded_filename in ("/usr/share/tcl8.6", "/usr/share/tcl8.5"):
                continue
            if loaded_filename in (
                    "/usr/share/tcl8.6/init.tcl",
                    "/usr/share/tcl8.5/init.tcl",
            ):
                continue
            if loaded_filename in (
                    "/usr/share/tcl8.6/encoding",
                    "/usr/share/tcl8.5/encoding",
            ):
                continue

            # System SSL config on Linux. TODO: Should this not be included and
            # read from dist folder.
            if loaded_basename == "openssl.cnf":
                continue

            # Taking these from system is harmless and desirable
            if loaded_basename.startswith(("libz.so", "libgcc_s.so")):
                continue

            # System C libraries are to be expected.
            if loaded_basename.startswith((
                    "ld-linux-x86-64.so",
                    "libc.so.",
                    "libpthread.so.",
                    "libm.so.",
                    "libdl.so.",
                    "libBrokenLocale.so.",
                    "libSegFault.so",
                    "libanl.so.",
                    "libcidn.so.",
                    "libcrypt.so.",
                    "libmemusage.so",
                    "libmvec.so.",
                    "libnsl.so.",
                    "libnss_compat.so.",
                    "libnss_db.so.",
                    "libnss_dns.so.",
                    "libnss_files.so.",
                    "libnss_hesiod.so.",
                    "libnss_nis.so.",
                    "libnss_nisplus.so.",
                    "libpcprofile.so",
                    "libresolv.so.",
                    "librt.so.",
                    "libthread_db-1.0.so",
                    "libthread_db.so.",
                    "libutil.so.",
            )):
                continue

            # Loaded by C library potentially for DNS lookups.
            if loaded_basename.startswith((
                    "libnss_",
                    "libnsl",
                    # Some systems load a lot more, this is CentOS 7 on OBS
                    "libattr.so.",
                    "libbz2.so.",
                    "libcap.so.",
                    "libdw.so.",
                    "libelf.so.",
                    "liblzma.so.",
                    # Some systems load a lot more, this is Fedora 26 on OBS
                    "libselinux.so.",
                    "libpcre.so.",
                    # And this is Fedora 29 on OBS
                    "libblkid.so.",
                    "libmount.so.",
                    "libpcre2-8.so.",
                    # CentOS 8 on OBS
                    "libuuid.so.",
            )):
                continue

            # Loaded by dtruss on macOS X.
            if loaded_filename.startswith("/usr/lib/dtrace/"):
                continue

            # Loaded by cowbuilder and pbuilder on Debian
            if loaded_basename == ".ilist":
                continue
            if "cowdancer" in loaded_filename:
                continue
            if "eatmydata" in loaded_filename:
                continue

            # Loading from home directories is OK too.
            if (loaded_filename.startswith("/home/")
                    or loaded_filename.startswith("/data/")
                    or loaded_filename.startswith("/root/")
                    or loaded_filename in ("/home", "/data", "/root")):
                continue

            # For Debian builders, /build is OK too.
            if loaded_filename.startswith(
                    "/build/") or loaded_filename == "/build":
                continue

            # TODO: Unclear, loading gconv from filesystem of installed system
            # may be OK or not. I think it should be.
            if loaded_basename == "gconv-modules.cache":
                continue
            if "/gconv/" in loaded_filename:
                continue
            if loaded_basename.startswith("libicu"):
                continue
            if loaded_filename.startswith("/usr/share/icu/"):
                continue

            # Loading from caches is OK.
            if loaded_filename.startswith("/var/cache/"):
                continue

            # PySide accesses its directory.
            if (loaded_filename == "/usr/lib/python" + python_version[:3] +
                    "/dist-packages/PySide"):
                continue

            # GTK accesses package directories only.
            if (loaded_filename == "/usr/lib/python" + python_version[:3] +
                    "/dist-packages/gtk-2.0/gtk"):
                continue
            if (loaded_filename == "/usr/lib/python" + python_version[:3] +
                    "/dist-packages/glib"):
                continue
            if (loaded_filename == "/usr/lib/python" + python_version[:3] +
                    "/dist-packages/gtk-2.0/gio"):
                continue
            if (loaded_filename == "/usr/lib/python" + python_version[:3] +
                    "/dist-packages/gobject"):
                continue

            # PyQt5 seems to do this, but won't use contents then.
            if loaded_filename in (
                    "/usr/lib/qt5/plugins",
                    "/usr/lib/qt5",
                    "/usr/lib64/qt5/plugins",
                    "/usr/lib64/qt5",
                    "/usr/lib/x86_64-linux-gnu/qt5/plugins",
                    "/usr/lib/x86_64-linux-gnu/qt5",
                    "/usr/lib/x86_64-linux-gnu",
                    "/usr/lib",
            ):
                continue

            # Can look at the interpreter of the system.
            if loaded_basename == "python3":
                continue

            # Current Python executable can actually be a symlink and
            # the real executable which it points to will be on the
            # loaded_filenames list. This is all fine, let's ignore it.
            # Also, because the loaded_filename can be yet another symlink
            # (this is weird, but it's true), let's better resolve its real
            # path too.
            if os.path.realpath(loaded_filename) == os.path.realpath(
                    sys.executable):
                continue

            # Accessing SE-Linux is OK.
            if loaded_filename in ("/sys/fs/selinux", "/selinux"):
                continue

            # Allow reading time zone info of local system.
            if loaded_filename.startswith("/usr/share/zoneinfo/"):
                continue

            # The access to .pth files has no effect.
            if loaded_filename.endswith(".pth"):
                continue

            # Looking at site-package dir alone is alone.
            if loaded_filename.endswith(("site-packages", "dist-packages")):
                continue

            # QtNetwork insist on doing this it seems.
            if loaded_basename.startswith(("libcrypto.so", "libssl.so")):
                continue

            # macOS uses these:
            if loaded_basename in (
                    "libcrypto.1.0.0.dylib",
                    "libssl.1.0.0.dylib",
                    "libcrypto.1.1.dylib",
            ):
                continue

            # MSVC run time DLLs, seem to sometimes come from system.
            if loaded_basename.upper() in ("MSVCRT.DLL", "MSVCR90.DLL"):
                continue

            my_print("Should not access '%s'." % loaded_filename)
            illegal_access = True

        if illegal_access:
            if os.name != "nt":
                my_print("Listing of dist folder:")
                os.system("ls -Rla %s" % filename[:-3] + ".dist")

                my_print("strace:")
                os.system("strace -s4096 -e file %s" % binary_filename)

            search_mode.onErrorDetected(1)

        removeDirectory(filename[:-3] + ".dist", ignore_errors=True)

        if search_mode.abortIfExecuted():
            break

    search_mode.finish()
def main():
    # pylint: disable=broad-except,too-many-branches,too-many-locals,too-many-statements

    setup()

    # cache_dir is where the git clones are cached
    cache_dir = os.path.join(getCacheDir(), "pypi-git-clones")
    base_dir = os.getcwd()

    if not os.path.isdir(cache_dir):
        os.mkdir(cache_dir)

    search_mode = createSearchMode()

    results = []

    # load json
    with open("packages.json", "r") as f:
        packages = json.load(f)

    for package_name, details in sorted(packages.items()):
        active = search_mode.consider(dirname=None, filename=package_name)

        if not active:
            continue

        if str is not bytes:
            # running on python3
            if package_name in ("futures", "future"):
                reportSkip("Does not run on Python3", ".", package_name)
                if search_mode.abortIfExecuted():
                    break
                continue

        if os.name == "nt":
            if package_name in ("cryptography",):
                reportSkip("Not working on Windows", ".", package_name)
                if search_mode.abortIfExecuted():
                    break
                continue

        if package_name == "pyyaml":
            reportSkip("Not yet supported, see Issue #476", ".", package_name)
            if search_mode.abortIfExecuted():
                break
            continue

        if package_name in ("pycparser", "numpy"):
            reportSkip("Not yet supported, see Issue #477", ".", package_name)
            if search_mode.abortIfExecuted():
                break
            continue

        if package_name in (
            "google-auth",  # bdist_nuitka fails AttributeError: single_version_externally_managed
            "jinja2",  # ModuleNotFoundError: No module named 'jinja2.tests'
            "pandas",  # ModuleNotFoundError: No module named 'Cython'
            "pytz",  # need to 'make build'
            "rsa",  # Now uses Poetry (no setup.py)
        ):
            if search_mode.abortIfExecuted():
                break
            continue

        package_dir = os.path.join(cache_dir, package_name)

        try:
            gitClone(package_name, details["url"], cache_dir)

            os.chdir(base_dir)
            with withVirtualenv(
                "venv_%s" % package_name, delete=False, style="blue"
            ) as venv:
                dist_dir = os.path.join(package_dir, "dist")

                # delete ignored tests if any
                if details["ignored_tests"]:
                    for test in details["ignored_tests"]:
                        venv.runCommand("rm -rf %s" % os.path.join(package_dir, test))

                # setup for pytest
                cmds = [
                    "python -m pip install pytest",
                    "cd %s" % os.path.join(os.path.dirname(nuitka.__file__), ".."),
                    "python setup.py develop",
                    "cd %s" % package_dir,
                ]

                if details["requirements_file"]:
                    cmds.append(
                        "python -m pip install -r %s" % details["requirements_file"]
                    )

                if details.get("extra_commands"):
                    cmds += details["extra_commands"]

                # build uncompiled .whl
                cmds.append("python setup.py bdist_wheel")

                venv.runCommand(commands=cmds)

                # install and print out if the active .whl is compiled or not
                venv.runCommand(
                    commands=[
                        "python -m pip install -U %s"
                        % os.path.join(dist_dir, os.listdir(dist_dir)[0]),
                        # use triple quotes for linux
                        """python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """
                        % details.get("package_name", package_name),
                    ]
                )

                # get uncompiled pytest results
                uncompiled_stdout, uncompiled_stderr = venv.runCommandWithOutput(
                    commands=[
                        "cd %s" % package_dir,
                        "python -m pytest --disable-warnings",
                    ],
                    style="blue",
                )

                # clean up before building compiled .whl
                cmds = ["cd %s" % package_dir, "git clean -dfx"]

                if details.get("extra_commands"):
                    cmds += details["extra_commands"]

                # build nuitka compiled .whl
                cmds.append("python setup.py bdist_nuitka")

                venv.runCommand(commands=cmds)

                # install and print out if the active .whl is compiled or not
                venv.runCommand(
                    commands=[
                        "python -m pip install -U %s"
                        % os.path.join(dist_dir, os.listdir(dist_dir)[0]),
                        # use triple quotes for linux
                        """python -c "print(getattr(__import__('%s'),'__compiled__','__uncompiled_version__'))" """
                        % details.get("package_name", package_name),
                    ]
                )

                # get compiled pytest results
                compiled_stdout, compiled_stderr = venv.runCommandWithOutput(
                    commands=[
                        "cd %s" % package_dir,
                        "python -m pytest --disable-warnings",
                    ],
                    style="blue",
                )

                venv.runCommand(commands=["cd %s" % package_dir, "git clean -q -dfx"])

        except Exception as e:
            my_print(
                "Package",
                package_name,
                "ran into an exception during execution, traceback: ",
            )
            my_print(e)
            results.append((package_name, "ERROR", "ERROR"))

            if search_mode.abortIfExecuted():
                break
            continue

        # compare outputs
        stdout_diff = compareOutput(
            "stdout",
            uncompiled_stdout,
            compiled_stdout,
            ignore_warnings=True,
            syntax_errors=True,
        )

        stderr_diff = compareOutput(
            "stderr",
            uncompiled_stderr,
            compiled_stderr,
            ignore_warnings=True,
            syntax_errors=True,
        )

        results.append((package_name, stdout_diff, stderr_diff))

        exit_code = stdout_diff or stderr_diff

        my_print(
            "\n=================================================================================",
            "\n--- %s ---" % package_name,
            "exit_stdout:",
            stdout_diff,
            "exit_stderr:",
            stderr_diff,
            "\nError, outputs differed for package %s." % package_name
            if exit_code
            else "\nNo differences found for package %s." % package_name,
            "\n=================================================================================\n",
            style="red" if exit_code else "green",
        )

        if exit_code != 0 and search_mode.abortOnFinding(
            dirname=None, filename=package_name
        ):
            break

        if search_mode.abortIfExecuted():
            break

    search_mode.finish()

    # give a summary of all packages
    my_print(
        "\n\n=====================================SUMMARY=====================================",
        style="yellow",
    )

    for package_name, stdout_diff, stderr_diff in results:
        my_print(
            package_name,
            "-",
            end=" ",
            style="red" if (stdout_diff or stderr_diff) else "green",
        )

        my_print(
            "stdout:", stdout_diff, end=" ", style="red" if stdout_diff else "green"
        )

        my_print(
            "stderr:", stderr_diff, end="", style="red" if stderr_diff else "green"
        )

        my_print(
            "\n---------------------------------------------------------------------------------"
        )

    my_print("TOTAL NUMBER OF PACKAGES TESTED: %s" % len(results), style="yellow")

    num_failed = 0
    num_errors = 0

    # tally the number of errors and failed
    for _, y, z in results:
        if type(y) is str:
            # this means the package ran into an exception
            num_errors += 1
        elif y or z:
            num_failed += 1

    my_print(
        "TOTAL PASSED: %s" % (len(results) - num_failed - num_errors), style="green"
    )

    my_print("TOTAL FAILED (differences): %s" % num_failed, style="red")

    my_print("TOTAL ERRORS (exceptions): %s" % num_errors, style="red")
Example #35
0
def main():
    python_version = setup(suite="onefile", needs_io_encoding=True)

    search_mode = createSearchMode()

    if getOS() == "Linux":
        addExtendedExtraOptions(
            "--linux-onefile-icon=../../doc/Logo/Nuitka-Logo-Symbol.svg"
        )

    for filename in sorted(os.listdir(".")):
        if not filename.endswith(".py"):
            continue

        if not decideFilenameVersionSkip(filename):
            continue

        active = search_mode.consider(dirname=None, filename=filename)

        if not active:
            test_logger.info("Skipping %s" % filename)
            continue

        extra_flags = [
            "expect_success",
            "remove_output",
            # Keep the binary, normally "remove_output" includes that.
            "--keep-binary",
            # Cache the CPython results for re-use, they will normally not change.
            "cpython_cache",
            # To understand what is slow.
            "timing",
            # The onefile can warn about zstandard not being installed.
            "ignore_warnings",
        ]

        if filename == "KeyboardInterruptTest.py":
            if getOS() == "Darwin":
                reportSkip(
                    "Exit code from KeyboardInterrupt on macOS is not yet good.",
                    ".",
                    filename,
                )
                continue

            if python_version < (3,):
                reportSkip(
                    "Python2 reports KeyboardInterrupt, but too late",
                    ".",
                    filename,
                )
                continue

            if os.name == "nt":
                reportSkip(
                    "Testing cannot send KeyboardInterrupt on Windows yet",
                    ".",
                    filename,
                )
                continue

            extra_flags.append("--send-ctrl-c")

        # skip each test if their respective requirements are not met
        requirements_met, error_message = checkRequirements(filename)
        if not requirements_met:
            reportSkip(error_message, ".", filename)
            continue

        test_logger.info(
            "Consider output of onefile mode compiled program: %s" % filename
        )

        # First compare so we know the program behaves identical.
        compareWithCPython(
            dirname=None,
            filename=filename,
            extra_flags=extra_flags,
            search_mode=search_mode,
            needs_2to3=False,
            on_error=displayError,
        )

        binary_filename = filename[:-3] + (".exe" if os.name == "nt" else ".bin")

        if filename == "KeyboardInterruptTest.py":
            continue

        # Then use "strace" on the result.
        with TimerReport(
            "Determining run time loaded files took %.2f", logger=test_logger
        ):
            loaded_filenames = getRuntimeTraceOfLoadedFiles(
                logger=test_logger, command=[binary_filename]
            )

        illegal_accesses = checkLoadedFileAccesses(
            loaded_filenames=loaded_filenames, current_dir=os.getcwd()
        )

        if illegal_accesses:
            displayError(None, filename)
            displayRuntimeTraces(test_logger, binary_filename)

            test_logger.warning(
                "Should not access these file(s): '%s'." % ",".join(illegal_accesses)
            )

            search_mode.onErrorDetected(1)

        os.unlink(binary_filename)

    search_mode.finish()
Example #36
0
def main():
    python_version = setup()

    os.chdir("subject")

    nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
    tmp_dir = getTempDir()

    command = [
        os.environ["PYTHON"],
        nuitka_main_path,
        "--plugin-enable=pylint-warnings",
        "--output-dir=%s" % tmp_dir,
        "--follow-imports",
        "--include-package=package",
        "--nofollow-import-to=*.tests",
        "--python-flag=-v",
        "--debug",
        "--module",
        "package",
    ]

    result = subprocess.call(command)

    if result != 0:
        sys.exit(result)

    os.makedirs(os.path.join(tmp_dir, "package.ext"))
    copyTree("package", os.path.join(tmp_dir, "package.ext/package"))

    os.chdir(tmp_dir)

    # We compile the package non-closed, so we can smuggle in tests
    # and user code. This is going to be the example code.
    with open("package.ext/package/user_provided.py", "w") as output:
        # TODO: Maybe assert that the type name of a local function and one from
        # the package are not the same, i.e. we are running inside the compiled
        # package.
        output.write("""
from __future__ import print_function

import package
print("__name__:",    package.__name__)
print("__package__:", package.__package__)
print("__path__:",    package.__path__)
print("__file__:",    package.__file__)
# print("__loader__:",    package.__loader__)

import package.sub_package1
print("__name__:",    package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:",    package.sub_package1.__path__)
print("__file__:",    package.sub_package1.__file__)
# print("__loader__:",    package.sub_package1.__loader__)

import package.sub_package1.tests;
print("__name__:",    package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:",    package.sub_package1.tests.__path__)
print("__file__:",    package.sub_package1.tests.__file__)
# print("__loader__:",    package.sub_package1.tests.__loader__)
""")

    os.makedirs("nose")
    with open("nose/usage.txt", "w") as output:
        pass

    os.system("find | sort")

    # Inform about the extra path, format is NUITKA_PACKAGE_fullname where
    # dots become "_" and should point to the directory where external code
    # to be loaded will live under. Probably should be an absolute path, but
    # we avoid it here.
    os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"

    # Lets make sure these to not work. These will be used in the compiled
    # form only.
    for module_path in ("__init__.py", "sub_package1__init__.py"):
        with open(os.path.join("./package.ext/package", module_path),
                  "w") as output:
            output.write("assert False")

    # Check the compiled package is functional for importing.
    my_print("Running package as basic test:")
    command = [os.environ["PYTHON"], "-c", "import package"]

    result = subprocess.call(command)

    if result != 0:
        sys.exit(result)

    my_print("Running nose tests:")
    # assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0

    my_print("Running py.test tests:")
    command = [
        os.environ["PYTHON"],
        "-m",
        "pytest",
        "-v",
        "--pyargs",
        "package.sub_package1.tests",
    ]

    result = subprocess.call(command)

    if result != 0:
        sys.exit(result)
Example #37
0
def main():
    python_version = setup()

    os.chdir("subject")

    nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
    tmp_dir = getTempDir()

    command = [
        os.environ["PYTHON"],
        nuitka_main_path,
        "--plugin-enable=pylint-warnings",
        "--output-dir=%s" % tmp_dir,
        "--follow-imports",
        "--include-package=package",
        "--nofollow-import-to=*.tests",
        "--python-flag=-v",
        "--debug",
        "--module",
        "package"
    ]

    result = subprocess.call(
        command
    )

    if result != 0:
        sys.exit(result)

    os.makedirs(os.path.join(tmp_dir, "package.ext"))
    shutil.copytree(
        "package",
        os.path.join(tmp_dir, "package.ext/package")
    )


    os.chdir(tmp_dir)

    # We compile the package non-closed, so we can smuggle in tests
    # and user code. This is going to be the example code.
    with open("package.ext/package/user_provided.py", "w") as output:
        # TODO: Maybe assert that the type name of a local function and one from
        # the package are not the same, i.e. we are running inside the compiled
        # package.
        output.write("""
from __future__ import print_function

import package
print("__name__:",    package.__name__)
print("__package__:", package.__package__)
print("__path__:",    package.__path__)
print("__file__:",    package.__file__)
# print("__loader__:",    package.__loader__)

import package.sub_package1
print("__name__:",    package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:",    package.sub_package1.__path__)
print("__file__:",    package.sub_package1.__file__)
# print("__loader__:",    package.sub_package1.__loader__)

import package.sub_package1.tests;
print("__name__:",    package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:",    package.sub_package1.tests.__path__)
print("__file__:",    package.sub_package1.tests.__file__)
# print("__loader__:",    package.sub_package1.tests.__loader__)
"""
        )

    os.makedirs("nose")
    with open("nose/usage.txt", "w") as output:
        pass

    os.system("find | sort")

    # Inform about the extra path, format is NUITKA_PACKAGE_fullname where
    # dots become "_" and should point to the directory where external code
    # to be loaded will live under. Probably should be an absolute path, but
    # we avoid it here.
    os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"

    # Lets make sure these to not work. These will be used in the compiled
    # form only.
    for module_path in ("__init__.py", "sub_package1__init__.py"):
        with open(os.path.join("./package.ext/package", module_path), "w") as output:
            output.write("assert False")


    # Check the compiled package is functional for importing.
    my_print("Running package as basic test:")
    assert os.system(os.environ["PYTHON"] + " -c 'import package'" ) == 0

    my_print("Running nose tests:")
    # assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0

    my_print("Running py.test tests:")
    assert os.system(os.environ["PYTHON"] + " -m pytest --pyargs package.sub_package1.tests" ) == 0