def main():
_python_version = setup()
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not os.path.isdir(filename) or filename.endswith(".build"):
continue
extra_flags = [
"expect_success",
"remove_output",
"module_mode",
"two_step_execution",
]
# The use of "__main__" in the test package gives a warning.
if filename == "sub_package":
extra_flags.append("ignore_warnings")
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of recursively compiled program:",
filename)
filename_main = None
for filename_main in os.listdir(filename):
if not os.path.isdir(os.path.join(filename, filename_main)):
continue
if filename_main not in ("..", "."):
break
else:
search_mode.onErrorDetected("""\
Error, no package in dir '%s' found, incomplete test case.""" % filename)
extensions = [
"--include-package=%s" % os.path.basename(filename_main)
]
if "--output-dir" not in os.environ.get("NUITKA_EXTRA_OPTIONS",
""):
extensions.append("--output-dir=%s" % getTempDir())
with withExtendedExtraOptions(*extensions):
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def executePyLint(filenames, show_todos, verbose, one_by_one):
filenames = list(filenames)
if verbose:
my_print("Checking", filenames, "...")
pylint_options = getOptions()
if not show_todos:
pylint_options.append("--notes=")
def hasPyLintBugTrigger(filename):
# Stack overflow core dumps with 1.9.x unfortunately.
if pylint_version < "2.0.0":
if os.path.basename(filename) in (
"ReformulationContractionExpressions.py",
"TreeHelpers.py",
):
return True
return False
filenames = [
filename for filename in filenames if not hasPyLintBugTrigger(filename)
]
extra_options = os.environ.get("PYLINT_EXTRA_OPTIONS", "").split()
if "" in extra_options:
extra_options.remove("")
if one_by_one:
for filename in filenames:
my_print("Checking", filename, ":")
_executePylint([filename], pylint_options, extra_options)
else:
_executePylint(filenames, pylint_options, extra_options)
def executePyLint(filenames, show_todos, verbose, one_by_one):
filenames = list(filenames)
if verbose:
my_print("Checking", filenames, "...")
pylint_options = getOptions()
if not show_todos:
pylint_options.append("--notes=")
filenames = [
filename for filename in filenames if not hasPyLintBugTrigger(filename)
if not isSpecificPythonOnly(filename)
]
extra_options = os.environ.get("PYLINT_EXTRA_OPTIONS", "").split()
if "" in extra_options:
extra_options.remove("")
if one_by_one:
for filename in filenames:
my_print("Checking", filename, ":")
_executePylint([filename], pylint_options, extra_options)
else:
_executePylint(filenames, pylint_options, extra_options)
def checkVersion():
# pylint: disable=global-statement
global pylint_version
if not hasModule("pylint"):
sys.exit(
"Error, pylint is not installed for this interpreter %r version."
% os.environ["PYTHON"]
)
if pylint_version is None:
with open(os.devnull, "w") as devnull:
pylint_version = Execution.check_output(
[os.environ["PYTHON"], "-m", "pylint", "--version"], stderr=devnull
)
if str is not bytes:
pylint_version = pylint_version.decode("utf8")
pylint_version = pylint_version.split("\n")[0].split()[-1].strip(",")
if pylint_version < "1.6.5":
sys.exit("Error, needs PyLint 1.6.5 or higher not %r." % pylint_version)
my_print("Using PyLint version:", pylint_version)
def action(stage_dir, root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"), "--module", "--output-dir",
stage_dir, "--remove-output", "--plugin-enable=pylint-warnings"
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(path)
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
my_print("Falling back to full comparison due to error exit.")
checkCompilesNotWithCPython(
dirname=None,
filename=path,
search_mode=search_mode,
)
else:
my_print("OK")
if os.name == "nt":
suffix = "pyd"
else:
suffix = "so"
target_filename = os.path.basename(path).replace(".py", '.' + suffix)
target_filename = target_filename.replace('(', "").replace(')', "")
os.unlink(os.path.join(stage_dir, target_filename))
def executePASS5():
my_print(
"PASS 5: Compiling the compiler 'nuitka' package to single '.so' file."
)
path = os.path.join("..", "..", "nuitka")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--recurse-all",
"--recurse-not-to=PyQt5",
"--recurse-not-to=nuitka.build.inline_copy",
"--recurse-not-to=nuitka.build.include",
"--recurse-dir=%s" % path,
"--module",
path
]
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
os.unlink(os.path.join(tmp_dir, "nuitka.so"))
os.unlink(os.path.join(tmp_dir, "nuitka.pyi"))
shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
def _executePylint(filenames, pylint_options, extra_options):
# This is kind of a singleton module, pylint: disable=global-statement
global our_exit_code
command = ([os.environ["PYTHON"], "-m", "pylint"] + pylint_options +
extra_options + filenames)
stdout, stderr, exit_code = executeProcess(command)
if exit_code == -11:
sys.exit("Error, segfault from pylint.")
stdout = _cleanupPylintOutput(stdout)
stderr = _cleanupPylintOutput(stderr)
if stderr:
our_exit_code = 1
for line in stderr:
my_print(line)
if stdout:
# If we filtered everything away, remove the leading file name reports.
while stdout and stdout[-1].startswith("******"):
del stdout[-1]
for line in stdout:
my_print(line)
if stdout:
our_exit_code = 1
sys.stdout.flush()
def main():
python_version = setup(needs_io_encoding=True)
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
if filename == "TryFinallyContinue.py" and python_version >= "3.8":
continue
active = search_mode.consider(dirname=None, filename=filename)
if active:
extra_flags = ["expect_failure", "remove_output", "syntax_errors"]
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def action(stage_dir, root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"), "--stand", "--run",
"--output-dir", stage_dir, "--remove-output"
]
filename = os.path.join(stage_dir, "importer.py")
assert path.startswith(root)
module_name = path[len(root) + 1:]
module_name = module_name.split(".")[0]
module_name = module_name.replace(os.path.sep, ".")
with open(filename, "w") as output:
output.write("import " + module_name + "\n")
output.write("print('OK')")
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(filename)
try:
output = subprocess.check_output(command).splitlines()
assert output[-1] == "OK", output
except Exception as e:
print(e)
sys.exit(1)
else:
my_print("OK")
shutil.rmtree(filename[:-3] + ".dist")
def executePASS5():
my_print("PASS 5: Compiling the compiler 'nuitka' package to single '.so' file.")
path = os.path.join("..", "..", "nuitka")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--include-plugin-dir=%s" % path,
"--nofollow-import-to=nuitka.build.inline_copy",
"--nofollow-import-to=nuitka.build.include",
"--nofollow-import-to=nuitka.build.static_src",
"--module",
path,
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
os.unlink(os.path.join(tmp_dir, "nuitka" + getSharedLibrarySuffix(preferred=True)))
os.unlink(os.path.join(tmp_dir, "nuitka.pyi"))
shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
def executePASS5():
my_print(
"PASS 5: Compiling the compiler 'nuitka' package to single '.so' file."
)
path = os.path.join("..", "..", "nuitka")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--include-plugin-dir=%s" % path,
"--nofollow-import-to=nuitka.build.inline_copy",
"--nofollow-import-to=nuitka.build.include",
"--nofollow-import-to=nuitka.build.static_src",
"--module",
path
]
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
os.unlink(os.path.join(tmp_dir, "nuitka.so"))
os.unlink(os.path.join(tmp_dir, "nuitka.pyi"))
shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
def main():
setup(suite="packages")
search_mode = createSearchMode()
for filename in sorted(os.listdir(".")):
if not os.path.isdir(filename) or filename.endswith(".build"):
continue
extra_flags = [
"--module",
"expect_success",
"remove_output",
"two_step_execution",
]
active = search_mode.consider(dirname=None, filename=filename)
if active:
my_print("Consider output of compiled package:", filename)
filename_main = None
filename_main = os.path.join(
filename, "".join(part.title() for part in filename.split("_")) + ".py"
)
if os.path.exists(filename_main):
filename_main = os.path.basename(filename_main)
else:
filename_main = None
if filename_main is None:
for filename_main in os.listdir(filename):
if not os.path.isdir(os.path.join(filename, filename_main)):
continue
if filename_main not in ("..", "."):
break
else:
search_mode.onErrorDetected(
"""\
Error, no package in test directory '%s' found, incomplete test case."""
% filename
)
extra_flags.append(
"--include-package=%s" % os.path.basename(filename_main)
)
extra_flags.append("--output-dir=%s" % getTempDir())
compareWithCPython(
dirname=filename,
filename=filename_main,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=False,
)
search_mode.finish()
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(
path = target_dir,
ignore_errors = False
)
command = [
nuitka,
"--module",
"--recurse-none",
"--plugin-enable=pylint-warnings",
"--output-dir=%s"% tmp_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
diffRecursive(os.path.join(package, target), target_dir)
shutil.rmtree(target_dir)
if os.name == "nt":
target_filename = filename.replace(".py", ".pyd")
else:
target_filename = filename.replace(".py", ".so")
os.unlink(os.path.join(tmp_dir, target_filename))
def runValgrind(descr, test_case, args):
my_print(descr, file=sys.stderr, end="... ")
log_base = test_case[:-3] if test_case.endswith(".py") else test_case
log_file = log_base + ".log"
valgrind_options = "-q --tool=callgrind --callgrind-out-file=%s" % log_file
command = ["valgrind"] + valgrind_options.split() + list(args)
process = subprocess.Popen(args=command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_stdout_valgrind, stderr_valgrind = process.communicate()
exit_valgrind = process.returncode
assert exit_valgrind == 0, stderr_valgrind
my_print("OK", file=sys.stderr)
try:
for line in open(log_file):
if line.startswith("summary:"):
return int(line.split()[1])
else:
assert False
finally:
os.unlink(log_file)
def diffRecursive(dir1, dir2):
done = set()
result = False
for path1, filename in listDir(dir1):
path2 = os.path.join(dir2, filename)
done.add(path1)
# Skip these binary files and scons build database of course.
if filename.endswith(
(".o", ".os", ".obj", ".dblite", ".tmp", ".sconsign", ".txt")):
continue
if not os.path.exists(path2):
sys.exit("Only in %s: %s" % (dir1, filename))
if os.path.isdir(path1):
r = diffRecursive(path1, path2)
if r:
result = True
elif os.path.isfile(path1):
fromdate = time.ctime(os.stat(path1).st_mtime)
todate = time.ctime(os.stat(path2).st_mtime)
diff = difflib.unified_diff(
a=readSource(path1).splitlines(),
b=readSource(path2).splitlines(),
fromfile=path1,
tofile=path2,
fromfiledate=fromdate,
tofiledate=todate,
n=3,
)
diff_list = list(diff)
if diff_list:
for line in diff_list:
try:
my_print(line)
except UnicodeEncodeError:
my_print(repr(line))
result = True
else:
assert False, path1
for path1, filename in listDir(dir2):
path2 = os.path.join(dir2, filename)
if path1 in done:
continue
if not os.path.exists(path1):
sys.exit("Only in %s: %s" % (dir2, filename))
return result
def diffRecursive(dir1, dir2):
done = set()
result = False
for path1, filename in listDir(dir1):
path2 = os.path.join(dir2, filename)
done.add(path1)
# Skip these binary files and scons build database of course.
if filename.endswith((".o", ".os", ".obj", ".dblite")):
continue
# Skip
if filename == ".sconsign":
continue
if not os.path.exists(path2):
sys.exit("Only in %s: %s" % (dir1, filename))
if os.path.isdir(path1):
r = diffRecursive(path1, path2)
if r:
result = True
elif os.path.isfile(path1):
fromdate = time.ctime(os.stat(path1).st_mtime)
todate = time.ctime(os.stat(path2).st_mtime)
diff = difflib.unified_diff(
a = readSource(path1).splitlines(),
b = readSource(path2).splitlines(),
fromfile = path1,
tofile = path2,
fromfiledate = fromdate,
tofiledate = todate,
n = 3
)
diff_list = list(diff)
if diff_list :
for line in diff_list :
my_print(line)
result = True
else:
assert False, path1
for path1, filename in listDir(dir2):
path2 = os.path.join(dir2, filename)
if path1 in done:
continue
if not os.path.exists(path1):
sys.exit("Only in %s: %s" % (dir2, filename))
return result
def action(stage_dir, root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"),
"--stand",
"--run",
"--output-dir",
stage_dir,
"--remove-output",
"--plugin-enable=pylint-warnings",
]
filename = os.path.join(stage_dir, "importer.py")
assert path.startswith(root)
module_name = path[len(root) + 1:]
module_name = module_name.split(".")[0]
module_name = module_name.replace(os.path.sep, ".")
with open(filename, "w") as output:
output.write("import " + module_name + "\n")
output.write("print('OK')")
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(filename)
if checkSucceedsWithCPython(filename):
try:
output = check_output(command).splitlines()
except Exception: # only trying to check for no exception, pylint: disable=try-except-raise
raise
else:
assert os.path.exists(filename[:-3] + ".dist")
loaded_filenames = getRuntimeTraceOfLoadedFiles(
logger=test_logger,
path=os.path.join(filename[:-3] + ".dist", "importer.exe"),
)
outside_accesses = checkRuntimeLoadedFilesForOutsideAccesses(
loaded_filenames,
[
filename[:-3] + ".dist", current_dir,
os.path.expanduser("~/.config")
],
)
if output[-1] != b"OK":
sys.exit("FAIL")
my_print("OK")
assert not outside_accesses, outside_accesses
shutil.rmtree(filename[:-3] + ".dist")
else:
my_print("SKIP (does not work with CPython)")
def executePASS4():
my_print("PASS 4: Compiling the compiler running from single exe.")
exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix)
compileAndCompareWith(exe_path)
my_print("OK.")
def executePASS4():
my_print("PASS 4: Compiling the compiler running from single exe.")
exe_path = os.path.join(tmp_dir, "nuitka.exe")
compileAndCompareWith(exe_path)
my_print("OK.")
def diffRecursive(dir1, dir2):
done = set()
for path1, filename in listDir(dir1):
path2 = os.path.join(dir2, filename)
done.add(path1)
# Skip these binary files of course.
if filename.endswith(".o") or \
filename.endswith(".os") or \
filename.endswith(".obj"):
continue
# Skip scons build database
if filename == ".sconsign.dblite":
continue
if not os.path.exists(path2):
sys.exit("Only in %s: %s" % (dir1, filename))
if os.path.isdir(path1):
diffRecursive(path1, path2)
elif os.path.isfile(path1):
fromdate = time.ctime(os.stat(path1).st_mtime)
todate = time.ctime(os.stat(path2).st_mtime)
diff = difflib.unified_diff(
a = readSource(path1).splitlines(),
b = readSource(path2).splitlines(),
fromfile = path1,
tofile = path2,
fromfiledate = fromdate,
tofiledate = todate,
n = 3
)
result = list(diff)
if result:
for line in result:
my_print(line)
sys.exit(1)
else:
assert False, path1
for path1, filename in listDir(dir2):
path2 = os.path.join(dir2, filename)
if path1 in done:
continue
if not os.path.exists(path1):
sys.exit("Only in %s: %s" % (dir2, filename))
def displayError(dirname, filename):
assert dirname is None
my_print("Listing of dist folder:")
if os.name == "nt":
command = "dir /b /s /a:-D %s"
else:
command = "ls -Rla %s"
os.system(command % filename)
def displayError(dirname, filename):
assert dirname is None
filename = filename[:-3] + ".dist"
my_print("Listing of dist folder '%s':" % filename)
if os.name == "nt":
command = "dir /b /s /a:-D %s" % filename
else:
command = "ls -Rla %s" % filename
os.system(command)
def executePyLint(filenames, show_todos, verbose, one_by_one):
filenames = list(filenames)
if verbose:
my_print("Checking", filenames, "...")
pylint_options = getOptions()
if not show_todos:
pylint_options.append("--notes=")
def hasPyLintBugTrigger(filename):
# Stack overflow core dumps with 1.9.x unfortunately.
if pylint_version < "2.0.0":
if os.path.basename(filename) in (
"ReformulationContractionExpressions.py",
"TreeHelpers.py",
):
return True
# Slot warning that is impossible to disable
if pylint_version < "2.0.0":
if os.path.basename(filename) in ("Variables.py",):
return True
return False
def isSpecificPythonOnly(filename):
if str is bytes and "TracebackEncryptionPlugin" in filename:
# This is Python3 only code.
return True
return False
filenames = [
filename
for filename in filenames
if not hasPyLintBugTrigger(filename)
if not isSpecificPythonOnly(filename)
]
extra_options = os.environ.get("PYLINT_EXTRA_OPTIONS", "").split()
if "" in extra_options:
extra_options.remove("")
if one_by_one:
for filename in filenames:
my_print("Checking", filename, ":")
_executePylint([filename], pylint_options, extra_options)
else:
_executePylint(filenames, pylint_options, extra_options)
def main():
python_version = setup(suite="basics", needs_io_encoding=True)
search_mode = createSearchMode()
filenames = _createTests(python_version)
# Now run all the tests in this directory.
for filename in filenames:
assert filename.endswith(".py")
if not decideFilenameVersionSkip(filename):
continue
extra_flags = [
# No error exits normally, unless we break tests, and that we would
# like to know.
"expect_success",
# Keep no temporary files.
"remove_output",
# Include imported files, mostly nothing though.
"recurse_all",
# Use the original __file__ value, at least one case warns about things
# with filename included.
"original_file",
# Cache the CPython results for re-use, they will normally not change.
"cpython_cache",
# We annotate some tests, use that to lower warnings.
"plugin_enable:pylint-warnings",
]
# This test should be run with the debug Python, and makes outputs to
active = search_mode.consider(dirname=None, filename=filename)
if active:
compareWithCPython(
dirname=None,
filename=filename,
extra_flags=extra_flags,
search_mode=search_mode,
needs_2to3=decideNeeds2to3(filename),
)
if search_mode.abortIfExecuted():
break
else:
my_print("Skipping", filename)
search_mode.finish()
def executePASS2():
my_print("PASS 2: Compiling from compiler running from .exe and many .so files.")
# Windows will load the compiled modules (pyd) only from PYTHONPATH, so we
# have to add it.
if os.name == "nt":
os.environ["PYTHONPATH"] = ":".join(PACKAGE_LIST)
compileAndCompareWith(os.path.join(".", "nuitka" + exe_suffix))
# Undo the damage from above.
if os.name == "nt":
del os.environ["PYTHONPATH"]
my_print("OK.")
def executePASS2():
my_print(
"PASS 2: Compiling from compiler running from .exe and many .so files."
)
# Windows will load the compiled modules (pyd) only from PYTHONPATH, so we
# have to add it.
if os.name == "nt":
os.environ["PYTHONPATH"] = ':'.join(PACKAGE_LIST)
compileAndCompareWith(os.path.join('.', "nuitka" + exe_suffix))
# Undo the damage from above.
if os.name == "nt":
del os.environ["PYTHONPATH"]
my_print("OK.")
def _executePylint(filenames, pylint_options, extra_options):
# This is kind of a singleton module, pylint: disable=global-statement
global our_exit_code
command = (
[os.environ["PYTHON"], "-m", "pylint"]
+ pylint_options
+ extra_options
+ filenames
)
process = subprocess.Popen(
args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False
)
stdout, stderr = process.communicate()
exit_code = process.returncode
if exit_code == -11:
sys.exit("Error, segfault from pylint.")
stdout = _cleanupPylintOutput(stdout)
stderr = _cleanupPylintOutput(stderr)
if stderr:
our_exit_code = 1
for line in stderr:
my_print(line)
if stdout:
# If we filtered everything away, remove the leading file name report.
if len(stdout) == 1:
assert stdout[0].startswith("*****")
stdout = []
for line in stdout:
my_print(line)
if stdout:
our_exit_code = 1
sys.stdout.flush()
def _executePylint(filenames, pylint_options, extra_options):
# This is kind of a singleton module, pylint: disable=global-statement
global our_exit_code
command = (
[os.environ["PYTHON"], "-m", "pylint"]
+ pylint_options
+ extra_options
+ filenames
)
process = subprocess.Popen(
args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False
)
stdout, stderr = process.communicate()
exit_code = process.returncode
if exit_code == -11:
sys.exit("Error, segfault from pylint.")
stdout = _cleanupPylintOutput(stdout)
stderr = _cleanupPylintOutput(stderr)
if stderr:
our_exit_code = 1
for line in stderr:
my_print(line)
if stdout:
# If we filtered everything away, remove the leading file name report.
if len(stdout) == 1:
assert stdout[0].startswith("*****")
stdout = []
for line in stdout:
my_print(line)
if stdout:
our_exit_code = 1
sys.stdout.flush()
def checkVersion():
# pylint: disable=global-statement
global pylint_version
if not hasModule("pylint"):
sys.exit(
"Error, pylint is not installed for this interpreter %r version." %
os.environ["PYTHON"])
if pylint_version is None:
pylint_version = check_output(
[os.environ["PYTHON"], "-m", "pylint", "--version"],
stderr=getNullOutput())
if str is not bytes:
pylint_version = pylint_version.decode("utf8")
pylint_version = pylint_version.split("\n")[0].split()[-1].strip(",")
my_print("Using PyLint version:", pylint_version)
def _executeSubTest(command, hide_output):
if options.coverage and "search" in command:
command = command.replace("search", "coverage")
parts = command.split()
parts[0] = parts[0].replace("/", os.path.sep)
# The running Python will be good enough, on some platforms there is no
# "python", and we need to pass this alone then.
parts.insert(0, sys.executable)
my_print("Run '%s' in '%s'." % (" ".join(parts), os.getcwd()))
if hide_output:
result = subprocess.call(parts, stdout=getNullOutput())
else:
result = subprocess.call(parts)
if result != 0:
sys.exit(result)
def action(stage_dir, _root, path):
command = [
sys.executable,
os.path.join("..", "..", "bin", "nuitka"),
"--module",
"--output-dir",
stage_dir,
"--remove-output",
"--plugin-enable=pylint-warnings",
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(path)
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
if os.path.basename(path) in nosyntax_errors:
my_print("Syntax error is known unreliable with file file.")
else:
my_print("Falling back to full comparison due to error exit.")
checkCompilesNotWithCPython(dirname=None,
filename=path,
search_mode=search_mode)
else:
my_print("OK")
suffix = getSharedLibrarySuffix(preferred=True)
target_filename = os.path.basename(path)[:-3] + suffix
target_filename = target_filename.replace("(", "").replace(")", "")
os.unlink(os.path.join(stage_dir, target_filename))
def executePASS3():
my_print(
"PASS 3: Compiling from compiler running from .py files to single .exe."
)
exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix)
if os.path.exists(exe_path):
os.unlink(exe_path)
build_path = os.path.join(tmp_dir, "nuitka.build")
if os.path.exists(build_path):
shutil.rmtree(build_path)
path = os.path.join("..", "..", "bin", "nuitka")
my_print("Compiling '%s'." % path)
command = [
os.environ["PYTHON"],
nuitka_main_path,
path,
"--output-dir=%s" % tmp_dir,
"--python-flag=-S",
"--follow-imports",
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
shutil.rmtree(build_path)
my_print("OK.")
def executePyLint(filenames, show_todos, verbose, one_by_one):
filenames = list(filenames)
if verbose:
my_print("Checking", filenames, "...")
pylint_options = getOptions()
if not show_todos:
pylint_options.append("--notes=")
def hasPyLintBugTrigger(filename):
# Stack overflow core dumps with 1.9.x unfortunately.
if pylint_version < "2.0.0":
if os.path.basename(filename) in (
"ReformulationContractionExpressions.py",
"TreeHelpers.py",
):
return True
if pylint_version == "2.0.0":
if os.path.basename(filename) in ("LocalsDictCodes.py", "FrameCodes.py"):
return True
return False
filenames = [
filename for filename in filenames if not hasPyLintBugTrigger(filename)
]
extra_options = os.environ.get("PYLINT_EXTRA_OPTIONS", "").split()
if "" in extra_options:
extra_options.remove("")
if one_by_one:
for filename in filenames:
my_print("Checking", filename, ":")
_executePylint([filename], pylint_options, extra_options)
else:
_executePylint(filenames, pylint_options, extra_options)
def checkVersion():
# pylint: disable=global-statement
global pylint_version
if not hasModule("pylint"):
sys.exit("Error, pylint is not installed for this interpreter version.")
if pylint_version is None:
with open(os.devnull, "w") as devnull:
pylint_version = Execution.check_output(
[os.environ["PYTHON"], "-m", "pylint", "--version"], stderr=devnull
)
if str is not bytes:
pylint_version = pylint_version.decode("utf8")
pylint_version = pylint_version.split("\n")[0].split()[-1].strip(",")
if pylint_version < "1.6.5":
sys.exit("Error, needs PyLint 1.6.5 or higher not %r." % pylint_version)
my_print("Using PyLint version:", pylint_version)
def executePASS3():
test_logger.info(
"PASS 3: Compiling from compiler running from .py files to single .exe."
)
exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix)
if os.path.exists(exe_path):
os.unlink(exe_path)
build_path = os.path.join(tmp_dir, "nuitka.build")
if os.path.exists(build_path):
shutil.rmtree(build_path)
path = os.path.join("..", "..", "bin", "nuitka")
_traceCompilation(path=path, pass_number=3)
command = [
os.environ["PYTHON"],
nuitka_main_path,
path,
"--output-dir=%s" % tmp_dir,
"--python-flag=-S",
"--follow-imports",
# "--include-package=nuitka.plugins",
]
my_print("Command: ", " ".join(command))
result = subprocess.call(command)
if result != 0:
sys.exit(result)
shutil.rmtree(build_path)
test_logger.info("OK.")
def executePASS3():
my_print(
"PASS 3: Compiling from compiler running from .py files to single .exe."
)
exe_path = os.path.join(tmp_dir, "nuitka" + exe_suffix)
if os.path.exists(exe_path):
os.unlink(exe_path)
build_path = os.path.join(tmp_dir, "nuitka.build")
if os.path.exists(build_path):
shutil.rmtree(build_path)
path = os.path.join("..", "..", "bin", "nuitka")
my_print("Compiling '%s'." % path)
command = [
os.environ["PYTHON"],
nuitka_main_path,
path,
"--output-dir=%s" % tmp_dir,
"--python-flag=-S",
"--follow-imports"
]
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
shutil.rmtree(build_path)
my_print("OK.")
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option(
"--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "")
)
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option("--code-diff", action="store", dest="diff_filename", default="")
parser.add_option("--copy-source-to", action="store", dest="target_dir", default="")
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit("Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
python_version = setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % python_version)
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
with open(test_case, "rb") as f:
my_print("TEST_CASE_HASH='%s'" % hashlib.md5(f.read()).hexdigest())
needs_2to3 = (
python_version.startswith("3")
and not test_case.endswith("32.py")
and not test_case.endswith("33.py")
)
if options.target_dir:
shutil.copyfile(
test_case, os.path.join(options.target_dir, os.path.basename(test_case))
)
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case))
with open(test_case) as f:
case_1_source, case_2_source = generateConstructCases(f.read())
with open(test_case_1, "w") as case_1_file:
case_1_file.write(case_1_source)
with open(test_case_2, "w") as case_2_file:
case_2_file.write(case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = "0"
if nuitka:
nuitka_id = check_output(
"cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True
)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3,):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
if nuitka:
nuitka_call = [
os.environ["PYTHON"],
nuitka,
"--python-flag=-S",
os.path.basename(test_case),
]
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
shutil.copyfile(test_case_1, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix),
)
shutil.copyfile(test_case_2, os.path.basename(test_case))
subprocess.check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix),
)
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(
test_case_1.replace(".py", ".build"), "module.__main__" + suffix
)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(
test_case_2.replace(".py", ".build"), "module.__main__" + suffix
)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
with open(options.diff_filename, "w") as f:
with open(cpp_1) as cpp1:
with open(cpp_2) as cpp2:
f.write(
difflib.HtmlDiff().make_table(
cpp1.readlines(),
cpp2.readlines(),
"Construct",
"Baseline",
True,
)
)
nuitka_1 = runValgrind(
"Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix),),
include_startup=True,
)
nuitka_2 = runValgrind(
"Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix),),
include_startup=True,
)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_1 = runValgrind(
"CPython construct",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_1),
include_startup=True,
)
cpython_2 = runValgrind(
"CPython baseline",
"callgrind",
(os.environ["PYTHON"], "-S", test_case_2),
include_startup=True,
)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
continue
filename = os.path.relpath(filename)
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(
dirname = None,
filename = filename
)
if active:
my_print("Consider distutils example:", filename)
case_dir = os.path.join(os.getcwd(), filename)
removeDirectory(os.path.join(case_dir, "build"), ignore_errors = False)
removeDirectory(os.path.join(case_dir, "dist"), ignore_errors = False)
with withVirtualenv("venv_cpython") as venv:
venv.runCommand(
commands = [
'cd "%s"' % case_dir,
"python setup.py bdist_wheel",
]
)
dist_dir = os.path.join(case_dir, "dist")
active = search_mode.consider(
dirname = None,
filename = filename
)
extra_flags = ["expect_success"]
if active:
# Apply 2to3 conversion if necessary.
if python_version.startswith('3'):
filename, changed = convertUsing2to3(filename)
else:
changed = False
my_print("Consider", filename, end = ' ')
command = [
os.environ["PYTHON"],
os.path.abspath(os.path.join("..", "..", "bin", "nuitka")),
"--xml",
"--module",
filename
]
if search_mode.isCoverage():
# To avoid re-execution, which is not acceptable to coverage.
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
outside_accesses = checkRuntimeLoadedFilesForOutsideAccesses(
loaded_filenames,
[
filename[:-3] + ".dist",
current_dir,
os.path.expanduser("~/.config")
]
)
if output[-1] != b"OK":
sys.exit("FAIL")
my_print("OK")
assert not outside_accesses, outside_accesses
shutil.rmtree(filename[:-3] + ".dist")
else:
my_print("SKIP (does not work with CPython)")
compileLibraryTest(
search_mode = search_mode,
stage_dir = os.path.join(tmp_dir, "compile_extensions"),
decide = decide,
action = action
)
my_print("FINISHED, all extension modules compiled.")
command
)
if result != 0:
sys.exit(result)
os.unlink(os.path.join(tmp_dir, "nuitka.so"))
os.unlink(os.path.join(tmp_dir, "nuitka.pyi"))
shutil.rmtree(os.path.join(tmp_dir, "nuitka.build"))
cross_compilation = False
executePASS1()
if cross_compilation:
my_print("PASS 2: Skipped for cross-compilation case.")
else:
executePASS2()
executePASS3()
if cross_compilation:
my_print("PASS 4: Skipped for cross-compilation case.")
else:
executePASS4()
shutil.rmtree("nuitka")
executePASS5()
os.unlink(os.path.join(tmp_dir, "nuitka" + exe_suffix))
def execute_tests(where, use_python, flags):
# Many cases, pylint: disable=too-many-branches,too-many-statements
my_print(
"Executing test case called '%s' with CPython '%s' and extra flags '%s'."
% (where, use_python, flags))
intended_version = use_python[6:]
if sys.version.startswith(intended_version):
os.environ["PYTHON"] = sys.executable
else:
if os.name == "nt":
os.environ["PYTHON"] = getPythonExePathWindows(
search=intended_version, arch=None)
else:
os.environ["PYTHON"] = getExecutablePath(use_python)
if options.basic_tests:
my_print("Running the basic tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "basics", flags)
executeSubTest("./tests/basics/run_all.py search")
if options.syntax_tests:
my_print("Running the syntax tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "syntax", flags)
executeSubTest("./tests/syntax/run_all.py search")
if options.program_tests:
my_print("Running the program tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "programs", flags)
executeSubTest("./tests/programs/run_all.py search")
if options.package_tests:
my_print("Running the package tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "packages", flags)
executeSubTest("./tests/packages/run_all.py search")
if options.plugin_tests:
my_print("Running the plugin tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "plugins", flags)
executeSubTest("./tests/plugins/run_all.py search")
# At least one Debian Jessie, these versions won't have lxml installed, so
# don't run them there. Also these won't be very version dependent in their
# results.
if use_python != "python2.6":
if options.optimization_tests:
my_print(
"Running the optimizations tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "optimizations", flags)
executeSubTest("./tests/optimizations/run_all.py search")
if options.standalone_tests and not options.coverage:
my_print(
"Running the standalone tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(None, "standalone", flags)
executeSubTest("./tests/standalone/run_all.py search")
if options.reflection_test and not options.coverage:
my_print(
"Running the reflection test with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(None, "reflected", flags)
executeSubTest("./tests/reflected/compile_itself.py search")
if not use_python.startswith("python3"):
if os.path.exists("./tests/CPython26/run_all.py"):
if options.cpython26:
my_print(
"Running the CPython 2.6 tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "26tests", flags)
executeSubTest("./tests/CPython26/run_all.py search")
else:
my_print("The CPython2.6 tests are not present, not run.")
# Running the Python 2.7 test suite with CPython 2.6 gives little
# insight, because "importlib" will not be there and that's it.
if use_python != "python2.6":
if os.path.exists("./tests/CPython27/run_all.py"):
if options.cpython27:
my_print(
"Running the CPython 2.7 tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "27tests", flags)
executeSubTest("./tests/CPython27/run_all.py search")
else:
my_print("The CPython2.7 tests are not present, not run.")
if "--debug" not in flags:
# Not running the Python 3.2 test suite with CPython2.6, as that's about
# the same as CPython2.7 and won't have any new insights.
if use_python not in ("python2.6",
"python2.7") or not options.coverage:
if os.path.exists("./tests/CPython32/run_all.py"):
if options.cpython32:
setExtraFlags(where, "32tests", flags)
executeSubTest("./tests/CPython32/run_all.py search")
else:
my_print("The CPython3.2 tests are not present, not run.")
# Running the Python 3.3 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython33/run_all.py"):
if options.cpython33:
setExtraFlags(where, "33tests", flags)
executeSubTest("./tests/CPython33/run_all.py search")
else:
my_print("The CPython3.3 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython34/run_all.py"):
if options.cpython34:
setExtraFlags(where, "34tests", flags)
executeSubTest("./tests/CPython34/run_all.py search")
else:
my_print("The CPython3.4 tests are not present, not run.")
# Running the Python 3.5 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython35/run_all.py"):
if options.cpython35:
setExtraFlags(where, "35tests", flags)
executeSubTest("./tests/CPython35/run_all.py search")
else:
my_print("The CPython3.5 tests are not present, not run.")
# Running the Python 3.6 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "36tests", flags)
executeSubTest("./tests/CPython36/run_all.py search")
else:
my_print("The CPython3.6 tests are not present, not run.")
# Running the Python 3.7 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython37/run_all.py"):
if options.cpython36:
setExtraFlags(where, "37tests", flags)
executeSubTest("./tests/CPython37/run_all.py search")
else:
my_print("The CPython3.7 tests are not present, not run.")
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython38/run_all.py"):
if options.cpython36:
setExtraFlags(where, "38tests", flags)
executeSubTest("./tests/CPython38/run_all.py search")
else:
my_print("The CPython3.8 tests are not present, not run.")
if "NUITKA_EXTRA_OPTIONS" in os.environ:
del os.environ["NUITKA_EXTRA_OPTIONS"]
python_version = setup(needs_io_encoding = True)
search_mode = createSearchMode()
for filename in sorted(os.listdir('.')):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
active = search_mode.consider(
dirname = None,
filename = filename
)
if active:
extra_flags = ["expect_failure", "remove_output", "syntax_errors"]
compareWithCPython(
dirname = None,
filename = filename,
extra_flags = extra_flags,
search_mode = search_mode,
needs_2to3 = False
)
else:
my_print("Skipping", filename)
search_mode.finish()
"MFCM140U.DLL",
)
for filename in sorted(os.listdir(".")):
if not filename.endswith(".py"):
continue
if not decideFilenameVersionSkip(filename):
continue
path = os.path.relpath(filename)
active = search_mode.consider(dirname=None, filename=filename)
if not active:
my_print("Skipping", filename)
continue
extra_flags = ["expect_success", "standalone", "remove_output"]
if filename == "PySideUsing.py":
# Don't test on platforms not supported by current Debian testing, and
# which should be considered irrelevant by now.
if python_version.startswith("2.6") or python_version.startswith("3.2"):
reportSkip("irrelevant Python version", ".", filename)
continue
if not hasModule("PySide.QtCore"):
reportSkip(
"PySide not installed for this Python version, but test needs it",
".",
def compileAndCompareWith(nuitka):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = '0'
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(
path = target_dir,
ignore_errors = False
)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s"% tmp_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
has_diff = diffRecursive(os.path.join(package, target), target_dir)
if has_diff:
sys.exit("There were differences!")
shutil.rmtree(target_dir)
if os.name == "nt":
target_filename = filename.replace(".py", ".pyd")
else:
target_filename = filename.replace(".py", ".so")
os.unlink(os.path.join(tmp_dir, target_filename))
def executePASS1():
my_print("PASS 1: Compiling from compiler running from .py files.")
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace('/', os.path.sep)
source_dir = os.path.join(base_dir, package)
target_dir = package
removeDirectory(
path = target_dir,
ignore_errors = False
)
os.mkdir(target_dir)
for path, filename in listDir(target_dir):
if filename.endswith(".so"):
os.unlink(path)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
if filename != "__init__.py":
my_print("Compiling '%s'." % path)
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % target_dir,
"--no-pyi-file",
path
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
else:
shutil.copyfile(path, os.path.join(target_dir, filename))
my_print("Compiling '%s'." % nuitka_main_path)
shutil.copyfile(nuitka_main_path, "nuitka-runner.py")
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--nofollow-imports",
"--plugin-enable=pylint-warnings",
"--output-dir=.",
"--python-flag=-S",
"nuitka-runner.py"
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
shutil.move("nuitka-runner" + exe_suffix, "nuitka" + exe_suffix)
scons_inline_copy_path = os.path.join(
base_dir,
"nuitka",
"build",
"inline_copy"
)
if os.path.exists(scons_inline_copy_path):
shutil.copytree(
scons_inline_copy_path,
os.path.join("nuitka", "build", "inline_copy")
)
shutil.copyfile(
os.path.join(base_dir, "nuitka", "build", "SingleExe.scons"),
os.path.join("nuitka", "build", "SingleExe.scons")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "static_src"),
os.path.join("nuitka", "build", "static_src")
)
shutil.copytree(
os.path.join(base_dir, "nuitka", "build", "include"),
os.path.join("nuitka", "build", "include")
)
def main():
python_version = setup()
os.chdir("subject")
nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--follow-imports",
"--include-package=package",
"--nofollow-import-to=*.tests",
"--python-flag=-v",
"--debug",
"--module",
"package"
]
result = subprocess.call(
command
)
if result != 0:
sys.exit(result)
os.makedirs(os.path.join(tmp_dir, "package.ext"))
shutil.copytree(
"package",
os.path.join(tmp_dir, "package.ext/package")
)
os.chdir(tmp_dir)
# We compile the package non-closed, so we can smuggle in tests
# and user code. This is going to be the example code.
with open("package.ext/package/user_provided.py", "w") as output:
# TODO: Maybe assert that the type name of a local function and one from
# the package are not the same, i.e. we are running inside the compiled
# package.
output.write("""
from __future__ import print_function
import package
print("__name__:", package.__name__)
print("__package__:", package.__package__)
print("__path__:", package.__path__)
print("__file__:", package.__file__)
# print("__loader__:", package.__loader__)
import package.sub_package1
print("__name__:", package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:", package.sub_package1.__path__)
print("__file__:", package.sub_package1.__file__)
# print("__loader__:", package.sub_package1.__loader__)
import package.sub_package1.tests;
print("__name__:", package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:", package.sub_package1.tests.__path__)
print("__file__:", package.sub_package1.tests.__file__)
# print("__loader__:", package.sub_package1.tests.__loader__)
"""
)
os.makedirs("nose")
with open("nose/usage.txt", "w") as output:
pass
os.system("find | sort")
# Inform about the extra path, format is NUITKA_PACKAGE_fullname where
# dots become "_" and should point to the directory where external code
# to be loaded will live under. Probably should be an absolute path, but
# we avoid it here.
os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"
# Lets make sure these to not work. These will be used in the compiled
# form only.
for module_path in ("__init__.py", "sub_package1__init__.py"):
with open(os.path.join("./package.ext/package", module_path), "w") as output:
output.write("assert False")
# Check the compiled package is functional for importing.
my_print("Running package as basic test:")
assert os.system(os.environ["PYTHON"] + " -c 'import package'" ) == 0
my_print("Running nose tests:")
# assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0
my_print("Running py.test tests:")
assert os.system(os.environ["PYTHON"] + " -m pytest --pyargs package.sub_package1.tests" ) == 0
elif filename == "multiprocessing_using":
if os.name == "nt":
extra_flags += [
"plugin_enable:multiprocessing",
"ignore_infos"
]
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = extra_options
active = search_mode.consider(
dirname = None,
filename = filename
)
if active:
my_print("Consider output of recursively compiled program:", filename)
for filename_main in os.listdir(filename):
if filename_main.endswith("Main.py"):
break
if filename_main.endswith("Main"):
break
else:
sys.exit(
"""\
Error, no file ends with 'Main.py' or 'Main' in %s, incomplete test case.""" % (
filename
)
)
def main():
python_version = setup()
os.chdir("subject")
nuitka_main_path = os.path.join("..", "..", "..", "bin", "nuitka")
tmp_dir = getTempDir()
command = [
os.environ["PYTHON"],
nuitka_main_path,
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--follow-imports",
"--include-package=package",
"--nofollow-import-to=*.tests",
"--python-flag=-v",
"--debug",
"--module",
"package",
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
os.makedirs(os.path.join(tmp_dir, "package.ext"))
copyTree("package", os.path.join(tmp_dir, "package.ext/package"))
os.chdir(tmp_dir)
# We compile the package non-closed, so we can smuggle in tests
# and user code. This is going to be the example code.
with open("package.ext/package/user_provided.py", "w") as output:
# TODO: Maybe assert that the type name of a local function and one from
# the package are not the same, i.e. we are running inside the compiled
# package.
output.write("""
from __future__ import print_function
import package
print("__name__:", package.__name__)
print("__package__:", package.__package__)
print("__path__:", package.__path__)
print("__file__:", package.__file__)
# print("__loader__:", package.__loader__)
import package.sub_package1
print("__name__:", package.sub_package1.__name__)
print("__package__:", package.sub_package1.__package__)
print("__path__:", package.sub_package1.__path__)
print("__file__:", package.sub_package1.__file__)
# print("__loader__:", package.sub_package1.__loader__)
import package.sub_package1.tests;
print("__name__:", package.sub_package1.tests.__name__)
print("__package__:", package.sub_package1.tests.__package__)
print("__path__:", package.sub_package1.tests.__path__)
print("__file__:", package.sub_package1.tests.__file__)
# print("__loader__:", package.sub_package1.tests.__loader__)
""")
os.makedirs("nose")
with open("nose/usage.txt", "w") as output:
pass
os.system("find | sort")
# Inform about the extra path, format is NUITKA_PACKAGE_fullname where
# dots become "_" and should point to the directory where external code
# to be loaded will live under. Probably should be an absolute path, but
# we avoid it here.
os.environ["NUITKA_PACKAGE_package"] = "./package.ext/package"
# Lets make sure these to not work. These will be used in the compiled
# form only.
for module_path in ("__init__.py", "sub_package1__init__.py"):
with open(os.path.join("./package.ext/package", module_path),
"w") as output:
output.write("assert False")
# Check the compiled package is functional for importing.
my_print("Running package as basic test:")
command = [os.environ["PYTHON"], "-c", "import package"]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
my_print("Running nose tests:")
# assert os.system(os.environ["PYTHON"] + " -m nose --first-package-wins -s package.sub_package1.tests" ) == 0
my_print("Running py.test tests:")
command = [
os.environ["PYTHON"],
"-m",
"pytest",
"-v",
"--pyargs",
"package.sub_package1.tests",
]
result = subprocess.call(command)
if result != 0:
sys.exit(result)
def action(stage_dir, root, path):
command = [
sys.executable,
os.path.join(
"..",
"..",
"bin",
"nuitka"
),
"--stand",
"--run",
"--output-dir",
stage_dir,
"--remove-output",
"--plugin-enable=pylint-warnings"
]
filename = os.path.join(stage_dir, "importer.py")
assert path.startswith(root)
module_name = path[len(root)+1:]
module_name = module_name.split(".")[0]
module_name = module_name.replace(os.path.sep, ".")
with open(filename, "w") as output:
output.write("import " + module_name + "\n")
output.write("print('OK')")
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
command.append(filename)
if checkSucceedsWithCPython(filename):
try:
output = check_output(command).splitlines()
except Exception:
raise
else:
assert os.path.exists(filename[:-3] + ".dist")
loaded_filenames = getRuntimeTraceOfLoadedFiles(
path = os.path.join(
filename[:-3] + ".dist",
"importer.exe"
)
)
outside_accesses = checkRuntimeLoadedFilesForOutsideAccesses(
loaded_filenames,
[
filename[:-3] + ".dist",
current_dir,
os.path.expanduser("~/.config")
]
)
if output[-1] != b"OK":
sys.exit("FAIL")
my_print("OK")
assert not outside_accesses, outside_accesses
shutil.rmtree(filename[:-3] + ".dist")
else:
my_print("SKIP (does not work with CPython)")
