def main():
parser = OptionParser()
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["nuitka/plugins/standard/*.yml"]
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
goHome()
filenames = list(scanTargets(
positional_args,
suffixes=(".yaml", ),
))
if not filenames:
sys.exit("No files found.")
for filename in filenames:
checkYamllint(filename)
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Default is %default.""",
)
# TODO: No actual options yet.
_options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
print("Running doxygen:")
subprocess.check_call(["doxygen", "doc/Doxyfile"])
# Update the repository on the web site.
assert (
os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
)
== 0
)
print("Finished.")
def main():
if len(sys.argv) < 2:
goHome()
checkReleaseDocumentation()
else:
for document in sys.argv[1:]:
checkRstLint(document)
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Show what it is doing. Default is %default.""",
)
parser.add_option(
"--write",
"-w",
action="store_true",
dest="write",
default=False,
help="""Write changes to the files. Default is %default.""",
)
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = [
"bin",
"nuitka",
"rpm",
"misc",
"tests/*/run_all.py",
"*.rst",
]
goHome()
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
filenames = list(
scanTargets(
positional_args,
suffixes=(".py", ".scons", ".rst", ".txt", ".j2", ".md", ".c",
".h"),
))
if not filenames:
sys.exit("No files found.")
result = runCodespell(filenames=filenames,
verbose=options.verbose,
write=options.write)
if result:
my_print("OK.")
else:
sys.exit(
"\nError, please correct the spelling problems found or extend 'misc/codespell-ignore.txt' if applicable."
)
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Default is %default.""",
)
parser.add_option(
"--from-commit",
action="store_true",
dest="from_commit",
default=False,
help=
"""From commit hook, do not descend into directories. Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""Stop if an error occurs, or continue. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.from_commit:
assert not positional_args
for desc in getStagedFileChangeDesc():
autoformat(desc["src_path"], git_stage=desc, abort=options.abort)
else:
if not positional_args:
positional_args = [
"bin", "nuitka", "setup.py", "tests/*/run_all.py"
]
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
goHome()
filenames = list(
scanTargets(
positional_args,
(".py", ".scons", ".rst", ".txt", ".j2", ".md", ".c", ".h"),
))
if not filenames:
sys.exit("No files found.")
for filename in filenames:
autoformat(filename, git_stage=False, abort=options.abort)
def main():
parser = OptionParser()
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Default is %default.""")
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["bin", "nuitka"]
print("Working on:", positional_args)
positional_args = [
os.path.abspath(positional_arg) for positional_arg in positional_args
]
goHome()
found = False
for filename in scanTargets(positional_args):
autoformat(filename)
found = True
if not found:
sys.exit("No files found.")
def main():
goHome()
msi_filename = createMSIPackage()
if not os.path.exists("msi"):
os.makedirs("msi")
shutil.move(msi_filename, "msi")
def main():
goHome()
msi_filename = createMSIPackage()
if not os.path.exists("msi"):
os.makedirs("msi")
shutil.move(msi_filename, "msi")
def main():
parser = OptionParser()
parser.add_option(
"--mode",
action="store",
dest="mode",
default=None,
help="""\
The mode of update, prerelease, hotfix, release, auto (default auto determines from branch).""",
)
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# Go its own directory, to have it easy with path knowledge.
goHome()
with open("nuitka/Version.py") as f:
option_lines = f.readlines()
(version_line, ) = [
line for line in option_lines if line.startswith("Nuitka V")
]
old_version = version_line[8:].rstrip()
mode = options.mode
branch_name = getBranchName()
if mode is None:
if branch_name.startswith("hotfix/"):
mode = "hotfix"
elif branch_name == "master" or branch_name.startswith("release/"):
mode = "release"
elif branch_name == "develop":
mode = "prerelease"
else:
sys.exit("Error, cannot detect mode from branch name '%s'." %
branch_name)
new_version = getBumpedVersion(mode, old_version)
print("Bumped", mode, old_version, "->", new_version)
with open("nuitka/Version.py", "w") as options_file:
for line in option_lines:
if line.startswith("Nuitka V"):
line = "Nuitka V" + new_version + "\n"
options_file.write(line)
# Debian is currently not in freeze, change to "experimental" once that changes.
updateDebianChangelog(old_version, new_version, "unstable")
def main():
parser = OptionParser()
parser.add_option(
"--mode",
action="store",
dest="mode",
default=None,
help="""\
The mode of update, prerelease, hotfix, release, auto (default auto determines from branch).""",
)
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# Go its own directory, to have it easy with path knowledge.
goHome()
with open("nuitka/Version.py") as f:
option_lines = f.readlines()
version_line, = [line for line in option_lines if line.startswith("Nuitka V")]
old_version = version_line[8:].rstrip()
mode = options.mode
branch_name = getBranchName()
if mode is None:
if branch_name.startswith("hotfix/"):
mode = "hotfix"
elif branch_name == "master" or branch_name.startswith("release/"):
mode = "release"
elif branch_name == "develop":
mode = "prerelease"
else:
sys.exit("Error, cannot detect mode from branch name '%s'." % branch_name)
new_version = getBumpedVersion(mode, old_version)
print("Bumped", mode, old_version, "->", new_version)
with open("nuitka/Version.py", "w") as options_file:
for line in option_lines:
if line.startswith("Nuitka V"):
line = "Nuitka V" + new_version + "\n"
options_file.write(line)
updateDebianChangelog(old_version, new_version)
def main():
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option("--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""")
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be version in output. Default is %default.""")
parser.add_option("--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""")
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["-m", "nuitka"]
print("Working on:", positional_args)
blacklist = (
"oset.py",
"odict.py",
"SyntaxHighlighting.py",
)
filenames = scanTargets(positional_args, (".py", ), blacklist)
PyLint.executePyLint(filenames, options.todos, options.verbose,
options.one_by_one)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
goHome()
# So isort finds nuitka package.
addPYTHONPATH(os.getcwd())
parser = OptionParser()
parser.add_option(
"--verbose",
action = "store_true",
dest = "verbose",
default = False,
help = """\
Default is %default."""
)
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["-m", "nuitka", "tests/reflected/compile_itself.py"]
target_files = []
for filename in scanTargets(positional_args, (".py",)):
package_name = os.path.dirname(filename)
if package_name.startswith("nuitka" + os.path.sep):
package_name = package_name.replace(os.path.sep, '.')
source_code = open(filename).read()
updated_code = re.sub(r"from %s import" % package_name, "from . import", source_code)
updated_code = re.sub(r"from %s\." % package_name, "from .", source_code)
if source_code != updated_code:
with open(filename, 'w') as out_file:
out_file.write(updated_code)
target_files.append(filename)
target_files.append("nuitka/build/SingleExe.scons")
setupPATH()
subprocess.check_call(
[
"isort",
"-ot",
"-m3",
"-ns",
"__init__.py"
] + target_files
)
def main():
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
target_files = []
for filename in scanTargets(["nuitka", "bin"]):
target_files.append(filename)
target_files.append("nuitka/build/SingleExe.scons")
setupPATH()
subprocess.check_call(["isort", "-ot", "-m3", "-ns", "__init__.py"] +
target_files)
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Show what it is doing. Default is %default.""",
)
parser.add_option(
"--write",
action="store_true",
dest="write",
default=False,
help="""Write changes to the files. Default is %default.""",
)
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = [
"bin", "nuitka", "rpm", "misc", "tests/*/run_all.py"
]
my_print("Working on:", positional_args)
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
goHome()
filenames = list(
scanTargets(
positional_args,
(".py", ".scons", ".rst", ".txt", ".j2", ".md", ".c", ".h")))
if not filenames:
sys.exit("No files found.")
codespell(filenames=filenames,
verbose=options.verbose,
write=options.write)
def main():
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option("--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Default is %default.""")
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Default is %default.""")
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["bin", "nuitka"]
print("Working on:", positional_args)
blacklist = (
"oset.py",
"odict.py",
"SyntaxHighlighting.py",
)
filenames = list(scanTargets(positional_args, blacklist))
PyLint.executePyLint(filenames, options.todos, options.verbose)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
parser = OptionParser()
parser.add_option("--mode",
action="store",
dest="mode",
default="release",
help="""\
The mode of update, prerelease, hotfix, or final.""")
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
# Go its own directory, to have it easy with path knowledge.
goHome()
option_lines = [line for line in open("nuitka/Version.py")]
version_line, = [
line for line in open("nuitka/Version.py")
if line.startswith("Nuitka V")
]
old_version = version_line[8:].rstrip()
new_version = getBumpedVersion(options.mode, old_version)
print("Bumping", old_version, "->", new_version)
with open("nuitka/Version.py", 'w') as options_file:
for line in option_lines:
if line.startswith("Nuitka V"):
line = "Nuitka V" + new_version + '\n'
options_file.write(line)
updateDebianChangelog(old_version, new_version)
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""\
Default is %default.""",
)
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["bin", "nuitka"]
my_print("Working on:", positional_args)
positional_args = [
os.path.abspath(positional_arg) for positional_arg in positional_args
]
goHome()
found = False
for filename in scanTargets(positional_args, (".py", ".scons")):
autoformat(filename, abort=options.abort)
found = True
if not found:
sys.exit("No files found.")
def main():
goHome()
# So isort finds nuitka package.
addPYTHONPATH(os.getcwd())
parser = OptionParser()
parser.add_option(
"--verbose",
action = "store_true",
dest = "verbose",
default = False,
help = """\
Default is %default."""
)
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = ["bin", "nuitka", "tests/reflected/compile_itself.py"]
target_files = []
for filename in scanTargets(positional_args):
target_files.append(filename)
target_files.append("nuitka/build/SingleExe.scons")
setupPATH()
subprocess.check_call(
[
"isort",
"-ot",
"-m3",
"-ns",
"__init__.py"
] + target_files
)
def main():
goHome()
checkReleaseDocumentation()
import os
import pipes
import sys
import stat
# Unchanged, running from checkout, use the parent directory, the nuitka
# package ought be there.
sys.path.insert(
0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
from nuitka.tools.Basics import goHome # isort:skip
from nuitka.utils.Execution import getExecutablePath # isort:skip
from nuitka.utils.FileOperations import getFileContents # isort:skip
goHome()
if os.name == "nt":
git_path = getExecutablePath("git")
if git_path is None:
git_path = r"C:\Program Files\Git\bin\sh.exe"
if not os.path.exists(git_path):
git_path = None
if git_path is None:
sys.exit("""\
Error, cannot locate 'git.exe' which we need to install git hooks. Add it to
PATH while executing this will be sufficient.""")
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Default is %default.""",
)
parser.add_option(
"--from-commit",
action="store_true",
dest="from_commit",
default=False,
help="""From commit hook, do not descend into directories. Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""Stop if an error occurs, or continue. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.from_commit:
assert not positional_args
for desc in getStagedFileChangeDesc():
autoformat(desc["src_path"], git_stage=desc, abort=options.abort)
else:
if not positional_args:
positional_args = [
"bin",
"nuitka",
# "tests/*/run_all.py"
]
my_print("Working on:", positional_args)
positional_args = sum(
(
resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args
),
[],
)
goHome()
filenames = list(
scanTargets(positional_args, (".py", ".scons", ".rst", ".txt"))
)
if not filenames:
sys.exit("No files found.")
for filename in filenames:
autoformat(filename, git_stage=False, abort=options.abort)
def main():
goHome()
checkReleaseDocumentation()
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to http://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit(
"Error, need to install Doxygen and add it to PATH for this to work."
)
try:
import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh",
delete=False) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" %
sys.executable)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" %
sys.executable)
doxy_batch_filename = doxy_batch_file.name
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
subprocess.check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
) == 0)
print("Finished.")
def main():
goHome()
if os.name == "nt":
git_path = getExecutablePath("git")
if git_path is None:
git_path = r"C:\Program Files\Git\bin\sh.exe"
if not os.path.exists(git_path):
git_path = None
if git_path is None:
sys.exit(
"""\
Error, cannot locate 'git.exe' which we need to install git hooks. Add it to
PATH while executing this will be sufficient."""
)
sh_path = os.path.join(os.path.dirname(git_path), "sh.exe")
if not os.path.exists(sh_path):
sh_path = os.path.join(
os.path.dirname(git_path), "..", "..", "bin", "sh.exe"
)
sh_path = os.path.normpath(sh_path)
if not os.path.exists(sh_path):
sys.exit(
"""\
Error, cannot locate 'sh.exe' near 'git.exe' which we need to install git hooks,
please improve this script."""
)
# For MinGW and #! we will need a path without spaces, so use this
# code to find the short name, that won't have it.
sh_path = getWindowsShortPathName(sh_path)
for hook in os.listdir(".githooks"):
full_path = os.path.join(".githooks", hook)
hook_contents = getFileContents(full_path)
if hook_contents.startswith("#!/bin/sh"):
if os.name == "nt":
# Correct shebang for Windows git to work.
hook_contents = "#!%s\n%s" % (
sh_path.replace("\\", "/").replace(" ", r"\ "),
hook_contents[10:],
)
# Also use sys.executable to make sure we find autoformat.
hook_contents = hook_contents.replace(
"./bin/autoformat-nuitka-source",
"'%s' ./bin/autoformat-nuitka-source" % sys.executable,
)
else:
sys.exit("Error, unknown hook contents.")
hook_target = os.path.join(".git/hooks/", hook)
with open(hook_target, "wb") as out_file:
out_file.write(hook_contents.encode("utf8"))
st = os.stat(hook_target)
os.chmod(hook_target, st.st_mode | stat.S_IEXEC)
def main():
goHome()
# So isort finds nuitka package.
addPYTHONPATH(os.getcwd())
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Default is %default.""",
)
_options, positional_args = parser.parse_args()
if not positional_args:
positional_args = [
"bin", "nuitka", "tests/reflected/compile_itself.py"
]
target_files = []
for filename in scanTargets(positional_args, (".py", ".scons")):
# This breaks the PyLint annotations currently.
if os.path.basename(filename) == "Autoformat.py":
continue
package_name = os.path.dirname(filename)
# Make imports local if possible.
if package_name.startswith("nuitka" + os.path.sep):
package_name = package_name.replace(os.path.sep, ".")
source_code = open(filename).read()
updated_code = re.sub(r"from %s import" % package_name,
"from . import", source_code)
updated_code = re.sub(r"from %s\." % package_name, "from .",
source_code)
if source_code != updated_code:
with open(filename, "w") as out_file:
out_file.write(updated_code)
target_files.append(filename)
setupPATH()
subprocess.check_call([
"isort",
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-up", # Prefer braces () over \ for line continuation.
"-ns", # Do not ignore those:
"__init__.py",
] + target_files)
# For Windows, work around that isort changes encoding.
if os.name == "nt":
for filename in target_files:
cleanupWindowsNewlines(filename)
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to http://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit("Error, need to install Doxygen and add it to PATH for this to work.")
try:
import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh", delete=False
) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write(
"%s -m doxypypy.doxypypy -a -c %%1" % sys.executable
)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1"
% sys.executable
)
doxy_batch_filename = doxy_batch_file.name
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
subprocess.check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (
os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
)
== 0
)
print("Finished.")
def main():
parser = OptionParser()
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""Default is %default.""",
)
parser.add_option(
"--from-commit",
action="store_true",
dest="from_commit",
default=False,
help=
"""From commit hook, do not descend into directories. Default is %default.""",
)
parser.add_option(
"--abort-on-parsing-error",
action="store_true",
dest="abort",
default=False,
help="""Stop if an error occurs, or continue. Default is %default.""",
)
options, positional_args = parser.parse_args()
if not positional_args:
positional_args = [
"bin",
"nuitka",
# "tests/*/run_all.py"
]
if options.from_commit:
positional_args = [
positional_arg for positional_arg in positional_args
if not os.path.isdir(positional_arg)
]
my_print("Working on:", positional_args)
if not options.from_commit:
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
positional_args = [
os.path.abspath(positional_arg) for positional_arg in positional_args
]
goHome()
filenames = list(
scanTargets(positional_args, (".py", ".scons", ".rst", ".txt")))
if not filenames:
sys.exit("No files found.")
for filename in filenames:
autoformat(filename, abort=options.abort)
def main():
setup(go_main=False)
# So PyLint finds nuitka package.
addPYTHONPATH(getHomePath())
setupPATH()
parser = OptionParser()
parser.add_option(
"--diff",
action="store_true",
dest="diff",
default=False,
help="""\
Analyse the changed files in git. Default is %default.""",
)
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be verbose in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if positional_args:
if options.diff:
sys.exit("Error, no filenames argument allowed in git diff mode.")
else:
goHome()
if options.diff:
positional_args = [
filename for filename in getModifiedPaths()
if isPythonFile(filename)
]
else:
positional_args = [
"bin", "nuitka", "setup.py", "tests/*/run_all.py"
]
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
if not positional_args:
sys.exit("No files found.")
print("Working on:", positional_args)
ignore_list = []
# Avoid checking the Python2 runner along with the one for Python3, it has name collisions.
if python_version >= 0x300:
ignore_list.append("nuitka")
filenames = list(
scanTargets(positional_args,
suffixes=(".py", ".scons"),
ignore_list=ignore_list))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
goHome()
if os.name == "nt":
git_path = getExecutablePath("git")
if git_path is None:
git_path = r"C:\Program Files\Git\bin\sh.exe"
if not os.path.exists(git_path):
git_path = None
if git_path is None:
sys.exit("""\
Error, cannot locate 'git.exe' which we need to install git hooks. Add it to
PATH while executing this will be sufficient.""")
sh_path = os.path.join(os.path.dirname(git_path), "sh.exe")
if not os.path.exists(sh_path):
sh_path = os.path.join(os.path.dirname(git_path), "..", "..",
"bin", "sh.exe")
sh_path = os.path.normpath(sh_path)
if not os.path.exists(sh_path):
sys.exit("""\
Error, cannot locate 'sh.exe' near 'git.exe' which we need to install git hooks,
please improve this script.""")
# For MinGW and #! we will need a path without spaces, so use this
# code to find the short name, that won't have it.
sh_path = getWindowsShortPathName(sh_path)
for hook in os.listdir(".githooks"):
full_path = os.path.join(".githooks", hook)
hook_contents = getFileContents(full_path)
if hook_contents.startswith("#!/bin/sh"):
if os.name == "nt":
# Correct shebang for Windows git to work.
hook_contents = "#!%s\n%s" % (
sh_path.replace("\\", "/").replace(" ", r"\ "),
hook_contents[10:],
)
# Also use sys.executable to make sure we find autoformat.
hook_contents = hook_contents.replace(
"./bin/autoformat-nuitka-source",
"'%s' ./bin/autoformat-nuitka-source" % sys.executable,
)
else:
sys.exit("Error, unknown hook contents.")
hook_target = os.path.join(".git/hooks/", hook)
with open(hook_target, "wb") as out_file:
out_file.write(hook_contents.encode("utf8"))
st = os.stat(hook_target)
os.chmod(hook_target, st.st_mode | stat.S_IEXEC)
def main():
setup()
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be version in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if not positional_args:
positional_args = ["bin", "nuitka"]
print("Working on:", positional_args)
blacklist = ["oset.py", "odict.py", "SyntaxHighlighting.py"]
# Avoid checking the Python2 runner with Python3, it has name collisions.
if python_version >= 300:
blacklist.append("nuitka")
filenames = list(scanTargets(positional_args, (".py",), blacklist))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
setup()
goHome()
# So PyLint finds nuitka package.
addPYTHONPATH(os.getcwd())
setupPATH()
parser = OptionParser()
parser.add_option(
"--show-todos",
"--todos",
action="store_true",
dest="todos",
default=False,
help="""\
Show TODO items. Default is %default.""",
)
parser.add_option(
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="""\
Be verbose in output. Default is %default.""",
)
parser.add_option(
"--one-by-one",
action="store_true",
dest="one_by_one",
default=False,
help="""\
Check files one by one. Default is %default.""",
)
parser.add_option(
"--not-installed-is-no-error",
action="store_true",
dest="not_installed_is_no_error",
default=False,
help="""\
Insist on PyLint to be installed. Default is %default.""",
)
options, positional_args = parser.parse_args()
if options.not_installed_is_no_error and not hasModule("pylint"):
print("PyLint is not installed for this interpreter version: SKIPPED")
sys.exit(0)
if not positional_args:
positional_args = ["bin", "nuitka", "tests/*/run_all.py"]
positional_args = sum(
(resolveShellPatternToFilenames(positional_arg)
for positional_arg in positional_args),
[],
)
print("Working on:", positional_args)
blacklist = ["oset.py", "odict.py"]
# Avoid checking the Python2 runner with Python3, it has name collisions.
if python_version >= 300:
blacklist.append("nuitka")
filenames = list(scanTargets(positional_args, (".py", ), blacklist))
PyLint.executePyLint(
filenames=filenames,
show_todos=options.todos,
verbose=options.verbose,
one_by_one=options.one_by_one,
)
if not filenames:
sys.exit("No files found.")
sys.exit(PyLint.our_exit_code)
def main():
# There are many cases to deal with,
# pylint: disable=too-many-branches,too-many-statements
# Lets honor this Debian option here.
if "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split():
print("Skipped all tests as per DEB_BUILD_OPTIONS environment.")
sys.exit(0)
# Make sure our resolving of "python2" to "python" doesn't get in the way.
os.environ["PYTHON_DISALLOW_AMBIGUOUS_VERSION"] = "0"
goHome()
options = parseOptions()
# Add the local bin directory to search path start.
os.environ["PATH"] = (
os.path.join(os.getcwd(), "bin") + os.pathsep + os.environ["PATH"]
)
def checkExecutableCommand(command):
""" Check if a command is executable. """
# Many cases, pylint: disable=too-many-branches,too-many-return-statements
# Do respect given options to disable specific Python versions
if command == "python2.6" and options.no26:
return False
if command == "python2.7" and options.no27:
return False
if command == "python3.3" and options.no33:
return False
if command == "python3.4" and options.no34:
return False
if command == "python3.5" and options.no35:
return False
if command == "python3.6" and options.no36:
return False
if command == "python3.7" and options.no37:
return False
# Shortcuts for python versions, also needed for Windows as it won't have
# the version number in the Python binaries at all.
if command == "python2.6" and sys.version_info[0:2] == (2, 6):
return True
if command == "python2.7" and sys.version_info[0:2] == (2, 7):
return True
if command == "python3.3" and sys.version_info[0:2] == (3, 3):
return True
if command == "python3.4" and sys.version_info[0:2] == (3, 4):
return True
if command == "python3.5" and sys.version_info[0:2] == (3, 5):
return True
if command == "python3.6" and sys.version_info[0:2] == (3, 6):
return True
if command == "python3.7" and sys.version_info[0:2] == (3, 7):
return True
path = os.environ["PATH"]
suffixes = (".exe",) if os.name == "nt" else ("",)
for part in path.split(os.pathsep):
if not part:
continue
for suffix in suffixes:
if os.path.exists(os.path.join(part, command + suffix)):
return True
if os.name == "nt":
if command.startswith("python"):
remainder = command[6:]
if len(remainder) == 3 and remainder[1] == ".":
command = getPythonExePathWindows(search=remainder, arch=None)
return True
return False
def setExtraFlags(where, name, flags):
if where is not None:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
where = os.path.join(tmp_dir, name, where)
if not os.path.exists(where):
os.makedirs(where)
os.environ["NUITKA_EXTRA_OPTIONS"] = flags + " --output-dir=" + where
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = flags
def executeSubTest(command, hide_output=False):
if options.coverage and "search" in command:
command = command.replace("search", "coverage")
parts = command.split()
parts[0] = parts[0].replace("/", os.path.sep)
# The running Python will be good enough, on some platforms there is no
# "python", and we need to pass this alone then.
parts.insert(0, sys.executable)
print("Run '%s' in '%s'." % (" ".join(parts), os.getcwd()))
sys.stdout.flush()
if hide_output:
with open(os.devnull, "w") as devnull:
result = subprocess.call(parts, stdout=devnull)
else:
result = subprocess.call(parts)
if result != 0:
sys.exit(result)
def execute_tests(where, use_python, flags):
# Many cases, pylint: disable=too-many-branches,too-many-statements
print(
"Executing test case called %s with CPython %s and extra flags '%s'."
% (where, use_python, flags)
)
intended_version = use_python[6:]
if sys.version.startswith(intended_version):
os.environ["PYTHON"] = sys.executable
else:
if os.name == "nt":
os.environ["PYTHON"] = getPythonExePathWindows(
search=intended_version, arch=None
)
else:
os.environ["PYTHON"] = getExecutablePath(use_python)
if options.basic_tests:
print(
"Running the basic tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "basics", flags)
executeSubTest("./tests/basics/run_all.py search")
if options.syntax_tests:
print(
"Running the syntax tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "syntax", flags)
executeSubTest("./tests/syntax/run_all.py search")
if options.program_tests:
print(
"Running the program tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "programs", flags)
executeSubTest("./tests/programs/run_all.py search")
if options.package_tests:
print(
"Running the package tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "packages", flags)
executeSubTest("./tests/packages/run_all.py search")
# At least one Debian Jessie, these versions won't have lxml installed, so
# don't run them there. Also these won't be very version dependent in their
# results.
if use_python != "python2.6":
if options.optimization_tests:
print(
"Running the optimizations tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "optimizations", flags)
executeSubTest("./tests/optimizations/run_all.py search")
if options.standalone_tests and not options.coverage:
print(
"Running the standalone tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(None, "standalone", flags)
executeSubTest("./tests/standalone/run_all.py search")
if options.reflection_test and not options.coverage:
print(
"Running the reflection test with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(None, "reflected", flags)
executeSubTest("./tests/reflected/compile_itself.py search")
if not use_python.startswith("python3"):
if os.path.exists("./tests/CPython26/run_all.py"):
if options.cpython26:
print(
"Running the CPython 2.6 tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "26tests", flags)
executeSubTest("./tests/CPython26/run_all.py search")
else:
print("The CPython2.6 tests are not present, not run.")
# Running the Python 2.7 test suite with CPython 2.6 gives little
# insight, because "importlib" will not be there and that's it.
if use_python != "python2.6":
if os.path.exists("./tests/CPython27/run_all.py"):
if options.cpython27:
print(
"Running the CPython 2.7 tests with options '%s' with %s:"
% (flags, use_python)
)
setExtraFlags(where, "27tests", flags)
executeSubTest("./tests/CPython27/run_all.py search")
else:
print("The CPython2.7 tests are not present, not run.")
if "--debug" not in flags:
# Not running the Python 3.2 test suite with CPython2.6, as that's about
# the same as CPython2.7 and won't have any new insights.
if use_python not in ("python2.6", "python2.7") or not options.coverage:
if os.path.exists("./tests/CPython32/run_all.py"):
if options.cpython32:
setExtraFlags(where, "32tests", flags)
executeSubTest("./tests/CPython32/run_all.py search")
else:
print("The CPython3.2 tests are not present, not run.")
# Running the Python 3.3 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython33/run_all.py"):
if options.cpython33:
setExtraFlags(where, "33tests", flags)
executeSubTest("./tests/CPython33/run_all.py search")
else:
print("The CPython3.3 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython34/run_all.py"):
if options.cpython34:
setExtraFlags(where, "34tests", flags)
executeSubTest("./tests/CPython34/run_all.py search")
else:
print("The CPython3.4 tests are not present, not run.")
# Running the Python 3.5 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython35/run_all.py"):
if options.cpython35:
setExtraFlags(where, "35tests", flags)
executeSubTest("./tests/CPython35/run_all.py search")
else:
print("The CPython3.5 tests are not present, not run.")
# Running the Python 3.6 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "36tests", flags)
executeSubTest("./tests/CPython36/run_all.py search")
else:
print("The CPython3.6 tests are not present, not run.")
# Running the Python 3.7 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "37tests", flags)
executeSubTest("./tests/CPython37/run_all.py search")
else:
print("The CPython3.7 tests are not present, not run.")
if "NUITKA_EXTRA_OPTIONS" in os.environ:
del os.environ["NUITKA_EXTRA_OPTIONS"]
assert (
checkExecutableCommand("python2.6")
or checkExecutableCommand("python2.7")
or checkExecutableCommand("python3.3")
or checkExecutableCommand("python3.4")
or checkExecutableCommand("python3.5")
or checkExecutableCommand("python3.6")
or checkExecutableCommand("python3.7")
)
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-debug", "python2.6", "--debug")
else:
print("Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-debug", "python2.7", "--debug")
else:
print("Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-nodebug", "python2.6", "")
else:
print("Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-nodebug", "python2.7", "")
else:
print("Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.3"):
execute_tests("python3.3-nodebug", "python3.3", "")
else:
print("Cannot execute tests with Python 3.3, disabled or not installed.")
if checkExecutableCommand("python3.4"):
execute_tests("python3.4-nodebug", "python3.4", "")
else:
print("Cannot execute tests with Python 3.4, disabled or not installed.")
if checkExecutableCommand("python3.5"):
execute_tests("python3.5-nodebug", "python3.5", "")
else:
print("Cannot execute tests with Python 3.5, disabled or not installed.")
if checkExecutableCommand("python3.6"):
execute_tests("python3.6-nodebug", "python3.6", "")
else:
print("Cannot execute tests with Python 3.6, disabled or not installed.")
if checkExecutableCommand("python3.7"):
execute_tests("python3.7-nodebug", "python3.7", "")
else:
print("Cannot execute tests with Python 3.7, disabled or not installed.")
if options.coverage:
publishCoverageData()
print("OK.")
def main():
# There are many cases to deal with,
# pylint: disable=too-many-branches,too-many-statements
# Lets honor this Debian option here.
if "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split():
my_print("Skipped all tests as per DEB_BUILD_OPTIONS environment.")
sys.exit(0)
# Make sure our resolving of "python2" to "python" doesn't get in the way.
os.environ["PYTHON_DISALLOW_AMBIGUOUS_VERSION"] = "0"
goHome()
options = parseOptions()
# Add the local bin directory to search path start.
os.environ["PATH"] = (os.path.join(os.getcwd(), "bin") + os.pathsep +
os.environ["PATH"])
def checkExecutableCommand(command):
""" Check if a command is executable. """
# Many cases, pylint: disable=too-many-branches,too-many-return-statements
# Do respect given options to disable specific Python versions
if command == "python2.6" and options.no26:
return False
if command == "python2.7" and options.no27:
return False
if command == "python3.3" and options.no33:
return False
if command == "python3.4" and options.no34:
return False
if command == "python3.5" and options.no35:
return False
if command == "python3.6" and options.no36:
return False
if command == "python3.7" and options.no37:
return False
if command == "python3.8" and options.no38:
return False
# Shortcuts for python versions, also needed for Windows as it won't have
# the version number in the Python binaries at all.
if command == "python2.6" and sys.version_info[0:2] == (2, 6):
return True
if command == "python2.7" and sys.version_info[0:2] == (2, 7):
return True
if command == "python3.3" and sys.version_info[0:2] == (3, 3):
return True
if command == "python3.4" and sys.version_info[0:2] == (3, 4):
return True
if command == "python3.5" and sys.version_info[0:2] == (3, 5):
return True
if command == "python3.6" and sys.version_info[0:2] == (3, 6):
return True
if command == "python3.7" and sys.version_info[0:2] == (3, 7):
return True
if command == "python3.8" and sys.version_info[0:2] == (3, 8):
return True
path = os.environ["PATH"]
suffixes = (".exe", ) if os.name == "nt" else ("", )
for part in path.split(os.pathsep):
if not part:
continue
for suffix in suffixes:
if os.path.exists(os.path.join(part, command + suffix)):
return True
if os.name == "nt":
if command.startswith("python"):
remainder = command[6:]
if len(remainder) == 3 and remainder[1] == ".":
command = getPythonExePathWindows(search=remainder,
arch=None)
return True
return False
def setExtraFlags(where, name, flags):
if (os.name == "nt" and name == "standalone"
and options.assume_yes_for_downloads):
if flags:
flags += " "
flags += "--assume-yes-for-downloads"
if where is not None:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
where = os.path.join(tmp_dir, name, where)
if not os.path.exists(where):
os.makedirs(where)
if flags:
flags += " "
flags += "--output-dir=" + where
os.environ["NUITKA_EXTRA_OPTIONS"] = flags
def executeSubTest(command, hide_output=False):
with TimerReport(message="Overall execution of %r took %%.2f seconds" %
command):
_executeSubTest(command, hide_output)
def _executeSubTest(command, hide_output):
if options.coverage and "search" in command:
command = command.replace("search", "coverage")
parts = command.split()
parts[0] = parts[0].replace("/", os.path.sep)
# The running Python will be good enough, on some platforms there is no
# "python", and we need to pass this alone then.
parts.insert(0, sys.executable)
my_print("Run '%s' in '%s'." % (" ".join(parts), os.getcwd()))
if hide_output:
with open(os.devnull, "w") as devnull:
result = subprocess.call(parts, stdout=devnull)
else:
result = subprocess.call(parts)
if result != 0:
sys.exit(result)
def execute_tests(where, use_python, flags):
# Many cases, pylint: disable=too-many-branches,too-many-statements
my_print(
"Executing test case called '%s' with CPython '%s' and extra flags '%s'."
% (where, use_python, flags))
intended_version = use_python[6:]
if sys.version.startswith(intended_version):
os.environ["PYTHON"] = sys.executable
else:
if os.name == "nt":
os.environ["PYTHON"] = getPythonExePathWindows(
search=intended_version, arch=None)
else:
os.environ["PYTHON"] = getExecutablePath(use_python)
if options.basic_tests:
my_print("Running the basic tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "basics", flags)
executeSubTest("./tests/basics/run_all.py search")
if options.syntax_tests:
my_print("Running the syntax tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "syntax", flags)
executeSubTest("./tests/syntax/run_all.py search")
if options.program_tests:
my_print("Running the program tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "programs", flags)
executeSubTest("./tests/programs/run_all.py search")
if options.package_tests:
my_print("Running the package tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "packages", flags)
executeSubTest("./tests/packages/run_all.py search")
if options.plugin_tests:
my_print("Running the plugin tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(where, "plugins", flags)
executeSubTest("./tests/plugins/run_all.py search")
# At least one Debian Jessie, these versions won't have lxml installed, so
# don't run them there. Also these won't be very version dependent in their
# results.
if use_python != "python2.6":
if options.optimization_tests:
my_print(
"Running the optimizations tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "optimizations", flags)
executeSubTest("./tests/optimizations/run_all.py search")
if options.standalone_tests and not options.coverage:
my_print(
"Running the standalone tests with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(None, "standalone", flags)
executeSubTest("./tests/standalone/run_all.py search")
if options.reflection_test and not options.coverage:
my_print(
"Running the reflection test with options '%s' with '%s':" %
(flags, use_python))
setExtraFlags(None, "reflected", flags)
executeSubTest("./tests/reflected/compile_itself.py search")
if not use_python.startswith("python3"):
if os.path.exists("./tests/CPython26/run_all.py"):
if options.cpython26:
my_print(
"Running the CPython 2.6 tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "26tests", flags)
executeSubTest("./tests/CPython26/run_all.py search")
else:
my_print("The CPython2.6 tests are not present, not run.")
# Running the Python 2.7 test suite with CPython 2.6 gives little
# insight, because "importlib" will not be there and that's it.
if use_python != "python2.6":
if os.path.exists("./tests/CPython27/run_all.py"):
if options.cpython27:
my_print(
"Running the CPython 2.7 tests with options '%s' with '%s':"
% (flags, use_python))
setExtraFlags(where, "27tests", flags)
executeSubTest("./tests/CPython27/run_all.py search")
else:
my_print("The CPython2.7 tests are not present, not run.")
if "--debug" not in flags:
# Not running the Python 3.2 test suite with CPython2.6, as that's about
# the same as CPython2.7 and won't have any new insights.
if use_python not in ("python2.6",
"python2.7") or not options.coverage:
if os.path.exists("./tests/CPython32/run_all.py"):
if options.cpython32:
setExtraFlags(where, "32tests", flags)
executeSubTest("./tests/CPython32/run_all.py search")
else:
my_print("The CPython3.2 tests are not present, not run.")
# Running the Python 3.3 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython33/run_all.py"):
if options.cpython33:
setExtraFlags(where, "33tests", flags)
executeSubTest("./tests/CPython33/run_all.py search")
else:
my_print("The CPython3.3 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython34/run_all.py"):
if options.cpython34:
setExtraFlags(where, "34tests", flags)
executeSubTest("./tests/CPython34/run_all.py search")
else:
my_print("The CPython3.4 tests are not present, not run.")
# Running the Python 3.5 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython35/run_all.py"):
if options.cpython35:
setExtraFlags(where, "35tests", flags)
executeSubTest("./tests/CPython35/run_all.py search")
else:
my_print("The CPython3.5 tests are not present, not run.")
# Running the Python 3.6 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "36tests", flags)
executeSubTest("./tests/CPython36/run_all.py search")
else:
my_print("The CPython3.6 tests are not present, not run.")
# Running the Python 3.7 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython37/run_all.py"):
if options.cpython36:
setExtraFlags(where, "37tests", flags)
executeSubTest("./tests/CPython37/run_all.py search")
else:
my_print("The CPython3.7 tests are not present, not run.")
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython38/run_all.py"):
if options.cpython36:
setExtraFlags(where, "38tests", flags)
executeSubTest("./tests/CPython38/run_all.py search")
else:
my_print("The CPython3.8 tests are not present, not run.")
if "NUITKA_EXTRA_OPTIONS" in os.environ:
del os.environ["NUITKA_EXTRA_OPTIONS"]
assert (checkExecutableCommand("python2.6")
or checkExecutableCommand("python2.7")
or checkExecutableCommand("python3.3")
or checkExecutableCommand("python3.4")
or checkExecutableCommand("python3.5")
or checkExecutableCommand("python3.6")
or checkExecutableCommand("python3.7")
or checkExecutableCommand("python3.8"))
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-debug", "python2.6", "--debug")
else:
my_print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-debug", "python2.7", "--debug")
else:
my_print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-nodebug", "python2.6", "")
else:
my_print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-nodebug", "python2.7", "")
else:
my_print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.3"):
execute_tests("python3.3-nodebug", "python3.3", "")
else:
my_print(
"Cannot execute tests with Python 3.3, disabled or not installed.")
if checkExecutableCommand("python3.4"):
execute_tests("python3.4-nodebug", "python3.4", "")
else:
my_print(
"Cannot execute tests with Python 3.4, disabled or not installed.")
if checkExecutableCommand("python3.5"):
execute_tests("python3.5-nodebug", "python3.5", "")
else:
my_print(
"Cannot execute tests with Python 3.5, disabled or not installed.")
if checkExecutableCommand("python3.6"):
execute_tests("python3.6-nodebug", "python3.6", "")
else:
my_print(
"Cannot execute tests with Python 3.6, disabled or not installed.")
if checkExecutableCommand("python3.7"):
execute_tests("python3.7-nodebug", "python3.7", "")
else:
my_print(
"Cannot execute tests with Python 3.7, disabled or not installed.")
if checkExecutableCommand("python3.8"):
execute_tests("python3.8-nodebug", "python3.8", "")
else:
my_print(
"Cannot execute tests with Python 3.8, disabled or not installed.")
if options.coverage:
publishCoverageData()
my_print("OK.")
def main():
# There are freaking many options to honor, pylint: disable=too-many-branches
# Lets honor this Debian option here.
if "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split():
print("Skipped all tests as per DEB_BUILD_OPTIONS environment.")
sys.exit(0)
goHome()
parser = OptionParser()
parser.add_option("--skip-basic-tests",
action="store_false",
dest="basic_tests",
default=True,
help="""\
The basic tests, execute these to check if Nuitka is healthy.
Default is %default.""")
parser.add_option("--skip-syntax-tests",
action="store_false",
dest="syntax_tests",
default=True,
help="""\
The syntax tests, execute these to check if Nuitka handles Syntax errors fine.
Default is %default.""")
parser.add_option("--skip-program-tests",
action="store_false",
dest="program_tests",
default=True,
help="""\
The programs tests, execute these to check if Nuitka handles programs, e.g.
import recursions, etc. fine. Default is %default.""")
parser.add_option("--skip-package-tests",
action="store_false",
dest="package_tests",
default=True,
help="""\
The packages tests, execute these to check if Nuitka handles packages, e.g.
import recursions, etc. fine. Default is %default.""")
parser.add_option("--skip-optimizations-tests",
action="store_false",
dest="optimization_tests",
default=True,
help="""\
The optimization tests, execute these to check if Nuitka does optimize certain
constructs fully away. Default is %default.""")
parser.add_option(
"--skip-standalone-tests",
action="store_false",
dest="standalone_tests",
default=os.name != "posix"
or os.uname()[0] != "NetBSD", # @UndefinedVariable
help="""\
The standalone tests, execute these to check if Nuitka standalone mode, e.g.
not referring to outside, important 3rd library packages like PyQt fine.
Default is %default.""")
parser.add_option("--skip-reflection-test",
action="store_false",
dest="reflection_test",
default=True,
help="""\
The reflection test compiles Nuitka with Nuitka, and then Nuitka with the
compile Nuitka and compares the outputs. Default is %default.""")
parser.add_option("--skip-cpython26-tests",
action="store_false",
dest="cpython26",
default=True,
help="""\
The standard CPython2.6 test suite. Execute this for all corner cases to be
covered. With Python 2.7 this covers exception behavior quite well. Default
is %default.""")
parser.add_option("--skip-cpython27-tests",
action="store_false",
dest="cpython27",
default=True,
help="""\
The standard CPython2.7 test suite. Execute this for all corner cases to be
covered. With Python 2.6 these are not run. Default is %default.""")
parser.add_option("--skip-cpython32-tests",
action="store_false",
dest="cpython32",
default=True,
help="""\
The standard CPython3.2 test suite. Execute this for all corner cases to be
covered. With Python 2.6 these are not run. Default is %default.""")
parser.add_option("--skip-cpython33-tests",
action="store_false",
dest="cpython33",
default=True,
help="""\
The standard CPython3.3 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython34-tests",
action="store_false",
dest="cpython34",
default=True,
help="""\
The standard CPython3.4 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython35-tests",
action="store_false",
dest="cpython35",
default=True,
help="""\
The standard CPython3.5 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-cpython36-tests",
action="store_false",
dest="cpython36",
default=True,
help="""\
The standard CPython3.6 test suite. Execute this for all corner cases to be
covered. With Python 2.x these are not run. Default is %default.""")
parser.add_option("--skip-other-cpython-tests",
action="store_true",
dest="cpython_no_other",
default=False,
help="""\
Do not execute any CPython test suite other than the one matching the running
Python. Default is %default.""")
parser.add_option("--skip-all-cpython-tests",
action="store_true",
dest="cpython_none",
default=False,
help="""\
Do not execute any CPython test suite other than the one matching the running
Python. Default is %default.""")
parser.add_option("--no-other-python",
action="store_true",
dest="no_other",
default=False,
help="""\
Do not use any other Python than the one running, even if available on
the system. Default is %default.""")
parser.add_option("--no-python2.6",
action="store_true",
dest="no26",
default=False,
help="""\
Do not use Python2.6 even if available on the system. Default is %default.""")
parser.add_option("--no-python2.7",
action="store_true",
dest="no27",
default=False,
help="""\
Do not use Python2.7 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.2",
action="store_true",
dest="no32",
default=False,
help="""\
Do not use Python3.2 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.3",
action="store_true",
dest="no33",
default=False,
help="""\
Do not use Python3.3 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.4",
action="store_true",
dest="no34",
default=False,
help="""\
Do not use Python3.4 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.5",
action="store_true",
dest="no35",
default=False,
help="""\
Do not use Python3.5 even if available on the system. Default is %default.""")
parser.add_option("--no-python3.6",
action="store_true",
dest="no36",
default=False,
help="""\
Do not use Python3.6 even if available on the system. Default is %default.""")
parser.add_option("--coverage",
action="store_true",
dest="coverage",
default=False,
help="""\
Make a coverage analysis, that does not really check. Default is %default.""")
options, positional_args = parser.parse_args()
if positional_args:
parser.print_help()
sys.exit("\nError, no positional argument allowed.")
if options.no_other:
if sys.version_info[0:2] != (2, 6):
options.no26 = True
if sys.version_info[0:2] != (2, 7):
options.no27 = True
if sys.version_info[0:2] != (3, 2):
options.no32 = True
if sys.version_info[0:2] != (3, 3):
options.no33 = True
if sys.version_info[0:2] != (3, 4):
options.no34 = True
if sys.version_info[0:2] != (3, 5):
options.no35 = True
if sys.version_info[0:2] != (3, 6):
options.no36 = True
if options.cpython_no_other:
if sys.version_info[0:2] != (2, 6):
options.cpython26 = False
if sys.version_info[0:2] != (2, 7):
options.cpython27 = False
if sys.version_info[0:2] != (3, 2):
options.cpython32 = False
if sys.version_info[0:2] != (3, 3):
options.cpython33 = False
if sys.version_info[0:2] != (3, 4):
options.cpython34 = False
if sys.version_info[0:2] != (3, 5):
options.cpython35 = False
if sys.version_info[0:2] != (3, 6):
options.cpython36 = False
if options.cpython_none:
options.cpython26 = False
options.cpython27 = False
options.cpython32 = False
options.cpython33 = False
options.cpython34 = False
options.cpython35 = False
options.cpython36 = False
if options.coverage and os.path.exists(".coverage"):
os.unlink(".coverage")
# Add the local bin directory to search path start.
os.environ["PATH"] = \
os.path.join(
os.getcwd(),
"bin" ) + \
os.pathsep + \
os.environ["PATH"]
def checkExecutableCommand(command):
""" Check if a command is executable. """
# Many cases, pylint: disable=too-many-branches,too-many-return-statements
# Do respect given options to disable specific Python versions
if command == "python2.6" and options.no26:
return False
if command == "python2.7" and options.no27:
return False
if command == "python3.2" and options.no32:
return False
if command == "python3.3" and options.no33:
return False
if command == "python3.4" and options.no34:
return False
if command == "python3.5" and options.no35:
return False
if command == "python3.6" and options.no36:
return False
# Shortcuts for python versions, also needed for Windows as it won't have
# the version number in the Python binaries at all.
if command == "python2.6" and sys.version_info[0:2] == (2, 6):
return True
if command == "python2.7" and sys.version_info[0:2] == (2, 7):
return True
if command == "python3.2" and sys.version_info[0:2] == (3, 2):
return True
if command == "python3.3" and sys.version_info[0:2] == (3, 3):
return True
if command == "python3.4" and sys.version_info[0:2] == (3, 4):
return True
if command == "python3.5" and sys.version_info[0:2] == (3, 5):
return True
if command == "python3.6" and sys.version_info[0:2] == (3, 6):
return True
path = os.environ["PATH"]
suffixes = (".exe", ) if os.name == "nt" else ("", )
for part in path.split(os.pathsep):
if not part:
continue
for suffix in suffixes:
if os.path.exists(os.path.join(part, command + suffix)):
return True
if os.name == "nt":
if command.startswith("python"):
remainder = command[6:]
if len(remainder) == 3 and remainder[1] == '.':
command = getPythonExePathWindows(search=remainder,
arch=None)
return True
return False
def setExtraFlags(where, name, flags):
if where is not None:
tmp_dir = tempfile.gettempdir()
# Try to avoid RAM disk /tmp and use the disk one instead.
if tmp_dir == "/tmp" and os.path.exists("/var/tmp"):
tmp_dir = "/var/tmp"
where = os.path.join(tmp_dir, name, where)
if not os.path.exists(where):
os.makedirs(where)
os.environ[
"NUITKA_EXTRA_OPTIONS"] = flags + " --output-dir=" + where
else:
os.environ["NUITKA_EXTRA_OPTIONS"] = flags
def executeSubTest(command, hide_output=False):
if options.coverage and "search" in command:
command = command.replace("search", "coverage")
parts = command.split()
parts[0] = parts[0].replace('/', os.path.sep)
# The running Python will be good enough, on some platforms there is no
# "python", and we need to pass this alone then.
parts.insert(0, sys.executable)
print("Run '%s' in '%s'." % (' '.join(parts), os.getcwd()))
sys.stdout.flush()
if hide_output:
result = subprocess.call(parts, stdout=open(os.devnull, 'w'))
else:
result = subprocess.call(parts)
if result != 0:
sys.exit(result)
def execute_tests(where, use_python, flags):
# Many cases, pylint: disable=too-many-branches,too-many-statements
print(
"Executing test case called %s with CPython %s and extra flags '%s'."
% (where, use_python, flags))
intended_version = use_python[6:]
if sys.version.startswith(intended_version):
os.environ["PYTHON"] = sys.executable
else:
if os.name == "nt":
os.environ["PYTHON"] = getPythonExePathWindows(
search=intended_version, arch=None)
else:
os.environ["PYTHON"] = getExecutablePath(use_python)
if options.basic_tests:
print("Running the basic tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "basics", flags)
executeSubTest("./tests/basics/run_all.py search")
if options.syntax_tests:
print("Running the syntax tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "syntax", flags)
executeSubTest("./tests/syntax/run_all.py search")
if options.program_tests:
print("Running the program tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "programs", flags)
executeSubTest("./tests/programs/run_all.py search")
if options.package_tests:
print("Running the package tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(where, "packages", flags)
executeSubTest("./tests/packages/run_all.py search")
# At least one Debian Jessie, these versions won't have lxml installed, so
# don't run them there. Also these won't be very version dependent in their
# results.
if use_python != "python2.6" and use_python != "python3.2":
if options.optimization_tests:
print(
"Running the optimizations tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "optimizations", flags)
executeSubTest("./tests/optimizations/run_all.py search")
if options.standalone_tests and not options.coverage:
print("Running the standalone tests with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(None, "standalone", flags)
executeSubTest("./tests/standalone/run_all.py search")
if options.reflection_test and not options.coverage:
print("Running the reflection test with options '%s' with %s:" %
(flags, use_python))
setExtraFlags(None, "reflected", flags)
executeSubTest("./tests/reflected/compile_itself.py search")
if not use_python.startswith("python3"):
if os.path.exists("./tests/CPython26/run_all.py"):
if options.cpython26:
print(
"Running the CPython 2.6 tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "26tests", flags)
executeSubTest("./tests/CPython26/run_all.py search")
else:
print("The CPython2.6 tests are not present, not run.")
# Running the Python 2.7 test suite with CPython 2.6 gives little
# insight, because "importlib" will not be there and that's it.
if use_python != "python2.6":
if os.path.exists("./tests/CPython27/run_all.py"):
if options.cpython27:
print(
"Running the CPython 2.7 tests with options '%s' with %s:"
% (flags, use_python))
setExtraFlags(where, "27tests", flags)
executeSubTest("./tests/CPython27/run_all.py search")
else:
print("The CPython2.7 tests are not present, not run.")
if "--debug" not in flags:
# Not running the Python 3.2 test suite with CPython2.6, as that's about
# the same as CPython2.7 and won't have any new insights.
if use_python != "python2.6" and \
use_python != "python2.7" or not options.coverage:
if os.path.exists("./tests/CPython32/run_all.py"):
if options.cpython32:
setExtraFlags(where, "32tests", flags)
executeSubTest("./tests/CPython32/run_all.py search")
else:
print("The CPython3.2 tests are not present, not run.")
# Running the Python 3.3 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython33/run_all.py"):
if options.cpython33:
setExtraFlags(where, "33tests", flags)
executeSubTest("./tests/CPython33/run_all.py search")
else:
print("The CPython3.3 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython34/run_all.py"):
if options.cpython34:
setExtraFlags(where, "34tests", flags)
executeSubTest("./tests/CPython34/run_all.py search")
else:
print("The CPython3.4 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython35/run_all.py"):
if options.cpython35:
setExtraFlags(where, "35tests", flags)
executeSubTest("./tests/CPython35/run_all.py search")
else:
print("The CPython3.5 tests are not present, not run.")
# Running the Python 3.4 test suite only with CPython3.x.
if not use_python.startswith("python2"):
if os.path.exists("./tests/CPython36/run_all.py"):
if options.cpython36:
setExtraFlags(where, "36tests", flags)
executeSubTest("./tests/CPython36/run_all.py search")
else:
print("The CPython3.6 tests are not present, not run.")
if "NUITKA_EXTRA_OPTIONS" in os.environ:
del os.environ["NUITKA_EXTRA_OPTIONS"]
assert checkExecutableCommand("python2.6") or \
checkExecutableCommand("python2.7") or \
checkExecutableCommand("python3.2") or \
checkExecutableCommand("python3.3") or \
checkExecutableCommand("python3.4") or \
checkExecutableCommand("python3.5") or \
checkExecutableCommand("python3.6")
# Just the quick syntax test, full tests are run later.
if checkExecutableCommand("python3.2"):
executeSubTest("./bin/nuitka --python-version=3.2 --version",
hide_output=True)
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-debug", "python2.6", "--debug")
else:
print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-debug", "python2.7", "--debug")
else:
print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.2"):
execute_tests("python3.2-debug", "python3.2", "--debug")
else:
print(
"Cannot execute tests with Python 3.2, disabled or not installed.")
if checkExecutableCommand("python2.6"):
execute_tests("python2.6-nodebug", "python2.6", "")
else:
print(
"Cannot execute tests with Python 2.6, disabled or not installed.")
if checkExecutableCommand("python2.7"):
execute_tests("python2.7-nodebug", "python2.7", "")
else:
print(
"Cannot execute tests with Python 2.7, disabled or not installed.")
if checkExecutableCommand("python3.2"):
execute_tests("python3.2-nodebug", "python3.2", "")
else:
print(
"Cannot execute tests with Python 3.2, disabled or not installed.")
if checkExecutableCommand("python3.3"):
execute_tests("python3.3-nodebug", "python3.3", "")
else:
print(
"Cannot execute tests with Python 3.3, disabled or not installed.")
if checkExecutableCommand("python3.4"):
execute_tests("python3.4-nodebug", "python3.4", "")
else:
print(
"Cannot execute tests with Python 3.4, disabled or not installed.")
if checkExecutableCommand("python3.5"):
execute_tests("python3.5-nodebug", "python3.5", "")
else:
print(
"Cannot execute tests with Python 3.5, disabled or not installed.")
if checkExecutableCommand("python3.6"):
execute_tests("python3.6-nodebug", "python3.6", "")
else:
print(
"Cannot execute tests with Python 3.6, disabled or not installed.")
if options.coverage:
def copyToGlobalCoverageData(source, target):
coverage_dir = os.environ.get("COVERAGE_DIR", None)
if coverage_dir is None:
return
assert subprocess.call(
("C:\\MinGW\\msys\\1.0\\bin\\scp.exe" if os.name == "nt" else
"scp", source, os.path.join(coverage_dir, target))) == 0
if os.name == "nt":
suffix = "win"
else:
import platform
suffix = platform.uname()[0] + '.' + platform.uname()[4]
with open("data.coverage", 'w') as data_file:
source_dir = os.path.abspath(
os.path.dirname(os.path.dirname(__file__)))
nuitka_id = check_output("cd '%s'; git rev-parse HEAD" %
source_dir,
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
data_file.write("NUITKA_COMMIT='%s'\n" % nuitka_id)
copyToGlobalCoverageData("data.coverage", "data.coverage." + suffix)
def makeCoverageRelative(filename):
""" Normalize coverage data.
"""
with open(filename) as input_file:
data = input_file.read()
data = data.replace(os.path.abspath('.') + os.path.sep, "")
if os.path.sep != '/':
data.replace(os.path.sep, '/')
with open(filename, 'w') as output_file:
output_file.write(data)
coverage_file = os.environ.get("COVERAGE_FILE", ".coverage")
makeCoverageRelative(coverage_file)
copyToGlobalCoverageData(coverage_file, ".coverage." + suffix)
print("OK.")
