def _cleanupRstFmt(filename):
updated_contents = contents = getFileContents(filename, mode="rb")
for keyword in extra_rst_keywords:
updated_contents = updated_contents.replace(b".. %s::" % keyword,
b".. raw:: %s" % keyword)
if updated_contents != contents:
with open(filename, "wb") as out_file:
out_file.write(updated_contents)
rstfmt_call = _getPythonBinaryCall("rstfmt")
check_call(rstfmt_call + [
filename,
],
# stdout=devnull,
)
cleanupWindowsNewlines(filename)
contents = getFileContents(filename, mode="rb")
# Enforce choice between "bash" and "sh" for code directive. Use bash as
# more people will know it.
updated_contents = contents.replace(b".. code:: sh\n", b".. code:: bash\n")
for keyword in extra_rst_keywords:
updated_contents = updated_contents.replace(b".. raw:: %s" % keyword,
b".. %s::" % keyword)
lines = []
inside = False
needs_empty = False
for line in updated_contents.splitlines():
if line.startswith(b"-"):
if inside and needs_empty:
lines.append(b"")
inside = True
needs_empty = True
lines.append(line)
elif inside and line == b"":
needs_empty = False
lines.append(line)
elif inside and line.startswith(b" "):
needs_empty = True
lines.append(line)
else:
inside = False
lines.append(line)
updated_contents = b"\n".join(lines) + b"\n"
if updated_contents != contents:
with open(filename, "wb") as out_file:
out_file.write(updated_contents)
def autoformat(filename, abort=False):
filename = os.path.normpath(filename)
my_print("Consider", filename, end=": ")
old_code = getFileContents(filename)
is_python = _isPythonFile(filename)
is_c = filename.endswith((".c", ".h"))
# Some parts of Nuitka must not be re-formatted with black or clang-format
# as they have different intentions.
if _shouldNotFormatCode(filename):
is_python = is_c = False
# Work on a temporary copy
tmp_filename = filename + ".tmp"
shutil.copy(filename, tmp_filename)
try:
if is_python:
_cleanupPyLintComments(tmp_filename, abort)
_cleanupImportSortOrder(tmp_filename)
if is_python:
black_call = _getPythonBinaryCall("black")
subprocess.call(black_call + ["-q", tmp_filename])
elif is_c:
_cleanupClangFormat(filename)
else:
_cleanupTrailingWhitespace(tmp_filename)
if getOS() == "Windows":
_cleanupWindowsNewlines(tmp_filename)
changed = False
if old_code != getFileContents(tmp_filename):
my_print("Updated.")
renameFile(tmp_filename, filename)
changed = True
else:
my_print("OK.")
return changed
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
def getFileContents(self):
if self.kind == "data_file":
return getFileContents(filename=self.source_path, mode="rb")
elif self.kind == "data_blob":
return self.data
else:
assert False
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def _cleanupPyLintComments(filename, abort):
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, )
old_code = getFileContents(filename)
# Baron does assertions too, and all kinds of strange errors, pylint: disable=broad-except
try:
red = RedBaron(old_code)
except Exception:
if abort:
raise
return
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def getCachedImportedModulesNames(module_name, source_code):
cache_name = makeCacheName(module_name, source_code)
return [
ModuleName(line) for line in getFileContents(
_getCacheFilename(cache_name, "txt")).strip().split("\n")
]
def setupCacheHashSalt(test_code_path):
assert os.path.exists(test_code_path)
if os.path.exists(os.path.join(test_code_path, ".git")):
git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path]
process = subprocess.Popen(
args=git_cmd,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_git, stderr_git = process.communicate()
assert process.returncode == 0, stderr_git
salt_value = hashlib.md5(stdout_git)
else:
salt_value = hashlib.md5()
for filename in getFileList(test_code_path):
if filename.endswith(".py"):
salt_value.update(getFileContents(filename, mode="rb"))
os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest()
def __init__(self, tests_path):
SearchModeBase.__init__(self)
tests_path = os.path.normcase(os.path.abspath(tests_path))
version = sys.version
if str is not bytes:
tests_path = tests_path.encode("utf8")
version = version.encode("utf8")
case_hash = hashlib.md5(tests_path)
case_hash.update(version)
from .Common import getTestingCacheDir
cache_filename = os.path.join(getTestingCacheDir(),
case_hash.hexdigest())
self.cache_filename = cache_filename
if os.path.exists(cache_filename):
self.resume_from = getFileContents(cache_filename) or None
else:
self.resume_from = None
self.active = not self.resume_from
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def _getCacheFilename(dependency_tool, is_main_executable, source_dir,
original_dir, binary_filename):
original_filename = os.path.join(original_dir,
os.path.basename(binary_filename))
original_filename = os.path.normcase(original_filename)
if is_main_executable:
# Normalize main program name for caching as well, but need to use the
# scons information to distinguish different compilers, so we use
# different libs there.
hashed_value = getFileContents(
os.path.join(source_dir, "scons-report.txt"))
else:
hashed_value = original_filename
# Have different values for different Python major versions.
hashed_value += sys.version + sys.executable
if str is not bytes:
hashed_value = hashed_value.encode("utf8")
cache_dir = os.path.join(getCacheDir(), "library_deps", dependency_tool)
makePath(cache_dir)
return os.path.join(cache_dir, hashlib.md5(hashed_value).hexdigest())
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir=source_dir,
key="CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)",
"cache hit (preprocessed)"):
result = "cache hit"
# Newer ccache has these, but they duplicate:
if result in (
"direct_cache_hit",
"direct_cache_miss",
"preprocessed_cache_hit",
"preprocessed_cache_miss",
"primary_storage_miss",
):
continue
if result == "primary_storage_hit":
result = "cache hit"
if result == "cache_miss":
result = "cache miss"
# Usage of incbin causes this for the constants blob integration.
if result in ("unsupported code directive", "disabled"):
continue
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d" %
(result, count))
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir=source_dir,
key="CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename))
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss))
def _cleanupPyLintComments(filename):
new_code = old_code = getFileContents(filename, encoding="utf8")
def replacer(part):
def renamer(pylint_token):
# pylint: disable=too-many-branches,too-many-return-statements
if pylint_token == "E0602":
return "undefined-variable"
elif pylint_token in ("E0401", "F0401"):
return "import-error"
elif pylint_token == "E1102":
return "not-callable"
elif pylint_token == "E1133":
return " not-an-iterable"
elif pylint_token == "E1128":
return "assignment-from-none"
# Save line length for this until isort is better at long lines.
elif pylint_token == "useless-suppression":
return "I0021"
elif pylint_token == "R0911":
return "too-many-return-statements"
elif pylint_token == "R0201":
return "no-self-use"
elif pylint_token == "R0902":
return "too-many-instance-attributes"
elif pylint_token == "R0912":
return "too-many-branches"
elif pylint_token == "R0914":
return "too-many-locals"
elif pylint_token == "R0915":
return "too-many-statements"
elif pylint_token == "W0123":
return "eval-used"
elif pylint_token == "W0603":
return "global-statement"
elif pylint_token == "W0613":
return "unused-argument"
elif pylint_token == "W0622":
return "redefined-builtin"
elif pylint_token == "W0703":
return "broad-except"
else:
return pylint_token
return part.group(1) + ",".join(
sorted(
set(
renamer(token)
for token in part.group(2).split(",") if token)))
new_code = re.sub(r"(pylint\: disable=)(.*)",
replacer,
new_code,
flags=re.M)
if new_code != old_code:
putTextFileContents(filename, new_code)
def _cleanupImportSortOrder(filename):
_cleanupImportRelative(filename)
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename)
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1 :])
with open(filename, "w") as out_file:
out_file.write(contents)
with open(os.devnull, "w") as devnull:
check_call(
isort_call
+ [
"-q", # quiet, but stdout is still garbage
"--overwrite-in-place", # avoid using another temp file, this is already on one.
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-tc", # Trailing commas
"-p", # make sure nuitka is first party package in import sorting.
"nuitka",
"-o",
"SCons",
filename,
],
stdout=devnull,
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[: start_index + 1]) + "\n" + contents
with open(filename, "w") as out_file:
out_file.write(contents)
def _discardDebianChangelogLastEntry():
changelog_lines = getFileContents("debian/changelog").splitlines()
with openTextFile("debian/changelog", "w") as output:
first = True
for line in changelog_lines[1:]:
if line.startswith("nuitka") and first:
first = False
if not first:
output.write(line + "\n")
def mungeFile(filename):
# Read the filename and modify it so that it can be bundled with the
# other Pmw files.
filename = "Pmw" + filename + ".py"
text = getFileContents(os.path.join(srcdir, filename))
text = re.sub(r"import Pmw\>", "", text)
text = re.sub("INITOPT = Pmw.INITOPT", "", text)
text = re.sub(r"\<Pmw\.", "", text)
text = "\n" + ("#" *
70) + "\n" + "### File: " + filename + "\n" + text
return text
def _cleanupImportSortOrder(filename):
_cleanupImportRelative(filename)
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename)
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1:])
with open(filename, "w") as out_file:
out_file.write(contents)
with open(os.devnull, "w") as devnull:
check_call(
isort_call + [
"-q", # quiet, but stdout is still garbage
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-up", # Prefer braces () over \ for line continuation.
"-tc", # Trailing commas
"-p", # make sure nuitka is first party package in import sorting.
"nuitka",
"-ns", # Do not ignore those:
"__init__.py",
filename,
],
stdout=devnull,
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[:start_index + 1]) + "\n" + contents
with open(filename, "w") as out_file:
out_file.write(contents)
def makeCoverageRelative(filename):
"""Normalize coverage data."""
data = getFileContents(filename)
data = data.replace(
(os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), "")
if os.path.sep != "/":
data.replace(os.path.sep, "/")
putTextFileContents(filename, contents=data)
def runCodespell(filenames, verbose, write):
if verbose:
my_print("Consider", " ".join(filenames))
command = [
"codespell",
"-f",
"-I",
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"misc/codespell-ignore.txt",
),
]
if write:
command.append("-w")
command += filenames
if os.name == "nt":
extra_path = os.path.join(sys.prefix, "Scripts")
else:
extra_path = None
with withEnvironmentPathAdded("PATH", extra_path):
result = subprocess.call(command)
if result == 0:
for filename in filenames:
if areSamePaths(__file__, filename):
continue
contents = getFileContents(filename)
old_contents = contents
for word, replacement in replacements:
contents = contents.replace(word, replacement)
contents = contents.replace(word.title(), replacement.title())
if old_contents != contents:
putTextFileContents(filename, contents)
cleanupWindowsNewlines(filename)
if verbose:
if result != 0:
my_print("FAILED.")
else:
my_print("OK.")
return result == 0
def updateDebianChangelog(old_version, new_version, distribution):
debian_version = new_version.replace("rc", "~rc") + "+ds-1"
os.environ["DEBEMAIL"] = "Kay Hayen <*****@*****.**>"
if "rc" in new_version:
if "rc" in old_version:
# Initial final release after pre-releases.
_discardDebianChangelogLastEntry()
message = "New upstream pre-release."
if checkNuitkaChangelog() != "draft":
changelog = getFileContents("Changelog.rst")
title = "Nuitka Release " + new_version[:-3] + " (Draft)"
found = False
with openTextFile("Changelog.rst", "w") as changelog_file:
for line in changelog.splitlines():
if not found:
if line.startswith("***") and line.endswith("***"):
found = True
marker = "*" * (len(title) + 2)
changelog_file.write(
marker + "\n " + title + "\n" + marker + "\n\n"
)
changelog_file.write("This release is not done yet.\n\n")
changelog_file.write(line + "\n")
continue
changelog_file.write(line + "\n")
assert found
else:
if "rc" in old_version:
# Initial final release after pre-releases.
_discardDebianChangelogLastEntry()
message = "New upstream release."
else:
# Initial final release after pre-releases.
# Hotfix release after previous final or hotfix release.
message = "New upstream hotfix release."
_callDebchange("--newversion=%s" % debian_version, message)
_callDebchange("-r", "--distribution", distribution, "")
def checkAtHome(expected="Nuitka Staging"):
assert os.path.isfile("setup.py")
if os.path.isdir(".git"):
git_dir = ".git"
else:
line = getFileFirstLine(".git", "r").strip()
git_dir = line[8:]
git_description_filename = os.path.join(git_dir, "description")
description = getFileContents(git_description_filename).strip()
assert description == expected, (expected, description)
def _cleanupImportSortOrder(filename):
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename)
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1 :])
with open(filename, "w") as out_file:
out_file.write(contents)
with open(os.devnull, "w") as devnull:
subprocess.check_call(
isort_call
+ [
"-q", # quiet, but stdout is still garbage
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-up", # Prefer braces () over \ for line continuation.
"-tc", # Trailing commas
"-ns", # Do not ignore those:
"__init__.py",
filename,
],
stdout=devnull,
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[: start_index + 1]) + "\n" + contents
with open(filename, "w") as out_file:
out_file.write(contents)
def _cleanupImportRelative(filename):
package_name = os.path.dirname(filename).replace(os.path.sep, ".")
# Make imports local if possible.
if package_name.startswith("nuitka."):
source_code = getFileContents(filename)
updated_code = re.sub(r"from %s import" % package_name,
"from . import", source_code)
updated_code = re.sub(r"from %s\." % package_name, "from .",
source_code)
if source_code != updated_code:
with open(filename, "w") as out_file:
out_file.write(updated_code)
def _cleanupImportRelative(filename):
package_name = os.path.dirname(filename)
# Make imports local if possible.
if package_name.startswith("nuitka" + os.path.sep):
package_name = package_name.replace(os.path.sep, ".")
source_code = getFileContents(filename)
updated_code = re.sub(
r"from %s import" % package_name, "from . import", source_code
)
updated_code = re.sub(r"from %s\." % package_name, "from .", source_code)
if source_code != updated_code:
with open(filename, "w") as out_file:
out_file.write(updated_code)
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir, "CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)", "cache hit (preprocessed)"):
result = "cache hit"
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d"
% (result, count)
)
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(source_dir, "CLCACHE_STATS")
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename
):
stats = eval( # lazy, pylint: disable=eval-used
getFileContents(clcache_stats_filename)
)
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss)
)
def _cleanupImportRelative(filename):
"""Make imports of Nuitka package when possible."""
# Avoid doing it for "__main__" packages, because for those the Visual Code
# IDE doesn't like it and it may not run
if os.path.basename(filename) == "__main__.py.tmp":
return
package_name = os.path.dirname(filename).replace(os.path.sep, ".")
# Make imports local if possible.
if not package_name.startswith("nuitka."):
return
source_code = getFileContents(filename)
updated_code = re.sub(r"from %s import" % package_name, "from . import",
source_code)
updated_code = re.sub(r"from %s\." % package_name, "from .", source_code)
if source_code != updated_code:
putTextFileContents(filename, contents=updated_code)
def makeManpage(python, suffix):
cmd = [
"help2man",
"-n",
"the Python compiler",
"--no-discard-stderr",
"--no-info",
"--include",
"doc/nuitka-man-include.txt",
"%s ./bin/nuitka" % python,
]
with openTextFile("doc/nuitka%s.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
cmd[-1] += "-run"
with openTextFile("doc/nuitka%s-run.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
for manpage in ("doc/nuitka%s.1" % suffix,
"doc/nuitka%s-run.1" % suffix):
manpage_contents = getFileContents(manpage).splitlines()
new_contents = []
mark = False
for count, line in enumerate(manpage_contents):
if mark:
line = ".SS " + line + ".BR\n"
mark = False
elif line == ".IP\n" and manpage_contents[count +
1].endswith(":\n"):
mark = True
continue
if line == r"\fB\-\-g\fR++\-only" + "\n":
line = r"\fB\-\-g\++\-only\fR" + "\n"
new_contents.append(line)
putTextFileContents(manpage, contents=new_contents)
def checkRstLint(document):
contents = getFileContents(document, mode="rb")
for keyword in extra_rst_keywords:
contents = contents.replace(b".. %s::" % keyword,
b".. raw:: %s" % keyword)
import restructuredtext_lint # pylint: disable=I0021,import-error
my_print("Checking %r for proper restructured text ..." % document,
style="blue")
lint_results = restructuredtext_lint.lint(
contents.decode("utf8"),
document,
)
lint_error = False
for lint_result in lint_results:
# Not an issue.
if lint_result.message.startswith("Duplicate implicit target name:"):
continue
# We switched to raw, but attributes will still bne unknown.
if lint_result.message.startswith(
'Error in "raw" directive:\nunknown option: "hidden"'):
continue
if lint_result.message.startswith(
'Error in "raw" directive:\nunknown option: "excerpts"'):
continue
if lint_result.message.startswith(
'Error in "raw" directive:\nunknown option: "members"'):
continue
my_print(lint_result, style="yellow")
lint_error = True
if lint_error:
sys.exit("Error, no lint clean rest.")
my_print("OK.", style="blue")
def main():
goHome()
if os.name == "nt":
git_path = getExecutablePath("git")
if git_path is None:
git_path = r"C:\Program Files\Git\bin\sh.exe"
if not os.path.exists(git_path):
git_path = None
if git_path is None:
sys.exit(
"""\
Error, cannot locate 'git.exe' which we need to install git hooks. Add it to
PATH while executing this will be sufficient."""
)
sh_path = os.path.join(os.path.dirname(git_path), "sh.exe")
if not os.path.exists(sh_path):
sh_path = os.path.join(
os.path.dirname(git_path), "..", "..", "bin", "sh.exe"
)
sh_path = os.path.normpath(sh_path)
if not os.path.exists(sh_path):
sys.exit(
"""\
Error, cannot locate 'sh.exe' near 'git.exe' which we need to install git hooks,
please improve this script."""
)
# For MinGW and #! we will need a path without spaces, so use this
# code to find the short name, that won't have it.
sh_path = getWindowsShortPathName(sh_path)
for hook in os.listdir(".githooks"):
full_path = os.path.join(".githooks", hook)
hook_contents = getFileContents(full_path)
if hook_contents.startswith("#!/bin/sh"):
if os.name == "nt":
# Correct shebang for Windows git to work.
hook_contents = "#!%s\n%s" % (
sh_path.replace("\\", "/").replace(" ", r"\ "),
hook_contents[10:],
)
# Also use sys.executable to make sure we find autoformat.
hook_contents = hook_contents.replace(
"./bin/autoformat-nuitka-source",
"'%s' ./bin/autoformat-nuitka-source" % sys.executable,
)
else:
sys.exit("Error, unknown hook contents.")
hook_target = os.path.join(".git/hooks/", hook)
with open(hook_target, "wb") as out_file:
out_file.write(hook_contents.encode("utf8"))
st = os.stat(hook_target)
os.chmod(hook_target, st.st_mode | stat.S_IEXEC)
def main():
goHome()
if os.name == "nt":
git_path = getExecutablePath("git")
if git_path is None:
git_path = r"C:\Program Files\Git\bin\sh.exe"
if not os.path.exists(git_path):
git_path = None
if git_path is None:
sys.exit("""\
Error, cannot locate 'git.exe' which we need to install git hooks. Add it to
PATH while executing this will be sufficient.""")
sh_path = os.path.join(os.path.dirname(git_path), "sh.exe")
if not os.path.exists(sh_path):
sh_path = os.path.join(os.path.dirname(git_path), "..", "..",
"bin", "sh.exe")
sh_path = os.path.normpath(sh_path)
if not os.path.exists(sh_path):
sys.exit("""\
Error, cannot locate 'sh.exe' near 'git.exe' which we need to install git hooks,
please improve this script.""")
# For MinGW and #! we will need a path without spaces, so use this
# code to find the short name, that won't have it.
sh_path = getWindowsShortPathName(sh_path)
for hook in os.listdir(".githooks"):
full_path = os.path.join(".githooks", hook)
hook_contents = getFileContents(full_path)
if hook_contents.startswith("#!/bin/sh"):
if os.name == "nt":
# Correct shebang for Windows git to work.
hook_contents = "#!%s\n%s" % (
sh_path.replace("\\", "/").replace(" ", r"\ "),
hook_contents[10:],
)
# Also use sys.executable to make sure we find autoformat.
hook_contents = hook_contents.replace(
"./bin/autoformat-nuitka-source",
"'%s' ./bin/autoformat-nuitka-source" % sys.executable,
)
else:
sys.exit("Error, unknown hook contents.")
hook_target = os.path.join(".git/hooks/", hook)
with open(hook_target, "wb") as out_file:
out_file.write(hook_contents.encode("utf8"))
st = os.stat(hook_target)
os.chmod(hook_target, st.st_mode | stat.S_IEXEC)
def autoformat(filename, git_stage, abort):
# This does a lot of distinctions, pylint:disable=too-many-branches
filename = os.path.normpath(filename)
my_print("Consider", filename, end=": ")
is_python = _isPythonFile(filename)
is_c = filename.endswith((".c", ".h"))
# Some parts of Nuitka must not be re-formatted with black or clang-format
# as they have different intentions.
if _shouldNotFormatCode(filename):
is_python = is_c = False
# Work on a temporary copy
tmp_filename = filename + ".tmp"
if git_stage:
old_code = getFileHashContent(git_stage["dst_hash"])
else:
old_code = getFileContents(filename)
with open(tmp_filename, "wb") as output_file:
output_file.write(old_code)
try:
_cleanupWindowsNewlines(tmp_filename)
if is_python:
_cleanupPyLintComments(tmp_filename, abort)
_cleanupImportSortOrder(tmp_filename)
if is_python:
black_call = _getPythonBinaryCall("black")
subprocess.call(black_call + ["-q", tmp_filename])
elif is_c:
_cleanupClangFormat(filename)
else:
_cleanupTrailingWhitespace(tmp_filename)
_cleanupWindowsNewlines(tmp_filename)
changed = False
if old_code != getFileContents(tmp_filename):
my_print("Updated.")
if git_stage:
new_hash_value = putFileHashContent(tmp_filename)
updateFileIndex(git_stage, new_hash_value)
updateWorkingFile(filename, git_stage["dst_hash"], new_hash_value)
else:
renameFile(tmp_filename, filename)
changed = True
else:
my_print("OK.")
return changed
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
def executePostProcessing():
# These is a bunch of stuff to consider, pylint: disable=too-many-branches
result_filename = OutputDirectories.getResultFullpath()
if not os.path.exists(result_filename):
postprocessing_logger.sysexit(
"Error, scons failed to create the expected file %r. " %
result_filename)
if isWin32Windows():
if not Options.shallMakeModule():
needs_manifest = False
manifest = None
if python_version < 0x300:
# Copy the Windows manifest from the CPython binary to the created
# executable, so it finds "MSCRT.DLL". This is needed for Python2
# only, for Python3 newer MSVC doesn't hide the C runtime.
manifest = getWindowsExecutableManifest(sys.executable)
if manifest is not None:
needs_manifest = True
if (Options.shallAskForWindowsAdminRights()
or Options.shallAskForWindowsUIAccessRights()):
needs_manifest = True
if manifest is None:
manifest = getDefaultWindowsExecutableManifest()
if Options.shallAskForWindowsAdminRights():
manifest.addUacAdmin()
if Options.shallAskForWindowsUIAccessRights():
manifest.addUacUiAccess()
if needs_manifest:
manifest.addResourceToFile(result_filename,
logger=postprocessing_logger)
if (Options.getWindowsVersionInfoStrings()
or Options.getWindowsProductVersion()
or Options.getWindowsFileVersion()):
version_resources.update(
addVersionInfoResource(
string_values=Options.getWindowsVersionInfoStrings(),
product_version=Options.getWindowsProductVersion(),
file_version=Options.getWindowsFileVersion(),
file_date=(0, 0),
is_exe=not Options.shallMakeModule(),
result_filename=result_filename,
logger=postprocessing_logger,
))
source_dir = OutputDirectories.getSourceDirectoryPath()
# Attach the binary blob as a Windows resource.
addResourceToFile(
target_filename=result_filename,
data=getFileContents(getConstantBlobFilename(source_dir), "rb"),
resource_kind=RT_RCDATA,
res_name=3,
lang_id=0,
logger=postprocessing_logger,
)
# Attach icons from template file if given.
template_exe = Options.getWindowsIconExecutablePath()
if template_exe is not None:
res_copied = copyResourcesFromFileToFile(
template_exe,
target_filename=result_filename,
resource_kinds=(RT_ICON, RT_GROUP_ICON),
)
if res_copied == 0:
postprocessing_logger.warning(
"The specified icon template executable %r didn't contain anything to copy."
% template_exe)
else:
postprocessing_logger.warning(
"Copied %d icon resources from %r." %
(res_copied, template_exe))
else:
addWindowsIconFromIcons()
# On macOS, we update the executable path for searching the "libpython"
# library.
if (getOS() == "Darwin" and not Options.shallMakeModule()
and not Options.shallUseStaticLibPython()):
python_abi_version = python_version_str + getPythonABI()
python_dll_filename = "libpython" + python_abi_version + ".dylib"
python_lib_path = os.path.join(sys.prefix, "lib")
# Note: For CPython and potentially others, the rpath for the Python
# library needs to be set.
callInstallNameTool(
filename=result_filename,
mapping=(
(
python_dll_filename,
os.path.join(python_lib_path, python_dll_filename),
),
(
"@rpath/Python3.framework/Versions/%s/Python3" %
python_version_str,
os.path.join(python_lib_path, python_dll_filename),
),
),
rpath=python_lib_path,
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if not isWin32Windows() and Options.shallMakeModule():
removeFileExecutablePermission(result_filename)
if isWin32Windows() and Options.shallMakeModule():
candidate = os.path.join(
os.path.dirname(result_filename),
"lib" + os.path.basename(result_filename)[:-4] + ".a",
)
if os.path.exists(candidate):
os.unlink(candidate)
if isWin32Windows() and Options.shallTreatUninstalledPython():
shutil.copy(getTargetPythonDLLPath(),
os.path.dirname(result_filename) or ".")
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit(
"""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux."""
)
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
with open(apprun_filename, "w") as output_file:
output_file.write(
"""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s $@"""
% os.path.basename(binary_filename)
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
shutil.copyfile(icon_paths[0], getResultBasepath() + extension)
with open(getResultBasepath() + ".desktop", "w") as output_file:
output_file.write(
"""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;"""
% {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
}
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while."
)
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = open(stdout_filename, "wb")
stderr_file = open(stderr_filename, "wb")
# Starting the process while locked, so file handles are not duplicated.
appimagetool_process = subprocess.Popen(
(
_getAppImageToolPath(
for_operation=True, assume_yes_for_downloads=assumeYesForDownloads()
),
dist_dir,
"--comp",
"xz",
"-n",
onefile_output_filename,
),
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs %r and %r."
% (onefile_output_filename, stdout_filename, stderr_filename)
)
if result != 0:
# Useless now.
os.unlink(onefile_output_filename)
if b"Text file busy" in getFileContents(stderr_filename, mode="rb"):
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs %r and %r."
% (stdout_filename, stderr_filename)
)
os.unlink(stdout_filename)
os.unlink(stderr_filename)
postprocessing_logger.info("Completed onefile creation.")
def __init__(self):
""" Read the JSON file and enable any standard plugins.
"""
if not getNuitkaVersion() >= "0.6.6":
sys.exit("Need Nuitka v0.6.6+ for hinted compilation.")
# start a timer
self.timer = StopWatch()
self.timer.start()
self.implicit_imports = OrderedSet() # speed up repeated lookups
self.ignored_modules = OrderedSet() # speed up repeated lookups
options = Options.options
fin_name = self.getPluginOptions()[0] # the JSON file name
import_info = json.loads(
getFileContents(fin_name)) # read it and extract the two lists
self.import_calls = import_info["calls"]
self.import_files = import_info["files"]
self.msg_count = dict() # to limit keep messages
self.msg_limit = 21
# suppress pytest / _pytest / unittest?
self.accept_test = self.getPluginOptionBool("test", False)
"""
Check if we should enable any (optional) standard plugins. This code
must be modified whenever more standard plugin become available.
"""
show_msg = False # only show info if one ore more detected
# indicators for found packages
tk = np = qt = sc = mp = pmw = torch = sklearn = False
eventlet = tflow = gevent = mpl = trio = False
msg = "'%s' is adding the following options:" % self.plugin_name
# detect required standard plugins and request enabling them
for m in self.import_calls: # scan thru called items
if m in ("numpy", "numpy.*"):
np = True
show_msg = True
if m in ("matplotlib", "matplotlib.*"):
mpl = True
show_msg = True
elif m in ("tkinter", "Tkinter", "tkinter.*", "Tkinter.*"):
tk = True
show_msg = True
elif m.startswith(("PyQt", "PySide")):
qt = True
show_msg = True
elif m in ("scipy", "scipy.*"):
sc = True
show_msg = True
elif m in ("multiprocessing",
"multiprocessing.*") and getOS() == "Windows":
mp = True
show_msg = True
elif m in ("Pmw", "Pmw.*"):
pmw = True
show_msg = True
elif m == "torch":
torch = True
show_msg = True
elif m in ("sklearn", "sklearn.*"):
sklearn = True
show_msg = True
elif m in ("tensorflow", "tensorflow.*"):
tflow = True
show_msg = True
elif m in ("gevent", "gevent.*"):
gevent = True
show_msg = True
elif m in ("eventlet", "eventlet.*"):
eventlet = True
show_msg = True
# elif m in ("trio", "trio.*"):
# trio = True
# show_msg = True
if show_msg is True:
info(msg)
if np:
o = ["numpy="]
if mpl:
o.append("matplotlib")
if sc:
o.append("scipy")
if sklearn:
o.append("sklearn")
o = ",".join(o).replace("=,", "=")
if o.endswith("="):
o = o[:-1]
options.plugins_enabled.append(o) # enable numpy
info("--enable-plugin=" + o)
if tk:
options.plugins_enabled.append("tk-inter") # enable tk-inter
info("--enable-plugin=tk-inter")
if qt:
# TODO more scrutiny for the qt options!
options.plugins_enabled.append("qt-plugins=sensible")
info("--enable-plugin=qt-plugins=sensible")
if mp:
options.plugins_enabled.append("multiprocessing")
info("--enable-plugin=multiprocessing")
if pmw:
options.plugins_enabled.append("pmw-freezer")
info("--enable-plugin=pmw-freezer")
if torch:
options.plugins_enabled.append("torch")
info("--enable-plugin=torch")
if tflow:
options.plugins_enabled.append("tensorflow")
info("--enable-plugin=tensorflow")
if gevent:
options.plugins_enabled.append("gevent")
info("--enable-plugin=gevent")
if eventlet:
options.plugins_enabled.append("eventlet")
info("--enable-plugin=eventlet")
# if trio:
# options.plugins_enabled.append("trio")
# info("--enable-plugin=trio")
recurse_count = 0
for f in self.import_files: # request recursion to called modules
if self.accept_test is False and f.startswith(
("pytest", "_pytest", "unittest")):
continue
options.recurse_modules.append(f)
recurse_count += 1
# no plugin detected, but recursing to modules?
if show_msg is False and recurse_count > 0:
info(msg)
msg = "--recurse-to for %i imported modules." % recurse_count
if len(self.import_files) > 0:
info(msg)
info("")
self.ImplicitImports = None # the 'implicit-imports' plugin object
return None
def autoformat(filename, git_stage, abort, effective_filename=None):
""" Format source code with external tools
Args:
filename: filename to work on
git_stage: indicate if this is to be done on staged content
abort: error exit in case a tool shows a problem
effective_filename: derive type of file from this name
Notes:
The effective filename can be used in case this is already a
temporary filename intended to replace another.
Returns:
None
"""
# This does a lot of distinctions, pylint: disable=too-many-branches,too-many-statements
if effective_filename is None:
effective_filename = filename
if os.path.isdir(effective_filename):
return
filename = os.path.normpath(filename)
effective_filename = os.path.normpath(effective_filename)
my_print("Consider", filename, end=": ")
is_python = _isPythonFile(effective_filename)
is_c = effective_filename.endswith((".c", ".h"))
is_txt = effective_filename.endswith((
".patch",
".txt",
".rst",
".sh",
".in",
".md",
".yml",
".stylesheet",
".j2",
".gitignore",
".json",
".spec",
"-rpmlintrc",
)) or os.path.basename(filename) in (
"changelog",
"compat",
"control",
"lintian-overrides",
)
# Some parts of Nuitka must not be re-formatted with black or clang-format
# as they have different intentions.
if not (is_python or is_c or is_txt):
my_print("Ignored file type.")
return
# Work on a temporary copy
tmp_filename = filename + ".tmp"
if git_stage:
old_code = getFileHashContent(git_stage["dst_hash"])
else:
old_code = getFileContents(filename, "rb")
with open(tmp_filename, "wb") as output_file:
output_file.write(old_code)
try:
if is_python:
cleanupWindowsNewlines(tmp_filename)
if not _shouldNotFormatCode(effective_filename):
_cleanupImportSortOrder(tmp_filename)
_cleanupPyLintComments(tmp_filename, abort)
black_call = _getPythonBinaryCall("black")
subprocess.call(black_call + ["-q", "--fast", tmp_filename])
cleanupWindowsNewlines(tmp_filename)
elif is_c:
cleanupWindowsNewlines(tmp_filename)
_cleanupClangFormat(filename)
cleanupWindowsNewlines(tmp_filename)
elif is_txt:
cleanupWindowsNewlines(tmp_filename)
_cleanupTrailingWhitespace(tmp_filename)
cleanupWindowsNewlines(tmp_filename)
_transferBOM(filename, tmp_filename)
changed = False
if old_code != getFileContents(tmp_filename, "rb"):
my_print("Updated.")
with withPreserveFileMode(filename):
if git_stage:
new_hash_value = putFileHashContent(tmp_filename)
updateFileIndex(git_stage, new_hash_value)
updateWorkingFile(filename, git_stage["dst_hash"],
new_hash_value)
else:
renameFile(tmp_filename, filename)
changed = True
else:
my_print("OK.")
return changed
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to http://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit("Error, need to install Doxygen and add it to PATH for this to work.")
try:
import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh", delete=False
) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write(
"%s -m doxypypy.doxypypy -a -c %%1" % sys.executable
)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1"
% sys.executable
)
doxy_batch_filename = doxy_batch_file.name
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
subprocess.check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (
os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
)
== 0
)
print("Finished.")
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to http://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit(
"Error, need to install Doxygen and add it to PATH for this to work."
)
try:
import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh",
delete=False) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" %
sys.executable)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" %
sys.executable)
doxy_batch_filename = doxy_batch_file.name
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
subprocess.check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
) == 0)
print("Finished.")