def detectDLLsWithDependencyWalker(binary_filename, scan_dirs):
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with withFileLock(
"Finding out dependency walker path and creating DWP file for %s"
% binary_filename
):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
putTextFileContents(
dwp_filename,
contents="""\
%(scan_dirs)s
SxS
"""
% {
"scan_dirs": "\n".join(
"UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs
)
},
)
# Starting the process while locked, so file handles are not duplicated.
# TODO: At least exit code should be checked, output goes to a filename,
# but errors might be interesting potentially.
_stdout, _stderr, _exit_code = executeProcess(
command=(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
external_cwd=True,
)
if not os.path.exists(output_filename):
inclusion_logger.sysexit(
"Error, 'depends.exe' failed to produce expected output."
)
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
result = parseDependsExeOutput(output_filename)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
return result
def writeSourceCode(filename, source_code):
# Prevent accidental overwriting. When this happens the collision detection
# or something else has failed.
assert not os.path.isfile(filename), filename
putTextFileContents(filename=filename,
contents=source_code,
encoding="latin1")
def _cleanupPyLintComments(filename):
new_code = old_code = getFileContents(filename, encoding="utf8")
def replacer(part):
def renamer(pylint_token):
# pylint: disable=too-many-branches,too-many-return-statements
if pylint_token == "E0602":
return "undefined-variable"
elif pylint_token in ("E0401", "F0401"):
return "import-error"
elif pylint_token == "E1102":
return "not-callable"
elif pylint_token == "E1133":
return " not-an-iterable"
elif pylint_token == "E1128":
return "assignment-from-none"
# Save line length for this until isort is better at long lines.
elif pylint_token == "useless-suppression":
return "I0021"
elif pylint_token == "R0911":
return "too-many-return-statements"
elif pylint_token == "R0201":
return "no-self-use"
elif pylint_token == "R0902":
return "too-many-instance-attributes"
elif pylint_token == "R0912":
return "too-many-branches"
elif pylint_token == "R0914":
return "too-many-locals"
elif pylint_token == "R0915":
return "too-many-statements"
elif pylint_token == "W0123":
return "eval-used"
elif pylint_token == "W0603":
return "global-statement"
elif pylint_token == "W0613":
return "unused-argument"
elif pylint_token == "W0622":
return "redefined-builtin"
elif pylint_token == "W0703":
return "broad-except"
else:
return pylint_token
return part.group(1) + ",".join(
sorted(
set(
renamer(token)
for token in part.group(2).split(",") if token)))
new_code = re.sub(r"(pylint\: disable=)(.*)",
replacer,
new_code,
flags=re.M)
if new_code != old_code:
putTextFileContents(filename, new_code)
def copyMplDataFiles(self, dist_dir):
""" Write matplotlib data files ('mpl-data')."""
matplotlib_info = self._getMatplotlibInfo()
if not os.path.isdir(matplotlib_info.data_path):
self.sysexit(
"mpl-data missing, matplotlib installation appears to be broken"
)
# Copy data files to dist folder
for fullname in getFileList(matplotlib_info.data_path):
filename = os.path.relpath(fullname, matplotlib_info.data_path)
if filename.endswith("matplotlibrc"): # handle config separately
continue
target_filename = os.path.join(dist_dir, "matplotlib", "mpl-data",
filename)
makePath(os.path.dirname(
target_filename)) # create intermediate folders
shutil.copyfile(fullname, target_filename)
old_lines = (open(
matplotlib_info.matplotlibrc_filename).read().splitlines()
) # old config file lines
new_lines = [] # new config file lines
found = False # checks whether backend definition encountered
for line in old_lines:
line = line.rstrip()
# omit meaningless lines
if line.startswith(
"#") and matplotlib_info.matplotlib_version < "3":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
# old config file has a backend definition
found = True
if not found and matplotlib_info.matplotlib_version < "3":
# Set the backend, so even if it was run time determined, we now enforce it.
new_lines.append("backend: %s" % matplotlib_info.backend)
matplotlibrc_filename = os.path.join(dist_dir, "matplotlib",
"mpl-data", "matplotlibrc")
putTextFileContents(filename=matplotlibrc_filename, contents=new_lines)
self.info("Copied 'matplotlib/mpl-data'.")
def makeCoverageRelative(filename):
"""Normalize coverage data."""
data = getFileContents(filename)
data = data.replace(
(os.path.abspath(".") + os.path.sep).replace("\\", "\\\\"), "")
if os.path.sep != "/":
data.replace(os.path.sep, "/")
putTextFileContents(filename, contents=data)
def _createTriggerLoadedModule(module, trigger_name, code, flags):
"""Create a "trigger" for a module to be imported.
Notes:
The trigger will incorpaorate the code to be prepended / appended.
Called by @onModuleDiscovered.
Args:
module: the module object (serves as dict key)
trigger_name: string ("-preload"/"-postload")
code: the code string
Returns
trigger_module
"""
from nuitka.nodes.ModuleNodes import CompiledPythonModule
from nuitka.tree.Building import createModuleTree
from .Plugins import Plugins
module_name = ModuleName(module.getFullName() + trigger_name)
source_ref = fromFilename(module.getCompileTimeFilename() +
trigger_name)
mode = Plugins.decideCompilation(module_name, source_ref)
trigger_module = CompiledPythonModule(
module_name=module_name,
is_top=False,
mode=mode,
future_spec=None,
source_ref=source_ref,
)
createModuleTree(
module=trigger_module,
source_ref=module.getSourceReference(),
source_code=code,
is_main=False,
)
if mode == "bytecode":
trigger_module.setSourceCode(code)
# In debug mode, put the files in the build folder, so they can be looked up easily.
if Options.is_debug and "HIDE_SOURCE" not in flags:
source_path = os.path.join(
OutputDirectories.getSourceDirectoryPath(),
module_name + ".py")
putTextFileContents(filename=source_path, contents=code)
return trigger_module
def runCodespell(filenames, verbose, write):
if verbose:
my_print("Consider", " ".join(filenames))
command = [
"codespell",
"-f",
"-I",
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"misc/codespell-ignore.txt",
),
]
if write:
command.append("-w")
command += filenames
if os.name == "nt":
extra_path = os.path.join(sys.prefix, "Scripts")
else:
extra_path = None
with withEnvironmentPathAdded("PATH", extra_path):
result = subprocess.call(command)
if result == 0:
for filename in filenames:
if areSamePaths(__file__, filename):
continue
contents = getFileContents(filename)
old_contents = contents
for word, replacement in replacements:
contents = contents.replace(word, replacement)
contents = contents.replace(word.title(), replacement.title())
if old_contents != contents:
putTextFileContents(filename, contents)
cleanupWindowsNewlines(filename)
if verbose:
if result != 0:
my_print("FAILED.")
else:
my_print("OK.")
return result == 0
def _cleanupTrailingWhitespace(filename):
"""Remove trailing white spaces from a file."""
source_lines = list(getFileContentByLine(filename, encoding="utf8"))
clean_lines = [
line.rstrip().replace("\t", " ") for line in source_lines
]
while clean_lines and clean_lines[-1] == "":
del clean_lines[-1]
if clean_lines != source_lines or (clean_lines and clean_lines[-1] != ""):
putTextFileContents(filename, contents=clean_lines, encoding="utf8")
def _createTriggerLoadedModule(cls, module, trigger_name, code, flags):
"""Create a "trigger" for a module to be imported.
Notes:
The trigger will incorporate the code to be prepended / appended.
Called by @onModuleDiscovered.
Args:
module: the module object (serves as dict key)
trigger_name: string ("preload"/"postload")
code: the code string
Returns
trigger_module
"""
from nuitka.tree.Building import buildModule
module_name = makeTriggerModuleName(module.getFullName(), trigger_name)
# In debug mode, put the files in the build folder, so they can be looked up easily.
if Options.is_debug and "HIDE_SOURCE" not in flags:
source_path = os.path.join(
OutputDirectories.getSourceDirectoryPath(),
module_name + ".py")
putTextFileContents(filename=source_path, contents=code)
try:
trigger_module, _added = buildModule(
module_filename=os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
module_name.asPath() + ".py",
),
module_name=module_name,
source_code=code,
is_top=False,
is_main=False,
is_extension=False,
is_fake=module_name,
hide_syntax_error=False,
)
except SyntaxError:
plugins_logger.sysexit(
"SyntaxError in plugin provided source code for '%s'." %
module_name)
if trigger_module.getCompilationMode() == "bytecode":
trigger_module.setSourceCode(code)
return trigger_module
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is the caching mechanism and plugin handling for DLL imports.
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
result = OrderedSet()
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
general.info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs)
if update_cache:
putTextFileContents(filename=cache_filename, contents=result)
return result
def consider(self, dirname, filename):
parts = [dirname, filename]
while None in parts:
parts.remove(None)
assert parts
path = os.path.join(*parts)
if self.active:
putTextFileContents(self.cache_filename, contents=path)
return True
if areSamePaths(path, self.resume_from):
self.active = True
return self.active
def writeCompilationReport(report_filename):
active_modules_info = getModuleInclusionInfos()
root = TreeXML.Element("nuitka-compilation-report")
for module in getDoneModules():
active_module_info = active_modules_info[module]
root.append(
TreeXML.Element(
"module",
name=module.getFullName(),
kind=module.__class__.__name__,
reason=active_module_info.reason,
))
putTextFileContents(filename=report_filename,
contents=TreeXML.toString(root))
general.info("Compilation report in file %r." % report_filename)
def _cleanupImportRelative(filename):
"""Make imports of Nuitka package when possible."""
# Avoid doing it for "__main__" packages, because for those the Visual Code
# IDE doesn't like it and it may not run
if os.path.basename(filename) == "__main__.py.tmp":
return
package_name = os.path.dirname(filename).replace(os.path.sep, ".")
# Make imports local if possible.
if not package_name.startswith("nuitka."):
return
source_code = getFileContents(filename)
updated_code = re.sub(r"from %s import" % package_name, "from . import",
source_code)
updated_code = re.sub(r"from %s\." % package_name, "from .", source_code)
if source_code != updated_code:
putTextFileContents(filename, contents=updated_code)
def makeManpage(python, suffix):
cmd = [
"help2man",
"-n",
"the Python compiler",
"--no-discard-stderr",
"--no-info",
"--include",
"doc/nuitka-man-include.txt",
"%s ./bin/nuitka" % python,
]
with openTextFile("doc/nuitka%s.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
cmd[-1] += "-run"
with openTextFile("doc/nuitka%s-run.1" % suffix, "wb") as output:
check_call(cmd, stdout=output)
for manpage in ("doc/nuitka%s.1" % suffix,
"doc/nuitka%s-run.1" % suffix):
manpage_contents = getFileContents(manpage).splitlines()
new_contents = []
mark = False
for count, line in enumerate(manpage_contents):
if mark:
line = ".SS " + line + ".BR\n"
mark = False
elif line == ".IP\n" and manpage_contents[count +
1].endswith(":\n"):
mark = True
continue
if line == r"\fB\-\-g\fR++\-only" + "\n":
line = r"\fB\-\-g\++\-only\fR" + "\n"
new_contents.append(line)
putTextFileContents(manpage, contents=new_contents)
def readSourceCodeFromFilename(module_name, source_filename):
if python_version < 0x300:
source_code = _readSourceCodeFromFilename2(source_filename)
else:
source_code = _readSourceCodeFromFilename3(source_filename)
# Allow plugins to mess with source code. Test code calls this
# without a module and doesn't want changes from plugins.
if module_name is not None:
source_code_modified = Plugins.onModuleSourceCode(module_name, source_code)
else:
source_code_modified = source_code
if Options.shallPersistModifications() and source_code_modified != source_code:
orig_source_filename = source_filename + ".orig"
if not os.path.exists(orig_source_filename):
putTextFileContents(filename=orig_source_filename, contents=source_code)
putTextFileContents(filename=source_filename, contents=source_code_modified)
return source_code_modified
def writeCompilationReport(report_filename):
active_modules_info = getModuleInclusionInfos()
root = TreeXML.Element("nuitka-compilation-report")
for module in getDoneModules():
active_module_info = active_modules_info[module]
root.append(
TreeXML.Element(
"module",
name=module.getFullName(),
kind=module.__class__.__name__,
reason=active_module_info.reason,
))
for included_datafile in getIncludedDataFiles():
if included_datafile.kind == "data_file":
root.append(
TreeXML.Element(
"data_file",
name=included_datafile.dest_path,
source=included_datafile.source_path,
reason=included_datafile.reason,
tags=",".join(included_datafile.tags),
))
elif included_datafile.kind == "data_blob":
root.append(
TreeXML.Element(
"data_blob",
name=included_datafile.dest_path,
reason=included_datafile.reason,
tags=",".join(included_datafile.tags),
))
putTextFileContents(filename=report_filename,
contents=TreeXML.toString(root))
general.info("Compilation report in file %r." % report_filename)
def _cleanupImportSortOrder(filename):
_cleanupImportRelative(filename)
isort_call = _getPythonBinaryCall("isort")
contents = getFileContents(filename, encoding="utf8")
start_index = None
if "\n# isort:start" in contents:
parts = contents.splitlines()
start_index = parts.index("# isort:start")
contents = "\n".join(parts[start_index + 1:]) + "\n"
putTextFileContents(filename, contents=contents)
check_call(
isort_call + [
"-q", # quiet, but stdout is still garbage
"--overwrite-in-place", # avoid using another temp file, this is already on one.
"-ot", # Order imports by type in addition to alphabetically
"-m3", # "vert-hanging"
"-tc", # Trailing commas
"-p", # make sure nuitka is first party package in import sorting.
"nuitka",
"-o",
"SCons",
filename,
],
stdout=getNullOutput(),
)
if start_index is not None:
contents = getFileContents(filename)
contents = "\n".join(parts[:start_index + 1]) + "\n" + contents
putTextFileContents(filename, contents=contents)
def _handleDataFile(included_datafile):
"""Handle a data file."""
tracer = included_datafile.tracer
if not isinstance(included_datafile,
(IncludedDataFile, IncludedDataDirectory)):
tracer.sysexit(
"Error, can only accept 'IncludedData*' objects from plugins.")
if not isStandaloneMode():
tracer.sysexit(
"Error, package data files are only included in standalone or onefile mode."
)
dist_dir = getStandaloneDirectoryPath()
if included_datafile.kind == "empty_dirs":
tracer.info("Included empty directories '%s' due to %s." % (
",".join(included_datafile.dest_path),
included_datafile.reason,
))
for sub_dir in included_datafile.dest_path:
created_dir = os.path.join(dist_dir, sub_dir)
makePath(created_dir)
putTextFileContents(filename=os.path.join(created_dir,
".keep_dir.txt"),
contents="")
elif included_datafile.kind == "data_blob":
dest_path = os.path.join(dist_dir, included_datafile.dest_path)
makePath(os.path.dirname(dest_path))
putTextFileContents(filename=dest_path,
contents=included_datafile.data)
tracer.info("Included data file '%s' due to %s." % (
included_datafile.dest_path,
included_datafile.reason,
))
elif included_datafile.kind == "data_file":
dest_path = os.path.join(dist_dir, included_datafile.dest_path)
tracer.info("Included data file '%s' due to %s." % (
included_datafile.dest_path,
included_datafile.reason,
))
makePath(os.path.dirname(dest_path))
copyFileWithPermissions(source_path=included_datafile.source_path,
dest_path=dest_path)
elif included_datafile.kind == "data_dir":
dest_path = os.path.join(dist_dir, included_datafile.dest_path)
makePath(os.path.dirname(dest_path))
copied = []
for filename in getFileList(
included_datafile.source_path,
ignore_dirs=included_datafile.ignore_dirs,
ignore_filenames=included_datafile.ignore_filenames,
ignore_suffixes=included_datafile.ignore_suffixes,
only_suffixes=included_datafile.only_suffixes,
normalize=included_datafile.normalize,
):
filename_relative = os.path.relpath(filename,
included_datafile.source_path)
filename_dest = os.path.join(dest_path, filename_relative)
makePath(os.path.dirname(filename_dest))
copyFileWithPermissions(source_path=filename,
dest_path=filename_dest)
copied.append(filename_relative)
tracer.info("Included data dir %r with %d files due to: %s." % (
included_datafile.dest_path,
len(copied),
included_datafile.reason,
))
else:
assert False, included_datafile
def main():
# Complex stuff, pylint: disable=too-many-statements
# Make sure error messages are in English.
os.environ["LANG"] = "C"
parser = OptionParser()
parser.add_option(
"--no-pbuilder-update",
action="store_false",
dest="update_pbuilder",
default=True,
help="""\
Update the pbuilder chroot before building. Default %default.""",
)
options, positional_args = parser.parse_args()
assert len(positional_args) == 1, positional_args
codename = positional_args[0]
nuitka_version = getNuitkaVersion()
branch_name = checkBranchName()
category = getBranchCategory(branch_name)
if category == "stable":
if nuitka_version.count(".") == 1:
assert checkChangeLog("New upstream release.")
else:
assert checkChangeLog("New upstream hotfix release.")
else:
assert checkChangeLog("New upstream pre-release.")
shutil.rmtree("dist", ignore_errors=True)
shutil.rmtree("build", ignore_errors=True)
createReleaseDocumentation()
assert os.system("python setup.py sdist --formats=gztar") == 0
os.chdir("dist")
# Clean the stage for the debian package. The name "deb_dist" is what "py2dsc"
# uses for its output later on.
shutil.rmtree("deb_dist", ignore_errors=True)
# Provide a re-packed tar.gz for the Debian package as input.
# Create it as a "+ds" file, removing:
# - the benchmarks (too many sources, not useful to end users, potential license
# issues)
# - the inline copy of scons (not wanted for Debian)
for filename in os.listdir("."):
if filename.endswith(".tar.gz"):
new_name = filename[:-7] + "+ds.tar.gz"
cleanupTarfileForDebian(filename, new_name)
entry = runPy2dsc(filename, new_name)
break
else:
assert False
os.chdir("deb_dist")
os.chdir(entry)
# Build the debian package, but disable the running of tests, will be done later
# in the pbuilder test steps.
assert os.system("debuild --set-envvar=DEB_BUILD_OPTIONS=nocheck") == 0
os.chdir("../../..")
assert os.path.exists("dist/deb_dist")
# Check with pylint in pedantic mode and don't proceed if there were any
# warnings given. Nuitka is lintian clean and shall remain that way. For
# hotfix releases, i.e. "stable" builds, we skip this test though.
if category == "stable":
my_print("Skipped lintian checks for stable releases.", style="blue")
else:
assert os.system(
"lintian --pedantic --allow-root dist/deb_dist/*.changes") == 0
# Move the created debian package files out.
os.system("cp dist/deb_dist/*.deb dist/")
# Build inside the pbuilder chroot, and output to dedicated directory.
shutil.rmtree("package", ignore_errors=True)
os.makedirs("package")
# Now update the pbuilder.
if options.update_pbuilder:
command = """\
sudo /usr/sbin/pbuilder --update --basetgz /var/cache/pbuilder/%s.tgz""" % (
codename)
assert os.system(command) == 0, codename
(dsc_filename, ) = resolveShellPatternToFilenames("dist/deb_dist/*.dsc")
# Execute the package build in the pbuilder with tests.
command = (
"sudo",
"/usr/sbin/pbuilder",
"--build",
"--basetgz",
"/var/cache/pbuilder/%s.tgz" % codename,
"--hookdir",
"debian/pbuilder-hookdir",
"--debemail",
"Kay Hayen <*****@*****.**>",
"--buildresult",
"package",
dsc_filename,
)
check_call(command, shell=False)
# Cleanup the build directory, not needed anymore.
shutil.rmtree("build", ignore_errors=True)
# Now build the repository.
my_print("Building repository ...", style="blue")
os.chdir("package")
os.makedirs("repo")
os.chdir("repo")
os.makedirs("conf")
putTextFileContents(
"conf/distributions",
contents="""\
Origin: Nuitka
Label: Nuitka
Codename: %(codename)s
Architectures: i386 amd64 armel armhf powerpc
Components: main
Description: Apt repository for project Nuitka %(codename)s
SignWith: D96ADCA1377F1CEB6B5103F11BFC33752912B99C
""" % {"codename": codename},
)
command = ["reprepro", "includedeb", codename]
command.extend(resolveShellPatternToFilenames("../*.deb"))
check_call(command, shell=False)
my_print("Uploading ...", style="blue")
# Create repo folder unless already done. This is needed for the first
# build only.
assert (os.system("ssh [email protected] mkdir -p /var/www/deb/%s/%s/" %
(category, codename)) == 0)
# Update the repository on the web site.
assert (os.system(
"rsync -avz --delete dists pool --chown www-data [email protected]:/var/www/deb/%s/%s/"
% (category, codename)) == 0)
my_print("Finished.", style="blue")
def addConstantBlobFile(env, resource_desc, source_dir, target_arch):
resource_mode, reason = resource_desc
constants_bin_filename = getConstantBlobFilename(source_dir)
scons_details_logger.info("Using resource mode: '%s' (%s)." %
(resource_mode, reason))
if resource_mode == "win_resource":
# On Windows constants can be accessed as a resource by Nuitka runtime afterwards.
env.Append(CPPDEFINES=["_NUITKA_CONSTANTS_FROM_RESOURCE"])
elif resource_mode == "incbin":
env.Append(CPPDEFINES=["_NUITKA_CONSTANTS_FROM_INCBIN"])
constants_generated_filename = os.path.join(source_dir,
"__constants_data.c")
putTextFileContents(
constants_generated_filename,
contents=r"""
#define INCBIN_PREFIX
#define INCBIN_STYLE INCBIN_STYLE_SNAKE
#define INCBIN_LOCAL
#ifdef _NUITKA_EXPERIMENTAL_WRITEABLE_CONSTANTS
#define INCBIN_OUTPUT_SECTION ".data"
#endif
#include "nuitka/incbin.h"
INCBIN(constant_bin, "%(constants_bin_filename)s");
unsigned char const *getConstantsBlobData(void) {
return constant_bin_data;
}
""" % {"constants_bin_filename":
os.path.join(source_dir, "__constants.bin")},
)
elif resource_mode == "linker":
# Indicate "linker" resource mode.
env.Append(CPPDEFINES=["_NUITKA_CONSTANTS_FROM_LINKER"])
env.Append(LINKFLAGS=[
"-Wl,-b",
"-Wl,binary",
"-Wl,%s" % constants_bin_filename,
"-Wl,-b",
"-Wl,%s" %
getLinkerArch(target_arch=target_arch, mingw_mode=env.mingw_mode),
"-Wl,-defsym",
"-Wl,%sconstant_bin_data=_binary_%s___constants_bin_start" % (
"_" if env.mingw_mode else "",
"".join(re.sub("[^a-zA-Z0-9_]", "_", c) for c in source_dir),
),
])
elif resource_mode == "code":
# Indicate "code" resource mode.
env.Append(CPPDEFINES=["_NUITKA_CONSTANTS_FROM_CODE"])
constants_generated_filename = os.path.join(source_dir,
"__constants_data.c")
def writeConstantsDataSource():
with openTextFile(constants_generated_filename, "w") as output:
if not env.c11_mode:
output.write('extern "C" {')
output.write("""
// Constant data for the program.
#if !defined(_NUITKA_EXPERIMENTAL_WRITEABLE_CONSTANTS)
const
#endif
unsigned char constant_bin_data[] =\n{\n
""")
with open(constants_bin_filename, "rb") as f:
content = f.read()
for count, stream_byte in enumerate(content):
if count % 16 == 0:
if count > 0:
output.write("\n")
output.write(" ")
if str is bytes:
stream_byte = ord(stream_byte)
output.write(" 0x%02x," % stream_byte)
output.write("\n};\n")
if not env.c11_mode:
output.write("}")
writeConstantsDataSource()
else:
scons_logger.sysexit("Error, illegal resource mode %r specified" %
resource_mode)
def executePostProcessing():
"""Postprocessing of the resulting binary.
These are in part required steps, not usable after failure.
"""
result_filename = OutputDirectories.getResultFullpath(onefile=False)
if not os.path.exists(result_filename):
postprocessing_logger.sysexit(
"Error, scons failed to create the expected file %r. " %
result_filename)
if isWin32Windows():
if not Options.shallMakeModule():
if python_version < 0x300:
# Copy the Windows manifest from the CPython binary to the created
# executable, so it finds "MSCRT.DLL". This is needed for Python2
# only, for Python3 newer MSVC doesn't hide the C runtime.
manifest = getWindowsExecutableManifest(sys.executable)
else:
manifest = None
executePostProcessingResources(manifest=manifest, onefile=False)
source_dir = OutputDirectories.getSourceDirectoryPath()
# Attach the binary blob as a Windows resource.
addResourceToFile(
target_filename=result_filename,
data=getFileContents(getConstantBlobFilename(source_dir), "rb"),
resource_kind=RT_RCDATA,
res_name=3,
lang_id=0,
logger=postprocessing_logger,
)
# On macOS, we update the executable path for searching the "libpython"
# library.
if (isMacOS() and not Options.shallMakeModule()
and not Options.shallUseStaticLibPython()):
python_abi_version = python_version_str + getPythonABI()
python_dll_filename = "libpython" + python_abi_version + ".dylib"
python_lib_path = os.path.join(sys.prefix, "lib")
# Note: For CPython and potentially others, the rpath for the Python
# library needs to be set.
callInstallNameTool(
filename=result_filename,
mapping=(
(
python_dll_filename,
os.path.join(python_lib_path, python_dll_filename),
),
(
"@rpath/Python3.framework/Versions/%s/Python3" %
python_version_str,
os.path.join(python_lib_path, python_dll_filename),
),
),
id_path=None,
rpath=python_lib_path,
)
if Options.shallCreateAppBundle():
createPlistInfoFile(logger=postprocessing_logger, onefile=False)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if not isWin32Windows() and Options.shallMakeModule():
removeFileExecutablePermission(result_filename)
if isWin32Windows() and Options.shallMakeModule():
candidate = os.path.join(
os.path.dirname(result_filename),
"lib" + os.path.basename(result_filename)[:-4] + ".a",
)
if os.path.exists(candidate):
os.unlink(candidate)
# Might have to create a CMD file, potentially with debugger run.
if Options.shallCreateCmdFileForExecution():
dll_directory = getExternalUsePath(
os.path.dirname(getTargetPythonDLLPath()))
cmd_filename = OutputDirectories.getResultRunFilename(onefile=False)
cmd_contents = """
@echo off
rem This script was created by Nuitka to execute '%(exe_filename)s' with Python DLL being found.
set PATH=%(dll_directory)s;%%PATH%%
set PYTHONHOME=%(dll_directory)s
%(debugger_call)s"%%~dp0.\\%(exe_filename)s" %%*
""" % {
"debugger_call": (" ".join(wrapCommandForDebuggerForExec()) +
" ") if Options.shallRunInDebugger() else "",
"dll_directory":
dll_directory,
"exe_filename":
os.path.basename(result_filename),
}
putTextFileContents(cmd_filename, cmd_contents)
# Create a ".pyi" file for created modules
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = OutputDirectories.getResultBasepath() + ".pyi"
putTextFileContents(
filename=pyi_filename,
contents="""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
""" % {
"imports":
"\n".join("import %s" % module_name
for module_name in getImportedNames())
},
)
def main():
"""Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches,too-many-statements
if not Options.shallDumpBuiltTreeXML():
general.info("Starting Python compilation.")
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Turn that source code into a node tree structure.
try:
main_module = _createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree()
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing()
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = OutputDirectories.getResultBasepath() + ".pyi"
putTextFileContents(
filename=pyi_filename,
contents="""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name for module_name in getImportedNames()
)
},
)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
setMainEntryPoint(binary_filename)
dist_dir = OutputDirectories.getStandaloneDirectoryPath()
for module in ModuleRegistry.getDoneModules():
addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(
source_dir=OutputDirectories.getSourceDirectoryPath(),
dist_dir=dist_dir,
standalone_entry_points=getStandaloneEntryPoints(),
)
copyDataFiles(dist_dir=dist_dir)
Plugins.onStandaloneDistributionFinished(dist_dir)
if Options.isOnefileMode():
packDistFolderToOnefile(dist_dir, binary_filename)
if Options.isRemoveBuildDir():
general.info("Removing dist folder %r." % dist_dir)
removeDirectory(path=dist_dir, ignore_errors=False)
else:
general.info(
"Keeping dist folder %r for inspection, no need to use it."
% dist_dir
)
# Remove the source directory (now build directory too) if asked to.
source_dir = OutputDirectories.getSourceDirectoryPath()
if Options.isRemoveBuildDir():
general.info("Removing build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping build directory %r." % source_dir)
final_filename = OutputDirectories.getResultFullpath(
onefile=Options.isOnefileMode()
)
Plugins.onFinalResult(final_filename)
general.info("Successfully created %r." % final_filename)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
general.info("Launching %r." % final_filename)
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=final_filename,
clean_path=Options.shallClearPythonPathEnvironment(),
)
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option("--nuitka",
action="store",
dest="nuitka",
default=os.environ.get("NUITKA", ""))
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option("--code-diff",
action="store",
dest="diff_filename",
default="")
parser.add_option("--copy-source-to",
action="store",
dest="target_dir",
default="")
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit(
"Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % getPythonVersionString())
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
my_print("TEST_CASE_HASH='%s'" %
hashlib.md5(getFileContents(test_case, "rb")).hexdigest())
needs_2to3 = decideNeeds2to3(test_case)
if options.target_dir:
copyFile(test_case,
os.path.join(options.target_dir, os.path.basename(test_case)))
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir,
"Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir,
"Variant2_" + os.path.basename(test_case))
case_1_source, case_2_source = generateConstructCases(
getFileContents(test_case))
putTextFileContents(test_case_1, case_1_source)
putTextFileContents(test_case_2, case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = "0"
if nuitka:
nuitka_id = check_output("cd %s; git rev-parse HEAD" %
os.path.dirname(nuitka),
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
case_name = os.path.basename(test_case)
no_site = "Numpy" not in case_name
if nuitka:
nuitka_call = [
os.environ["PYTHON"],
nuitka,
"--quiet",
"--no-progress",
]
if no_site:
nuitka_call.append("--python-flag=-S")
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
nuitka_call.append(case_name)
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
copyFile(test_case_1, case_name)
check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix),
)
copyFile(test_case_2, os.path.basename(test_case))
check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix),
)
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(test_case_1.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(test_case_2.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
putTextFileContents(
options.diff_filename,
difflib.HtmlDiff().make_table(
getFileContentByLine(cpp_1),
getFileContentByLine(cpp_2),
"Construct",
"Baseline",
True,
),
)
nuitka_1 = runValgrind(
"Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_2 = runValgrind(
"Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call),
file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_call = [os.environ["PYTHON"], "-S", test_case_1]
if not no_site:
cpython_call.remove("-S")
cpython_1 = runValgrind(
"CPython construct",
"callgrind",
cpython_call,
include_startup=True,
)
cpython_call = [os.environ["PYTHON"], "-S", test_case_2]
if not no_site:
cpython_call.remove("-S")
cpython_2 = runValgrind(
"CPython baseline",
"callgrind",
cpython_call,
include_startup=True,
)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir,
binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit("""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux.""")
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
putTextFileContents(
apprun_filename,
contents="""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename),
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
copyFile(icon_paths[0], getResultBasepath() + extension)
putTextFileContents(
getResultBasepath() + ".desktop",
contents="""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
},
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = openTextFile(stdout_filename, "wb")
stderr_file = openTextFile(stderr_filename, "wb")
command = (
_getAppImageToolPath(for_operation=True,
assume_yes_for_downloads=assumeYesForDownloads()),
dist_dir,
"--comp",
getAppImageCompression(),
"-n",
onefile_output_filename,
)
stderr_file.write(b"Executed %r\n" % " ".join(command))
# Starting the process while locked, so file handles are not duplicated, we
# need fine grained control over process here, therefore we cannot use the
# Execution.executeProcess() function without making it too complex and not
# all Python versions allow using with, pylint: disable=consider-using-with
# pylint: disable
appimagetool_process = subprocess.Popen(
command,
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if result != 0:
# Useless result if there were errors, so now remove it.
deleteFile(onefile_output_filename, must_exist=False)
stderr = getFileContents(stderr_filename, mode="rb")
if b"Text file busy" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
if b"modprobe fuse" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because fuse kernel module was not loaded."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs '%s' and '%s'."
% (stdout_filename, stderr_filename))
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'."
% (onefile_output_filename, stdout_filename, stderr_filename))
deleteFile(stdout_filename, must_exist=True)
deleteFile(stderr_filename, must_exist=True)
postprocessing_logger.info("Completed onefile creation.")
