def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
# Many cases to deal with, pylint: disable=too-many-branches
if isWin32Windows():
sys.exit("Error, valgrind is not available on Windows.")
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
_stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess(
command)
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
copyFile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def considerExtraDlls(self, dist_dir, module):
module_name = module.getFullName()
if module_name == "zmq.libzmq" and isWin32Windows():
# TODO: Very strange thing for zmq on Windows, needs the .pyd file in wrong dir too. Have
# this done in a dedicated form somewhere.
copyFile(
os.path.join(dist_dir, "zmq\\libzmq.pyd"),
os.path.join(
dist_dir, "libzmq" + getSharedLibrarySuffix(preferred=True)
),
)
return NuitkaPluginBase.considerExtraDlls(
self, dist_dir=dist_dir, module=module
)
def considerExtraDlls(self, dist_dir, module):
"""Provide a tuple of names of binaries to be included.
Args:
dist_dir: the distribution folder
module: the module object needing the binaries
Returns:
tuple
"""
# TODO: This should no longer be here, as this API is obsolete, pylint: disable=unused-argument
for included_entry_point in self.getExtraDlls(module):
# Copy to the dist directory, which normally should not be a plugin task, but is for now.
makePath(os.path.dirname(included_entry_point.dest_path))
copyFile(included_entry_point.source_path,
included_entry_point.dest_path)
yield included_entry_point
def _compressFile(self, filename, use_cache):
upx_options = ["-q", "--no-progress"]
if os.path.basename(filename).startswith("vcruntime140"):
return
if use_cache:
if self.upx_binary_hash is None:
self.upx_binary_hash = getFileContentsHash(self.upx_binary,
as_string=False)
upx_hash = Hash()
upx_hash.updateFromBytes(self.upx_binary_hash)
upx_hash.updateFromValues(*upx_options)
upx_hash.updateFromFile(filename)
# TODO: Repeating pattern
upx_cache_dir = os.path.join(getCacheDir(), "upx")
makePath(upx_cache_dir)
upx_cache_filename = os.path.join(upx_cache_dir,
upx_hash.asHexDigest() + ".bin")
if os.path.exists(upx_cache_filename):
copyFile(upx_cache_filename, filename)
return
if use_cache:
self.info("Uncached file, compressing '%s' may take a while." %
os.path.basename(filename))
else:
self.info("Compressing '%s'." % filename)
command = [self.upx_binary] + upx_options + [filename]
executeToolChecked(
logger=self,
command=command,
absence_message=None,
stderr_filter=self._filterUpxError,
)
if use_cache:
copyFile(filename, upx_cache_filename)
def considerExtraDlls(dist_dir, module):
"""Ask plugins to provide extra DLLs.
Notes:
These will be of type nuitka.freezer.IncludedEntryPoints.IncludedEntryPoint
and currently there is a backward compatibility for old style plugins that do
provide tuples of 3 elements. But plugins are really supposed to provide the
stuff created from factory functions for that type.
"""
result = []
for plugin in getActivePlugins():
for extra_dll in plugin.considerExtraDlls(dist_dir, module):
# Backward compatibility with plugins not yet migrated to getExtraDlls usage.
if len(extra_dll) == 3:
extra_dll = makeDllEntryPointOld(
source_path=extra_dll[0],
dest_path=extra_dll[1],
package_name=extra_dll[2],
)
if not os.path.isfile(extra_dll.dest_path):
plugin.sysexit(
"Error, copied filename %r for module %r that is not a file."
% (extra_dll.dest_path, module.getFullName()))
else:
if not os.path.isfile(extra_dll.source_path):
plugin.sysexit(
"Error, attempting to copy plugin determined filename %r for module %r that is not a file."
% (extra_dll.source_path, module.getFullName()))
makePath(os.path.dirname(extra_dll.dest_path))
copyFile(extra_dll.source_path, extra_dll.dest_path)
if extra_dll.executable:
addFileExecutablePermission(extra_dll.dest_path)
result.append(extra_dll)
return result
def cleanupTarfileForDebian(filename, new_name):
"""Remove files that shouldn't be in Debian.
The inline copies should definitely not be there. Also remove the
PDF files.
"""
copyFile(filename, new_name)
check_call(["gunzip", new_name])
check_call(
[
"tar",
"--wildcards",
"--delete",
"--file",
new_name[:-3],
"Nuitka*/*.pdf",
"Nuitka*/build/inline_copy",
],
)
check_call(["gzip", "-9", "-n", new_name[:-3]])
def _compressFile(self, filename, use_cache):
upx_options = ["-q", "--no-progress"]
if use_cache:
if self.upx_binary_hash is None:
self.upx_binary_hash = getFileContentsHash(self.upx_binary,
as_string=False)
upx_hash = Hash()
upx_hash.updateFromBytes(self.upx_binary_hash)
upx_hash.updateFromValues(*upx_options)
upx_hash.updateFromFile(filename)
# TODO: Repeating pattern
upx_cache_dir = os.path.join(getCacheDir(), "upx")
makePath(upx_cache_dir)
upx_cache_filename = os.path.join(upx_cache_dir,
upx_hash.asHexDigest() + ".bin")
if os.path.exists(upx_cache_filename):
copyFile(upx_cache_filename, filename)
return
if use_cache:
self.info("Uncached file, compressing '%s' may take a while." %
os.path.basename(filename))
else:
self.info("Compressing '%s'." % filename)
check_call(
[self.upx_binary] + upx_options + [filename],
stdout=getNullOutput(),
shell=False,
)
if use_cache:
copyFile(filename, upx_cache_filename)
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir,
binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit("""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux.""")
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
putTextFileContents(
apprun_filename,
contents="""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename),
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
copyFile(icon_paths[0], getResultBasepath() + extension)
putTextFileContents(
getResultBasepath() + ".desktop",
contents="""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
},
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = openTextFile(stdout_filename, "wb")
stderr_file = openTextFile(stderr_filename, "wb")
command = (
_getAppImageToolPath(for_operation=True,
assume_yes_for_downloads=assumeYesForDownloads()),
dist_dir,
"--comp",
getAppImageCompression(),
"-n",
onefile_output_filename,
)
stderr_file.write(b"Executed %r\n" % " ".join(command))
# Starting the process while locked, so file handles are not duplicated, we
# need fine grained control over process here, therefore we cannot use the
# Execution.executeProcess() function without making it too complex and not
# all Python versions allow using with, pylint: disable=consider-using-with
# pylint: disable
appimagetool_process = subprocess.Popen(
command,
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if result != 0:
# Useless result if there were errors, so now remove it.
deleteFile(onefile_output_filename, must_exist=False)
stderr = getFileContents(stderr_filename, mode="rb")
if b"Text file busy" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
if b"modprobe fuse" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because fuse kernel module was not loaded."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs '%s' and '%s'."
% (stdout_filename, stderr_filename))
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'."
% (onefile_output_filename, stdout_filename, stderr_filename))
deleteFile(stdout_filename, must_exist=True)
deleteFile(stderr_filename, must_exist=True)
postprocessing_logger.info("Completed onefile creation.")
def main():
# Complex stuff, not broken down yet
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
parser = OptionParser()
parser.add_option("--nuitka",
action="store",
dest="nuitka",
default=os.environ.get("NUITKA", ""))
parser.add_option(
"--cpython",
action="store",
dest="cpython",
default=os.environ.get("PYTHON", sys.executable),
)
parser.add_option("--code-diff",
action="store",
dest="diff_filename",
default="")
parser.add_option("--copy-source-to",
action="store",
dest="target_dir",
default="")
options, positional_args = parser.parse_args()
if len(positional_args) != 1:
sys.exit(
"Error, need to give test case file name as positional argument.")
test_case = positional_args[0]
if os.path.exists(test_case):
test_case = os.path.abspath(test_case)
if options.cpython == "no":
options.cpython = ""
nuitka = options.nuitka
if os.path.exists(nuitka):
nuitka = os.path.abspath(nuitka)
elif nuitka:
sys.exit("Error, nuitka binary '%s' not found." % nuitka)
setup(silent=True, go_main=False)
assert os.path.exists(test_case), (test_case, os.getcwd())
my_print("PYTHON='%s'" % getPythonVersionString())
my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"])
my_print("TEST_CASE_HASH='%s'" %
hashlib.md5(getFileContents(test_case, "rb")).hexdigest())
needs_2to3 = decideNeeds2to3(test_case)
if options.target_dir:
copyFile(test_case,
os.path.join(options.target_dir, os.path.basename(test_case)))
# First produce two variants.
temp_dir = getTempDir()
test_case_1 = os.path.join(temp_dir,
"Variant1_" + os.path.basename(test_case))
test_case_2 = os.path.join(temp_dir,
"Variant2_" + os.path.basename(test_case))
case_1_source, case_2_source = generateConstructCases(
getFileContents(test_case))
putTextFileContents(test_case_1, case_1_source)
putTextFileContents(test_case_2, case_2_source)
if needs_2to3:
test_case_1, _needs_delete = convertUsing2to3(test_case_1)
test_case_2, _needs_delete = convertUsing2to3(test_case_2)
os.environ["PYTHONHASHSEED"] = "0"
if nuitka:
nuitka_id = check_output("cd %s; git rev-parse HEAD" %
os.path.dirname(nuitka),
shell=True)
nuitka_id = nuitka_id.strip()
if sys.version_info > (3, ):
nuitka_id = nuitka_id.decode()
my_print("NUITKA_COMMIT='%s'" % nuitka_id)
os.chdir(getTempDir())
case_name = os.path.basename(test_case)
no_site = "Numpy" not in case_name
if nuitka:
nuitka_call = [
os.environ["PYTHON"],
nuitka,
"--quiet",
"--no-progress",
]
if no_site:
nuitka_call.append("--python-flag=-S")
nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split())
nuitka_call.append(case_name)
# We want to compile under the same filename to minimize differences, and
# then copy the resulting files afterwards.
copyFile(test_case_1, case_name)
check_call(nuitka_call)
if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")):
exe_suffix = ".exe"
else:
exe_suffix = ".bin"
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_1).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_1).replace(".py", exe_suffix),
)
copyFile(test_case_2, os.path.basename(test_case))
check_call(nuitka_call)
os.rename(
os.path.basename(test_case).replace(".py", ".build"),
os.path.basename(test_case_2).replace(".py", ".build"),
)
os.rename(
os.path.basename(test_case).replace(".py", exe_suffix),
os.path.basename(test_case_2).replace(".py", exe_suffix),
)
if options.diff_filename:
suffixes = [".c", ".cpp"]
for suffix in suffixes:
cpp_1 = os.path.join(test_case_1.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_1):
break
else:
assert False
for suffix in suffixes:
cpp_2 = os.path.join(test_case_2.replace(".py", ".build"),
"module.__main__" + suffix)
if os.path.exists(cpp_2):
break
else:
assert False
import difflib
putTextFileContents(
options.diff_filename,
difflib.HtmlDiff().make_table(
getFileContentByLine(cpp_1),
getFileContentByLine(cpp_2),
"Construct",
"Baseline",
True,
),
)
nuitka_1 = runValgrind(
"Nuitka construct",
"callgrind",
(test_case_1.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_2 = runValgrind(
"Nuitka baseline",
"callgrind",
(test_case_2.replace(".py", exe_suffix), ),
include_startup=True,
)
nuitka_diff = nuitka_1 - nuitka_2
my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call),
file=sys.stderr)
my_print("NUITKA_RAW=%s" % nuitka_1)
my_print("NUITKA_BASE=%s" % nuitka_2)
my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff)
if options.cpython:
cpython_call = [os.environ["PYTHON"], "-S", test_case_1]
if not no_site:
cpython_call.remove("-S")
cpython_1 = runValgrind(
"CPython construct",
"callgrind",
cpython_call,
include_startup=True,
)
cpython_call = [os.environ["PYTHON"], "-S", test_case_2]
if not no_site:
cpython_call.remove("-S")
cpython_2 = runValgrind(
"CPython baseline",
"callgrind",
cpython_call,
include_startup=True,
)
cpython_diff = cpython_1 - cpython_2
my_print("CPYTHON_RAW=%d" % cpython_1)
my_print("CPYTHON_BASE=%d" % cpython_2)
my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff)
if options.cpython and options.nuitka:
if nuitka_diff == 0:
nuitka_gain = float("inf")
else:
nuitka_gain = float(100 * cpython_diff) / nuitka_diff
my_print("NUITKA_GAIN=%.3f" % nuitka_gain)
my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1))
my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))