def runValgrind(descr, tool, args, include_startup, save_logfilename=None): # Many cases to deal with, pylint: disable=too-many-branches if isWin32Windows(): sys.exit("Error, valgrind is not available on Windows.") if descr: my_print(descr, tool, file=sys.stderr, end="... ") with withTemporaryFile() as log_file: log_filename = log_file.name command = ["valgrind", "-q"] if tool == "callgrind": command += ("--tool=callgrind", "--callgrind-out-file=%s" % log_filename) elif tool == "massif": command += ("--tool=massif", "--massif-out-file=%s" % log_filename) else: sys.exit("Error, no support for tool '%s' yet." % tool) # Do not count things before main module starts its work. if not include_startup: command += ( "--zero-before=init__main__()", "--zero-before=init__main__", "--zero-before=PyInit___main__", "--zero-before=PyInit___main__()", ) command.extend(args) _stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess( command) assert exit_valgrind == 0, stderr_valgrind if descr: my_print("OK", file=sys.stderr) if save_logfilename is not None: copyFile(log_filename, save_logfilename) max_mem = None for line in getFileContentByLine(log_filename): if tool == "callgrind" and line.startswith("summary:"): return int(line.split()[1]) elif tool == "massif" and line.startswith("mem_heap_B="): mem = int(line.split("=")[1]) if max_mem is None: max_mem = 0 max_mem = max(mem, max_mem) if tool == "massif" and max_mem is not None: return max_mem sys.exit("Error, didn't parse Valgrind log file successfully.")
def considerExtraDlls(self, dist_dir, module): module_name = module.getFullName() if module_name == "zmq.libzmq" and isWin32Windows(): # TODO: Very strange thing for zmq on Windows, needs the .pyd file in wrong dir too. Have # this done in a dedicated form somewhere. copyFile( os.path.join(dist_dir, "zmq\\libzmq.pyd"), os.path.join( dist_dir, "libzmq" + getSharedLibrarySuffix(preferred=True) ), ) return NuitkaPluginBase.considerExtraDlls( self, dist_dir=dist_dir, module=module )
def considerExtraDlls(self, dist_dir, module): """Provide a tuple of names of binaries to be included. Args: dist_dir: the distribution folder module: the module object needing the binaries Returns: tuple """ # TODO: This should no longer be here, as this API is obsolete, pylint: disable=unused-argument for included_entry_point in self.getExtraDlls(module): # Copy to the dist directory, which normally should not be a plugin task, but is for now. makePath(os.path.dirname(included_entry_point.dest_path)) copyFile(included_entry_point.source_path, included_entry_point.dest_path) yield included_entry_point
def _compressFile(self, filename, use_cache): upx_options = ["-q", "--no-progress"] if os.path.basename(filename).startswith("vcruntime140"): return if use_cache: if self.upx_binary_hash is None: self.upx_binary_hash = getFileContentsHash(self.upx_binary, as_string=False) upx_hash = Hash() upx_hash.updateFromBytes(self.upx_binary_hash) upx_hash.updateFromValues(*upx_options) upx_hash.updateFromFile(filename) # TODO: Repeating pattern upx_cache_dir = os.path.join(getCacheDir(), "upx") makePath(upx_cache_dir) upx_cache_filename = os.path.join(upx_cache_dir, upx_hash.asHexDigest() + ".bin") if os.path.exists(upx_cache_filename): copyFile(upx_cache_filename, filename) return if use_cache: self.info("Uncached file, compressing '%s' may take a while." % os.path.basename(filename)) else: self.info("Compressing '%s'." % filename) command = [self.upx_binary] + upx_options + [filename] executeToolChecked( logger=self, command=command, absence_message=None, stderr_filter=self._filterUpxError, ) if use_cache: copyFile(filename, upx_cache_filename)
def considerExtraDlls(dist_dir, module): """Ask plugins to provide extra DLLs. Notes: These will be of type nuitka.freezer.IncludedEntryPoints.IncludedEntryPoint and currently there is a backward compatibility for old style plugins that do provide tuples of 3 elements. But plugins are really supposed to provide the stuff created from factory functions for that type. """ result = [] for plugin in getActivePlugins(): for extra_dll in plugin.considerExtraDlls(dist_dir, module): # Backward compatibility with plugins not yet migrated to getExtraDlls usage. if len(extra_dll) == 3: extra_dll = makeDllEntryPointOld( source_path=extra_dll[0], dest_path=extra_dll[1], package_name=extra_dll[2], ) if not os.path.isfile(extra_dll.dest_path): plugin.sysexit( "Error, copied filename %r for module %r that is not a file." % (extra_dll.dest_path, module.getFullName())) else: if not os.path.isfile(extra_dll.source_path): plugin.sysexit( "Error, attempting to copy plugin determined filename %r for module %r that is not a file." % (extra_dll.source_path, module.getFullName())) makePath(os.path.dirname(extra_dll.dest_path)) copyFile(extra_dll.source_path, extra_dll.dest_path) if extra_dll.executable: addFileExecutablePermission(extra_dll.dest_path) result.append(extra_dll) return result
def cleanupTarfileForDebian(filename, new_name): """Remove files that shouldn't be in Debian. The inline copies should definitely not be there. Also remove the PDF files. """ copyFile(filename, new_name) check_call(["gunzip", new_name]) check_call( [ "tar", "--wildcards", "--delete", "--file", new_name[:-3], "Nuitka*/*.pdf", "Nuitka*/build/inline_copy", ], ) check_call(["gzip", "-9", "-n", new_name[:-3]])
def _compressFile(self, filename, use_cache): upx_options = ["-q", "--no-progress"] if use_cache: if self.upx_binary_hash is None: self.upx_binary_hash = getFileContentsHash(self.upx_binary, as_string=False) upx_hash = Hash() upx_hash.updateFromBytes(self.upx_binary_hash) upx_hash.updateFromValues(*upx_options) upx_hash.updateFromFile(filename) # TODO: Repeating pattern upx_cache_dir = os.path.join(getCacheDir(), "upx") makePath(upx_cache_dir) upx_cache_filename = os.path.join(upx_cache_dir, upx_hash.asHexDigest() + ".bin") if os.path.exists(upx_cache_filename): copyFile(upx_cache_filename, filename) return if use_cache: self.info("Uncached file, compressing '%s' may take a while." % os.path.basename(filename)) else: self.info("Compressing '%s'." % filename) check_call( [self.upx_binary] + upx_options + [filename], stdout=getNullOutput(), shell=False, ) if use_cache: copyFile(filename, upx_cache_filename)
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ if not locateDLL("fuse"): postprocessing_logger.sysexit("""\ Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed for onefile creation to work on Linux.""") # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") putTextFileContents( apprun_filename, contents="""\ #!/bin/bash exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename), ) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() assert icon_paths extension = os.path.splitext(icon_paths[0])[1].lower() copyFile(icon_paths[0], getResultBasepath() + extension) putTextFileContents( getResultBasepath() + ".desktop", contents="""\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), }, ) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") stdout_filename = binary_filename + ".appimage.stdout.txt" stderr_filename = binary_filename + ".appimage.stderr.txt" stdout_file = openTextFile(stdout_filename, "wb") stderr_file = openTextFile(stderr_filename, "wb") command = ( _getAppImageToolPath(for_operation=True, assume_yes_for_downloads=assumeYesForDownloads()), dist_dir, "--comp", getAppImageCompression(), "-n", onefile_output_filename, ) stderr_file.write(b"Executed %r\n" % " ".join(command)) # Starting the process while locked, so file handles are not duplicated, we # need fine grained control over process here, therefore we cannot use the # Execution.executeProcess() function without making it too complex and not # all Python versions allow using with, pylint: disable=consider-using-with # pylint: disable appimagetool_process = subprocess.Popen( command, shell=False, stdin=getNullInput(), stdout=stdout_file, stderr=stderr_file, ) result = appimagetool_process.wait() stdout_file.close() stderr_file.close() if result != 0: # Useless result if there were errors, so now remove it. deleteFile(onefile_output_filename, must_exist=False) stderr = getFileContents(stderr_filename, mode="rb") if b"Text file busy" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because target file is locked." ) if b"modprobe fuse" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because fuse kernel module was not loaded." ) postprocessing_logger.sysexit( "Error, error exit from AppImage, check its outputs '%s' and '%s'." % (stdout_filename, stderr_filename)) if not os.path.exists(onefile_output_filename): postprocessing_logger.sysexit( "Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'." % (onefile_output_filename, stdout_filename, stderr_filename)) deleteFile(stdout_filename, must_exist=True) deleteFile(stderr_filename, must_exist=True) postprocessing_logger.info("Completed onefile creation.")
def main(): # Complex stuff, not broken down yet # pylint: disable=too-many-branches,too-many-locals,too-many-statements parser = OptionParser() parser.add_option("--nuitka", action="store", dest="nuitka", default=os.environ.get("NUITKA", "")) parser.add_option( "--cpython", action="store", dest="cpython", default=os.environ.get("PYTHON", sys.executable), ) parser.add_option("--code-diff", action="store", dest="diff_filename", default="") parser.add_option("--copy-source-to", action="store", dest="target_dir", default="") options, positional_args = parser.parse_args() if len(positional_args) != 1: sys.exit( "Error, need to give test case file name as positional argument.") test_case = positional_args[0] if os.path.exists(test_case): test_case = os.path.abspath(test_case) if options.cpython == "no": options.cpython = "" nuitka = options.nuitka if os.path.exists(nuitka): nuitka = os.path.abspath(nuitka) elif nuitka: sys.exit("Error, nuitka binary '%s' not found." % nuitka) setup(silent=True, go_main=False) assert os.path.exists(test_case), (test_case, os.getcwd()) my_print("PYTHON='%s'" % getPythonVersionString()) my_print("PYTHON_BINARY='%s'" % os.environ["PYTHON"]) my_print("TEST_CASE_HASH='%s'" % hashlib.md5(getFileContents(test_case, "rb")).hexdigest()) needs_2to3 = decideNeeds2to3(test_case) if options.target_dir: copyFile(test_case, os.path.join(options.target_dir, os.path.basename(test_case))) # First produce two variants. temp_dir = getTempDir() test_case_1 = os.path.join(temp_dir, "Variant1_" + os.path.basename(test_case)) test_case_2 = os.path.join(temp_dir, "Variant2_" + os.path.basename(test_case)) case_1_source, case_2_source = generateConstructCases( getFileContents(test_case)) putTextFileContents(test_case_1, case_1_source) putTextFileContents(test_case_2, case_2_source) if needs_2to3: test_case_1, _needs_delete = convertUsing2to3(test_case_1) test_case_2, _needs_delete = convertUsing2to3(test_case_2) os.environ["PYTHONHASHSEED"] = "0" if nuitka: nuitka_id = check_output("cd %s; git rev-parse HEAD" % os.path.dirname(nuitka), shell=True) nuitka_id = nuitka_id.strip() if sys.version_info > (3, ): nuitka_id = nuitka_id.decode() my_print("NUITKA_COMMIT='%s'" % nuitka_id) os.chdir(getTempDir()) case_name = os.path.basename(test_case) no_site = "Numpy" not in case_name if nuitka: nuitka_call = [ os.environ["PYTHON"], nuitka, "--quiet", "--no-progress", ] if no_site: nuitka_call.append("--python-flag=-S") nuitka_call.extend(os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()) nuitka_call.append(case_name) # We want to compile under the same filename to minimize differences, and # then copy the resulting files afterwards. copyFile(test_case_1, case_name) check_call(nuitka_call) if os.path.exists(os.path.basename(test_case).replace(".py", ".exe")): exe_suffix = ".exe" else: exe_suffix = ".bin" os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_1).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_1).replace(".py", exe_suffix), ) copyFile(test_case_2, os.path.basename(test_case)) check_call(nuitka_call) os.rename( os.path.basename(test_case).replace(".py", ".build"), os.path.basename(test_case_2).replace(".py", ".build"), ) os.rename( os.path.basename(test_case).replace(".py", exe_suffix), os.path.basename(test_case_2).replace(".py", exe_suffix), ) if options.diff_filename: suffixes = [".c", ".cpp"] for suffix in suffixes: cpp_1 = os.path.join(test_case_1.replace(".py", ".build"), "module.__main__" + suffix) if os.path.exists(cpp_1): break else: assert False for suffix in suffixes: cpp_2 = os.path.join(test_case_2.replace(".py", ".build"), "module.__main__" + suffix) if os.path.exists(cpp_2): break else: assert False import difflib putTextFileContents( options.diff_filename, difflib.HtmlDiff().make_table( getFileContentByLine(cpp_1), getFileContentByLine(cpp_2), "Construct", "Baseline", True, ), ) nuitka_1 = runValgrind( "Nuitka construct", "callgrind", (test_case_1.replace(".py", exe_suffix), ), include_startup=True, ) nuitka_2 = runValgrind( "Nuitka baseline", "callgrind", (test_case_2.replace(".py", exe_suffix), ), include_startup=True, ) nuitka_diff = nuitka_1 - nuitka_2 my_print("NUITKA_COMMAND='%s'" % " ".join(nuitka_call), file=sys.stderr) my_print("NUITKA_RAW=%s" % nuitka_1) my_print("NUITKA_BASE=%s" % nuitka_2) my_print("NUITKA_CONSTRUCT=%s" % nuitka_diff) if options.cpython: cpython_call = [os.environ["PYTHON"], "-S", test_case_1] if not no_site: cpython_call.remove("-S") cpython_1 = runValgrind( "CPython construct", "callgrind", cpython_call, include_startup=True, ) cpython_call = [os.environ["PYTHON"], "-S", test_case_2] if not no_site: cpython_call.remove("-S") cpython_2 = runValgrind( "CPython baseline", "callgrind", cpython_call, include_startup=True, ) cpython_diff = cpython_1 - cpython_2 my_print("CPYTHON_RAW=%d" % cpython_1) my_print("CPYTHON_BASE=%d" % cpython_2) my_print("CPYTHON_CONSTRUCT=%d" % cpython_diff) if options.cpython and options.nuitka: if nuitka_diff == 0: nuitka_gain = float("inf") else: nuitka_gain = float(100 * cpython_diff) / nuitka_diff my_print("NUITKA_GAIN=%.3f" % nuitka_gain) my_print("RAW_GAIN=%.3f" % (float(100 * cpython_1) / nuitka_1)) my_print("BASE_GAIN=%.3f" % (float(100 * cpython_2) / nuitka_2))