def _detectBinaryPathDLLsMacOS(original_dir, binary_filename, keep_unresolved): result = set() process = subprocess.Popen( args=["otool", "-L", binary_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _stderr = process.communicate() system_paths = (b"/usr/lib/", b"/System/Library/Frameworks/") for line in stdout.split(b"\n")[1:]: if not line: continue filename = line.split(b" (")[0].strip() stop = False for w in system_paths: if filename.startswith(w): stop = True break if not stop: if python_version >= 0x300: filename = filename.decode("utf-8") # print("adding", filename) result.add(filename) resolved_result = _resolveBinaryPathDLLsMacOS(original_dir, binary_filename, result, keep_unresolved) return resolved_result
def _detectBinaryRPathsMacOS(original_dir, binary_filename): result = set() process = subprocess.Popen( args=["otool", "-l", binary_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _stderr = process.communicate() lines = stdout.split(b"\n") for i, o in enumerate(lines): if o.endswith(b"cmd LC_RPATH"): line = lines[i + 2] if python_version >= 0x300: line = line.decode("utf-8") line = line.split("path ")[1] line = line.split(" (offset")[0] if line.startswith("@loader_path"): line = os.path.join(original_dir, line[13:]) elif line.startswith("@executable_path"): continue result.add(line) return result
def getCompilerArch(mingw_mode, msvc_mode, the_cc_name, compiler_path): if mingw_mode: if compiler_path not in _compiler_arch: _compiler_arch[compiler_path] = _getBinaryArch( binary=compiler_path, mingw_mode=mingw_mode) elif msvc_mode: cmdline = [compiler_path] if "-cl" in the_cc_name: cmdline.append("--version") proc = subprocess.Popen( cmdline, stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, ) # The cl.exe without further args will give error output indicating # arch, while clang outputs in stdout. stdout, stderr = proc.communicate() _rv = proc.wait() if b"x86" in stderr or b"i686" in stdout: _compiler_arch[compiler_path] = "pei-i386" elif b"x64" in stderr or b"x86_64" in stdout: _compiler_arch[compiler_path] = "pei-x86-64" else: assert False, (stdout, stderr) else: assert False, compiler_path return _compiler_arch[compiler_path]
def setupCacheHashSalt(test_code_path): assert os.path.exists(test_code_path) if os.path.exists(os.path.join(test_code_path, ".git")): git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path] process = subprocess.Popen( args=git_cmd, stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout_git, stderr_git = process.communicate() assert process.returncode == 0, stderr_git salt_value = hashlib.md5(stdout_git) else: salt_value = hashlib.md5() for filename in getFileList(test_code_path): if filename.endswith(".py"): salt_value.update(getFileContents(filename, mode="rb")) os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest()
def _getBinaryArch(binary, mingw_mode): if "linux" in sys.platform or mingw_mode: assert os.path.exists(binary), binary command = ["objdump", "-f", binary] try: proc = subprocess.Popen( command, stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, ) except OSError: return None data, _err = proc.communicate() rv = proc.wait() if rv != 0: return None if str is not bytes: data = decodeData(data) for line in data.splitlines(): if " file format " in line: return line.split(" file format ")[-1] else: # TODO: Missing for macOS, FreeBSD, other Linux return None
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs): dwp_filename = binary_filename + ".dwp" output_filename = binary_filename + ".depends" # User query should only happen once if at all. with withFileLock( "Finding out dependency walker path and creating DWP file for %s" % binary_filename): depends_exe = getDependsExePath() # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. with open(dwp_filename, "w") as dwp_file: dwp_file.write( """\ %(scan_dirs)s SxS """ % { "scan_dirs": "\n".join("UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs) }) # Starting the process while locked, so file handles are not duplicated. depends_exe_process = subprocess.Popen( ( depends_exe, "-c", "-ot%s" % output_filename, "-d:%s" % dwp_filename, "-f1", "-pa1", "-ps1", binary_filename, ), stdin=getNullInput(), cwd=getExternalUsePath(os.getcwd()), ) # TODO: Exit code should be checked. depends_exe_process.wait() if not os.path.exists(output_filename): inclusion_logger.sysexit( "Error, depends.exe failed to produce expected output.") # Opening the result under lock, so it is not getting locked by new processes. # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. result = _parseDependsExeOutput(output_filename) deleteFile(output_filename, must_exist=True) deleteFile(dwp_filename, must_exist=True) return result
def _executePylint(filenames, pylint_options, extra_options): # This is kind of a singleton module, pylint: disable=global-statement global our_exit_code command = ([os.environ["PYTHON"], "-m", "pylint"] + pylint_options + extra_options + filenames) process = subprocess.Popen( args=command, stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, ) stdout, stderr = process.communicate() exit_code = process.returncode if exit_code == -11: sys.exit("Error, segfault from pylint.") stdout = _cleanupPylintOutput(stdout) stderr = _cleanupPylintOutput(stderr) if stderr: our_exit_code = 1 for line in stderr: my_print(line) if stdout: # If we filtered everything away, remove the leading file name reports. while stdout and stdout[-1].startswith("******"): del stdout[-1] for line in stdout: my_print(line) if stdout: our_exit_code = 1 sys.stdout.flush()
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( "$ORIGIN", os.path.dirname(sys.executable)) ld_library_path = OrderedSet() if _detected_python_rpath: ld_library_path.add(_detected_python_rpath) ld_library_path.update(getPackageSpecificDLLDirectories(package_name)) if original_dir is not None: ld_library_path.add(original_dir) # ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",))) with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path): process = subprocess.Popen( args=["ldd", dll_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() stderr = b"\n".join( line for line in stderr.splitlines() if not line.startswith( b"ldd: warning: you do not have execution permission for")) inclusion_logger.debug("ldd output for %s is:\n%s" % (dll_filename, stdout)) if stderr: inclusion_logger.debug("ldd error for %s is:\n%s" % (dll_filename, stderr)) for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 0x300: filename = filename.decode("utf-8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Do not include kernel / glibc specific libraries. This list has been # assembled by looking what are the most common .so files provided by # glibc packages from ArchLinux, Debian Stretch and CentOS. # # Online sources: # - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html # - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html # - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html # - https://packages.debian.org/stretch/amd64/libc6/filelist # # Note: This list may still be incomplete. Some additional libraries # might be provided by glibc - it may vary between the package versions # and between Linux distros. It might or might not be a problem in the # future, but it should be enough for now. if os.path.basename(filename).startswith(( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", )): continue result.add(filename) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsPosix( dll_filename=sub_dll_filename, package_name=package_name, original_dir=original_dir, )) return sub_result
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command import tempfile tmp_file, tmp_filename = tempfile.mkstemp() try: if python_version >= 0x300: command = command.encode("utf8") os.write(tmp_file, command) os.close(tmp_file) process = subprocess.Popen( args=[sys.executable, "-s", "-S", "-v", tmp_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=dict(os.environ, PYTHONIOENCODING="utf_8"), ) _stdout, stderr = process.communicate() finally: os.unlink(tmp_filename) # Don't let errors here go unnoticed. if process.returncode != 0: general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): # print(line) parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf-8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300: if module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "shlib", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "shlib", filename)) for module_name, _prio, kind, filename in sorted(detections): if kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "shlib": _detectedShlibFile(filename=filename, module_name=module_name) else: assert False, kind return result
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ if not locateDLL("fuse"): postprocessing_logger.sysexit( """\ Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed for onefile creation to work on Linux.""" ) # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") with open(apprun_filename, "w") as output_file: output_file.write( """\ #!/bin/bash exec -a $ARGV0 $APPDIR/%s $@""" % os.path.basename(binary_filename) ) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() assert icon_paths extension = os.path.splitext(icon_paths[0])[1].lower() shutil.copyfile(icon_paths[0], getResultBasepath() + extension) with open(getResultBasepath() + ".desktop", "w") as output_file: output_file.write( """\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), } ) postprocessing_logger.info( "Creating single file from dist folder, this may take a while." ) stdout_filename = binary_filename + ".appimage.stdout.txt" stderr_filename = binary_filename + ".appimage.stderr.txt" stdout_file = open(stdout_filename, "wb") stderr_file = open(stderr_filename, "wb") # Starting the process while locked, so file handles are not duplicated. appimagetool_process = subprocess.Popen( ( _getAppImageToolPath( for_operation=True, assume_yes_for_downloads=assumeYesForDownloads() ), dist_dir, "--comp", "xz", "-n", onefile_output_filename, ), shell=False, stdin=getNullInput(), stdout=stdout_file, stderr=stderr_file, ) result = appimagetool_process.wait() stdout_file.close() stderr_file.close() if not os.path.exists(onefile_output_filename): postprocessing_logger.sysexit( "Error, expected output file %r not created by AppImage, check its outputs %r and %r." % (onefile_output_filename, stdout_filename, stderr_filename) ) if result != 0: # Useless now. os.unlink(onefile_output_filename) if b"Text file busy" in getFileContents(stderr_filename, mode="rb"): postprocessing_logger.sysexit( "Error, error exit from AppImage because target file is locked." ) postprocessing_logger.sysexit( "Error, error exit from AppImage, check its outputs %r and %r." % (stdout_filename, stderr_filename) ) os.unlink(stdout_filename) os.unlink(stderr_filename) postprocessing_logger.info("Completed onefile creation.")
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ if not locateDLL("fuse"): postprocessing_logger.sysexit("""\ Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed for onefile creation to work on Linux.""") # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") putTextFileContents( apprun_filename, contents="""\ #!/bin/bash exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename), ) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() assert icon_paths extension = os.path.splitext(icon_paths[0])[1].lower() copyFile(icon_paths[0], getResultBasepath() + extension) putTextFileContents( getResultBasepath() + ".desktop", contents="""\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), }, ) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") stdout_filename = binary_filename + ".appimage.stdout.txt" stderr_filename = binary_filename + ".appimage.stderr.txt" stdout_file = openTextFile(stdout_filename, "wb") stderr_file = openTextFile(stderr_filename, "wb") command = ( _getAppImageToolPath(for_operation=True, assume_yes_for_downloads=assumeYesForDownloads()), dist_dir, "--comp", getAppImageCompression(), "-n", onefile_output_filename, ) stderr_file.write(b"Executed %r\n" % " ".join(command)) # Starting the process while locked, so file handles are not duplicated, we # need fine grained control over process here, therefore we cannot use the # Execution.executeProcess() function without making it too complex and not # all Python versions allow using with, pylint: disable=consider-using-with # pylint: disable appimagetool_process = subprocess.Popen( command, shell=False, stdin=getNullInput(), stdout=stdout_file, stderr=stderr_file, ) result = appimagetool_process.wait() stdout_file.close() stderr_file.close() if result != 0: # Useless result if there were errors, so now remove it. deleteFile(onefile_output_filename, must_exist=False) stderr = getFileContents(stderr_filename, mode="rb") if b"Text file busy" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because target file is locked." ) if b"modprobe fuse" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because fuse kernel module was not loaded." ) postprocessing_logger.sysexit( "Error, error exit from AppImage, check its outputs '%s' and '%s'." % (stdout_filename, stderr_filename)) if not os.path.exists(onefile_output_filename): postprocessing_logger.sysexit( "Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'." % (onefile_output_filename, stdout_filename, stderr_filename)) deleteFile(stdout_filename, must_exist=True) deleteFile(stderr_filename, must_exist=True) postprocessing_logger.info("Completed onefile creation.")
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ if not locateDLL("fuse"): postprocessing_logger.sysexit( "Error, the fuse library (libfuse.so.x) must be installed for onefile to work on Linux." ) # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") with open(apprun_filename, "w") as output_file: output_file.write("""\ #!/bin/sh exec $APPDIR/%s $@""" % os.path.basename(binary_filename)) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() assert icon_paths extension = os.path.splitext(icon_paths[0])[1].lower() shutil.copyfile(icon_paths[0], getResultBasepath() + extension) with open(getResultBasepath() + ".desktop", "w") as output_file: output_file.write( """\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), }) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") # Starting the process while locked, so file handles are not duplicated. appimagetool_process = subprocess.Popen( ( getAppImageToolPath(), dist_dir, "--comp", "xz", "-n", onefile_output_filename, ), shell=False, stdin=getNullInput(), stderr=getNullOutput(), stdout=getNullOutput(), ) # TODO: Exit code should be checked. result = appimagetool_process.wait() if not os.path.exists(onefile_output_filename): postprocessing_logger.sysexit( "Error, expected output file %s not created by AppImage." % onefile_output_filename) postprocessing_logger.info("Completed onefile creation.") assert result == 0, result
def _callDebchange(*args): args = ["debchange"] + list(args) os.environ["EDITOR"] = "" check_call(args, stdin=getNullInput())