def _cleanupClangFormat(filename): """ Call clang-format on a given filename to format C code. Args: filename: What file to re-format. """ # Using global here, as this is really a singleton, in # the form of a module, pylint: disable=global-statement global warned_clang_format clang_format_path = getExecutablePath("clang-format-7") # Extra ball on Windows, check default installation PATH too. if not clang_format_path and getOS() == "Windows": with withEnvironmentPathAdded( "PATH", r"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\Llvm\8.0.0\bin", ): with withEnvironmentPathAdded("PATH", r"C:\Program Files\LLVM\bin"): clang_format_path = getExecutablePath("clang-format") if clang_format_path: subprocess.call([ clang_format_path, "-i", "-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}", filename, ]) else: if not warned_clang_format: warning("Need to install LLVM for C files format.") warned_clang_format = True
def _cleanupClangFormat(filename): # Using global here, as this is really a singleton, in # the form of a module, pylint: disable=global-statement global warned_clang_format clang_format_path = getExecutablePath("clang-format") # Extra ball on Windows, check default installation PATH too. if not clang_format_path and getOS() == "Windows": with withEnvironmentPathAdded("PATH", r"C:\Program Files\LLVM\bin"): clang_format_path = getExecutablePath("clang-format") if clang_format_path: subprocess.call( [ clang_format_path, "-i", "-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}", filename, ] ) else: if not warned_clang_format: warning("Need to install LLVM for C files format.") warned_clang_format = True
def _detectBinaryPathDLLsLinuxBSD(dll_filename): # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None: _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( b"$ORIGIN", os.path.dirname(sys.executable).encode("utf-8")) with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath): process = subprocess.Popen(args=["ldd", dll_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _stderr = process.communicate() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 300: filename = filename.decode("utf-8") # Sometimes might use stuff not found. if filename == "not found": continue # Do not include kernel specific libraries. if os.path.basename(filename).startswith( ("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")): continue result.add(filename) return result
def runCodespell(filenames, verbose, write): if verbose: my_print("Consider", " ".join(filenames)) command = [ "codespell", "-f", "-I", os.path.join( os.path.dirname(__file__), "..", "..", "..", "..", "misc/codespell-ignore.txt", ), ] if write: command.append("-w") command += filenames if os.name == "nt": extra_path = os.path.join(sys.prefix, "Scripts") else: extra_path = None with withEnvironmentPathAdded("PATH", extra_path): result = subprocess.call(command) if result == 0: for filename in filenames: if areSamePaths(__file__, filename): continue contents = getFileContents(filename) old_contents = contents for word, replacement in replacements: contents = contents.replace(word, replacement) contents = contents.replace(word.title(), replacement.title()) if old_contents != contents: putTextFileContents(filename, contents) cleanupWindowsNewlines(filename) if verbose: if result != 0: my_print("FAILED.") else: my_print("OK.") return result == 0
def getOtoolDependencyOutput(filename, package_specific_dirs): key = filename, tuple(package_specific_dirs), os.environ.get( "DYLD_LIBRARY_PATH") if key not in _otool_dependency_cache: with withEnvironmentPathAdded("DYLD_LIBRARY_PATH", *package_specific_dirs): _otool_dependency_cache[key] = executeToolChecked( logger=inclusion_logger, command=("otool", "-L", filename), absence_message=otool_usage, ) return _otool_dependency_cache[key]
def _getPythonBinaryCall(binary_name): if binary_name not in _binary_calls: messages = [] # Try running Python installation. try: __import__(binary_name) except ImportError: pass else: call = [sys.executable, "-m", binary_name] ok, message = _checkRequiredVersion(binary_name, call) if ok: _binary_calls[binary_name] = call return _binary_calls[binary_name] else: messages.append(message) with withEnvironmentPathAdded( "PATH", os.path.join(sys.prefix, "Scripts"), os.path.join(sys.prefix, "bin") ): binary_path = getExecutablePath(binary_name) if binary_path: call = [binary_path] ok, message = _checkRequiredVersion(binary_name, call) if ok: _binary_calls[binary_name] = call return _binary_calls[binary_name] else: messages.append(message) if messages: my_print("ERROR") for message in messages: my_print(message, style="red") sys.exit( "Error, cannot find %r version %r, not installed or wrong version for this Python?" % (binary_name, _getRequiredVersion(binary_name)) ) return _binary_calls[binary_name]
def codespell(filenames, verbose, write): if verbose: my_print("Consider", " ".join(filenames)) command = ["codespell"] if write: command.append("-w") command += filenames if os.name == "nt": extra_path = os.path.join(sys.prefix, "Scripts") else: extra_path = None with withEnvironmentPathAdded("PATH", extra_path): result = subprocess.call(command) if verbose: if result != 0: my_print("FAILED.") else: my_print("OK.") return result == 0
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( "$ORIGIN", os.path.dirname(sys.executable)) ld_library_path = OrderedSet() if _detected_python_rpath: ld_library_path.add(_detected_python_rpath) ld_library_path.update(getPackageSpecificDLLDirectories(package_name)) if original_dir is not None: ld_library_path.add(original_dir) # ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",))) with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path): process = subprocess.Popen( args=["ldd", dll_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = process.communicate() stderr = b"\n".join( line for line in stderr.splitlines() if not line.startswith( b"ldd: warning: you do not have execution permission for")) inclusion_logger.debug("ldd output for %s is:\n%s" % (dll_filename, stdout)) if stderr: inclusion_logger.debug("ldd error for %s is:\n%s" % (dll_filename, stderr)) for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 0x300: filename = filename.decode("utf-8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Do not include kernel / glibc specific libraries. This list has been # assembled by looking what are the most common .so files provided by # glibc packages from ArchLinux, Debian Stretch and CentOS. # # Online sources: # - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html # - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html # - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html # - https://packages.debian.org/stretch/amd64/libc6/filelist # # Note: This list may still be incomplete. Some additional libraries # might be provided by glibc - it may vary between the package versions # and between Linux distros. It might or might not be a problem in the # future, but it should be enough for now. if os.path.basename(filename).startswith(( "ld-linux-x86-64.so", "libc.so.", "libpthread.so.", "libm.so.", "libdl.so.", "libBrokenLocale.so.", "libSegFault.so", "libanl.so.", "libcidn.so.", "libcrypt.so.", "libmemusage.so", "libmvec.so.", "libnsl.so.", "libnss_compat.so.", "libnss_db.so.", "libnss_dns.so.", "libnss_files.so.", "libnss_hesiod.so.", "libnss_nis.so.", "libnss_nisplus.so.", "libpcprofile.so", "libresolv.so.", "librt.so.", "libthread_db-1.0.so", "libthread_db.so.", "libutil.so.", )): continue result.add(filename) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsPosix( dll_filename=sub_dll_filename, package_name=package_name, original_dir=original_dir, )) return sub_result
def _detectBinaryPathDLLsLinuxBSD(dll_filename): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. result = set() # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None: _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( b"$ORIGIN", os.path.dirname(sys.executable).encode("utf-8")) with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath): process = subprocess.Popen(args=["ldd", dll_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _stderr = process.communicate() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 300: filename = filename.decode("utf-8") # Sometimes might use stuff not found. if filename == "not found": continue # Do not include kernel specific libraries. if os.path.basename(filename).startswith( ("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")): continue result.add(filename) # Allow plugins to prevent inclusion. blocked = Plugins.removeDllDependencies(dll_filename=dll_filename, dll_filenames=result) for to_remove in blocked: result.discard(to_remove) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsLinuxBSD(sub_dll_filename)) return sub_result
def main(): goHome() parser = OptionParser() parser.add_option( "--upload", action="store_true", dest="upload", default=False, help="""\ Upload to http://nuitka.net/apidoc requires access rights and is done by the official servers automatically only. Without this, create the local html folder only. Default is %default.""", ) options, _positional_args = parser.parse_args() shutil.rmtree("html", ignore_errors=True) doxygen_path = getExecutablePath("doxygen") # Extra ball on Windows, check default installation PATH too. if not doxygen_path and getOS() == "Windows": with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"): doxygen_path = getExecutablePath("doxygen") if not doxygen_path: sys.exit( "Error, need to install Doxygen and add it to PATH for this to work." ) try: import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable except ImportError: sys.exit("Error, needs to install doxypypy into this Python.") with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file: doxy_config = getFileContents("doc/Doxyfile.template") with withTemporaryFile( suffix=".bat" if getOS() == "Windows" else ".sh", delete=False) as doxy_batch_file: if getOS() == "Windows": doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" % sys.executable) else: doxy_batch_file.write( "#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" % sys.executable) doxy_batch_filename = doxy_batch_file.name doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename) doxy_file.write(doxy_config) doxy_filename = doxy_file.name print("Running doxygen:") try: subprocess.check_call([doxygen_path, doxy_filename]) finally: os.unlink(doxy_filename) os.unlink(doxy_batch_filename) # Update the repository on the web site. if options.upload: assert (os.system( "rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/" ) == 0) print("Finished.")
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-branches if ldd_result_cache.get(dll_filename): return ldd_result_cache[dll_filename] # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. # This is the rpath of the Python binary, which will be effective when # loading the other DLLs too. This happens at least for Python installs # on Travis. pylint: disable=global-statement global _detected_python_rpath if _detected_python_rpath is None and not Utils.isPosixWindows(): _detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False if _detected_python_rpath: _detected_python_rpath = _detected_python_rpath.replace( "$ORIGIN", os.path.dirname(sys.executable)) # TODO: Actually would be better to pass it as env to the created process instead. with withEnvironmentPathAdded( "LD_LIBRARY_PATH", *_getLdLibraryPath( package_name=package_name, python_rpath=_detected_python_rpath, original_dir=original_dir, )): # TODO: Check exit code, should never fail. stdout, stderr, _exit_code = executeProcess(command=("ldd", dll_filename)) stderr = b"\n".join( line for line in stderr.splitlines() if not line.startswith( b"ldd: warning: you do not have execution permission for")) inclusion_logger.debug("ldd output for %s is:\n%s" % (dll_filename, stdout)) if stderr: inclusion_logger.debug("ldd error for %s is:\n%s" % (dll_filename, stderr)) result = set() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(") - 1] else: filename = part if not filename: continue if python_version >= 0x300: filename = filename.decode("utf8") # Sometimes might use stuff not found or supplied by ldd itself. if filename in ("not found", "ldd"): continue # Normalize, sometimes the DLLs produce "something/../", this has # been seen with Qt at least. filename = os.path.normpath(filename) # Do not include kernel DLLs on the ignore list. filename_base = os.path.basename(filename) if any(filename_base == entry or filename_base.startswith(entry + ".") for entry in _linux_dll_ignore_list): continue result.add(filename) ldd_result_cache[dll_filename] = result sub_result = set(result) for sub_dll_filename in result: sub_result = sub_result.union( _detectBinaryPathDLLsPosix( dll_filename=sub_dll_filename, package_name=package_name, original_dir=original_dir, )) return sub_result