def _cleanupClangFormat(filename):
""" Call clang-format on a given filename to format C code.
Args:
filename: What file to re-format.
"""
# Using global here, as this is really a singleton, in
# the form of a module, pylint: disable=global-statement
global warned_clang_format
clang_format_path = getExecutablePath("clang-format-7")
# Extra ball on Windows, check default installation PATH too.
if not clang_format_path and getOS() == "Windows":
with withEnvironmentPathAdded(
"PATH",
r"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\Llvm\8.0.0\bin",
):
with withEnvironmentPathAdded("PATH",
r"C:\Program Files\LLVM\bin"):
clang_format_path = getExecutablePath("clang-format")
if clang_format_path:
subprocess.call([
clang_format_path,
"-i",
"-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}",
filename,
])
else:
if not warned_clang_format:
warning("Need to install LLVM for C files format.")
warned_clang_format = True
def _cleanupClangFormat(filename):
# Using global here, as this is really a singleton, in
# the form of a module, pylint: disable=global-statement
global warned_clang_format
clang_format_path = getExecutablePath("clang-format")
# Extra ball on Windows, check default installation PATH too.
if not clang_format_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\LLVM\bin"):
clang_format_path = getExecutablePath("clang-format")
if clang_format_path:
subprocess.call(
[
clang_format_path,
"-i",
"-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}",
filename,
]
)
else:
if not warned_clang_format:
warning("Need to install LLVM for C files format.")
warned_clang_format = True
def _detectBinaryPathDLLsLinuxBSD(dll_filename):
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None:
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
b"$ORIGIN",
os.path.dirname(sys.executable).encode("utf-8"))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
process = subprocess.Popen(args=["ldd", dll_filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _stderr = process.communicate()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found.
if filename == "not found":
continue
# Do not include kernel specific libraries.
if os.path.basename(filename).startswith(
("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")):
continue
result.add(filename)
return result
def runCodespell(filenames, verbose, write):
if verbose:
my_print("Consider", " ".join(filenames))
command = [
"codespell",
"-f",
"-I",
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"..",
"misc/codespell-ignore.txt",
),
]
if write:
command.append("-w")
command += filenames
if os.name == "nt":
extra_path = os.path.join(sys.prefix, "Scripts")
else:
extra_path = None
with withEnvironmentPathAdded("PATH", extra_path):
result = subprocess.call(command)
if result == 0:
for filename in filenames:
if areSamePaths(__file__, filename):
continue
contents = getFileContents(filename)
old_contents = contents
for word, replacement in replacements:
contents = contents.replace(word, replacement)
contents = contents.replace(word.title(), replacement.title())
if old_contents != contents:
putTextFileContents(filename, contents)
cleanupWindowsNewlines(filename)
if verbose:
if result != 0:
my_print("FAILED.")
else:
my_print("OK.")
return result == 0
def getOtoolDependencyOutput(filename, package_specific_dirs):
key = filename, tuple(package_specific_dirs), os.environ.get(
"DYLD_LIBRARY_PATH")
if key not in _otool_dependency_cache:
with withEnvironmentPathAdded("DYLD_LIBRARY_PATH",
*package_specific_dirs):
_otool_dependency_cache[key] = executeToolChecked(
logger=inclusion_logger,
command=("otool", "-L", filename),
absence_message=otool_usage,
)
return _otool_dependency_cache[key]
def _getPythonBinaryCall(binary_name):
if binary_name not in _binary_calls:
messages = []
# Try running Python installation.
try:
__import__(binary_name)
except ImportError:
pass
else:
call = [sys.executable, "-m", binary_name]
ok, message = _checkRequiredVersion(binary_name, call)
if ok:
_binary_calls[binary_name] = call
return _binary_calls[binary_name]
else:
messages.append(message)
with withEnvironmentPathAdded(
"PATH", os.path.join(sys.prefix, "Scripts"), os.path.join(sys.prefix, "bin")
):
binary_path = getExecutablePath(binary_name)
if binary_path:
call = [binary_path]
ok, message = _checkRequiredVersion(binary_name, call)
if ok:
_binary_calls[binary_name] = call
return _binary_calls[binary_name]
else:
messages.append(message)
if messages:
my_print("ERROR")
for message in messages:
my_print(message, style="red")
sys.exit(
"Error, cannot find %r version %r, not installed or wrong version for this Python?"
% (binary_name, _getRequiredVersion(binary_name))
)
return _binary_calls[binary_name]
def codespell(filenames, verbose, write):
if verbose:
my_print("Consider", " ".join(filenames))
command = ["codespell"]
if write:
command.append("-w")
command += filenames
if os.name == "nt":
extra_path = os.path.join(sys.prefix, "Scripts")
else:
extra_path = None
with withEnvironmentPathAdded("PATH", extra_path):
result = subprocess.call(command)
if verbose:
if result != 0:
my_print("FAILED.")
else:
my_print("OK.")
return result == 0
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
ld_library_path = OrderedSet()
if _detected_python_rpath:
ld_library_path.add(_detected_python_rpath)
ld_library_path.update(getPackageSpecificDLLDirectories(package_name))
if original_dir is not None:
ld_library_path.add(original_dir)
# ld_library_path.update(getSubDirectories(original_dir, ignore_dirs=("__pycache__",)))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", *ld_library_path):
process = subprocess.Popen(
args=["ldd", dll_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Do not include kernel / glibc specific libraries. This list has been
# assembled by looking what are the most common .so files provided by
# glibc packages from ArchLinux, Debian Stretch and CentOS.
#
# Online sources:
# - https://centos.pkgs.org/7/puias-computational-x86_64/glibc-aarch64-linux-gnu-2.24-2.sdl7.2.noarch.rpm.html
# - https://centos.pkgs.org/7/centos-x86_64/glibc-2.17-222.el7.x86_64.rpm.html
# - https://archlinux.pkgs.org/rolling/archlinux-core-x86_64/glibc-2.28-5-x86_64.pkg.tar.xz.html
# - https://packages.debian.org/stretch/amd64/libc6/filelist
#
# Note: This list may still be incomplete. Some additional libraries
# might be provided by glibc - it may vary between the package versions
# and between Linux distros. It might or might not be a problem in the
# future, but it should be enough for now.
if os.path.basename(filename).startswith((
"ld-linux-x86-64.so",
"libc.so.",
"libpthread.so.",
"libm.so.",
"libdl.so.",
"libBrokenLocale.so.",
"libSegFault.so",
"libanl.so.",
"libcidn.so.",
"libcrypt.so.",
"libmemusage.so",
"libmvec.so.",
"libnsl.so.",
"libnss_compat.so.",
"libnss_db.so.",
"libnss_dns.so.",
"libnss_files.so.",
"libnss_hesiod.so.",
"libnss_nis.so.",
"libnss_nisplus.so.",
"libpcprofile.so",
"libresolv.so.",
"librt.so.",
"libthread_db-1.0.so",
"libthread_db.so.",
"libutil.so.",
)):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(
dll_filename=sub_dll_filename,
package_name=package_name,
original_dir=original_dir,
))
return sub_result
def _detectBinaryPathDLLsLinuxBSD(dll_filename):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
result = set()
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None:
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
b"$ORIGIN",
os.path.dirname(sys.executable).encode("utf-8"))
with withEnvironmentPathAdded("LD_LIBRARY_PATH", _detected_python_rpath):
process = subprocess.Popen(args=["ldd", dll_filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _stderr = process.communicate()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 300:
filename = filename.decode("utf-8")
# Sometimes might use stuff not found.
if filename == "not found":
continue
# Do not include kernel specific libraries.
if os.path.basename(filename).startswith(
("libc.so.", "libpthread.so.", "libm.so.", "libdl.so.")):
continue
result.add(filename)
# Allow plugins to prevent inclusion.
blocked = Plugins.removeDllDependencies(dll_filename=dll_filename,
dll_filenames=result)
for to_remove in blocked:
result.discard(to_remove)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsLinuxBSD(sub_dll_filename))
return sub_result
def main():
goHome()
parser = OptionParser()
parser.add_option(
"--upload",
action="store_true",
dest="upload",
default=False,
help="""\
Upload to http://nuitka.net/apidoc requires access rights and is done by the
official servers automatically only. Without this, create the local html folder
only.
Default is %default.""",
)
options, _positional_args = parser.parse_args()
shutil.rmtree("html", ignore_errors=True)
doxygen_path = getExecutablePath("doxygen")
# Extra ball on Windows, check default installation PATH too.
if not doxygen_path and getOS() == "Windows":
with withEnvironmentPathAdded("PATH", r"C:\Program Files\Doxygen\bin"):
doxygen_path = getExecutablePath("doxygen")
if not doxygen_path:
sys.exit(
"Error, need to install Doxygen and add it to PATH for this to work."
)
try:
import doxypypy # @UnusedImport pylint: disable=I0021,unused-import,unused-variable
except ImportError:
sys.exit("Error, needs to install doxypypy into this Python.")
with withTemporaryFile(suffix=".doxyfile", delete=False) as doxy_file:
doxy_config = getFileContents("doc/Doxyfile.template")
with withTemporaryFile(
suffix=".bat" if getOS() == "Windows" else ".sh",
delete=False) as doxy_batch_file:
if getOS() == "Windows":
doxy_batch_file.write("%s -m doxypypy.doxypypy -a -c %%1" %
sys.executable)
else:
doxy_batch_file.write(
"#!/bin/sh\nexec '%s' -m doxypypy.doxypypy -a -c $1" %
sys.executable)
doxy_batch_filename = doxy_batch_file.name
doxy_config = doxy_config.replace("%DOXYPYPY%", doxy_batch_filename)
doxy_file.write(doxy_config)
doxy_filename = doxy_file.name
print("Running doxygen:")
try:
subprocess.check_call([doxygen_path, doxy_filename])
finally:
os.unlink(doxy_filename)
os.unlink(doxy_batch_filename)
# Update the repository on the web site.
if options.upload:
assert (os.system(
"rsync -avz --delete html/ --chown www-data [email protected]:/var/www/apidoc/"
) == 0)
print("Finished.")
def _detectBinaryPathDLLsPosix(dll_filename, package_name, original_dir):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches
if ldd_result_cache.get(dll_filename):
return ldd_result_cache[dll_filename]
# Ask "ldd" about the libraries being used by the created binary, these
# are the ones that interest us.
# This is the rpath of the Python binary, which will be effective when
# loading the other DLLs too. This happens at least for Python installs
# on Travis. pylint: disable=global-statement
global _detected_python_rpath
if _detected_python_rpath is None and not Utils.isPosixWindows():
_detected_python_rpath = getSharedLibraryRPATH(sys.executable) or False
if _detected_python_rpath:
_detected_python_rpath = _detected_python_rpath.replace(
"$ORIGIN", os.path.dirname(sys.executable))
# TODO: Actually would be better to pass it as env to the created process instead.
with withEnvironmentPathAdded(
"LD_LIBRARY_PATH",
*_getLdLibraryPath(
package_name=package_name,
python_rpath=_detected_python_rpath,
original_dir=original_dir,
)):
# TODO: Check exit code, should never fail.
stdout, stderr, _exit_code = executeProcess(command=("ldd",
dll_filename))
stderr = b"\n".join(
line for line in stderr.splitlines() if not line.startswith(
b"ldd: warning: you do not have execution permission for"))
inclusion_logger.debug("ldd output for %s is:\n%s" %
(dll_filename, stdout))
if stderr:
inclusion_logger.debug("ldd error for %s is:\n%s" %
(dll_filename, stderr))
result = set()
for line in stdout.split(b"\n"):
if not line:
continue
if b"=>" not in line:
continue
part = line.split(b" => ", 2)[1]
if b"(" in part:
filename = part[:part.rfind(b"(") - 1]
else:
filename = part
if not filename:
continue
if python_version >= 0x300:
filename = filename.decode("utf8")
# Sometimes might use stuff not found or supplied by ldd itself.
if filename in ("not found", "ldd"):
continue
# Normalize, sometimes the DLLs produce "something/../", this has
# been seen with Qt at least.
filename = os.path.normpath(filename)
# Do not include kernel DLLs on the ignore list.
filename_base = os.path.basename(filename)
if any(filename_base == entry or filename_base.startswith(entry + ".")
for entry in _linux_dll_ignore_list):
continue
result.add(filename)
ldd_result_cache[dll_filename] = result
sub_result = set(result)
for sub_dll_filename in result:
sub_result = sub_result.union(
_detectBinaryPathDLLsPosix(
dll_filename=sub_dll_filename,
package_name=package_name,
original_dir=original_dir,
))
return sub_result
