def _getCcacheStatistics(ccache_logfile):
data = {}
if os.path.exists(ccache_logfile):
re_command = re.compile(r"\[.*? (\d+) *\] Command line: (.*)$")
re_result = re.compile(r"\[.*? (\d+) *\] Result: (.*)$")
re_anything = re.compile(r"\[.*? (\d+) *\] (.*)$")
# Remember command from the pid, so later decision logged against pid
# can be matched against it.
commands = {}
for line in getFileContentByLine(ccache_logfile):
match = re_command.match(line)
if match:
pid, command = match.groups()
commands[pid] = command
match = re_result.match(line)
if match:
pid, result = match.groups()
result = result.strip()
try:
command = data[commands[pid]]
except KeyError:
# It seems writing to the file can be lossy, so we can have results for
# unknown commands, but we don't use the command yet anyway, so just
# be unique.
command = "unknown command leading to " + line
# Older ccache on e.g. RHEL6 wasn't explicit about linking.
if result == "unsupported compiler option":
if " -o " in command or "unknown command" in command:
result = "called for link"
# But still try to catch this with log output if it happens.
if result == "unsupported compiler option":
scons_logger.warning(
"Encountered unsupported compiler option for ccache in '%s'."
% command)
all_text = []
for line2 in getFileContentByLine(ccache_logfile):
match = re_anything.match(line2)
if match:
pid2, result = match.groups()
if pid == pid2:
all_text.append(result)
scons_logger.warning("Full scons output: %s" % all_text)
if result != "called for link":
data[command] = result
return data
def detectPthImportedPackages():
if not hasattr(sys.modules["site"], "getsitepackages"):
return ()
pth_imports = set()
for prefix in sys.modules["site"].getsitepackages():
if not os.path.isdir(prefix):
continue
for path, filename in listDir(prefix):
if filename.endswith(".pth"):
try:
for line in getFileContentByLine(path, "rU"):
if line.startswith("import "):
if ";" in line:
line = line[:line.find(";")]
for part in line[7:].split(","):
pth_imports.add(part.strip())
except OSError:
warning(
"Python installation problem, cannot read file '%s'.")
return tuple(sorted(pth_imports))
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
# Many cases to deal with, pylint: disable=too-many-branches
if isWin32Windows():
sys.exit("Error, valgrind is not available on Windows.")
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
_stdout_valgrind, stderr_valgrind, exit_valgrind = executeProcess(
command)
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
copyFile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def checkRequirements(filename):
for line in getFileContentByLine(filename):
if line.startswith("# nuitka-skip-unless-"):
if line[21:33] == "expression: ":
expression = line[33:]
with open(os.devnull, "w") as devnull:
result = subprocess.call(
(
os.environ["PYTHON"],
"-c",
"import sys, os; sys.exit(not bool(%s))" %
expression,
),
stdout=devnull,
stderr=subprocess.STDOUT,
)
if not result == 0:
return (False,
"Expression '%s' evaluated to false" % expression)
elif line[21:30] == "imports: ":
imports_needed = line[30:].rstrip().split(",")
for i in imports_needed:
if not hasModule(i):
return (
False,
i +
" not installed for this Python version, but test needs it",
)
# default return value
return (True, "")
def detectPthImportedPackages():
if not hasattr(sys.modules["site"], "getsitepackages"):
return ()
# TODO: Move hard import config to elsewhere.
from nuitka.nodes.ImportNodes import isHardModuleWithoutSideEffect
pth_imports = set()
for prefix in sys.modules["site"].getsitepackages():
if not os.path.isdir(prefix):
continue
for path, filename in listDir(prefix):
if filename.endswith(".pth"):
try:
for line in getFileContentByLine(path, "rU"):
if line.startswith("import "):
if ";" in line:
line = line[:line.find(";")]
for part in line[7:].split(","):
pth_import = part.strip()
if not isHardModuleWithoutSideEffect(
pth_import):
pth_imports.add(pth_import)
except OSError:
recursion_logger.warning(
"Python installation problem, cannot read file '%s'.")
return tuple(sorted(pth_imports))
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ("--tool=callgrind",
"--callgrind-out-file=%s" % log_filename)
elif tool == "massif":
command += ("--tool=massif", "--massif-out-file=%s" % log_filename)
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += (
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
)
command.extend(args)
process = subprocess.Popen(args=command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_stdout_valgrind, stderr_valgrind = process.communicate()
exit_valgrind = process.returncode
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
shutil.copyfile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def displayFileContents(name, path):
test_logger.info("Contents of %s %r:" % (name, path))
if os.path.exists(path):
for line in getFileContentByLine(path):
my_print(line)
else:
test_logger.info("Does not exist.")
def runValgrind(descr, tool, args, include_startup, save_logfilename=None):
if descr:
my_print(descr, tool, file=sys.stderr, end="... ")
with withTemporaryFile() as log_file:
log_filename = log_file.name
command = ["valgrind", "-q"]
if tool == "callgrind":
command += ["--tool=callgrind", "--callgrind-out-file=%s" % log_filename]
elif tool == "massif":
command += ["--tool=massif", "--massif-out-file=%s" % log_filename]
else:
sys.exit("Error, no support for tool '%s' yet." % tool)
# Do not count things before main module starts its work.
if not include_startup:
command += [
"--zero-before=init__main__()",
"--zero-before=init__main__",
"--zero-before=PyInit___main__",
"--zero-before=PyInit___main__()",
]
command.extend(args)
process = subprocess.Popen(
args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_stdout_valgrind, stderr_valgrind = process.communicate()
exit_valgrind = process.returncode
assert exit_valgrind == 0, stderr_valgrind
if descr:
my_print("OK", file=sys.stderr)
if save_logfilename is not None:
shutil.copyfile(log_filename, save_logfilename)
max_mem = None
for line in getFileContentByLine(log_filename):
if tool == "callgrind" and line.startswith("summary:"):
return int(line.split()[1])
elif tool == "massif" and line.startswith("mem_heap_B="):
mem = int(line.split("=")[1])
if max_mem is None:
max_mem = 0
max_mem = max(mem, max_mem)
if tool == "massif" and max_mem is not None:
return max_mem
sys.exit("Error, didn't parse Valgrind log file successfully.")
def checkChangeLog(message):
"""Check debian changelog for given message to be present."""
for line in getFileContentByLine("debian/changelog"):
if line.startswith(" --"):
return False
if message in line:
return True
sys.exit("Error, didn't find in debian/changelog: '%s'" % message)
def _cleanupTrailingWhitespace(filename):
"""Remove trailing white spaces from a file."""
source_lines = list(getFileContentByLine(filename, encoding="utf8"))
clean_lines = [
line.rstrip().replace("\t", " ") for line in source_lines
]
while clean_lines and clean_lines[-1] == "":
del clean_lines[-1]
if clean_lines != source_lines or (clean_lines and clean_lines[-1] != ""):
putTextFileContents(filename, contents=clean_lines, encoding="utf8")
def checkChangeLog(message):
""" Check debian changelog for given message to be present.
"""
for line in getFileContentByLine("debian/changelog"):
if line.startswith(" --"):
return False
if message in line:
return True
sys.exit("Error, didn't find in debian/changelog: '%s'" % message)
def main():
quote_start_re = re.compile("[Qq]uoting the ``(.*)`` documentation")
quote_end_re = re.compile("[Ed]nd quoting the ``(.*)`` documentation")
quoting = False
for line in getFileContentByLine("Developer_Manual.rst"):
if not quoting:
print(line, end="")
if not quoting:
match = quote_start_re.search(line)
if match:
quoting = match.group(1)
if "." in quoting:
import_from, import_value = quoting.rsplit(".", 1)
# Hopefully OK for us, pylint: disable=W0122
exec("from %s import %s" % (import_from, import_value))
item = getattr(sys.modules[import_from], import_value)
# Should potentially be derived from quoting line.
indentation = " " * line.find("Quoting")
# Empty line to separate
print()
for quote_line in inspect.getdoc(item).splitlines():
if quote_line:
print(indentation + quote_line)
else:
print()
print()
else:
assert False, quoting
if quoting:
match = quote_end_re.search(line)
if match:
assert quoting == match.group(1)
quoting = False
print(line, end="")
if quoting:
sys.exit("Error, waiting for end of quote for %s failed" % quoting)
def main():
quote_start_re = re.compile("[Qq]uoting the ``(.*)`` documentation")
quote_end_re = re.compile("(End|end) quoting the ``(.*)`` documentation")
quoting = False
for line in getFileContentByLine("Developer_Manual.rst"):
if not quoting:
print(line, end="")
if not quoting:
match = quote_start_re.search(line)
if match:
quoting = match.group(1)
if "." in quoting:
import_from, import_value = quoting.rsplit(".", 1)
# Hopefully OK for us, pylint: disable=W0122
exec("from %s import %s" % (import_from, import_value))
item = getattr(sys.modules[import_from], import_value)
# Should potentially be derived from quoting line.
indentation = " " * line.find("Quoting")
# Empty line to separate
print()
for quote_line in inspect.getdoc(item).splitlines():
if quote_line:
print(indentation + quote_line)
else:
print()
print()
else:
assert False, quoting
if quoting:
match = quote_end_re.search(line)
if match:
assert quoting == match.group(1)
quoting = False
print(line, end="")
if quoting:
sys.exit("Error, waiting for end of quote for %s failed" % quoting)
def considerDataFiles(self, module):
if module.getFullName() != "matplotlib":
return
matplotlib_info = self._getMatplotlibInfo()
if not os.path.isdir(matplotlib_info.data_path):
self.sysexit(
"mpl-data missing, matplotlib installation appears to be broken"
)
# Include the "mpl-data" files.
yield self.makeIncludedDataDirectory(
source_path=matplotlib_info.data_path,
dest_path=os.path.join("matplotlib", "mpl-data"),
ignore_dirs=("sample_data", ),
ignore_filenames=("matplotlibrc", ),
reason="package data for 'matplotlib",
tags="mpl-data",
)
# Handle the config file with an update.
new_lines = [] # new config file lines
found = False # checks whether backend definition encountered
for line in getFileContentByLine(
matplotlib_info.matplotlibrc_filename):
line = line.rstrip()
# omit meaningless lines
if line.startswith(
"#") and matplotlib_info.matplotlib_version < "3":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
# old config file has a backend definition
found = True
if not found and matplotlib_info.matplotlib_version < "3":
# Set the backend, so even if it was run time determined, we now enforce it.
new_lines.append("backend: %s" % matplotlib_info.backend)
yield self.makeIncludedGeneratedDataFile(
data=new_lines,
dest_path=os.path.join("matplotlib", "mpl-data", "matplotlibrc"),
reason="Updated matplotlib config file with backend to use.",
)
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith("("):
# No multiline imports please
assert imported.endswith(")")
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
name = name.strip()
pyi_deps.add(parts[1] + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple(
(pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
def _parseDependsExeOutput(filename, result):
inside = False
first = False
for line in getFileContentByLine(filename):
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[:line.find("]")]:
continue
# Skip DLLs that failed to load, apparently not needed anyway.
if "E" in line[:line.find("]")]:
continue
dll_filename = line[line.find("]") + 2:-1]
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), dll_filename
dll_name = os.path.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or dll_name.startswith(
"EXT-MS-WIN-"):
continue
if dll_name in _win_dll_whitelist:
continue
result.add(os.path.normcase(os.path.abspath(dll_filename)))
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
if parts[1] == "typing":
continue
pyi_deps.add(parts[1])
imported = parts[3]
if imported.startswith("("):
# No multiline imports please
assert imported.endswith(")")
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
name = name.strip()
pyi_deps.add(parts[1] + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
self.used_modules = tuple((pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
def readSconsReport(source_dir):
if source_dir not in _scons_reports:
scons_report = {}
for line in getFileContentByLine(
os.path.join(source_dir, "scons-report.txt")):
if "=" not in line:
continue
key, value = line.strip().split("=", 1)
scons_report[key] = value
_scons_reports[source_dir] = scons_report
return _scons_reports[source_dir]
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is the caching mechanism and plugin handling for DLL imports.
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
result = OrderedSet()
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
general.info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs)
if update_cache:
putTextFileContents(filename=cache_filename, contents=result)
return result
def getImplicitImports(self, module):
# Dealing with OpenGL is a bit detailed, pylint: disable=too-many-branches
if module.getFullName() == "OpenGL":
opengl_infos = self.queryRuntimeInformationSingle(
setup_codes="import OpenGL.plugins",
value=
"[(f.name, f.import_path) for f in OpenGL.plugins.FormatHandler.all()]",
)
# TODO: Filter by name.
for _name, import_path in opengl_infos:
yield ModuleName(import_path).getPackageName()
for line in getFileContentByLine(module.getCompileTimeFilename()):
if line.startswith("PlatformPlugin("):
os_part, plugin_name_part = line[15:-1].split(",")
os_part = os_part.strip("' ")
plugin_name_part = plugin_name_part.strip(") '")
plugin_name_part = plugin_name_part[:plugin_name_part.
rfind(".")]
if os_part == "nt":
if getOS() == "Windows":
yield plugin_name_part
elif os_part.startswith("linux"):
if isLinux():
yield plugin_name_part
elif os_part.startswith("darwin"):
if isMacOS():
yield plugin_name_part
elif os_part.startswith(("posix", "osmesa", "egl")):
if getOS() != "Windows":
yield plugin_name_part
else:
self.sysexit(
"Undetected OS, please report bug for '%s'." %
os_part)
def detectPthImportedPackages():
if not hasattr(sys.modules["site"], "getsitepackages"):
return ()
pth_imports = set()
for prefix in sys.modules["site"].getsitepackages():
if not os.path.isdir(prefix):
continue
for path, filename in listDir(prefix):
if filename.endswith(".pth"):
try:
for line in getFileContentByLine(path, "rU"):
if line.startswith("import "):
if ";" in line:
line = line[: line.find(";")]
for part in line[7:].split(","):
pth_imports.add(part.strip())
except OSError:
warning("Python installation problem, cannot read file '%s'.")
return tuple(sorted(pth_imports))
def getImplicitImports(self, module):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements
full_name = module.getFullName()
if module.isPythonShlibModule():
for used_module in module.getUsedModules():
yield used_module[0], False
# TODO: Move this out to some kind of configuration format.
elements = full_name.split(".")
if elements[0] in ("PyQt4", "PyQt5"):
if python_version < 300:
yield "atexit", True
# These are alternatives now:
# TODO: One day it should avoid including both.
yield "sip", False
if elements[0] == "PyQt5":
yield "PyQt5.sip", False
child = elements[1] if len(elements) > 1 else None
if child in (
"QtGui",
"QtAssistant",
"QtDBus",
"QtDeclarative",
"QtSql",
"QtDesigner",
"QtHelp",
"QtNetwork",
"QtScript",
"QtQml",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtOpenGL",
"QtXml",
"QtXmlPatterns",
"QtPrintSupport",
"QtNfc",
"QtWebKitWidgets",
"QtBluetooth",
"QtMultimediaWidgets",
"QtQuick",
"QtWebChannel",
"QtWebSockets",
"QtX11Extras",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtCore", True
if child in (
"QtDeclarative",
"QtWebKit",
"QtXmlPatterns",
"QtQml",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtQuick",
"QtQuickWidgets",
"QtWebSockets",
):
yield elements[0] + ".QtNetwork", True
if child == "QtScriptTools":
yield elements[0] + ".QtScript", True
if child in (
"QtWidgets",
"QtDeclarative",
"QtDesigner",
"QtHelp",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtOpenGL",
"QtQuick",
"QtQuickWidgets",
"QtSql",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtGui", True
if full_name in (
"PyQt5.QtDesigner",
"PyQt5.QtHelp",
"PyQt5.QtTest",
"PyQt5.QtPrintSupport",
"PyQt5.QtSvg",
"PyQt5.QtOpenGL",
"PyQt5.QtWebKitWidgets",
"PyQt5.QtMultimediaWidgets",
"PyQt5.QtQuickWidgets",
"PyQt5.QtSql",
):
yield "PyQt5.QtWidgets", True
if full_name in ("PyQt5.QtPrintSupport",):
yield "PyQt5.QtSvg", True
if full_name in ("PyQt5.QtWebKitWidgets",):
yield "PyQt5.QtWebKit", True
yield "PyQt5.QtPrintSupport", True
if full_name in ("PyQt5.QtMultimediaWidgets",):
yield "PyQt5.QtMultimedia", True
if full_name in ("PyQt5.QtQuick", "PyQt5.QtQuickWidgets"):
yield "PyQt5.QtQml", True
if full_name in ("PyQt5.QtQuickWidgets", "PyQt5.QtQml"):
yield "PyQt5.QtQuick", True
if full_name == "PyQt5.Qt":
yield "PyQt5.QtCore", True
yield "PyQt5.QtDBus", True
yield "PyQt5.QtGui", True
yield "PyQt5.QtNetwork", True
yield "PyQt5.QtNetworkAuth", False
yield "PyQt5.QtSensors", False
yield "PyQt5.QtSerialPort", False
yield "PyQt5.QtMultimedia", True
yield "PyQt5.QtQml", False
yield "PyQt5.QtWidgets", True
elif full_name == "sip" and python_version < 300:
yield "enum", False
elif full_name == "PySide.QtDeclarative":
yield "PySide.QtGui", True
elif full_name == "PySide.QtHelp":
yield "PySide.QtGui", True
elif full_name == "PySide.QtOpenGL":
yield "PySide.QtGui", True
elif full_name == "PySide.QtScriptTools":
yield "PySide.QtScript", True
yield "PySide.QtGui", True
elif full_name == "PySide.QtSql":
yield "PySide.QtGui", True
elif full_name == "PySide.QtSvg":
yield "PySide.QtGui", True
elif full_name == "PySide.QtTest":
yield "PySide.QtGui", True
elif full_name == "PySide.QtUiTools":
yield "PySide.QtGui", True
yield "PySide.QtXml", True
elif full_name == "PySide.QtWebKit":
yield "PySide.QtGui", True
elif full_name == "PySide.phonon":
yield "PySide.QtGui", True
elif full_name == "lxml.etree":
yield "gzip", True
yield "lxml._elementpath", True
elif full_name == "gtk._gtk":
yield "pangocairo", True
yield "pango", True
yield "cairo", True
yield "gio", True
yield "atk", True
elif full_name == "atk":
yield "gobject", True
elif full_name == "gtkunixprint":
yield "gobject", True
yield "cairo", True
yield "gtk", True
elif full_name == "pango":
yield "gobject", True
elif full_name == "pangocairo":
yield "pango", True
yield "cairo", True
elif full_name == "reportlab.rl_config":
yield "reportlab.rl_settings", True
elif full_name == "socket":
yield "_socket", False
elif full_name == "ctypes":
yield "_ctypes", True
elif full_name == "gi._gi":
yield "gi._error", True
elif full_name == "gi._gi_cairo":
yield "cairo", True
elif full_name == "cairo._cairo":
yield "gi._gobject", False
elif full_name in ("Tkinter", "tkinter"):
yield "_tkinter", False
elif full_name in (
"cryptography.hazmat.bindings._openssl",
"cryptography.hazmat.bindings._constant_time",
"cryptography.hazmat.bindings._padding",
):
yield "_cffi_backend", True
elif full_name.startswith("cryptography._Cryptography_cffi_"):
yield "_cffi_backend", True
elif full_name == "bcrypt._bcrypt":
yield "_cffi_backend", True
elif full_name == "nacl._sodium":
yield "_cffi_backend", True
elif full_name == "_dbus_glib_bindings":
yield "_dbus_bindings", True
elif full_name == "_mysql":
yield "_mysql_exceptions", True
elif full_name == "lxml.objectify":
yield "lxml.etree", True
elif full_name == "_yaml":
yield "yaml", True
elif full_name == "apt_inst":
yield "apt_pkg", True
# start of gevent imports --------------------------------------------
elif full_name == "gevent":
yield "_cffi_backend", True
yield "gevent._config", True
yield "gevent.core", True
yield "gevent.resolver_thread", True
yield "gevent.resolver_ares", True
yield "gevent.socket", True
yield "gevent.threadpool", True
yield "gevent.thread", True
yield "gevent.threading", True
yield "gevent.select", True
yield "gevent.subprocess", True
if getOS() == "Windows":
yield "gevent.libuv", True
else:
yield "gevent.libev", True
elif full_name == "gevent.hub":
yield "gevent._hub_primitives", True
yield "gevent._greenlet_primitives", True
yield "gevent._hub_local", True
yield "gevent._waiter", True
yield "gevent._util", True
yield "gevent._ident", True
yield "gevent.exceptions", True
elif full_name == "gevent.libev":
yield "gevent.libev.corecext", True
yield "gevent.libev.corecffi", True
yield "gevent.libev.watcher", True
elif full_name == "gevent.libuv":
yield "gevent._interfaces", True
yield "gevent._ffi", True
yield "gevent.libuv.loop", True
yield "gevent.libuv.watcher", True
elif full_name == "gevent.libuv.loop":
yield "gevent.libuv._corecffi", True
yield "gevent._interfaces", True
elif full_name == "gevent._ffi":
yield "gevent._ffi.loop", True
elif full_name == "gevent._waiter":
yield "gevent.__waiter", True
elif full_name == "gevent._hub_local":
yield "gevent.__hub_local", True
yield "gevent.__greenlet_primitives", True
elif full_name == "gevent._hub_primitives":
yield "gevent.__hub_primitives", True
elif full_name == "gevent.greenlet":
yield "gevent._hub_local", True
yield "gevent._greenlet", True
elif full_name == "gevent._greenlet":
yield "gevent.__ident", True
elif full_name == "gevent.monkey":
yield "gevent.builtins", True
yield "gevent.time", True
yield "gevent.local", True
yield "gevent.ssl", True
yield "gevent.events", True
elif full_name == "gevent._semaphore":
yield "gevent._abstract_linkable", True
yield "gevent.__semaphore", True
elif full_name == "gevent._abstract_linkable":
yield "gevent.__abstract_linkable", True
elif full_name == "gevent.local":
yield "gevent._local", True
elif full_name == "gevent.event":
yield "gevent._event", True
elif full_name == "gevent.queue":
yield "gevent._queue", True
elif full_name == "gevent.pool":
yield "gevent._imap", True
elif full_name == "gevent._imap":
yield "gevent.__imap", True
# end of gevent imports ----------------------------------------------
# start of tensorflow imports --------------------------------------------
elif full_name == "tensorflow":
yield "tensorflow._api", True
yield "tensorflow.python", True
yield "tensorflow.core", True
yield "tensorflow.lite.python.lite", True
yield "tensorflow_estimator.python.estimator.api", False
elif full_name == "tensorflow.lite.python":
yield "tensorflow.python.framework.importer", True
elif full_name == "tensorflow.lite.python.optimize":
yield "tensorflow.lite.python.optimize._tensorflow_lite_wrap_calibration_wrapper", True
elif full_name == "tensorflow.lite.toco.python":
yield "tensorflow.lite.toco.python._tensorflow_wrap_toco", True
elif full_name == "tensorflow.lite.python.interpreter_wrapper":
yield "tensorflow.lite.python.interpreter_wrapper._tensorflow_wrap_interpreter_wrapper", True
elif full_name == "tensorflow.python":
yield "tensorflow.python.pywrap_tensorflow", True
yield "tensorflow.python._pywrap_tensorflow_internal", True
yield "tensorflow.python.tools", True
yield "tensorflow.python.compat", True
yield "tensorflow.python.framework", True
yield "tensorflow.python.module", True
yield "tensorflow.python.ops", True
yield "tensorflow.python.platform", True
yield "tensorflow.python.lib.io", True
yield "tensorflow.python.util", True
yield "tensorflow.python.keras.api", False
# end of tensorflow imports --------------------------------------------
elif full_name == "numpy.core":
yield "numpy.core._dtype_ctypes", False
elif full_name == "scipy.special":
yield "scipy.special._ufuncs_cxx", True
elif full_name == "scipy.linalg":
yield "scipy.linalg.cython_blas", True
yield "scipy.linalg.cython_lapack", True
elif full_name == "scipy.sparse.csgraph":
yield "scipy.sparse.csgraph._validation", True
elif full_name == "scipy._lib":
yield "scipy._lib.messagestream", True
elif full_name == "sklearn.utils.sparsetools":
yield "sklearn.utils.sparsetools._graph_validation", True
yield "sklearn.utils.sparsetools._graph_tools", True
elif full_name == "sklearn.utils":
yield "sklearn.utils.lgamma", True
yield "sklearn.utils.weight_vector", True
yield "sklearn.utils._unittest_backport", True
elif full_name == "PIL._imagingtk":
yield "PIL._tkinter_finder", True
elif full_name == "pkg_resources.extern":
if self.pkg_utils_externals is None:
for line in getFileContentByLine(module.getCompileTimeFilename()):
if line.startswith("names"):
line = line.split("=")[-1].strip()
parts = line.split(",")
self.pkg_utils_externals = [part.strip("' ") for part in parts]
break
else:
self.pkg_utils_externals = ()
for pkg_util_external in self.pkg_utils_externals:
yield "pkg_resources._vendor." + pkg_util_external, False
elif full_name == "pkg_resources._vendor.packaging":
yield "pkg_resources._vendor.packaging.version", True
yield "pkg_resources._vendor.packaging.specifiers", True
yield "pkg_resources._vendor.packaging.requirements", True
elif full_name == "uvloop.loop":
yield "uvloop._noop", True
elif full_name == "fitz.fitz":
yield "fitz._fitz", True
elif full_name == "pandas._libs":
yield "pandas._libs.tslibs.np_datetime", False
yield "pandas._libs.tslibs.nattype", False
elif full_name == "pandas.core.window":
yield "pandas._libs.skiplist", False
elif full_name == "zmq.backend":
yield "zmq.backend.cython", True
elif full_name == "OpenGL":
if self.opengl_plugins is None:
self.opengl_plugins = []
for line in getFileContentByLine(module.getCompileTimeFilename()):
if line.startswith("PlatformPlugin("):
os_part, plugin_name_part = line[15:-1].split(",")
os_part = os_part.strip("' ")
plugin_name_part = plugin_name_part.strip(") '")
plugin_name_part = plugin_name_part[
: plugin_name_part.rfind(".")
]
if os_part == "nt":
if getOS() == "Windows":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("linux"):
if getOS() == "Linux":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("darwin"):
if getOS() == "Darwin":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith(("posix", "osmesa", "egl")):
if getOS() != "Windows":
self.opengl_plugins.append(plugin_name_part)
else:
assert False, os_part
for opengl_plugin in self.opengl_plugins:
yield opengl_plugin, True
# Support for both pycryotodome (module name Crypto) and pycyptodomex (module name Cryptodome)
elif full_name.split(".")[0] in ("Crypto", "Cryptodome"):
crypto_module_name = full_name.split(".")[0]
if full_name == crypto_module_name + ".Util._raw_api":
for module_name in (
"_raw_aes",
"_raw_aesni",
"_raw_arc2",
"_raw_blowfish",
"_raw_cast",
"_raw_cbc",
"_raw_cfb",
"_raw_ctr",
"_raw_des",
"_raw_des3",
"_raw_ecb",
"_raw_ocb",
"_raw_ofb",
):
if full_name == crypto_module_name + ".Util._raw_api":
yield crypto_module_name + ".Cipher." + module_name, True
elif full_name == crypto_module_name + ".Util.strxor":
yield crypto_module_name + ".Util._strxor", True
elif full_name == crypto_module_name + ".Util._cpu_features":
yield crypto_module_name + ".Util._cpuid_c", True
elif full_name == crypto_module_name + ".Hash.BLAKE2s":
yield crypto_module_name + ".Hash._BLAKE2s", True
elif full_name == crypto_module_name + ".Hash.SHA1":
yield crypto_module_name + ".Hash._SHA1", True
elif full_name == crypto_module_name + ".Hash.SHA256":
yield crypto_module_name + ".Hash._SHA256", True
elif full_name == crypto_module_name + ".Hash.MD5":
yield crypto_module_name + ".Hash._MD5", True
elif full_name == crypto_module_name + ".Protocol.KDF":
yield crypto_module_name + ".Cipher._Salsa20", True
yield crypto_module_name + ".Protocol._scrypt", True
elif full_name == crypto_module_name + ".Cipher._mode_gcm":
yield crypto_module_name + ".Hash._ghash_portable", True
elif full_name == "pycparser.c_parser":
yield "pycparser.yacctab", True
yield "pycparser.lextab", True
elif full_name == "passlib.hash":
yield "passlib.handlers.sha2_crypt", True
def _readPyPIFile(self):
""" Read the .pyi file if present and scan for dependencies. """
# Complex stuff, pylint: disable=too-many-branches,too-many-statements
if self.used_modules is None:
pyi_filename = self.getPyIFilename()
if os.path.exists(pyi_filename):
pyi_deps = OrderedSet()
# Flag signalling multiline import handling
in_import = False
in_import_part = ""
for line in getFileContentByLine(pyi_filename):
line = line.strip()
if not in_import:
if line.startswith("import "):
imported = line[7:]
pyi_deps.add(imported)
elif line.startswith("from "):
parts = line.split(None, 3)
assert parts[0] == "from"
assert parts[2] == "import"
origin_name = parts[1]
if origin_name == "typing":
continue
if origin_name == ".":
origin_name = self.getFullName()
# TODO: Might want to add full relative import handling.
if origin_name != self.getFullName():
pyi_deps.add(origin_name)
imported = parts[3]
if imported.startswith("("):
# Handle multiline imports
if not imported.endswith(")"):
in_import = True
imported = imported[1:]
in_import_part = origin_name
assert in_import_part, (
"Multiline part in file %s cannot be empty"
% pyi_filename)
else:
in_import = False
imported = imported[1:-1]
assert imported
if imported == "*":
continue
for name in imported.split(","):
if name:
name = name.strip()
pyi_deps.add(origin_name + "." + name)
else: # In import
imported = line
if imported.endswith(")"):
imported = imported[0:-1]
in_import = False
for name in imported.split(","):
name = name.strip()
if name:
pyi_deps.add(in_import_part + "." + name)
if "typing" in pyi_deps:
pyi_deps.discard("typing")
if "__future__" in pyi_deps:
pyi_deps.discard("__future__")
if self.getFullName() in pyi_deps:
pyi_deps.discard(self.getFullName())
if self.getFullName().getPackageName() in pyi_deps:
pyi_deps.discard(self.getFullName().getPackageName())
self.used_modules = tuple(
(pyi_dep, None) for pyi_dep in pyi_deps)
else:
self.used_modules = ()
def getImplicitImports(self, module):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements
full_name = module.getFullName()
if module.isPythonShlibModule():
for used_module in module.getUsedModules():
yield used_module[0], False
# TODO: Move this out to some kind of configuration format.
elements = full_name.split(".")
if elements[0] in ("PyQt4", "PyQt5"):
if python_version < 300:
yield "atexit", True
# These are alternatives now:
# TODO: One day it should avoid including both.
yield "sip", False
if elements[0] == "PyQt5":
yield "PyQt5.sip", False
child = elements[1] if len(elements) > 1 else None
if child in (
"QtGui",
"QtAssistant",
"QtDBus",
"QtDeclarative",
"QtSql",
"QtDesigner",
"QtHelp",
"QtNetwork",
"QtScript",
"QtQml",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtOpenGL",
"QtXml",
"QtXmlPatterns",
"QtPrintSupport",
"QtNfc",
"QtWebKitWidgets",
"QtBluetooth",
"QtMultimediaWidgets",
"QtQuick",
"QtWebChannel",
"QtWebSockets",
"QtX11Extras",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtCore", True
if child in (
"QtDeclarative",
"QtWebKit",
"QtXmlPatterns",
"QtQml",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtQuick",
"QtQuickWidgets",
"QtWebSockets",
):
yield elements[0] + ".QtNetwork", True
if child == "QtScriptTools":
yield elements[0] + ".QtScript", True
if child in (
"QtWidgets",
"QtDeclarative",
"QtDesigner",
"QtHelp",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtOpenGL",
"QtQuick",
"QtQuickWidgets",
"QtSql",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtGui", True
if full_name in (
"PyQt5.QtDesigner",
"PyQt5.QtHelp",
"PyQt5.QtTest",
"PyQt5.QtPrintSupport",
"PyQt5.QtSvg",
"PyQt5.QtOpenGL",
"PyQt5.QtWebKitWidgets",
"PyQt5.QtMultimediaWidgets",
"PyQt5.QtQuickWidgets",
"PyQt5.QtSql",
):
yield "PyQt5.QtWidgets", True
if full_name in ("PyQt5.QtPrintSupport",):
yield "PyQt5.QtSvg", True
if full_name in ("PyQt5.QtWebKitWidgets",):
yield "PyQt5.QtWebKit", True
yield "PyQt5.QtPrintSupport", True
if full_name in ("PyQt5.QtMultimediaWidgets",):
yield "PyQt5.QtMultimedia", True
if full_name in ("PyQt5.QtQuick", "PyQt5.QtQuickWidgets"):
yield "PyQt5.QtQml", True
if full_name in ("PyQt5.QtQuickWidgets", "PyQt5.QtQml"):
yield "PyQt5.QtQuick", True
if full_name == "PyQt5.Qt":
yield "PyQt5.QtCore", True
yield "PyQt5.QtDBus", True
yield "PyQt5.QtGui", True
yield "PyQt5.QtNetwork", True
yield "PyQt5.QtNetworkAuth", False
yield "PyQt5.QtSensors", False
yield "PyQt5.QtSerialPort", False
yield "PyQt5.QtMultimedia", True
yield "PyQt5.QtQml", False
yield "PyQt5.QtWidgets", True
elif full_name == "sip" and python_version < 300:
yield "enum", False
elif full_name == "PySide.QtDeclarative":
yield "PySide.QtGui", True
elif full_name == "PySide.QtHelp":
yield "PySide.QtGui", True
elif full_name == "PySide.QtOpenGL":
yield "PySide.QtGui", True
elif full_name == "PySide.QtScriptTools":
yield "PySide.QtScript", True
yield "PySide.QtGui", True
elif full_name == "PySide.QtSql":
yield "PySide.QtGui", True
elif full_name == "PySide.QtSvg":
yield "PySide.QtGui", True
elif full_name == "PySide.QtTest":
yield "PySide.QtGui", True
elif full_name == "PySide.QtUiTools":
yield "PySide.QtGui", True
yield "PySide.QtXml", True
elif full_name == "PySide.QtWebKit":
yield "PySide.QtGui", True
elif full_name == "PySide.phonon":
yield "PySide.QtGui", True
elif full_name == "lxml.etree":
yield "gzip", True
yield "lxml._elementpath", True
elif full_name == "gtk._gtk":
yield "pangocairo", True
yield "pango", True
yield "cairo", True
yield "gio", True
yield "atk", True
elif full_name == "atk":
yield "gobject", True
elif full_name == "gtkunixprint":
yield "gobject", True
yield "cairo", True
yield "gtk", True
elif full_name == "pango":
yield "gobject", True
elif full_name == "pangocairo":
yield "pango", True
yield "cairo", True
elif full_name == "reportlab.rl_config":
yield "reportlab.rl_settings", True
elif full_name == "socket":
yield "_socket", False
elif full_name == "ctypes":
yield "_ctypes", True
elif full_name == "gi._gi":
yield "gi._error", True
elif full_name == "gi._gi_cairo":
yield "cairo", True
elif full_name == "cairo._cairo":
yield "gi._gobject", False
elif full_name in ("Tkinter", "tkinter"):
yield "_tkinter", False
elif full_name in (
"cryptography.hazmat.bindings._openssl",
"cryptography.hazmat.bindings._constant_time",
"cryptography.hazmat.bindings._padding",
):
yield "_cffi_backend", True
elif full_name.startswith("cryptography._Cryptography_cffi_"):
yield "_cffi_backend", True
elif full_name == "bcrypt._bcrypt":
yield "_cffi_backend", True
elif full_name == "nacl._sodium":
yield "_cffi_backend", True
elif full_name == "_dbus_glib_bindings":
yield "_dbus_bindings", True
elif full_name == "_mysql":
yield "_mysql_exceptions", True
elif full_name == "lxml.objectify":
yield "lxml.etree", True
elif full_name == "_yaml":
yield "yaml", True
elif full_name == "apt_inst":
yield "apt_pkg", True
elif full_name == "numpy.core":
yield "numpy.core._dtype_ctypes", False
elif full_name == "scipy.special":
yield "scipy.special._ufuncs_cxx", False
elif full_name == "scipy.linalg":
yield "scipy.linalg.cython_blas", False
yield "scipy.linalg.cython_lapack", False
elif full_name == "PIL._imagingtk":
yield "PIL._tkinter_finder", True
elif full_name == "pkg_resources.extern":
if self.pkg_utils_externals is None:
for line in getFileContentByLine(module.getCompileTimeFilename()):
if line.startswith("names"):
line = line.split("=")[-1].strip()
parts = line.split(",")
self.pkg_utils_externals = [part.strip("' ") for part in parts]
break
else:
self.pkg_utils_externals = ()
for pkg_util_external in self.pkg_utils_externals:
yield "pkg_resources._vendor." + pkg_util_external, False
elif full_name == "pkg_resources._vendor.packaging":
yield "pkg_resources._vendor.packaging.version", True
yield "pkg_resources._vendor.packaging.specifiers", True
yield "pkg_resources._vendor.packaging.requirements", True
elif full_name == "uvloop.loop":
yield "uvloop._noop", True
elif full_name == "fitz.fitz":
yield "fitz._fitz", True
elif full_name == "pandas._libs":
yield "pandas._libs.tslibs.np_datetime", False
yield "pandas._libs.tslibs.nattype", False
elif full_name == "pandas.core.window":
yield "pandas._libs.skiplist", False
elif full_name == "zmq.backend":
yield "zmq.backend.cython", True
elif full_name == "OpenGL":
if self.opengl_plugins is None:
self.opengl_plugins = []
for line in getFileContentByLine(module.getCompileTimeFilename()):
if line.startswith("PlatformPlugin("):
os_part, plugin_name_part = line[15:-1].split(",")
os_part = os_part.strip("' ")
plugin_name_part = plugin_name_part.strip(") '")
plugin_name_part = plugin_name_part[
: plugin_name_part.rfind(".")
]
if os_part == "nt":
if getOS() == "Windows":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("linux"):
if getOS() == "Linux":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("darwin"):
if getOS() == "Darwin":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith(("posix", "osmesa", "egl")):
if getOS() != "Windows":
self.opengl_plugins.append(plugin_name_part)
else:
assert False, os_part
for opengl_plugin in self.opengl_plugins:
yield opengl_plugin, True
elif full_name == "Cryptodome.Util._raw_api":
for module_name in (
"_raw_aes",
"_raw_aesni",
"_raw_arc2",
"_raw_blowfish",
"_raw_cast",
"_raw_cbc",
"_raw_cfb",
"_raw_ctr",
"_raw_des",
"_raw_des3",
"_raw_ecb",
"_raw_ocb",
"_raw_ofb",
):
yield "Cryptodome.Cipher." + module_name, True
elif full_name == "Cryptodome.Util.strxor":
yield "Cryptodome.Util._strxor", True
elif full_name == "Cryptodome.Util._cpu_features":
yield "Cryptodome.Util._cpuid_c", True
elif full_name == "Cryptodome.Hash.BLAKE2s":
yield "Cryptodome.Hash._BLAKE2s", True
elif full_name == "Cryptodome.Hash.SHA1":
yield "Cryptodome.Hash._SHA1", True
elif full_name == "Cryptodome.Hash.SHA256":
yield "Cryptodome.Hash._SHA256", True
elif full_name == "Cryptodome.Hash.MD5":
yield "Cryptodome.Hash._MD5", True
elif full_name == "Cryptodome.Protocol.KDF":
yield "Cryptodome.Cipher._Salsa20", True
yield "Cryptodome.Protocol._scrypt", True
elif full_name == "Cryptodome.Cipher._mode_gcm":
yield "Cryptodome.Hash._ghash_portable", True
elif full_name == "pycparser.c_parser":
yield "pycparser.yacctab", True
yield "pycparser.lextab", True
def getImplicitImports(self, module):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches,too-many-statements
full_name = module.getFullName()
if module.isPythonShlibModule():
for used_module in module.getUsedModules():
yield used_module[0], False
# TODO: Move this out to some kind of configuration format.
elements = full_name.split(".")
if elements[0] in ("PyQt4", "PyQt5"):
if python_version < 300:
yield "atexit", True
# These are alternatives now:
# TODO: One day it should avoid including both.
yield "sip", False
if elements[0] == "PyQt5":
yield "PyQt5.sip", False
child = elements[1] if len(elements) > 1 else None
if child in (
"QtGui",
"QtAssistant",
"QtDBus",
"QtDeclarative",
"QtSql",
"QtDesigner",
"QtHelp",
"QtNetwork",
"QtScript",
"QtQml",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtOpenGL",
"QtXml",
"QtXmlPatterns",
"QtPrintSupport",
"QtNfc",
"QtWebKitWidgets",
"QtBluetooth",
"QtMultimediaWidgets",
"QtQuick",
"QtWebChannel",
"QtWebSockets",
"QtX11Extras",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtCore", True
if child in (
"QtDeclarative",
"QtWebKit",
"QtXmlPatterns",
"QtQml",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtQuick",
"QtQuickWidgets",
"QtWebSockets",
):
yield elements[0] + ".QtNetwork", True
if child == "QtScriptTools":
yield elements[0] + ".QtScript", True
if child in (
"QtWidgets",
"QtDeclarative",
"QtDesigner",
"QtHelp",
"QtScriptTools",
"QtSvg",
"QtTest",
"QtWebKit",
"QtPrintSupport",
"QtWebKitWidgets",
"QtMultimedia",
"QtMultimediaWidgets",
"QtOpenGL",
"QtQuick",
"QtQuickWidgets",
"QtSql",
"_QOpenGLFunctions_2_0",
"_QOpenGLFunctions_2_1",
"_QOpenGLFunctions_4_1_Core",
):
yield elements[0] + ".QtGui", True
if full_name in (
"PyQt5.QtDesigner",
"PyQt5.QtHelp",
"PyQt5.QtTest",
"PyQt5.QtPrintSupport",
"PyQt5.QtSvg",
"PyQt5.QtOpenGL",
"PyQt5.QtWebKitWidgets",
"PyQt5.QtMultimediaWidgets",
"PyQt5.QtQuickWidgets",
"PyQt5.QtSql",
):
yield "PyQt5.QtWidgets", True
if full_name in ("PyQt5.QtPrintSupport", ):
yield "PyQt5.QtSvg", True
if full_name in ("PyQt5.QtWebKitWidgets", ):
yield "PyQt5.QtWebKit", True
yield "PyQt5.QtPrintSupport", True
if full_name in ("PyQt5.QtMultimediaWidgets", ):
yield "PyQt5.QtMultimedia", True
if full_name in ("PyQt5.QtQuick", "PyQt5.QtQuickWidgets"):
yield "PyQt5.QtQml", True
if full_name in ("PyQt5.QtQuickWidgets", "PyQt5.QtQml"):
yield "PyQt5.QtQuick", True
if full_name == "PyQt5.Qt":
yield "PyQt5.QtCore", True
yield "PyQt5.QtDBus", True
yield "PyQt5.QtGui", True
yield "PyQt5.QtNetwork", True
yield "PyQt5.QtNetworkAuth", False
yield "PyQt5.QtSensors", False
yield "PyQt5.QtSerialPort", False
yield "PyQt5.QtMultimedia", True
yield "PyQt5.QtQml", False
yield "PyQt5.QtWidgets", True
elif full_name == "sip" and python_version < 300:
yield "enum", False
elif full_name == "PySide.QtDeclarative":
yield "PySide.QtGui", True
elif full_name == "PySide.QtHelp":
yield "PySide.QtGui", True
elif full_name == "PySide.QtOpenGL":
yield "PySide.QtGui", True
elif full_name == "PySide.QtScriptTools":
yield "PySide.QtScript", True
yield "PySide.QtGui", True
elif full_name == "PySide.QtSql":
yield "PySide.QtGui", True
elif full_name == "PySide.QtSvg":
yield "PySide.QtGui", True
elif full_name == "PySide.QtTest":
yield "PySide.QtGui", True
elif full_name == "PySide.QtUiTools":
yield "PySide.QtGui", True
yield "PySide.QtXml", True
elif full_name == "PySide.QtWebKit":
yield "PySide.QtGui", True
elif full_name == "PySide.phonon":
yield "PySide.QtGui", True
elif full_name == "lxml.etree":
yield "gzip", True
yield "lxml._elementpath", True
elif full_name == "gtk._gtk":
yield "pangocairo", True
yield "pango", True
yield "cairo", True
yield "gio", True
yield "atk", True
elif full_name == "atk":
yield "gobject", True
elif full_name == "gtkunixprint":
yield "gobject", True
yield "cairo", True
yield "gtk", True
elif full_name == "pango":
yield "gobject", True
elif full_name == "pangocairo":
yield "pango", True
yield "cairo", True
elif full_name == "reportlab.rl_config":
yield "reportlab.rl_settings", True
elif full_name == "socket":
yield "_socket", False
elif full_name == "ctypes":
yield "_ctypes", True
elif full_name == "gi._gi":
yield "gi._error", True
elif full_name == "gi._gi_cairo":
yield "cairo", True
elif full_name == "cairo._cairo":
yield "gi._gobject", False
elif full_name in ("Tkinter", "tkinter"):
yield "_tkinter", False
elif full_name in (
"cryptography.hazmat.bindings._openssl",
"cryptography.hazmat.bindings._constant_time",
"cryptography.hazmat.bindings._padding",
):
yield "_cffi_backend", True
elif full_name.startswith("cryptography._Cryptography_cffi_"):
yield "_cffi_backend", True
elif full_name == "bcrypt._bcrypt":
yield "_cffi_backend", True
elif full_name == "nacl._sodium":
yield "_cffi_backend", True
elif full_name == "_dbus_glib_bindings":
yield "_dbus_bindings", True
elif full_name == "_mysql":
yield "_mysql_exceptions", True
elif full_name == "lxml.objectify":
yield "lxml.etree", True
elif full_name == "_yaml":
yield "yaml", True
elif full_name == "apt_inst":
yield "apt_pkg", True
elif full_name == "numpy.core":
yield "numpy.core._dtype_ctypes", False
elif full_name == "scipy.special":
yield "scipy.special._ufuncs_cxx", False
elif full_name == "scipy.linalg":
yield "scipy.linalg.cython_blas", False
yield "scipy.linalg.cython_lapack", False
elif full_name == "PIL._imagingtk":
yield "PIL._tkinter_finder", True
elif full_name == "pkg_resources.extern":
if self.pkg_utils_externals is None:
for line in getFileContentByLine(
module.getCompileTimeFilename()):
if line.startswith("names"):
line = line.split("=")[-1].strip()
parts = line.split(",")
self.pkg_utils_externals = [
part.strip("' ") for part in parts
]
break
else:
self.pkg_utils_externals = ()
for pkg_util_external in self.pkg_utils_externals:
yield "pkg_resources._vendor." + pkg_util_external, False
elif full_name == "pkg_resources._vendor.packaging":
yield "pkg_resources._vendor.packaging.version", True
yield "pkg_resources._vendor.packaging.specifiers", True
yield "pkg_resources._vendor.packaging.requirements", True
elif full_name == "uvloop.loop":
yield "uvloop._noop", True
elif full_name == "fitz.fitz":
yield "fitz._fitz", True
elif full_name == "pandas._libs":
yield "pandas._libs.tslibs.np_datetime", False
yield "pandas._libs.tslibs.nattype", False
elif full_name == "pandas.core.window":
yield "pandas._libs.skiplist", False
elif full_name == "zmq.backend":
yield "zmq.backend.cython", True
elif full_name == "OpenGL":
if self.opengl_plugins is None:
self.opengl_plugins = []
for line in getFileContentByLine(
module.getCompileTimeFilename()):
if line.startswith("PlatformPlugin("):
os_part, plugin_name_part = line[15:-1].split(",")
os_part = os_part.strip("' ")
plugin_name_part = plugin_name_part.strip(") '")
plugin_name_part = plugin_name_part[:plugin_name_part.
rfind(".")]
if os_part == "nt":
if getOS() == "Windows":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("linux"):
if getOS() == "Linux":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("darwin"):
if getOS() == "Darwin":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith(("posix", "osmesa", "egl")):
if getOS() != "Windows":
self.opengl_plugins.append(plugin_name_part)
else:
assert False, os_part
for opengl_plugin in self.opengl_plugins:
yield opengl_plugin, True
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-locals
result = set()
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with _withLock(
"Finding out dependency walker path and creating DWP file for %s"
% binary_filename
):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
with open(dwp_filename, "w") as dwp_file:
dwp_file.write(
"""\
%(scan_dirs)s
SxS
"""
% {
"scan_dirs": "\n".join(
"UserDir %s" % dirname for dirname in scan_dirs
)
}
)
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
cwd=getExternalUsePath(os.getcwd()),
)
# TODO: Exit code should be checked.
depends_exe_process.wait()
if not os.path.exists(output_filename):
sys.exit("Error, depends.exe failed to product output.")
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
_parseDependsExeOutput(output_filename, result)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
if update_cache:
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def _getRequirementsContentsByLine():
return getFileContentByLine(
os.path.join(
os.path.dirname(__file__), "..", "..", "..", "..", "requirements-devel.txt"
)
)
def _parseDependsExeOutput(filename, result):
_parseDependsExeOutput2(getFileContentByLine(filename), result)
def getRuntimeTraceOfLoadedFiles(logger, path):
""" Returns the files loaded when executing a binary. """
# This will make a crazy amount of work,
# pylint: disable=I0021,too-many-branches,too-many-locals,too-many-statements
if not os.path.exists(path):
# TODO: Have a logger package passed.
logger.sysexit("Error, cannot find %r (%r)." % (path, os.path.abspath(path)))
result = []
if os.name == "posix":
if sys.platform == "darwin" or sys.platform.startswith("freebsd"):
if not isExecutableCommand("dtruss"):
sys.exit(
"""\
Error, needs 'dtruss' on your system to scan used libraries."""
)
if not isExecutableCommand("sudo"):
sys.exit(
"""\
Error, needs 'sudo' on your system to scan used libraries."""
)
args = ("sudo", "dtruss", "-t", "open", path)
else:
if not isExecutableCommand("strace"):
sys.exit(
"""\
Error, needs 'strace' on your system to scan used libraries."""
)
args = (
"strace",
"-e",
"file",
"-s4096", # Some paths are truncated in output otherwise
os.path.abspath(path),
)
# Ensure executable is not polluted with third party stuff,
# tests may fail otherwise due to unexpected libs being loaded
with withEnvironmentVarOverriden("LD_PRELOAD", None):
tracing_command = args[0] if args[0] != "sudo" else args[1]
process = subprocess.Popen(
args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_stdout_strace, stderr_strace = process.communicate()
exit_strace = process.returncode
if exit_strace != 0:
if str is not bytes:
stderr_strace = stderr_strace.decode("utf8")
logger.warning(stderr_strace)
logger.sysexit("Failed to run %r." % tracing_command)
with open(path + ".strace", "wb") as f:
f.write(stderr_strace)
for line in stderr_strace.split(b"\n"):
if process.returncode != 0:
logger.my_print(line)
if not line:
continue
# Don't consider files not found. The "site" module checks lots
# of things.
if b"ENOENT" in line:
continue
if line.startswith(b"stat(") and b"S_IFDIR" in line:
continue
# Allow stats on the python binary, and stuff pointing to the
# standard library, just not uses of it. It will search there
# for stuff.
if (
line.startswith(b"lstat(")
or line.startswith(b"stat(")
or line.startswith(b"readlink(")
):
filename = line[line.find(b"(") + 2 : line.find(b", ") - 1]
# At least Python3.7 considers the default Python3 path.
if filename == b"/usr/bin/python3":
continue
if filename in (
b"/usr/bin/python3." + version
for version in (b"5", b"6", b"7", b"8", b"9")
):
continue
binary_path = _python_executable
if str is not bytes:
binary_path = binary_path.encode("utf-8")
found = False
while binary_path:
if filename == binary_path:
found = True
break
if binary_path == os.path.dirname(binary_path):
break
binary_path = os.path.dirname(binary_path)
if filename == os.path.join(
binary_path,
b"python"
+ (
"%d%d" % (_python_version[0], _python_version[1])
).encode("utf8"),
):
found = True
continue
if found:
continue
result.extend(
os.path.abspath(match)
for match in re.findall(b'"(.*?)(?:\\\\0)?"', line)
)
if sys.version.startswith("3"):
result = [s.decode("utf-8") for s in result]
elif os.name == "nt":
command = (
getDependsExePath(),
"-c", # Console mode
"-ot%s" % path + ".depends",
"-f1",
"-pb",
"-pa1", # Turn on all profiling options.
"-ps1", # Simulate ShellExecute with app dirs in PATH.
"-pp1", # Do not long DllMain calls.
"-po1", # Log DllMain call for all other messages.
"-ph1", # Hook the process.
"-pl1", # Log LoadLibrary calls.
"-pt1", # Thread information.
"-pe1", # First chance exceptions.
"-pg1", # Log GetProcAddress calls.
"-pf1", # Use full paths.
"-pc1", # Profile child processes.
path,
)
subprocess.call(command)
inside = False
for line in getFileContentByLine(path + ".depends"):
if "| Module Dependency Tree |" in line:
inside = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[: line.find("]")]:
continue
dll_filename = line[line.find("]") + 2 :].rstrip()
dll_filename = os.path.normcase(dll_filename)
assert os.path.isfile(dll_filename), repr(dll_filename)
# The executable itself is of course exempted.
if dll_filename == os.path.normcase(os.path.abspath(path)):
continue
result.append(dll_filename)
os.unlink(path + ".depends")
result = list(sorted(set(result)))
return result
def _detectBinaryPathDLLsWindows(is_main_executable, source_dir, original_dir,
binary_filename, package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if (os.path.exists(cache_filename)
and not Options.shallNotUseDependsExeCachedResults()):
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
# User query should only happen once if at all.
with _withLock():
depends_exe = getDependsExePath()
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
with _withLock():
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", "w") as dwp_file:
dwp_file.write(
"""\
KnownDLLs
%(original_dirs)s
SysPath
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""" % {
"original_dirs":
"\n".join("UserDir %s" % dirname for dirname in scan_dirs
if not os.path.basename(dirname) == "__pycache__"
if any(entry[1].lower().endswith(".dll")
for entry in listDir(dirname)))
})
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen((
depends_exe,
"-c",
"-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp",
"-f1",
"-pa1",
"-ps1",
binary_filename,
))
# TODO: Exit code should be checked.
depends_exe_process.wait()
# Opening the result under lock, so it is not getting locked by new processes.
with _withLock():
_parseDependsExeOutput(binary_filename + ".depends", result)
deleteFile(binary_filename + ".depends", must_exist=True)
deleteFile(binary_filename + ".dwp", must_exist=True)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def getImplicitImports(self, module):
# Many variables, branches, due to the many cases, pylint: disable=too-many-branches
full_name = module.getFullName()
if module.isPythonShlibModule():
for used_module in module.getUsedModules():
yield used_module[0], False
if full_name == "pkg_resources.extern":
if self.pkg_utils_externals is None:
for line in getFileContentByLine(
module.getCompileTimeFilename()):
if line.startswith("names"):
line = line.split("=")[-1].strip()
parts = line.split(",")
self.pkg_utils_externals = [
part.strip("' ") for part in parts
]
break
else:
self.pkg_utils_externals = ()
for pkg_util_external in self.pkg_utils_externals:
yield "pkg_resources._vendor." + pkg_util_external, False
elif full_name == "OpenGL":
if self.opengl_plugins is None:
self.opengl_plugins = []
for line in getFileContentByLine(
module.getCompileTimeFilename()):
if line.startswith("PlatformPlugin("):
os_part, plugin_name_part = line[15:-1].split(",")
os_part = os_part.strip("' ")
plugin_name_part = plugin_name_part.strip(") '")
plugin_name_part = plugin_name_part[:plugin_name_part.
rfind(".")]
if os_part == "nt":
if getOS() == "Windows":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("linux"):
if getOS() == "Linux":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith("darwin"):
if getOS() == "Darwin":
self.opengl_plugins.append(plugin_name_part)
elif os_part.startswith(("posix", "osmesa", "egl")):
if getOS() != "Windows":
self.opengl_plugins.append(plugin_name_part)
else:
assert False, os_part
for opengl_plugin in self.opengl_plugins:
yield opengl_plugin, True
else:
# create a flattened import set for full_name and yield from it
for item in self.getImportsByFullname(full_name):
yield item
def _detectBinaryPathDLLsWindowsPE(is_main_executable, source_dir,
original_dir, binary_filename,
package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches,too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if (os.path.exists(cache_filename)
and not Options.shallNotUseDependsExeCachedResults()):
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if os.path.isdir(original_dir):
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if Options.isExperimental("use_pefile_fullrecurse"):
try:
scan_dirs.extend(getSubDirectories(get_python_lib()))
except OSError:
print(
"Cannot recurse into site-packages for dependencies. Path not found."
)
else:
# Fix for missing pywin32 inclusion when using pythonwin library, no way to detect that automagically
# Since there are more than one dependencies on pywintypes37.dll, let's include this anyway
# In recursive mode, using dirname(original_dir) won't always work, hence get_python_lib
try:
scan_dirs.append(os.path.join(get_python_lib(),
"pywin32_system32"))
except OSError:
pass
# Add native system directory based on pe file architecture and os architecture
# Python 32: system32 = syswow64 = 32 bits systemdirectory
# Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
binary_file_is_64bit = _isPE64(binary_filename)
python_is_64bit = getArchitecture() == "x86_64"
if binary_file_is_64bit is not python_is_64bit:
print("Warning: Using Python x64=%s with x64=%s binary dependencies" %
(binary_file_is_64bit, python_is_64bit))
if binary_file_is_64bit:
# This is actually not useful as of today since we don't compile 32 bits on 64 bits
if python_is_64bit:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"System32"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32"))
if Options.isExperimental("use_pefile_recurse"):
# Recursive one level scanning of all .pyd and .dll in the original_dir too
# This shall fix a massive list of missing dependencies that may come with included libraries which themselves
# need to be scanned for inclusions
for root, _, filenames in os.walk(original_dir):
for optional_libary in filenames:
if optional_libary.endswith(
".dll") or optional_libary.endswith(".pyd"):
_parsePEFileOutput(os.path.join(root, optional_libary),
scan_dirs, result)
_parsePEFileOutput(binary_filename, scan_dirs, result)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def parseDependsExeOutput(filename):
return _parseDependsExeOutput2(getFileContentByLine(filename, encoding="latin1"))
def _parseDependsExeOutput(filename, result):
_parseDependsExeOutput2(getFileContentByLine(filename, encoding="latin1"), result)
def _getProjectOptions(logger, filename_arg, module_mode):
# Complex stuff, pylint: disable=too-many-branches,too-many-locals
# Do it only once.
if os.environ.get("NUITKA_REEXECUTION", "0") == "1":
return
if os.path.isdir(filename_arg):
if module_mode:
filename_arg = os.path.join(filename_arg, "__init__.py")
else:
filename_arg = os.path.join(filename_arg, "__main__.py")
# The file specified may not exist, let the later parts of Nuitka handle this.
try:
contents_by_line = getFileContentByLine(filename_arg, "rb")
except (OSError, IOError):
return
def sysexit(count, message):
logger.sysexit("%s:%d %s" % (filename_arg, count + 1, message))
execute_block = True
expect_block = False
cond_level = -1
for count, line in enumerate(contents_by_line):
match = re.match(b"^\\s*#(\\s+)nuitka-project(.*?):(.*)", line)
if match:
level, command, arg = match.groups()
level = len(level)
# Check for empty conditional blocks.
if expect_block and level <= cond_level:
sysexit(
count,
"Error, 'nuitka-project-if' is expected to be followed by block start.",
)
expect_block = False
if level <= cond_level:
execute_block = True
if level > cond_level and not execute_block:
continue
if str is not bytes:
command = command.decode("utf8")
arg = arg.decode("utf8")
if command == "-if":
if not arg.endswith(":"):
sysexit(
count,
"Error, 'nuitka-project-if' needs to start a block with a colon.",
)
arg = arg[:-1]
expanded = _expandProjectArg(arg, filename_arg, for_eval=True)
r = eval( # We allow the user to run any code, pylint: disable=eval-used
expanded)
# Likely mistakes, e.g. with "in" tests.
if r is not True and r is not False:
sys.exit(
"Error, 'nuitka-project-if' condition %r (%r) does not yield boolean result %r"
% (arg, expanded, r))
execute_block = r
expect_block = True
cond_level = level
elif command == "":
arg = re.sub(r"""^([\w-]*=)(['"])(.*)\2$""", r"\1\3",
arg.lstrip())
yield _expandProjectArg(arg, filename_arg, for_eval=False)
else:
assert False, (command, line)
def getRuntimeTraceOfLoadedFiles(path, trace_error=True):
""" Returns the files loaded when executing a binary. """
# This will make a crazy amount of work,
# pylint: disable=I0021,too-many-branches,too-many-locals,too-many-statements
result = []
if os.name == "posix":
if sys.platform == "darwin" or sys.platform.startswith("freebsd"):
if not isExecutableCommand("dtruss"):
sys.exit("""\
Error, needs 'dtruss' on your system to scan used libraries.""")
if not isExecutableCommand("sudo"):
sys.exit("""\
Error, needs 'sudo' on your system to scan used libraries.""")
args = ("sudo", "dtruss", "-t", "open", path)
else:
if not isExecutableCommand("strace"):
sys.exit("""\
Error, needs 'strace' on your system to scan used libraries.""")
args = (
"strace",
"-e",
"file",
"-s4096", # Some paths are truncated otherwise.
path,
)
# Ensure executable is not polluted with third party stuff,
# tests may fail otherwise due to unexpected libs being loaded
with withEnvironmentVarOverriden("LD_PRELOAD", None):
process = subprocess.Popen(args=args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_stdout_strace, stderr_strace = process.communicate()
exit_strace = process.returncode
if exit_strace != 0:
if str is not bytes:
stderr_strace = stderr_strace.decode("utf8")
my_print(stderr_strace, file=sys.stderr)
sys.exit("Failed to run strace.")
with open(path + ".strace", "wb") as f:
f.write(stderr_strace)
for line in stderr_strace.split(b"\n"):
if process.returncode != 0 and trace_error:
my_print(line)
if not line:
continue
# Don't consider files not found. The "site" module checks lots
# of things.
if b"ENOENT" in line:
continue
if line.startswith(b"stat(") and b"S_IFDIR" in line:
continue
# Allow stats on the python binary, and stuff pointing to the
# standard library, just not uses of it. It will search there
# for stuff.
if (line.startswith(b"lstat(") or line.startswith(b"stat(")
or line.startswith(b"readlink(")):
filename = line[line.find(b"(") + 2:line.find(b", ") - 1]
# At least Python3.7 considers the default Python3 path.
if filename == b"/usr/bin/python3":
continue
if filename in (b"/usr/bin/python3." + version
for version in (b"5", b"6", b"7")):
continue
binary_path = _python_executable
if str is not bytes:
binary_path = binary_path.encode("utf-8")
found = False
while binary_path:
if filename == binary_path:
found = True
break
if binary_path == os.path.dirname(binary_path):
break
binary_path = os.path.dirname(binary_path)
if filename == os.path.join(
binary_path, b"python" +
_python_version[:3].encode("utf8")):
found = True
continue
if found:
continue
result.extend(
os.path.abspath(match)
for match in re.findall(b'"(.*?)(?:\\\\0)?"', line))
if sys.version.startswith("3"):
result = [s.decode("utf-8") for s in result]
elif os.name == "nt":
subprocess.call((
getDependsExePath(),
"-c",
"-ot%s" % path + ".depends",
"-f1",
"-pa1",
"-ps1",
"-pp0",
"-pl1",
path,
))
inside = False
for line in getFileContentByLine(path + ".depends"):
if "| Module Dependency Tree |" in line:
inside = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if "]" not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if "?" in line[:line.find("]")]:
continue
dll_filename = line[line.find("]") + 2:-1]
assert os.path.isfile(dll_filename), dll_filename
# The executable itself is of course exempted.
if os.path.normcase(dll_filename) == os.path.normcase(
os.path.abspath(path)):
continue
dll_filename = os.path.normcase(dll_filename)
result.append(dll_filename)
os.unlink(path + ".depends")
result = list(sorted(set(result)))
return result
