def test_manifest_from_res_file(tmp_path):
# This import only works on Windows. Place it here, protected by the
# `@pytest.mark.win32`` decorator.
from PyInstaller.utils.win32 import winmanifest
# Locate bootloader executable
bootloader_file = os.path.join(HOMEPATH, 'PyInstaller', 'bootloader',
PLATFORM, 'run.exe')
# Create a local copy
test_file = str(tmp_path / 'test_file.exe')
shutil.copyfile(bootloader_file, test_file)
# Create a manifest, ...
manifest_filename = test_file + '.manifest'
manifest = winmanifest.Manifest(
type_="win32",
name='test_file.exe',
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
manifest.filename = manifest_filename
manifest.requestedExecutionLevel = 'asInvoker'
manifest.uiAccess = True
# ... embed it, ...
manifest.update_resources(test_file, [1])
# ... and read it back
manifest2 = winmanifest.ManifestFromResFile(test_file)
assert manifest == manifest2
def getAssemblies(pth):
"""
On Windows return the dependent Side-by-Side (SxS) assemblies of a binary as a list of Manifest objects.
Dependent assemblies are required only by binaries compiled with MSVC 9.0. Python 2.7 and 3.2 are compiled with
MSVC 9.0 and thus depend on Microsoft Redistributable runtime libraries 9.0.
Python 3.3+ is compiled with version 10.0 and does not use SxS assemblies.
FIXME: Can this be removed since we now only support Python 3.5+?
FIXME: IS there some test-case covering this?
"""
if pth.lower().endswith(".manifest"):
return []
# check for manifest file
manifestnm = pth + ".manifest"
if os.path.isfile(manifestnm):
with open(manifestnm, "rb") as fd:
res = {winmanifest.RT_MANIFEST: {1: {0: fd.read()}}}
else:
# check the binary for embedded manifest
try:
res = winmanifest.GetManifestResources(pth)
except winresource.pywintypes.error as exc:
if exc.args[0] == winresource.ERROR_BAD_EXE_FORMAT:
logger.info('Cannot get manifest resource from non-PE file %s',
pth)
return []
raise
rv = []
if winmanifest.RT_MANIFEST in res and len(res[winmanifest.RT_MANIFEST]):
for name in res[winmanifest.RT_MANIFEST]:
for language in res[winmanifest.RT_MANIFEST][name]:
# check the manifest for dependent assemblies
try:
manifest = winmanifest.Manifest()
manifest.filename = ":".join([
pth,
str(winmanifest.RT_MANIFEST),
str(name),
str(language),
])
manifest.parse_string(
res[winmanifest.RT_MANIFEST][name][language], False)
except Exception:
logger.error(
"Cannot parse manifest resource %s, %s from %s",
name,
language,
pth,
exc_info=1)
else:
if manifest.dependentAssemblies:
logger.debug("Dependent assemblies of %s:", pth)
logger.debug(", ".join([
assembly.getid()
for assembly in manifest.dependentAssemblies
]))
rv.extend(manifest.dependentAssemblies)
return rv
def assemble(self):
"""
This method is the MAIN method for finding all necessary files to be bundled.
"""
from PyInstaller.config import CONF
for m in self.excludes:
logger.debug("Excluding module '%s'" % m)
self.graph = initialize_modgraph(excludes=self.excludes,
user_hook_dirs=self.hookspath)
# TODO Find a better place where to put 'base_library.zip' and when to created it.
# For Python 3 it is necessary to create file 'base_library.zip'
# containing core Python modules. In Python 3 some built-in modules
# are written in pure Python. base_library.zip is a way how to have
# those modules as "built-in".
libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip')
create_py3_base_library(libzip_filename, graph=self.graph)
# Bundle base_library.zip as data file.
# Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA')
self.datas.append(
(os.path.basename(libzip_filename), libzip_filename, 'DATA'))
# Expand sys.path of module graph.
# The attribute is the set of paths to use for imports: sys.path,
# plus our loader, plus other paths from e.g. --path option).
self.graph.path = self.pathex + self.graph.path
self.graph.set_setuptools_nspackages()
logger.info("running Analysis %s", self.tocbasename)
# Get paths to Python and, in Windows, the manifest.
python = compat.python_executable
if not is_win:
# Linux/MacOS: get a real, non-link path to the running Python executable.
while os.path.islink(python):
python = os.path.join(os.path.dirname(python),
os.readlink(python))
depmanifest = None
else:
# Windows: Create a manifest to embed into built .exe, containing the same
# dependencies as python.exe.
depmanifest = winmanifest.Manifest(
type_="win32",
name=CONF['specnm'],
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
depmanifest.filename = os.path.join(
CONF['workpath'], CONF['specnm'] + ".exe.manifest")
# We record "binaries" separately from the modulegraph, as there
# is no way to record those dependencies in the graph. These include
# the python executable and any binaries added by hooks later.
# "binaries" are not the same as "extensions" which are .so or .dylib
# that are found and recorded as extension nodes in the graph.
# Reset seen variable before running bindepend. We use bindepend only for
# the python executable.
bindepend.seen.clear()
# Add binary and assembly dependencies of Python.exe.
# This also ensures that its assembly depencies under Windows get added to the
# built .exe's manifest. Python 2.7 extension modules have no assembly
# dependencies, and rely on the app-global dependencies set by the .exe.
self.binaries.extend(
bindepend.Dependencies([('', python, '')],
manifest=depmanifest,
redirects=self.binding_redirects)[1:])
if is_win:
depmanifest.writeprettyxml()
### Module graph.
#
# Construct the module graph of import relationships between modules
# required by this user's application. For each entry point (top-level
# user-defined Python script), all imports originating from this entry
# point are recursively parsed into a subgraph of the module graph. This
# subgraph is then connected to this graph's root node, ensuring
# imported module nodes will be reachable from the root node -- which is
# is (arbitrarily) chosen to be the first entry point's node.
# List to hold graph nodes of scripts and runtime hooks in use order.
priority_scripts = []
# Assume that if the script does not exist, Modulegraph will raise error.
# Save the graph nodes of each in sequence.
for script in self.inputs:
logger.info("Analyzing %s", script)
priority_scripts.append(self.graph.add_script(script))
# Analyze the script's hidden imports (named on the command line)
self.graph.add_hiddenimports(self.hiddenimports)
### Post-graph hooks.
self.graph.process_post_graph_hooks(self)
# Update 'binaries' TOC and 'datas' TOC.
deps_proc = DependencyProcessor(self.graph,
self.graph._additional_files_cache)
self.binaries.extend(deps_proc.make_binaries_toc())
self.datas.extend(deps_proc.make_datas_toc())
self.zipped_data.extend(deps_proc.make_zipped_data_toc())
# Note: zipped eggs are collected below
### Look for dlls that are imported by Python 'ctypes' module.
# First get code objects of all modules that import 'ctypes'.
logger.info('Looking for ctypes DLLs')
# dict like: {'module1': code_obj, 'module2': code_obj}
ctypes_code_objs = self.graph.get_code_using("ctypes")
for name, co in ctypes_code_objs.items():
# Get dlls that might be needed by ctypes.
logger.debug('Scanning %s for shared libraries or dlls', name)
try:
ctypes_binaries = scan_code_for_ctypes(co)
self.binaries.extend(set(ctypes_binaries))
except Exception as ex:
raise RuntimeError(f"Failed to scan the module '{name}'. "
f"This is a bug. Please report it.") from ex
self.datas.extend((dest, source, "DATA")
for (dest, source) in format_binaries_and_datas(
self.graph.metadata_required()))
# Analyze run-time hooks.
# Run-time hooks has to be executed before user scripts. Add them
# to the beginning of 'priority_scripts'.
priority_scripts = self.graph.analyze_runtime_hooks(
self.custom_runtime_hooks) + priority_scripts
# 'priority_scripts' is now a list of the graph nodes of custom runtime
# hooks, then regular runtime hooks, then the PyI loader scripts.
# Further on, we will make sure they end up at the front of self.scripts
### Extract the nodes of the graph as TOCs for further processing.
# Initialize the scripts list with priority scripts in the proper order.
self.scripts = self.graph.nodes_to_toc(priority_scripts)
# Extend the binaries list with all the Extensions modulegraph has found.
self.binaries = self.graph.make_binaries_toc(self.binaries)
# Fill the "pure" list with pure Python modules.
assert len(self.pure) == 0
self.pure = self.graph.make_pure_toc()
# And get references to module code objects constructed by ModuleGraph
# to avoid writing .pyc/pyo files to hdd.
self.pure._code_cache = self.graph.get_code_objects()
# Add remaining binary dependencies - analyze Python C-extensions and what
# DLLs they depend on.
logger.info('Looking for dynamic libraries')
self.binaries.extend(
bindepend.Dependencies(self.binaries,
redirects=self.binding_redirects))
### Include zipped Python eggs.
logger.info('Looking for eggs')
self.zipfiles.extend(deps_proc.make_zipfiles_toc())
# Verify that Python dynamic library can be found.
# Without dynamic Python library PyInstaller cannot continue.
self._check_python_library(self.binaries)
if is_win:
# Remove duplicate redirects
self.binding_redirects[:] = list(set(self.binding_redirects))
logger.info("Found binding redirects: \n%s",
self.binding_redirects)
# Filter binaries to adjust path of extensions that come from
# python's lib-dynload directory. Prefix them with lib-dynload
# so that we'll collect them into subdirectory instead of
# directly into _MEIPASS
for idx, tpl in enumerate(self.binaries):
name, path, typecode = tpl
if typecode == 'EXTENSION' \
and not os.path.dirname(os.path.normpath(name)) \
and os.path.basename(os.path.dirname(path)) == 'lib-dynload':
name = os.path.join('lib-dynload', name)
self.binaries[idx] = (name, path, typecode)
# Place Python source in data files for the noarchive case.
if self.noarchive:
# Create a new TOC of ``(dest path for .pyc, source for .py, type)``.
new_toc = TOC()
for name, path, typecode in self.pure:
assert typecode == 'PYMODULE'
# Transform a python module name into a file name.
name = name.replace('.', os.sep)
# Special case: modules have an implied filename to add.
if os.path.splitext(os.path.basename(path))[0] == '__init__':
name += os.sep + '__init__'
# Append the extension for the compiled result.
# In python 3.5 (PEP-488) .pyo files were replaced by
# .opt-1.pyc and .opt-2.pyc. However, it seems that for
# bytecode-only module distribution, we always need to
# use the .pyc extension.
name += '.pyc'
new_toc.append((name, path, typecode))
# Put the result of byte-compiling this TOC in datas. Mark all entries as data.
for name, path, typecode in compile_py_files(
new_toc, CONF['workpath']):
self.datas.append((name, path, 'DATA'))
# Store no source in the archive.
self.pure = TOC()
# Write warnings about missing modules.
self._write_warnings()
# Write debug information about hte graph
self._write_graph_debug()
def checkCache(fnm,
strip=False,
upx=False,
upx_exclude=None,
dist_nm=None,
target_arch=None,
codesign_identity=None,
entitlements_file=None):
"""
Cache prevents preprocessing binary files again and again.
'dist_nm' Filename relative to dist directory. We need it on Mac to determine level of paths for @loader_path like
'@loader_path/../../' for qt4 plugins.
"""
from PyInstaller.config import CONF
# On Mac OS, a cache is required anyway to keep the libaries with relative install names.
# Caching on Mac OS does not work since we need to modify binary headers to use relative paths to dll depencies and
# starting with '@loader_path'.
if not strip and not upx and not is_darwin and not is_win:
return fnm
if dist_nm is not None and ":" in dist_nm:
# A file embedded in another PyInstaller build via multipackage.
# No actual file exists to process.
return fnm
if strip:
strip = True
else:
strip = False
upx_exclude = upx_exclude or []
upx = (upx and (is_win or is_cygwin)
and os.path.normcase(os.path.basename(fnm)) not in upx_exclude)
# Load cache index.
# Make cachedir per Python major/minor version.
# This allows parallel building of executables with different Python versions as one user.
pyver = 'py%d%s' % (sys.version_info[0], sys.version_info[1])
arch = platform.architecture()[0]
cachedir = os.path.join(CONF['cachedir'],
'bincache%d%d_%s_%s' % (strip, upx, pyver, arch))
if target_arch:
cachedir = os.path.join(cachedir, target_arch)
if is_darwin:
# Separate by codesign identity
if codesign_identity:
# Compute hex digest of codesign identity string to prevent issues with invalid characters.
csi_hash = hashlib.sha256(codesign_identity.encode('utf-8'))
cachedir = os.path.join(cachedir, csi_hash.hexdigest())
else:
cachedir = os.path.join(cachedir, 'adhoc') # ad-hoc signing
# Separate by entitlements
if entitlements_file:
# Compute hex digest of entitlements file contents
with open(entitlements_file, 'rb') as fp:
ef_hash = hashlib.sha256(fp.read())
cachedir = os.path.join(cachedir, ef_hash.hexdigest())
else:
cachedir = os.path.join(cachedir, 'no-entitlements')
if not os.path.exists(cachedir):
os.makedirs(cachedir)
cacheindexfn = os.path.join(cachedir, "index.dat")
if os.path.exists(cacheindexfn):
try:
cache_index = misc.load_py_data_struct(cacheindexfn)
except Exception:
# Tell the user they may want to fix their cache... However, do not delete it for them; if it keeps getting
# corrupted, we will never find out.
logger.warning(
"PyInstaller bincache may be corrupted; use pyinstaller --clean to fix it."
)
raise
else:
cache_index = {}
# Verify that the file we are looking for is present in the cache. Use the dist_mn if given to avoid different
# extension modules sharing the same basename get corrupted.
if dist_nm:
basenm = os.path.normcase(dist_nm)
else:
basenm = os.path.normcase(os.path.basename(fnm))
# Binding redirects should be taken into account to see if the file needs to be reprocessed. The redirects may
# change if the versions of dependent manifests change due to system updates.
redirects = CONF.get('binding_redirects', [])
digest = cacheDigest(fnm, redirects)
cachedfile = os.path.join(cachedir, basenm)
cmd = None
if basenm in cache_index:
if digest != cache_index[basenm]:
os.remove(cachedfile)
else:
return cachedfile
# Optionally change manifest and its dependencies to private assemblies.
if fnm.lower().endswith(".manifest"):
manifest = winmanifest.Manifest()
manifest.filename = fnm
with open(fnm, "rb") as f:
manifest.parse_string(f.read())
if CONF.get('win_private_assemblies', False):
if manifest.publicKeyToken:
logger.info("Changing %s into private assembly",
os.path.basename(fnm))
manifest.publicKeyToken = None
for dep in manifest.dependentAssemblies:
# Exclude common-controls which is not bundled
if dep.name != "Microsoft.Windows.Common-Controls":
dep.publicKeyToken = None
applyRedirects(manifest, redirects)
manifest.writeprettyxml(cachedfile)
return cachedfile
if upx:
if strip:
fnm = checkCache(fnm,
strip=True,
upx=False,
dist_nm=dist_nm,
target_arch=target_arch,
codesign_identity=codesign_identity,
entitlements_file=entitlements_file)
# We need to avoid using UPX with Windows DLLs that have Control Flow Guard enabled, as it breaks them.
if is_win and versioninfo.pefile_check_control_flow_guard(fnm):
logger.info('Disabling UPX for %s due to CFG!', fnm)
elif misc.is_file_qt_plugin(fnm):
logger.info('Disabling UPX for %s due to it being a Qt plugin!',
fnm)
else:
bestopt = "--best"
# FIXME: Linux builds of UPX do not seem to contain LZMA (they assert out).
# A better configure-time check is due.
if CONF["hasUPX"] >= (3, ) and os.name == "nt":
bestopt = "--lzma"
upx_executable = "upx"
if CONF.get('upx_dir'):
upx_executable = os.path.join(CONF['upx_dir'], upx_executable)
cmd = [upx_executable, bestopt, "-q", cachedfile]
else:
if strip:
strip_options = []
if is_darwin:
# The default strip behavior breaks some shared libraries under Mac OS.
strip_options = ["-S"] # -S = strip only debug symbols.
cmd = ["strip"] + strip_options + [cachedfile]
if not os.path.exists(os.path.dirname(cachedfile)):
os.makedirs(os.path.dirname(cachedfile))
# There are known some issues with 'shutil.copy2' on Mac OS 10.11 with copying st_flags. Issue #1650.
# 'shutil.copy' copies also permission bits and it should be sufficient for PyInstaller's purposes.
shutil.copy(fnm, cachedfile)
# TODO: find out if this is still necessary when no longer using shutil.copy2()
if hasattr(os, 'chflags'):
# Some libraries on FreeBSD have immunable flag (libthr.so.3, for example). If this flag is preserved,
# os.chmod() fails with: OSError: [Errno 1] Operation not permitted.
try:
os.chflags(cachedfile, 0)
except OSError:
pass
os.chmod(cachedfile, 0o755)
if os.path.splitext(fnm.lower())[1] in (".pyd", ".dll"):
# When shared assemblies are bundled into the app, they may optionally be changed into private assemblies.
try:
res = winmanifest.GetManifestResources(os.path.abspath(cachedfile))
except winresource.pywintypes.error as e:
if e.args[0] == winresource.ERROR_BAD_EXE_FORMAT:
# Not a win32 PE file
pass
else:
logger.error(os.path.abspath(cachedfile))
raise
else:
if winmanifest.RT_MANIFEST in res and len(
res[winmanifest.RT_MANIFEST]):
for name in res[winmanifest.RT_MANIFEST]:
for language in res[winmanifest.RT_MANIFEST][name]:
try:
manifest = winmanifest.Manifest()
manifest.filename = ":".join([
cachedfile,
str(winmanifest.RT_MANIFEST),
str(name),
str(language)
])
manifest.parse_string(
res[winmanifest.RT_MANIFEST][name][language],
False)
except Exception:
logger.error(
"Cannot parse manifest resource %s, =%s", name,
language)
logger.error("From file %s",
cachedfile,
exc_info=1)
else:
# optionally change manifest to private assembly
private = CONF.get('win_private_assemblies', False)
if private:
if manifest.publicKeyToken:
logger.info(
"Changing %s into a private assembly",
os.path.basename(fnm))
manifest.publicKeyToken = None
# Change dep to private assembly
for dep in manifest.dependentAssemblies:
# Exclude common-controls which is not bundled
if dep.name != "Microsoft.Windows.Common-Controls":
dep.publicKeyToken = None
redirecting = applyRedirects(manifest, redirects)
if redirecting or private:
try:
manifest.update_resources(
os.path.abspath(cachedfile), [name],
[language])
except Exception:
logger.error(os.path.abspath(cachedfile))
raise
if cmd:
logger.info("Executing - " + ' '.join(cmd))
# terminates if execution fails
compat.exec_command(*cmd)
# update cache index
cache_index[basenm] = digest
misc.save_py_data_struct(cacheindexfn, cache_index)
# On Mac OS we need relative paths to dll dependencies starting with @executable_path. While modifying
# the headers invalidates existing signatures, we avoid removing them in order to speed things up (and
# to avoid potential bugs in the codesign utility, like the one reported on Mac OS 10.13 in #6167).
# The forced re-signing at the end should take care of the invalidated signatures.
if is_darwin:
osxutils.binary_to_target_arch(cachedfile,
target_arch,
display_name=fnm)
#osxutils.remove_signature_from_binary(cachedfile) # Disabled as per comment above.
dylib.mac_set_relative_dylib_deps(cachedfile, dist_nm)
osxutils.sign_binary(cachedfile, codesign_identity, entitlements_file)
return cachedfile