def assemble(self):
from PyInstaller.config import CONF
logger.info("Building EXE from %s", self.tocbasename)
if os.path.exists(self.name):
os.remove(self.name)
if not os.path.exists(os.path.dirname(self.name)):
os.makedirs(os.path.dirname(self.name))
exe = self.exefiles[0][1] # pathname of bootloader
if not os.path.exists(exe):
raise SystemExit(_MISSING_BOOTLOADER_ERRORMSG)
# Step 1: copy the bootloader file, and perform any operations that need to be done prior to appending the PKG.
logger.info("Copying bootloader EXE to %s", self.name)
self._copyfile(exe, self.name)
os.chmod(self.name, 0o755)
if is_win:
# First, remove all resources from the file. This ensures that no manifest is embedded, even if bootloader
# was compiled with a toolchain that forcibly embeds a default manifest (e.g., mingw toolchain from msys2).
winresource.RemoveAllResources(self.name)
# Embed icon.
if self.icon != "NONE":
logger.info("Copying icon to EXE")
icon.CopyIcons(self.name, self.icon)
# Embed version info.
if self.versrsrc:
logger.info("Copying version information to EXE")
versioninfo.SetVersion(self.name, self.versrsrc)
# Embed other resources.
logger.info("Copying %d resources to EXE", len(self.resources))
for res in self.resources:
res = res.split(",")
for i in range(1, len(res)):
try:
res[i] = int(res[i])
except ValueError:
pass
resfile = res[0]
if not os.path.isabs(resfile):
resfile = os.path.join(CONF['specpath'], resfile)
restype = resname = reslang = None
if len(res) > 1:
restype = res[1]
if len(res) > 2:
resname = res[2]
if len(res) > 3:
reslang = res[3]
try:
winresource.UpdateResourcesFromResFile(
self.name, resfile, [restype or "*"], [resname or "*"], [reslang or "*"]
)
except winresource.pywintypes.error as exc:
if exc.args[0] != winresource.ERROR_BAD_EXE_FORMAT:
logger.error(
"Error while updating resources in %s from resource file %s!",
self.name,
resfile,
exc_info=1
)
continue
# Handle the case where the file contains no resources, and is intended as a single resource to be
# added to the exe.
if not restype or not resname:
logger.error("Resource type and/or name not specified!")
continue
if "*" in (restype, resname):
logger.error(
"No wildcards allowed for resource type and name when the source file does not contain "
"any resources!"
)
continue
try:
winresource.UpdateResourcesFromDataFile(self.name, resfile, restype, [resname], [reslang or 0])
except winresource.pywintypes.error:
logger.error(
"Error while updating resource %s %s in %s from data file %s!",
restype,
resname,
self.name,
resfile,
exc_info=1
)
# Embed the manifest into the executable.
if self.embed_manifest:
logger.info("Emedding manifest in EXE")
self.manifest.update_resources(self.name, [1])
elif is_darwin:
# Convert bootloader to the target arch
logger.info("Converting EXE to target arch (%s)", self.target_arch)
osxutils.binary_to_target_arch(self.name, self.target_arch, display_name='Bootloader EXE')
# Step 2: append the PKG, if necessary
if not self.append_pkg:
# In onefile mode, copy the stand-alone pkg next to the executable. In onedir, this will be done by the
# COLLECT() target.
if not self.exclude_binaries:
pkg_dst = os.path.join(os.path.dirname(self.name), os.path.basename(self.pkgname))
logger.info("Copying stand-alone PKG archive from %s to %s", self.pkg.name, pkg_dst)
self._copyfile(self.pkg.name, pkg_dst)
else:
logger.info("Stand-alone PKG archive will be handled by COLLECT")
elif is_linux:
# Linux: append PKG into ELF section using objcopy
logger.info("Appending PKG archive to ELF section in EXE")
retcode, stdout, stderr = exec_command_all(
'objcopy', '--add-section', 'pydata=%s' % self.pkg.name, self.name
)
logger.debug("objcopy returned %i", retcode)
if stdout:
logger.debug(stdout)
if stderr:
logger.debug(stderr)
if retcode != 0:
raise SystemError("objcopy Failure: %s" % stderr)
elif is_darwin:
# macOS: remove signature, append PKG, and fix-up headers so that PKG appears to be part of the executable.
# Strip signatures from all arch slices. Strictly speaking, we need to remove signature (if present) from
# the last slice, because we will be appending data to it. When building universal2 bootloaders natively on
# macOS, only arm64 slices have a (dummy) signature. However, when cross-compiling with osxcross, we seem to
# get dummy signatures on both x86_64 and arm64 slices. While the former should not have any impact, it does
# seem to cause issues with further binary signing using real identity. Therefore, we remove all signatures
# and re-sign the binary using dummy signature once the data is appended.
logger.info("Removing signature(s) from EXE")
osxutils.remove_signature_from_binary(self.name)
# Append the PKG data
logger.info("Appending PKG archive to EXE")
with open(self.name, 'ab') as outf:
with open(self.pkg.name, 'rb') as inf:
shutil.copyfileobj(inf, outf, length=64 * 1024)
# Fix Mach-O header for code signing
logger.info("Fixing EXE headers for code signing")
osxutils.fix_exe_for_code_signing(self.name)
else:
# Fall back to just appending PKG at the end of the file
logger.info("Appending PKG archive to EXE")
with open(self.name, 'ab') as outf:
with open(self.pkg.name, 'rb') as inf:
shutil.copyfileobj(inf, outf, length=64 * 1024)
# Step 3: post-processing
if is_win:
# Set checksum to appease antiviral software.
set_exe_checksum(self.name)
elif is_darwin:
# If the version of macOS SDK used to build bootloader exceeds that of macOS SDK used to built Python
# library (and, by extension, bundled Tcl/Tk libraries), force the version declared by the frozen executable
# to match that of the Python library.
# Having macOS attempt to enable new features (based on SDK version) for frozen application has no benefit
# if the Python library does not support them as well.
# On the other hand, there seem to be UI issues in tkinter due to failed or partial enablement of dark mode
# (i.e., the bootloader executable being built against SDK 10.14 or later, which causes macOS to enable dark
# mode, and Tk libraries being built against an earlier SDK version that does not support the dark mode).
# With python.org Intel macOS installers, this manifests as black Tk windows and UI elements (see issue
# #5827), while in Anaconda python, it may result in white text on bright background.
pylib_version = osxutils.get_macos_sdk_version(bindepend.get_python_library_path())
exe_version = osxutils.get_macos_sdk_version(self.name)
if pylib_version < exe_version:
logger.info(
"Rewriting the executable's macOS SDK version (%d.%d.%d) to match the SDK version of the Python "
"library (%d.%d.%d) in order to avoid inconsistent behavior and potential UI issues in the "
"frozen application.", *exe_version, *pylib_version
)
osxutils.set_macos_sdk_version(self.name, *pylib_version)
# Re-sign the binary (either ad-hoc or using real identity, if provided).
logger.info("Re-signing the EXE")
osxutils.sign_binary(self.name, self.codesign_identity, self.entitlements_file)
# Ensure executable flag is set
os.chmod(self.name, 0o755)
# Get mtime for storing into the guts
self.mtm = misc.mtime(self.name)
logger.info("Building EXE from %s completed successfully.", self.tocbasename)
def checkCache(fnm,
strip=False,
upx=False,
upx_exclude=None,
dist_nm=None,
target_arch=None,
codesign_identity=None,
entitlements_file=None):
"""
Cache prevents preprocessing binary files again and again.
'dist_nm' Filename relative to dist directory. We need it on Mac to determine level of paths for @loader_path like
'@loader_path/../../' for qt4 plugins.
"""
from PyInstaller.config import CONF
# On Mac OS, a cache is required anyway to keep the libaries with relative install names.
# Caching on Mac OS does not work since we need to modify binary headers to use relative paths to dll depencies and
# starting with '@loader_path'.
if not strip and not upx and not is_darwin and not is_win:
return fnm
if dist_nm is not None and ":" in dist_nm:
# A file embedded in another PyInstaller build via multipackage.
# No actual file exists to process.
return fnm
if strip:
strip = True
else:
strip = False
upx_exclude = upx_exclude or []
upx = (upx and (is_win or is_cygwin)
and os.path.normcase(os.path.basename(fnm)) not in upx_exclude)
# Load cache index.
# Make cachedir per Python major/minor version.
# This allows parallel building of executables with different Python versions as one user.
pyver = 'py%d%s' % (sys.version_info[0], sys.version_info[1])
arch = platform.architecture()[0]
cachedir = os.path.join(CONF['cachedir'],
'bincache%d%d_%s_%s' % (strip, upx, pyver, arch))
if target_arch:
cachedir = os.path.join(cachedir, target_arch)
if is_darwin:
# Separate by codesign identity
if codesign_identity:
# Compute hex digest of codesign identity string to prevent issues with invalid characters.
csi_hash = hashlib.sha256(codesign_identity.encode('utf-8'))
cachedir = os.path.join(cachedir, csi_hash.hexdigest())
else:
cachedir = os.path.join(cachedir, 'adhoc') # ad-hoc signing
# Separate by entitlements
if entitlements_file:
# Compute hex digest of entitlements file contents
with open(entitlements_file, 'rb') as fp:
ef_hash = hashlib.sha256(fp.read())
cachedir = os.path.join(cachedir, ef_hash.hexdigest())
else:
cachedir = os.path.join(cachedir, 'no-entitlements')
if not os.path.exists(cachedir):
os.makedirs(cachedir)
cacheindexfn = os.path.join(cachedir, "index.dat")
if os.path.exists(cacheindexfn):
try:
cache_index = misc.load_py_data_struct(cacheindexfn)
except Exception:
# Tell the user they may want to fix their cache... However, do not delete it for them; if it keeps getting
# corrupted, we will never find out.
logger.warning(
"PyInstaller bincache may be corrupted; use pyinstaller --clean to fix it."
)
raise
else:
cache_index = {}
# Verify that the file we are looking for is present in the cache. Use the dist_mn if given to avoid different
# extension modules sharing the same basename get corrupted.
if dist_nm:
basenm = os.path.normcase(dist_nm)
else:
basenm = os.path.normcase(os.path.basename(fnm))
# Binding redirects should be taken into account to see if the file needs to be reprocessed. The redirects may
# change if the versions of dependent manifests change due to system updates.
redirects = CONF.get('binding_redirects', [])
digest = cacheDigest(fnm, redirects)
cachedfile = os.path.join(cachedir, basenm)
cmd = None
if basenm in cache_index:
if digest != cache_index[basenm]:
os.remove(cachedfile)
else:
return cachedfile
# Optionally change manifest and its dependencies to private assemblies.
if fnm.lower().endswith(".manifest"):
manifest = winmanifest.Manifest()
manifest.filename = fnm
with open(fnm, "rb") as f:
manifest.parse_string(f.read())
if CONF.get('win_private_assemblies', False):
if manifest.publicKeyToken:
logger.info("Changing %s into private assembly",
os.path.basename(fnm))
manifest.publicKeyToken = None
for dep in manifest.dependentAssemblies:
# Exclude common-controls which is not bundled
if dep.name != "Microsoft.Windows.Common-Controls":
dep.publicKeyToken = None
applyRedirects(manifest, redirects)
manifest.writeprettyxml(cachedfile)
return cachedfile
if upx:
if strip:
fnm = checkCache(fnm,
strip=True,
upx=False,
dist_nm=dist_nm,
target_arch=target_arch,
codesign_identity=codesign_identity,
entitlements_file=entitlements_file)
# We need to avoid using UPX with Windows DLLs that have Control Flow Guard enabled, as it breaks them.
if is_win and versioninfo.pefile_check_control_flow_guard(fnm):
logger.info('Disabling UPX for %s due to CFG!', fnm)
elif misc.is_file_qt_plugin(fnm):
logger.info('Disabling UPX for %s due to it being a Qt plugin!',
fnm)
else:
bestopt = "--best"
# FIXME: Linux builds of UPX do not seem to contain LZMA (they assert out).
# A better configure-time check is due.
if CONF["hasUPX"] >= (3, ) and os.name == "nt":
bestopt = "--lzma"
upx_executable = "upx"
if CONF.get('upx_dir'):
upx_executable = os.path.join(CONF['upx_dir'], upx_executable)
cmd = [upx_executable, bestopt, "-q", cachedfile]
else:
if strip:
strip_options = []
if is_darwin:
# The default strip behavior breaks some shared libraries under Mac OS.
strip_options = ["-S"] # -S = strip only debug symbols.
cmd = ["strip"] + strip_options + [cachedfile]
if not os.path.exists(os.path.dirname(cachedfile)):
os.makedirs(os.path.dirname(cachedfile))
# There are known some issues with 'shutil.copy2' on Mac OS 10.11 with copying st_flags. Issue #1650.
# 'shutil.copy' copies also permission bits and it should be sufficient for PyInstaller's purposes.
shutil.copy(fnm, cachedfile)
# TODO: find out if this is still necessary when no longer using shutil.copy2()
if hasattr(os, 'chflags'):
# Some libraries on FreeBSD have immunable flag (libthr.so.3, for example). If this flag is preserved,
# os.chmod() fails with: OSError: [Errno 1] Operation not permitted.
try:
os.chflags(cachedfile, 0)
except OSError:
pass
os.chmod(cachedfile, 0o755)
if os.path.splitext(fnm.lower())[1] in (".pyd", ".dll"):
# When shared assemblies are bundled into the app, they may optionally be changed into private assemblies.
try:
res = winmanifest.GetManifestResources(os.path.abspath(cachedfile))
except winresource.pywintypes.error as e:
if e.args[0] == winresource.ERROR_BAD_EXE_FORMAT:
# Not a win32 PE file
pass
else:
logger.error(os.path.abspath(cachedfile))
raise
else:
if winmanifest.RT_MANIFEST in res and len(
res[winmanifest.RT_MANIFEST]):
for name in res[winmanifest.RT_MANIFEST]:
for language in res[winmanifest.RT_MANIFEST][name]:
try:
manifest = winmanifest.Manifest()
manifest.filename = ":".join([
cachedfile,
str(winmanifest.RT_MANIFEST),
str(name),
str(language)
])
manifest.parse_string(
res[winmanifest.RT_MANIFEST][name][language],
False)
except Exception:
logger.error(
"Cannot parse manifest resource %s, =%s", name,
language)
logger.error("From file %s",
cachedfile,
exc_info=1)
else:
# optionally change manifest to private assembly
private = CONF.get('win_private_assemblies', False)
if private:
if manifest.publicKeyToken:
logger.info(
"Changing %s into a private assembly",
os.path.basename(fnm))
manifest.publicKeyToken = None
# Change dep to private assembly
for dep in manifest.dependentAssemblies:
# Exclude common-controls which is not bundled
if dep.name != "Microsoft.Windows.Common-Controls":
dep.publicKeyToken = None
redirecting = applyRedirects(manifest, redirects)
if redirecting or private:
try:
manifest.update_resources(
os.path.abspath(cachedfile), [name],
[language])
except Exception:
logger.error(os.path.abspath(cachedfile))
raise
if cmd:
logger.info("Executing - " + ' '.join(cmd))
# terminates if execution fails
compat.exec_command(*cmd)
# update cache index
cache_index[basenm] = digest
misc.save_py_data_struct(cacheindexfn, cache_index)
# On Mac OS we need relative paths to dll dependencies starting with @executable_path. While modifying
# the headers invalidates existing signatures, we avoid removing them in order to speed things up (and
# to avoid potential bugs in the codesign utility, like the one reported on Mac OS 10.13 in #6167).
# The forced re-signing at the end should take care of the invalidated signatures.
if is_darwin:
osxutils.binary_to_target_arch(cachedfile,
target_arch,
display_name=fnm)
#osxutils.remove_signature_from_binary(cachedfile) # Disabled as per comment above.
dylib.mac_set_relative_dylib_deps(cachedfile, dist_nm)
osxutils.sign_binary(cachedfile, codesign_identity, entitlements_file)
return cachedfile
def assemble(self):
from PyInstaller.config import CONF
logger.info("Building EXE from %s", self.tocbasename)
trash = []
if os.path.exists(self.name):
os.remove(self.name)
if not os.path.exists(os.path.dirname(self.name)):
os.makedirs(os.path.dirname(self.name))
exe = self.exefiles[0][1] # pathname of bootloader
if not os.path.exists(exe):
raise SystemExit(_MISSING_BOOTLOADER_ERRORMSG)
if is_win:
fd, tmpnm = tempfile.mkstemp(prefix=os.path.basename(exe) + ".",
dir=CONF['workpath'])
# need to close the file, otherwise copying resources will fail
# with "the file [...] is being used by another process"
os.close(fd)
self._copyfile(exe, tmpnm)
os.chmod(tmpnm, 0o755)
if self.icon != "NONE":
icon.CopyIcons(tmpnm, self.icon)
if self.versrsrc:
versioninfo.SetVersion(tmpnm, self.versrsrc)
for res in self.resources:
res = res.split(",")
for i in range(1, len(res)):
try:
res[i] = int(res[i])
except ValueError:
pass
resfile = res[0]
if not os.path.isabs(resfile):
resfile = os.path.join(CONF['specpath'], resfile)
restype = resname = reslang = None
if len(res) > 1:
restype = res[1]
if len(res) > 2:
resname = res[2]
if len(res) > 3:
reslang = res[3]
try:
winresource.UpdateResourcesFromResFile(
tmpnm, resfile, [restype or "*"], [resname or "*"],
[reslang or "*"])
except winresource.pywintypes.error as exc:
if exc.args[0] != winresource.ERROR_BAD_EXE_FORMAT:
logger.error(
"Error while updating resources in %s"
" from resource file %s",
tmpnm,
resfile,
exc_info=1)
continue
# Handle the case where the file contains no resources, and is
# intended as a single resource to be added to the exe.
if not restype or not resname:
logger.error("resource type and/or name not specified")
continue
if "*" in (restype, resname):
logger.error("no wildcards allowed for resource type "
"and name when source file does not "
"contain resources")
continue
try:
winresource.UpdateResourcesFromDataFile(
tmpnm, resfile, restype, [resname], [reslang or 0])
except winresource.pywintypes.error:
logger.error(
"Error while updating resource %s %s in %s"
" from data file %s",
restype,
resname,
tmpnm,
resfile,
exc_info=1)
if self.manifest and not self.exclude_binaries:
self.manifest.update_resources(tmpnm, [1])
trash.append(tmpnm)
exe = tmpnm
# NOTE: Do not look up for bootloader file in the cache because it might
# get corrupted by UPX when UPX is available. See #1863 for details.
if not self.append_pkg:
logger.info("Copying bootloader exe to %s", self.name)
self._copyfile(exe, self.name)
logger.info("Copying archive to %s", self.pkgname)
self._copyfile(self.pkg.name, self.pkgname)
elif is_linux:
self._copyfile(exe, self.name)
logger.info("Appending archive to ELF section in EXE %s",
self.name)
retcode, stdout, stderr = exec_command_all(
'objcopy', '--add-section', 'pydata=%s' % self.pkg.name,
self.name)
logger.debug("objcopy returned %i", retcode)
if stdout:
logger.debug(stdout)
if stderr:
logger.debug(stderr)
if retcode != 0:
raise SystemError("objcopy Failure: %s" % stderr)
elif is_darwin:
import PyInstaller.utils.osx as osxutils
# Copy bootloader
logger.info("Copying bootloader exe to %s", self.name)
with open(self.name, 'wb') as outf:
with open(exe, 'rb') as inf:
shutil.copyfileobj(inf, outf, length=64 * 1024)
# Convert bootloader to target arch
logger.info("Converting EXE to target arch (%s)", self.target_arch)
osxutils.binary_to_target_arch(self.name,
self.target_arch,
display_name='Bootloader EXE')
# Strip signatures from all arch slices. Strictly speaking,
# we need to remove signature (if present) from the last
# slice, because we will be appending data to it. When
# building universal2 bootloaders natively on macOS, only
# arm64 slices have a (dummy) signature. However, when
# cross-compiling with osxcross, we seem to get dummy
# signatures on both x86_64 and arm64 slices. While the former
# should not have any impact, it does seem to cause issues
# with further binary signing using real identity. Therefore,
# we remove all signatures and re-sign the binary using
# dummy signature once the data is appended.
logger.info("Removing signature(s) from EXE")
osxutils.remove_signature_from_binary(self.name)
# Append the data
with open(self.name, 'ab') as outf:
with open(self.pkg.name, 'rb') as inf:
shutil.copyfileobj(inf, outf, length=64 * 1024)
# If the version of macOS SDK used to build bootloader exceeds
# that of macOS SDK used to built Python library (and, by
# extension, bundled Tcl/Tk libraries), force the version
# declared by the frozen executable to match that of the Python
# library.
# Having macOS attempt to enable new features (based on SDK
# version) for frozen application has no benefit if the Python
# library does not support them as well.
# On the other hand, there seem to be UI issues in tkinter
# due to failed or partial enablement of dark mode (i.e., the
# bootloader executable being built against SDK 10.14 or later,
# which causes macOS to enable dark mode, and Tk libraries being
# built against an earlier SDK version that does not support the
# dark mode). With python.org Intel macOS installers, this
# manifests as black Tk windows and UI elements (see issue #5827),
# while in Anaconda python, it may result in white text on bright
# background.
pylib_version = osxutils.get_macos_sdk_version(
bindepend.get_python_library_path())
exe_version = osxutils.get_macos_sdk_version(self.name)
if pylib_version < exe_version:
logger.info(
"Rewriting executable's macOS SDK version (%d.%d.%d) to "
"match the SDK version of the Python library (%d.%d.%d) "
"in order to avoid inconsistent behavior and potential UI "
"issues in the frozen application.", *exe_version,
*pylib_version)
osxutils.set_macos_sdk_version(self.name, *pylib_version)
# Fix Mach-O header for codesigning on OS X.
logger.info("Fixing EXE for code signing %s", self.name)
osxutils.fix_exe_for_code_signing(self.name)
# Re-sign the binary (either ad-hoc or using real identity,
# if provided)
logger.info("Re-signing the EXE")
osxutils.sign_binary(self.name, self.codesign_identity,
self.entitlements_file)
else:
# Fall back to just append on end of file
logger.info("Appending archive to EXE %s", self.name)
with open(self.name, 'wb') as outf:
# write the bootloader data
with open(exe, 'rb') as infh:
shutil.copyfileobj(infh, outf, length=64 * 1024)
# write the archive data
with open(self.pkg.name, 'rb') as infh:
shutil.copyfileobj(infh, outf, length=64 * 1024)
if is_win:
# Set checksum to appease antiviral software.
from PyInstaller.utils.win32.winutils import set_exe_checksum
set_exe_checksum(self.name)
os.chmod(self.name, 0o755)
# get mtime for storing into the guts
self.mtm = misc.mtime(self.name)
for item in trash:
os.remove(item)
logger.info("Building EXE from %s completed successfully.",
self.tocbasename)