def add_suffix_to_extensions(toc):
"""
Returns a new TOC with proper library suffix for EXTENSION items.
"""
# TODO: Fix this recursive import
from PyInstaller.building.datastruct import TOC
new_toc = TOC()
for inm, fnm, typ in toc:
if typ == 'EXTENSION':
# Change the dotted name into a relative path. This places C
# extensions in the Python-standard location.
inm = inm.replace('.', os.sep)
# In some rare cases extension might already contain a suffix.
# Skip it in this case.
if os.path.splitext(inm)[1] not in EXTENSION_SUFFIXES:
# Determine the base name of the file.
base_name = os.path.basename(inm)
assert '.' not in base_name
# Use this file's existing extension. For extensions such as
# ``libzmq.cp36-win_amd64.pyd``, we can't use
# ``os.path.splitext``, which would give only the ```.pyd`` part
# of the extension.
inm = inm + os.path.basename(fnm)[len(base_name):]
elif typ == 'DEPENDENCY':
# Use the suffix from the filename.
# TODO Verify what extensions are by DEPENDENCIES.
binext = os.path.splitext(fnm)[1]
if not os.path.splitext(inm)[1] == binext:
inm = inm + binext
new_toc.append((inm, fnm, typ))
return new_toc
def get_bootstrap_modules():
"""
Get TOC with the bootstrapping modules and their dependencies.
:return: TOC with modules
"""
# Import 'struct' modules to get real paths to module file names.
mod_struct = __import__('struct')
# Basic modules necessary for the bootstrap process.
loader_mods = TOC()
loaderpath = os.path.join(HOMEPATH, 'PyInstaller', 'loader')
# On some platforms (Windows, Debian/Ubuntu) '_struct' and zlib modules are
# built-in modules (linked statically) and thus does not have attribute __file__.
# 'struct' module is required for reading Python bytecode from executable.
# 'zlib' is required to decompress this bytecode.
for mod_name in ['_struct', 'zlib']:
mod = __import__(mod_name) # C extension.
if hasattr(mod, '__file__'):
loader_mods.append((mod_name, os.path.abspath(mod.__file__), 'EXTENSION'))
# NOTE:These modules should be kept simple without any complicated dependencies.
loader_mods +=[
('struct', os.path.abspath(mod_struct.__file__), 'PYMODULE'),
('pyimod01_os_path', os.path.join(loaderpath, 'pyimod01_os_path.pyc'), 'PYMODULE'),
('pyimod02_archive', os.path.join(loaderpath, 'pyimod02_archive.pyc'), 'PYMODULE'),
('pyimod03_importers', os.path.join(loaderpath, 'pyimod03_importers.pyc'), 'PYMODULE'),
('pyiboot01_bootstrap', os.path.join(loaderpath, 'pyiboot01_bootstrap.py'), 'PYSOURCE'),
]
return loader_mods
def test_insert_other_case_pymodule():
# python modules should not use C-I comparisons. Both 'encodings' and
# 'EnCodIngs' should be added.
toc = TOC(ELEMS1)
elem = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'PYMODULE')
toc.insert(1, elem)
expected = list(ELEMS1)
expected.insert(1, elem)
assert toc == expected
def test_insert_other_case_mixed():
# If a binary file is added with the same filename as an existing pymodule,
# it should not be added
toc = TOC(ELEMS1)
elem = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'BINARY')
toc.insert(1, elem)
expected = list(ELEMS1)
assert toc == expected
def test_radd_toc():
toc = TOC(ELEMS1)
other = TOC(ELEMS2)
result = other + toc
assert result is not toc
assert result is not other
assert isinstance(result, TOC)
expected = list(ELEMS2) + list(ELEMS1)
assert result == expected
def test_sub_toc():
toc = TOC(ELEMS1) + ELEMS2
other = TOC(ELEMS2)
result = toc - other
assert result is not toc
assert result is not other
assert isinstance(result, TOC)
expected = list(ELEMS1)
assert result == expected
def test_rsub_toc():
toc = TOC(ELEMS1)
other = TOC(ELEMS1) + ELEMS2
result = other - toc
assert result is not toc
assert result is not other
assert isinstance(result, TOC)
expected = list(ELEMS2)
assert result == expected
def test_insert_other_case_mixed():
# If a binary file is added with the same filename as an existing pymodule,
# it should not be added
toc = TOC(ELEMS1)
elem = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'BINARY')
toc.insert(1, elem)
expected = list(ELEMS1)
assert toc == expected
def test_insert_other_case_pymodule():
# python modules should not use C-I comparisons. Both 'encodings' and
# 'EnCodIngs' should be added.
toc = TOC(ELEMS1)
elem = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'PYMODULE')
toc.insert(1, elem)
expected = list(ELEMS1)
expected.insert(1, elem)
assert toc == expected
def __init__(self, *tocs, **kwargs):
"""
tocs
One or more TOCs (Tables of Contents), normally an
Analysis.pure.
If this TOC has an attribute `_code_cache`, this is
expected to be a dict of module code objects from
ModuleGraph.
kwargs
Possible keywork arguments:
name
A filename for the .pyz. Normally not needed, as the generated
name will do fine.
cipher
The block cipher that will be used to encrypt Python bytecode.
"""
from ..config import CONF
Target.__init__(self)
name = kwargs.get('name', None)
cipher = kwargs.get('cipher', None)
self.toc = TOC()
# If available, use code objects directly from ModuleGraph to
# speed up PyInstaller.
self.code_dict = {}
for t in tocs:
self.toc.extend(t)
self.code_dict.update(getattr(t, '_code_cache', {}))
# Paths to remove from filenames embedded in code objects
self.replace_paths = sys.path + CONF['pathex']
# Make sure paths end with os.sep
self.replace_paths = [os.path.join(f, '') for f in self.replace_paths]
self.name = name
if name is None:
self.name = os.path.splitext(self.tocfilename)[0] + '.pyz'
# PyInstaller bootstrapping modules.
self.dependencies = get_bootstrap_modules()
# Bundle the crypto key.
self.cipher = cipher
if cipher:
key_file = ('pyimod00_crypto_key',
os.path.join(CONF['workpath'], 'pyimod00_crypto_key.pyc'),
'PYMODULE')
# Insert the key as the first module in the list. The key module contains
# just variables and does not depend on other modules.
self.dependencies.insert(0, key_file)
# Compile the top-level modules so that they end up in the CArchive and can be
# imported by the bootstrap script.
self.dependencies = misc.compile_py_files(self.dependencies, CONF['workpath'])
self.__postinit__()
def __init__(self, *args, **kws):
"""
args
One or more arguments that are either TOCs Targets.
kws
Possible keywork arguments:
name
The name of the directory to be built.
"""
from PyInstaller.config import CONF
Target.__init__(self)
self.strip_binaries = kws.get('strip', False)
self.upx_exclude = kws.get("upx_exclude", [])
self.console = True
self.target_arch = None
self.codesign_identity = None
self.entitlements_file = None
if CONF['hasUPX']:
self.upx_binaries = kws.get('upx', False)
else:
self.upx_binaries = False
self.name = kws.get('name')
# Old .spec format included in 'name' the path where to collect files
# for the created app.
# app. New format includes only directory name.
#
# The 'name' directory is created in DISTPATH and necessary files are
# then collected to this directory.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append(
(os.path.basename(arg.name), arg.name, arg.typ))
if isinstance(arg, EXE):
self.console = arg.console
self.target_arch = arg.target_arch
self.codesign_identity = arg.codesign_identity
self.entitlements_file = arg.entitlements_file
for tocnm, fnm, typ in arg.toc:
if tocnm == os.path.basename(arg.name) + ".manifest":
self.toc.append((tocnm, fnm, typ))
if not arg.append_pkg:
self.toc.append((os.path.basename(arg.pkgname),
arg.pkgname, 'PKG'))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
self.__postinit__()
def make_datas_toc(self):
toc = TOC((x, y, 'DATA') for x, y in self._datas)
for dist in self._distributions:
if (dist._pyinstaller_info['egg']
and not dist._pyinstaller_info['zipped']
and not dist._pyinstaller_info['zip-safe']):
# this is a un-zipped, not-zip-safe egg
toplevel = dist.get_metadata('top_level.txt').strip()
basedir = dist.location
if toplevel:
os.path.join(basedir, toplevel)
tree = Tree(dist.location, excludes=PY_IGNORE_EXTENSIONS)
toc.extend(tree)
return toc
def _make_toc(self, typecode=None, existing_TOC=None):
"""
Return the name, path and type of selected nodes as a TOC, or appended
to a TOC. The selection is via a list of PyInstaller TOC typecodes.
If that list is empty we return the complete flattened graph as a TOC
with the ModuleGraph note types in place of typecodes -- meant for
debugging only. Normally we return ModuleGraph nodes whose types map
to the requested PyInstaller typecode(s) as indicated in the MODULE_TYPES_TO_TOC_DICT.
We use the ModuleGraph (really, ObjectGraph) flatten() method to
scan all the nodes. This is patterned after ModuleGraph.report().
"""
# Construct regular expression for matching modules that should be
# excluded because they are bundled in base_library.zip.
#
# This expression matches the base module name, optionally followed by
# a period and then any number of characters. This matches the module name and
# the fully qualified names of any of its submodules.
regex_str = '(' + '|'.join(PY3_BASE_MODULES) + r')(\.|$)'
module_filter = re.compile(regex_str)
result = existing_TOC or TOC()
for node in self.iter_graph(start=self._top_script_node):
# Skip modules that are in base_library.zip.
if module_filter.match(node.identifier):
continue
entry = self._node_to_toc(node, typecode)
if entry is not None:
# TOC.append the data. This checks for a pre-existing name
# and skips it if it exists.
result.append(entry)
return result
def test_add_tuple():
toc = TOC(ELEMS1)
other = ELEMS2
result = toc + other
assert result is not toc
assert result is not other
assert isinstance(result, TOC)
expected = list(ELEMS1) + list(ELEMS2)
assert result == expected
def test_rsub_non_existing():
toc = TOC(ELEMS3)
other = ELEMS1
result = other - toc
assert result is not toc
assert result is not other
assert isinstance(result, TOC)
expected = list(ELEMS1)
assert result == expected
def nodes_to_toc(self, node_list, existing_TOC=None):
"""
Given a list of nodes, create a TOC representing those nodes. This is mainly used to initialize a TOC of
scripts with the ones that are runtime hooks. The process is almost the same as _make_toc(), but the caller
guarantees the nodes are valid, so minimal checking.
"""
result = existing_TOC or TOC()
for node in node_list:
result.append(self._node_to_toc(node))
return result
def _check_guts(self, data, last_build):
if Target._check_guts(self, data, last_build):
return True
for fnm in self.inputs:
if mtime(fnm) > last_build:
logger.info("Building because %s changed", fnm)
return True
# Now we know that none of the input parameters and none of the input files has changed. So take the values
# calculated resp. analysed in the last run and store them in `self`.
self.scripts = TOC(data['scripts'])
self.pure = TOC(data['pure'])
self.binaries = TOC(data['binaries'])
self.zipfiles = TOC(data['zipfiles'])
self.zipped_data = TOC(data['zipped_data'])
self.datas = TOC(data['datas'])
# Store previously found binding redirects in CONF for later use by PKG/COLLECT
from PyInstaller.config import CONF
self.binding_redirects = CONF['binding_redirects'] = data[
'binding_redirects']
return False
def Datafiles(*filenames, **kw):
import os
def datafile(path, strip_path=True):
parts = path.split('/')
path = name = os.path.join(*parts)
if strip_path:
name = os.path.basename(path)
return name, path, 'DATA'
strip_path = kw.get('strip_path', True)
return TOC(
datafile(filename, strip_path=strip_path) for filename in filenames
if os.path.isfile(filename))
def __init__(self, *args, **kws):
"""
args
One or more arguments that are either TOCs Targets.
kws
Possible keywork arguments:
name
The name of the directory to be built.
"""
from ..config import CONF
Target.__init__(self)
self.strip_binaries = kws.get('strip', False)
self.console = True
if CONF['hasUPX']:
self.upx_binaries = kws.get('upx', False)
else:
self.upx_binaries = False
self.name = kws.get('name')
# Old .spec format included in 'name' the path where to collect files
# for the created app.
# app. New format includes only directory name.
#
# The 'name' directory is created in DISTPATH and necessary files are
# then collected to this directory.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append((os.path.basename(arg.name), arg.name, arg.typ))
if isinstance(arg, EXE):
self.console = arg.console
for tocnm, fnm, typ in arg.toc:
if tocnm == os.path.basename(arg.name) + ".manifest":
self.toc.append((tocnm, fnm, typ))
if not arg.append_pkg:
self.toc.append((os.path.basename(arg.pkgname), arg.pkgname, 'PKG'))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
self.__postinit__()
def make_zipped_data_toc(self):
toc = TOC()
logger.debug('Looking for egg data files...')
for dist in self._distributions:
if dist._pyinstaller_info['egg']:
# TODO: check in docs if top_level.txt always exists
toplevel = dist.get_metadata('top_level.txt').strip()
if dist._pyinstaller_info['zipped']:
# this is a zipped egg
tree = self.__collect_data_files_from_zip(dist.location)
toc.extend(tree)
elif dist._pyinstaller_info['zip-safe']:
# this is an un-zipped, zip-safe egg
basedir = dist.location
if toplevel:
os.path.join(basedir, toplevel)
tree = Tree(dist.location, excludes=PY_IGNORE_EXTENSIONS)
toc.extend(tree)
else:
# this is an un-zipped, not-zip-safe egg, handled in make_datas_toc()
pass
return toc
class EXE(Target):
"""
Creates the final executable of the frozen app.
This bundles all necessary files together.
"""
typ = 'EXECUTABLE'
def __init__(self, *args, **kwargs):
"""
args
One or more arguments that are either TOCs Targets.
kwargs
Possible keywork arguments:
console
On Windows or OSX governs whether to use the console executable
or the windowed executable. Always True on Linux/Unix (always
console executable - it does not matter there).
debug
Setting to True gives you progress mesages from the executable
(for console=False there will be annoying MessageBoxes on Windows).
name
The filename for the executable. On Windows suffix '.exe' is
appended.
exclude_binaries
Forwarded to the PKG the EXE builds.
icon
Windows or OSX only. icon='myicon.ico' to use an icon file or
icon='notepad.exe,0' to grab an icon resource.
version
Windows only. version='myversion.txt'. Use grab_version.py to get
a version resource from an executable and then edit the output to
create your own. (The syntax of version resources is so arcane
that I wouldn't attempt to write one from scratch).
uac_admin
Windows only. Setting to True creates a Manifest with will request
elevation upon application restart
uac_uiaccess
Windows only. Setting to True allows an elevated application to
work with Remote Desktop
"""
from ..config import CONF
Target.__init__(self)
# Available options for EXE in .spec files.
self.exclude_binaries = kwargs.get('exclude_binaries', False)
self.console = kwargs.get('console', True)
self.debug = kwargs.get('debug', False)
self.name = kwargs.get('name', None)
self.icon = kwargs.get('icon', None)
self.versrsrc = kwargs.get('version', None)
self.manifest = kwargs.get('manifest', None)
self.resources = kwargs.get('resources', [])
self.strip = kwargs.get('strip', False)
# If ``append_pkg`` is false, the archive will not be appended
# to the exe, but copied beside it.
self.append_pkg = kwargs.get('append_pkg', True)
# On Windows allows the exe to request admin privileges.
self.uac_admin = kwargs.get('uac_admin', False)
self.uac_uiaccess = kwargs.get('uac_uiaccess', False)
if CONF['hasUPX']:
self.upx = kwargs.get('upx', False)
else:
self.upx = False
# Old .spec format included in 'name' the path where to put created
# app. New format includes only exename.
#
# Ignore fullpath in the 'name' and prepend DISTPATH or WORKPATH.
# DISTPATH - onefile
# WORKPATH - onedir
if self.exclude_binaries:
# onedir mode - create executable in WORKPATH.
self.name = os.path.join(CONF['workpath'], os.path.basename(self.name))
else:
# onefile mode - create executable in DISTPATH.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
# Old .spec format included on Windows in 'name' .exe suffix.
if is_win or is_cygwin:
# Append .exe suffix if it is not already there.
if not self.name.endswith('.exe'):
self.name += '.exe'
base_name = os.path.splitext(os.path.basename(self.name))[0]
else:
base_name = os.path.basename(self.name)
self.pkgname = base_name + '.pkg'
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append((os.path.basename(arg.name), arg.name, arg.typ))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
if is_win:
filename = os.path.join(CONF['workpath'], CONF['specnm'] + ".exe.manifest")
self.manifest = winmanifest.create_manifest(filename, self.manifest,
self.console, self.uac_admin, self.uac_uiaccess)
manifest_filename = os.path.basename(self.name) + ".manifest"
self.toc.append((manifest_filename, filename, 'BINARY'))
if not self.exclude_binaries:
# Onefile mode: manifest file is explicitly loaded.
# Store name of manifest file as bootloader option. Allows
# the exe to be renamed.
self.toc.append(("pyi-windows-manifest-filename " + manifest_filename,
"", "OPTION"))
self.pkg = PKG(self.toc, cdict=kwargs.get('cdict', None),
exclude_binaries=self.exclude_binaries,
strip_binaries=self.strip, upx_binaries=self.upx,
)
self.dependencies = self.pkg.dependencies
# Get the path of the bootloader and store it in a TOC, so it
# can be checked for being changed.
exe = self._bootloader_file('run', '.exe' if is_win or is_cygwin else '')
self.exefiles = TOC([(os.path.basename(exe), exe, 'EXECUTABLE')])
self.__postinit__()
_GUTS = (# input parameters
('name', _check_guts_eq),
('console', _check_guts_eq),
('debug', _check_guts_eq),
('exclude_binaries', _check_guts_eq),
('icon', _check_guts_eq),
('versrsrc', _check_guts_eq),
('uac_admin', _check_guts_eq),
('uac_uiaccess', _check_guts_eq),
('manifest', _check_guts_eq),
('append_pkg', _check_guts_eq),
# for the case the directory ius shared between platforms:
('pkgname', _check_guts_eq),
('toc', _check_guts_eq),
('resources', _check_guts_eq),
('strip', _check_guts_eq),
('upx', _check_guts_eq),
('mtm', None,), # checked below
# no calculated/analysed values
('exefiles', _check_guts_toc),
)
def _check_guts(self, data, last_build):
if not os.path.exists(self.name):
logger.info("Rebuilding %s because %s missing",
self.tocbasename, os.path.basename(self.name))
return 1
if not self.append_pkg and not os.path.exists(self.pkgname):
logger.info("Rebuilding because %s missing",
os.path.basename(self.pkgname))
return 1
if Target._check_guts(self, data, last_build):
return True
if (data['versrsrc'] or data['resources']) and not is_win:
# todo: really ignore :-)
logger.warn('ignoring version, manifest and resources, platform not capable')
if data['icon'] and not (is_win or is_darwin):
logger.warn('ignoring icon, platform not capable')
mtm = data['mtm']
if mtm != misc.mtime(self.name):
logger.info("Rebuilding %s because mtimes don't match", self.tocbasename)
return True
if mtm < misc.mtime(self.pkg.tocfilename):
logger.info("Rebuilding %s because pkg is more recent", self.tocbasename)
return True
return False
def _bootloader_file(self, exe, extension=None):
"""
Pick up the right bootloader file - debug, console, windowed.
"""
# Having console/windowed bootolader makes sense only on Windows and
# Mac OS X.
if is_win or is_darwin:
if not self.console:
exe = exe + 'w'
# There are two types of bootloaders:
# run - release, no verbose messages in console.
# run_d - contains verbose messages in console.
if self.debug:
exe = exe + '_d'
if extension:
exe = exe + extension
bootloader_file = os.path.join(HOMEPATH, 'PyInstaller', 'bootloader', PLATFORM, exe)
logger.info('Bootloader %s' % bootloader_file)
return bootloader_file
def assemble(self):
logger.info("Building EXE from %s", self.tocbasename)
trash = []
if not os.path.exists(os.path.dirname(self.name)):
os.makedirs(os.path.dirname(self.name))
outf = open(self.name, 'wb')
exe = self.exefiles[0][1] # pathname of bootloader
if not os.path.exists(exe):
raise SystemExit(_MISSING_BOOTLOADER_ERRORMSG)
if is_win and (self.icon or self.versrsrc or self.resources):
tmpnm = tempfile.mktemp()
shutil.copy2(exe, tmpnm)
os.chmod(tmpnm, 0o755)
if self.icon:
icon.CopyIcons(tmpnm, self.icon)
if self.versrsrc:
versioninfo.SetVersion(tmpnm, self.versrsrc)
for res in self.resources:
res = res.split(",")
for i in range(1, len(res)):
try:
res[i] = int(res[i])
except ValueError:
pass
resfile = res[0]
restype = resname = reslang = None
if len(res) > 1:
restype = res[1]
if len(res) > 2:
resname = res[2]
if len(res) > 3:
reslang = res[3]
try:
winresource.UpdateResourcesFromResFile(tmpnm, resfile,
[restype or "*"],
[resname or "*"],
[reslang or "*"])
except winresource.pywintypes.error as exc:
if exc.args[0] != winresource.ERROR_BAD_EXE_FORMAT:
logger.exception(exc)
continue
if not restype or not resname:
logger.error("resource type and/or name not specified")
continue
if "*" in (restype, resname):
logger.error("no wildcards allowed for resource type "
"and name when source file does not "
"contain resources")
continue
try:
winresource.UpdateResourcesFromDataFile(tmpnm,
resfile,
restype,
[resname],
[reslang or 0])
except winresource.pywintypes.error as exc:
logger.exception(exc)
trash.append(tmpnm)
exe = tmpnm
exe = checkCache(exe, strip=self.strip, upx=self.upx)
self.copy(exe, outf)
if self.append_pkg:
logger.info("Appending archive to EXE %s", self.name)
self.copy(self.pkg.name, outf)
else:
logger.info("Copying archive to %s", self.pkgname)
shutil.copy2(self.pkg.name, self.pkgname)
outf.close()
if is_darwin:
# Fix Mach-O header for codesigning on OS X.
logger.info("Fixing EXE for code signing %s", self.name)
import PyInstaller.utils.osx as osxutils
osxutils.fix_exe_for_code_signing(self.name)
pass
os.chmod(self.name, 0o755)
# get mtime for storing into the guts
self.mtm = misc.mtime(self.name)
for item in trash:
os.remove(item)
def copy(self, fnm, outf):
inf = open(fnm, 'rb')
while 1:
data = inf.read(64 * 1024)
if not data:
break
outf.write(data)
def test_extend():
toc = TOC(ELEMS1)
toc.extend(ELEMS2)
expected = list(ELEMS1)
expected.extend(ELEMS2)
assert toc == expected
def test_extend_existing():
toc = TOC(ELEMS1)
toc.extend(ELEMS1)
expected = list(ELEMS1)
assert toc == expected
def test_append():
toc = TOC(ELEMS1)
toc.append(('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
expected = list(ELEMS1)
expected.append(('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
assert toc == expected
def test_insert_keep_filename():
# name in TOC should be the same as the one added
toc = TOC()
entry = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'BINARY')
toc.insert(1, entry)
assert toc[0][0] == entry[0]
class COLLECT(Target):
"""
In one-dir mode creates the output folder with all necessary files.
"""
def __init__(self, *args, **kws):
"""
args
One or more arguments that are either TOCs Targets.
kws
Possible keywork arguments:
name
The name of the directory to be built.
"""
from ..config import CONF
Target.__init__(self)
self.strip_binaries = kws.get('strip', False)
if CONF['hasUPX']:
self.upx_binaries = kws.get('upx', False)
else:
self.upx_binaries = False
self.name = kws.get('name')
# Old .spec format included in 'name' the path where to collect files
# for the created app.
# app. New format includes only directory name.
#
# The 'name' directory is created in DISTPATH and necessary files are
# then collected to this directory.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append((os.path.basename(arg.name), arg.name, arg.typ))
if isinstance(arg, EXE):
for tocnm, fnm, typ in arg.toc:
if tocnm == os.path.basename(arg.name) + ".manifest":
self.toc.append((tocnm, fnm, typ))
if not arg.append_pkg:
self.toc.append((os.path.basename(arg.pkgname), arg.pkgname, 'PKG'))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
self.__postinit__()
_GUTS = (
# COLLECT always builds, just want the toc to be written out
('toc', None),
)
def _check_guts(self, data, last_build):
# COLLECT always needs to be executed, since it will clean the output
# directory anyway to make sure there is no existing cruft accumulating
return 1
def assemble(self):
if _check_path_overlap(self.name) and os.path.isdir(self.name):
_rmtree(self.name)
logger.info("Building COLLECT %s", self.tocbasename)
os.makedirs(self.name)
toc = add_suffix_to_extensions(self.toc)
for inm, fnm, typ in toc:
if not os.path.exists(fnm) or not os.path.isfile(fnm) and is_path_to_egg(fnm):
# file is contained within python egg, it is added with the egg
continue
if os.pardir in os.path.normpath(inm) or os.path.isabs(inm):
raise SystemExit('Security-Alert: try to store file outside '
'of dist-directory. Aborting. %r' % inm)
tofnm = os.path.join(self.name, inm)
todir = os.path.dirname(tofnm)
if not os.path.exists(todir):
os.makedirs(todir)
if typ in ('EXTENSION', 'BINARY'):
fnm = checkCache(fnm, strip=self.strip_binaries,
upx=(self.upx_binaries and (is_win or is_cygwin)),
dist_nm=inm)
if typ != 'DEPENDENCY':
shutil.copy(fnm, tofnm)
try:
shutil.copystat(fnm, tofnm)
except OSError:
logger.warn("failed to copy flags of %s", fnm)
if typ in ('EXTENSION', 'BINARY'):
os.chmod(tofnm, 0o755)
def __init__(self,
scripts,
pathex=None,
binaries=None,
datas=None,
hiddenimports=None,
hookspath=None,
hooksconfig=None,
excludes=None,
runtime_hooks=None,
cipher=None,
win_no_prefer_redirects=False,
win_private_assemblies=False,
noarchive=False):
"""
scripts
A list of scripts specified as file names.
pathex
An optional list of paths to be searched before sys.path.
binaries
An optional list of additional binaries (dlls, etc.) to include.
datas
An optional list of additional data files to include.
hiddenimport
An optional list of additional (hidden) modules to include.
hookspath
An optional list of additional paths to search for hooks.
(hook-modules).
hooksconfig
An optional dict of config settings for hooks.
(hook-modules).
excludes
An optional list of module or package names (their Python names,
not path names) that will be ignored (as though they were not found).
runtime_hooks
An optional list of scripts to use as users' runtime hooks. Specified
as file names.
cipher
Add optional instance of the pyz_crypto.PyiBlockCipher class
(with a provided key).
win_no_prefer_redirects
If True, prefers not to follow version redirects when searching for
Windows SxS Assemblies.
win_private_assemblies
If True, changes all bundled Windows SxS Assemblies into Private
Assemblies to enforce assembly versions.
noarchive
If True, don't place source files in a archive, but keep them as
individual files.
"""
super(Analysis, self).__init__()
from PyInstaller.config import CONF
self.inputs = []
spec_dir = os.path.dirname(CONF['spec'])
for script in scripts:
# If path is relative, it is relative to the location of .spec file.
if not os.path.isabs(script):
script = os.path.join(spec_dir, script)
if absnormpath(script) in self._old_scripts:
logger.warning('Ignoring obsolete auto-added script %s',
script)
continue
# Normalize script path.
script = os.path.normpath(script)
if not os.path.exists(script):
raise SystemExit("script '%s' not found" % script)
self.inputs.append(script)
# Django hook requires this variable to find the script manage.py.
CONF['main_script'] = self.inputs[0]
self.pathex = self._extend_pathex(pathex, self.inputs)
# Set global config variable 'pathex' to make it available for
# PyInstaller.utils.hooks and import hooks. Path extensions for module
# search.
CONF['pathex'] = self.pathex
# Extend sys.path so PyInstaller could find all necessary modules.
logger.info('Extending PYTHONPATH with paths\n' +
pprint.pformat(self.pathex))
sys.path.extend(self.pathex)
# Set global variable to hold assembly binding redirects
CONF['binding_redirects'] = []
self.hiddenimports = hiddenimports or []
# Include modules detected when parsing options, like 'codecs' and encodings.
self.hiddenimports.extend(CONF['hiddenimports'])
self.hookspath = []
# Append directories in `hookspath` (`--additional-hooks-dir`) to
# take precedence over those from the entry points.
if hookspath:
self.hookspath.extend(hookspath)
# Add hook directories from PyInstaller entry points.
self.hookspath += discover_hook_directories()
self.hooksconfig = {}
if hooksconfig:
self.hooksconfig.update(hooksconfig)
# Custom runtime hook files that should be included and started before
# any existing PyInstaller runtime hooks.
self.custom_runtime_hooks = runtime_hooks or []
if cipher:
logger.info('Will encrypt Python bytecode with key: %s',
cipher.key)
# Create a Python module which contains the decryption key which will
# be used at runtime by pyi_crypto.PyiBlockCipher.
pyi_crypto_key_path = os.path.join(CONF['workpath'],
'pyimod00_crypto_key.py')
with open(pyi_crypto_key_path, 'w', encoding='utf-8') as f:
f.write('# -*- coding: utf-8 -*-\n' 'key = %r\n' % cipher.key)
self.hiddenimports.append('tinyaes')
self.excludes = excludes or []
self.scripts = TOC()
self.pure = TOC()
self.binaries = TOC()
self.zipfiles = TOC()
self.zipped_data = TOC()
self.datas = TOC()
self.dependencies = TOC()
self.binding_redirects = CONF['binding_redirects'] = []
self.win_no_prefer_redirects = win_no_prefer_redirects
self.win_private_assemblies = win_private_assemblies
self._python_version = sys.version
self.noarchive = noarchive
self.__postinit__()
# TODO create function to convert datas/binaries from 'hook format' to TOC.
# Initialise 'binaries' and 'datas' with lists specified in .spec file.
if binaries:
logger.info("Appending 'binaries' from .spec")
for name, pth in format_binaries_and_datas(binaries,
workingdir=spec_dir):
self.binaries.append((name, pth, 'BINARY'))
if datas:
logger.info("Appending 'datas' from .spec")
for name, pth in format_binaries_and_datas(datas,
workingdir=spec_dir):
self.datas.append((name, pth, 'DATA'))
def test_extend():
toc = TOC(ELEMS1)
toc.extend(ELEMS2)
expected = list(ELEMS1)
expected.extend(ELEMS2)
assert toc == expected
class EXE(Target):
"""
Creates the final executable of the frozen app.
This bundles all necessary files together.
"""
typ = 'EXECUTABLE'
def __init__(self, *args, **kwargs):
"""
args
One or more arguments that are either TOCs Targets.
kwargs
Possible keywork arguments:
bootloader_ignore_signals
Non-Windows only. If True, the bootloader process will ignore
all ignorable signals. If False (default), it will forward
all signals to the child process. Useful in situations where
e.g. a supervisor process signals both the bootloader and
child (e.g. via a process group) to avoid signalling the
child twice.
console
On Windows or OSX governs whether to use the console executable
or the windowed executable. Always True on Linux/Unix (always
console executable - it does not matter there).
debug
Setting to True gives you progress mesages from the executable
(for console=False there will be annoying MessageBoxes on Windows).
name
The filename for the executable. On Windows suffix '.exe' is
appended.
exclude_binaries
Forwarded to the PKG the EXE builds.
icon
Windows or OSX only. icon='myicon.ico' to use an icon file or
icon='notepad.exe,0' to grab an icon resource.
version
Windows only. version='myversion.txt'. Use grab_version.py to get
a version resource from an executable and then edit the output to
create your own. (The syntax of version resources is so arcane
that I wouldn't attempt to write one from scratch).
uac_admin
Windows only. Setting to True creates a Manifest with will request
elevation upon application restart
uac_uiaccess
Windows only. Setting to True allows an elevated application to
work with Remote Desktop
"""
from ..config import CONF
Target.__init__(self)
# Available options for EXE in .spec files.
self.exclude_binaries = kwargs.get('exclude_binaries', False)
self.bootloader_ignore_signals = kwargs.get(
'bootloader_ignore_signals', False)
self.console = kwargs.get('console', True)
self.debug = kwargs.get('debug', False)
self.name = kwargs.get('name', None)
self.icon = kwargs.get('icon', None)
self.versrsrc = kwargs.get('version', None)
self.manifest = kwargs.get('manifest', None)
self.resources = kwargs.get('resources', [])
self.strip = kwargs.get('strip', False)
self.runtime_tmpdir = kwargs.get('runtime_tmpdir', None)
# If ``append_pkg`` is false, the archive will not be appended
# to the exe, but copied beside it.
self.append_pkg = kwargs.get('append_pkg', True)
# On Windows allows the exe to request admin privileges.
self.uac_admin = kwargs.get('uac_admin', False)
self.uac_uiaccess = kwargs.get('uac_uiaccess', False)
if CONF['hasUPX']:
self.upx = kwargs.get('upx', False)
else:
self.upx = False
# Old .spec format included in 'name' the path where to put created
# app. New format includes only exename.
#
# Ignore fullpath in the 'name' and prepend DISTPATH or WORKPATH.
# DISTPATH - onefile
# WORKPATH - onedir
if self.exclude_binaries:
# onedir mode - create executable in WORKPATH.
self.name = os.path.join(CONF['workpath'],
os.path.basename(self.name))
else:
# onefile mode - create executable in DISTPATH.
self.name = os.path.join(CONF['distpath'],
os.path.basename(self.name))
# Old .spec format included on Windows in 'name' .exe suffix.
if is_win or is_cygwin:
# Append .exe suffix if it is not already there.
if not self.name.endswith('.exe'):
self.name += '.exe'
base_name = os.path.splitext(os.path.basename(self.name))[0]
else:
base_name = os.path.basename(self.name)
self.pkgname = base_name + '.pkg'
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append(
(os.path.basename(arg.name), arg.name, arg.typ))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
if self.runtime_tmpdir is not None:
self.toc.append(
("pyi-runtime-tmpdir " + self.runtime_tmpdir, "", "OPTION"))
if self.bootloader_ignore_signals:
# no value; presence means "true"
self.toc.append(("pyi-bootloader-ignore-signals", "", "OPTION"))
if is_win:
filename = os.path.join(CONF['workpath'],
CONF['specnm'] + ".exe.manifest")
self.manifest = winmanifest.create_manifest(
filename, self.manifest, self.console, self.uac_admin,
self.uac_uiaccess)
manifest_filename = os.path.basename(self.name) + ".manifest"
self.toc.append((manifest_filename, filename, 'BINARY'))
if not self.exclude_binaries:
# Onefile mode: manifest file is explicitly loaded.
# Store name of manifest file as bootloader option. Allows
# the exe to be renamed.
self.toc.append(
("pyi-windows-manifest-filename " + manifest_filename, "",
"OPTION"))
self.pkg = PKG(
self.toc,
cdict=kwargs.get('cdict', None),
exclude_binaries=self.exclude_binaries,
strip_binaries=self.strip,
upx_binaries=self.upx,
)
self.dependencies = self.pkg.dependencies
# Get the path of the bootloader and store it in a TOC, so it
# can be checked for being changed.
exe = self._bootloader_file('run',
'.exe' if is_win or is_cygwin else '')
self.exefiles = TOC([(os.path.basename(exe), exe, 'EXECUTABLE')])
self.__postinit__()
_GUTS = ( # input parameters
('name', _check_guts_eq),
('console', _check_guts_eq),
('debug', _check_guts_eq),
('exclude_binaries', _check_guts_eq),
('icon', _check_guts_eq),
('versrsrc', _check_guts_eq),
('uac_admin', _check_guts_eq),
('uac_uiaccess', _check_guts_eq),
('manifest', _check_guts_eq),
('append_pkg', _check_guts_eq),
# for the case the directory ius shared between platforms:
('pkgname', _check_guts_eq),
('toc', _check_guts_eq),
('resources', _check_guts_eq),
('strip', _check_guts_eq),
('upx', _check_guts_eq),
(
'mtm',
None,
), # checked below
# no calculated/analysed values
('exefiles', _check_guts_toc),
)
def _check_guts(self, data, last_build):
if not os.path.exists(self.name):
logger.info("Rebuilding %s because %s missing", self.tocbasename,
os.path.basename(self.name))
return 1
if not self.append_pkg and not os.path.exists(self.pkgname):
logger.info("Rebuilding because %s missing",
os.path.basename(self.pkgname))
return 1
if Target._check_guts(self, data, last_build):
return True
if (data['versrsrc'] or data['resources']) and not is_win:
# todo: really ignore :-)
logger.warning(
'ignoring version, manifest and resources, platform not capable'
)
if data['icon'] and not (is_win or is_darwin):
logger.warning('ignoring icon, platform not capable')
mtm = data['mtm']
if mtm != misc.mtime(self.name):
logger.info("Rebuilding %s because mtimes don't match",
self.tocbasename)
return True
if mtm < misc.mtime(self.pkg.tocfilename):
logger.info("Rebuilding %s because pkg is more recent",
self.tocbasename)
return True
return False
def _bootloader_file(self, exe, extension=None):
"""
Pick up the right bootloader file - debug, console, windowed.
"""
# Having console/windowed bootolader makes sense only on Windows and
# Mac OS X.
if is_win or is_darwin:
if not self.console:
exe = exe + 'w'
# There are two types of bootloaders:
# run - release, no verbose messages in console.
# run_d - contains verbose messages in console.
if self.debug:
exe = exe + '_d'
if extension:
exe = exe + extension
bootloader_file = os.path.join(HOMEPATH, 'PyInstaller', 'bootloader',
PLATFORM, exe)
logger.info('Bootloader %s' % bootloader_file)
return bootloader_file
def assemble(self):
logger.info("Building EXE from %s", self.tocbasename)
trash = []
if os.path.exists(self.name):
os.remove(self.name)
if not os.path.exists(os.path.dirname(self.name)):
os.makedirs(os.path.dirname(self.name))
exe = self.exefiles[0][1] # pathname of bootloader
if not os.path.exists(exe):
raise SystemExit(_MISSING_BOOTLOADER_ERRORMSG)
if is_win and (self.icon or self.versrsrc or self.resources):
tmpnm = tempfile.mktemp()
self._copyfile(exe, tmpnm)
os.chmod(tmpnm, 0o755)
if self.icon:
icon.CopyIcons(tmpnm, self.icon)
if self.versrsrc:
versioninfo.SetVersion(tmpnm, self.versrsrc)
for res in self.resources:
res = res.split(",")
for i in range(1, len(res)):
try:
res[i] = int(res[i])
except ValueError:
pass
resfile = res[0]
restype = resname = reslang = None
if len(res) > 1:
restype = res[1]
if len(res) > 2:
resname = res[2]
if len(res) > 3:
reslang = res[3]
try:
winresource.UpdateResourcesFromResFile(
tmpnm, resfile, [restype or "*"], [resname or "*"],
[reslang or "*"])
except winresource.pywintypes.error as exc:
if exc.args[0] != winresource.ERROR_BAD_EXE_FORMAT:
logger.error(
"Error while updating resources in %s"
" from resource file %s",
tmpnm,
resfile,
exc_info=1)
continue
# Handle the case where the file contains no resources, and is
# intended as a single resource to be added to the exe.
if not restype or not resname:
logger.error("resource type and/or name not specified")
continue
if "*" in (restype, resname):
logger.error("no wildcards allowed for resource type "
"and name when source file does not "
"contain resources")
continue
try:
winresource.UpdateResourcesFromDataFile(
tmpnm, resfile, restype, [resname], [reslang or 0])
except winresource.pywintypes.error:
logger.error(
"Error while updating resource %s %s in %s"
" from data file %s",
restype,
resname,
tmpnm,
resfile,
exc_info=1)
trash.append(tmpnm)
exe = tmpnm
# NOTE: Do not look up for bootloader file in the cache because it might
# get corrupted by UPX when UPX is available. See #1863 for details.
if not self.append_pkg:
logger.info("Copying bootloader exe to %s", self.name)
self._copyfile(exe, self.name)
logger.info("Copying archive to %s", self.pkgname)
self._copyfile(self.pkg.name, self.pkgname)
elif is_linux:
self._copyfile(exe, self.name)
logger.info("Appending archive to ELF section in EXE %s",
self.name)
retcode, stdout, stderr = exec_command_all(
'objcopy', '--add-section', 'pydata=%s' % self.pkg.name,
self.name)
logger.debug("objcopy returned %i", retcode)
if stdout:
logger.debug(stdout)
if stderr:
logger.debug(stderr)
if retcode != 0:
raise SystemError("objcopy Failure: %s" % stderr)
else:
# Fall back to just append on end of file
logger.info("Appending archive to EXE %s", self.name)
with open(self.name, 'wb') as outf:
# write the bootloader data
with open(exe, 'rb') as infh:
shutil.copyfileobj(infh, outf, length=64 * 1024)
# write the archive data
with open(self.pkg.name, 'rb') as infh:
shutil.copyfileobj(infh, outf, length=64 * 1024)
if is_darwin:
# Fix Mach-O header for codesigning on OS X.
logger.info("Fixing EXE for code signing %s", self.name)
import PyInstaller.utils.osx as osxutils
osxutils.fix_exe_for_code_signing(self.name)
os.chmod(self.name, 0o755)
# get mtime for storing into the guts
self.mtm = misc.mtime(self.name)
for item in trash:
os.remove(item)
logger.info("Building EXE from %s completed successfully.",
self.tocbasename)
def _copyfile(self, infile, outfile):
with open(infile, 'rb') as infh:
with open(outfile, 'wb') as outfh:
shutil.copyfileobj(infh, outfh, length=64 * 1024)
def test_insert():
toc = TOC(ELEMS1)
toc.insert(1, ('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
expected = list(ELEMS1)
expected.insert(1, ('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
assert toc == expected
def test_extend_existing():
toc = TOC(ELEMS1)
toc.extend(ELEMS1)
expected = list(ELEMS1)
assert toc == expected
def test_append_existing():
toc = TOC(ELEMS1)
toc.append(ELEMS1[-1])
expected = list(ELEMS1)
assert toc == expected
def test_insert_other_case_binary():
# binary files should use C-I comparisons. 'LiBrEADlInE.so.6' should not be added.
toc = TOC(ELEMS1)
toc.insert(1, ('LiBrEADlInE.so.6', '/lib64/libreadline.so.6', 'BINARY'))
expected = list(ELEMS1)
assert toc == expected
class PYZ(Target):
"""
Creates a ZlibArchive that contains all pure Python modules.
"""
typ = 'PYZ'
def __init__(self, *tocs, **kwargs):
"""
tocs
One or more TOCs (Tables of Contents), normally an
Analysis.pure.
If this TOC has an attribute `_code_cache`, this is
expected to be a dict of module code objects from
ModuleGraph.
kwargs
Possible keywork arguments:
name
A filename for the .pyz. Normally not needed, as the generated
name will do fine.
cipher
The block cipher that will be used to encrypt Python bytecode.
"""
from ..config import CONF
Target.__init__(self)
name = kwargs.get('name', None)
cipher = kwargs.get('cipher', None)
self.toc = TOC()
# If available, use code objects directly from ModuleGraph to
# speed up PyInstaller.
self.code_dict = {}
for t in tocs:
self.toc.extend(t)
self.code_dict.update(getattr(t, '_code_cache', {}))
self.name = name
if name is None:
self.name = os.path.splitext(self.tocfilename)[0] + '.pyz'
# PyInstaller bootstrapping modules.
self.dependencies = get_bootstrap_modules()
# Bundle the crypto key.
self.cipher = cipher
if cipher:
key_file = ('pyimod00_crypto_key',
os.path.join(CONF['workpath'],
'pyimod00_crypto_key.pyc'), 'PYMODULE')
# Insert the key as the first module in the list. The key module contains
# just variables and does not depend on other modules.
self.dependencies.insert(0, key_file)
# Compile the top-level modules so that they end up in the CArchive and can be
# imported by the bootstrap script.
self.dependencies = misc.compile_py_files(self.dependencies,
CONF['workpath'])
self.__postinit__()
_GUTS = ( # input parameters
('name', _check_guts_eq),
('toc', _check_guts_toc), # todo: pyc=1
# no calculated/analysed values
)
def _check_guts(self, data, last_build):
if Target._check_guts(self, data, last_build):
return True
return False
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc[:]:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
try:
self.code_dict[entry[0]] = get_code_object(
entry[0], entry[1])
except SyntaxError:
# Exclude the module in case this is code meant for a newer Python version.
toc.remove(entry)
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name,
toc,
code_dict=self.code_dict,
cipher=self.cipher)
logger.info("Building PYZ (ZlibArchive) %s completed successfully.",
self.name)
def __init__(self, *args, **kwargs):
"""
args
One or more arguments that are either TOCs Targets.
kwargs
Possible keywork arguments:
console
On Windows or OSX governs whether to use the console executable
or the windowed executable. Always True on Linux/Unix (always
console executable - it does not matter there).
debug
Setting to True gives you progress mesages from the executable
(for console=False there will be annoying MessageBoxes on Windows).
name
The filename for the executable. On Windows suffix '.exe' is
appended.
exclude_binaries
Forwarded to the PKG the EXE builds.
icon
Windows or OSX only. icon='myicon.ico' to use an icon file or
icon='notepad.exe,0' to grab an icon resource.
version
Windows only. version='myversion.txt'. Use grab_version.py to get
a version resource from an executable and then edit the output to
create your own. (The syntax of version resources is so arcane
that I wouldn't attempt to write one from scratch).
uac_admin
Windows only. Setting to True creates a Manifest with will request
elevation upon application restart
uac_uiaccess
Windows only. Setting to True allows an elevated application to
work with Remote Desktop
"""
from ..config import CONF
Target.__init__(self)
# Available options for EXE in .spec files.
self.exclude_binaries = kwargs.get('exclude_binaries', False)
self.console = kwargs.get('console', True)
self.debug = kwargs.get('debug', False)
self.name = kwargs.get('name', None)
self.icon = kwargs.get('icon', None)
self.versrsrc = kwargs.get('version', None)
self.manifest = kwargs.get('manifest', None)
self.resources = kwargs.get('resources', [])
self.strip = kwargs.get('strip', False)
# If ``append_pkg`` is false, the archive will not be appended
# to the exe, but copied beside it.
self.append_pkg = kwargs.get('append_pkg', True)
# On Windows allows the exe to request admin privileges.
self.uac_admin = kwargs.get('uac_admin', False)
self.uac_uiaccess = kwargs.get('uac_uiaccess', False)
if CONF['hasUPX']:
self.upx = kwargs.get('upx', False)
else:
self.upx = False
# Old .spec format included in 'name' the path where to put created
# app. New format includes only exename.
#
# Ignore fullpath in the 'name' and prepend DISTPATH or WORKPATH.
# DISTPATH - onefile
# WORKPATH - onedir
if self.exclude_binaries:
# onedir mode - create executable in WORKPATH.
self.name = os.path.join(CONF['workpath'], os.path.basename(self.name))
else:
# onefile mode - create executable in DISTPATH.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
# Old .spec format included on Windows in 'name' .exe suffix.
if is_win or is_cygwin:
# Append .exe suffix if it is not already there.
if not self.name.endswith('.exe'):
self.name += '.exe'
base_name = os.path.splitext(os.path.basename(self.name))[0]
else:
base_name = os.path.basename(self.name)
self.pkgname = base_name + '.pkg'
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append((os.path.basename(arg.name), arg.name, arg.typ))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
if is_win:
filename = os.path.join(CONF['workpath'], CONF['specnm'] + ".exe.manifest")
self.manifest = winmanifest.create_manifest(filename, self.manifest,
self.console, self.uac_admin, self.uac_uiaccess)
manifest_filename = os.path.basename(self.name) + ".manifest"
self.toc.append((manifest_filename, filename, 'BINARY'))
if not self.exclude_binaries:
# Onefile mode: manifest file is explicitly loaded.
# Store name of manifest file as bootloader option. Allows
# the exe to be renamed.
self.toc.append(("pyi-windows-manifest-filename " + manifest_filename,
"", "OPTION"))
self.pkg = PKG(self.toc, cdict=kwargs.get('cdict', None),
exclude_binaries=self.exclude_binaries,
strip_binaries=self.strip, upx_binaries=self.upx,
)
self.dependencies = self.pkg.dependencies
# Get the path of the bootloader and store it in a TOC, so it
# can be checked for being changed.
exe = self._bootloader_file('run', '.exe' if is_win or is_cygwin else '')
self.exefiles = TOC([(os.path.basename(exe), exe, 'EXECUTABLE')])
self.__postinit__()
def test_insert_keep_filename():
# name in TOC should be the same as the one added
toc = TOC()
entry = ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'BINARY')
toc.insert(1, entry)
assert toc[0][0] == entry[0]
class PYZ(Target):
"""
Creates a ZlibArchive that contains all pure Python modules.
"""
typ = 'PYZ'
def __init__(self, *tocs, **kwargs):
"""
tocs
One or more TOCs (Tables of Contents), normally an
Analysis.pure.
If this TOC has an attribute `_code_cache`, this is
expected to be a dict of module code objects from
ModuleGraph.
kwargs
Possible keywork arguments:
name
A filename for the .pyz. Normally not needed, as the generated
name will do fine.
cipher
The block cipher that will be used to encrypt Python bytecode.
"""
from ..config import CONF
Target.__init__(self)
name = kwargs.get('name', None)
cipher = kwargs.get('cipher', None)
self.toc = TOC()
# If available, use code objects directly from ModuleGraph to
# speed up PyInstaller.
self.code_dict = {}
for t in tocs:
self.toc.extend(t)
self.code_dict.update(getattr(t, '_code_cache', {}))
self.name = name
if name is None:
self.name = os.path.splitext(self.tocfilename)[0] + '.pyz'
# PyInstaller bootstrapping modules.
self.dependencies = get_bootstrap_modules()
# Bundle the crypto key.
self.cipher = cipher
if cipher:
key_file = ('pyimod00_crypto_key',
os.path.join(CONF['workpath'], 'pyimod00_crypto_key.pyc'),
'PYMODULE')
# Insert the key as the first module in the list. The key module contains
# just variables and does not depend on other modules.
self.dependencies.insert(0, key_file)
# Compile the top-level modules so that they end up in the CArchive and can be
# imported by the bootstrap script.
self.dependencies = misc.compile_py_files(self.dependencies, CONF['workpath'])
self.__postinit__()
_GUTS = (# input parameters
('name', _check_guts_eq),
('toc', _check_guts_toc), # todo: pyc=1
# no calculated/analysed values
)
def _check_guts(self, data, last_build):
if Target._check_guts(self, data, last_build):
return True
return False
# TODO Could this function be merged with 'PyInstaller.utils.misc:get_code_object()'?
def __get_code(self, modname, filename):
"""
Get the code-object for a module.
This is a extra-simple version for compiling a module. It's
not worth spending more effort here, as it is only used in the
rare case if outXX-Analysis.toc exists, but outXX-PYZ.toc does
not.
"""
def load_code(modname, filename):
path_item = os.path.dirname(filename)
if os.path.basename(filename).startswith('__init__.py'):
# this is a package
path_item = os.path.dirname(path_item)
if os.path.basename(path_item) == '__pycache__':
path_item = os.path.dirname(path_item)
importer = pkgutil.get_importer(path_item)
package, _, modname = modname.rpartition('.')
if sys.version_info >= (3,3) and hasattr(importer, 'find_loader'):
loader, portions = importer.find_loader(modname)
else:
loader = importer.find_module(modname)
portions = []
assert loader and hasattr(loader, 'get_code')
logger.debug('Compiling %s', filename)
return loader.get_code(modname)
try:
if filename in ('-', None):
# This is a NamespacePackage, modulegraph marks them
# by using the filename '-'. (But wants to use None,
# so check for None, too, to be forward-compatible.)
logger.debug('Compiling namespace package %s', modname)
txt = '#\n'
return compile(txt, filename, 'exec')
else:
logger.debug('Compiling %s', filename)
co = load_code(modname, filename)
if not co:
raise ValueError("Module file %s is missing" % filename)
return co
except SyntaxError as e:
print("Syntax error in ", filename)
print(e.args)
raise
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
self.code_dict[entry[0]] = self.__get_code(entry[0], entry[1])
pyz = ZlibArchiveWriter(code_dict=self.code_dict, cipher=self.cipher)
pyz.build(self.name, toc)
def test_insert():
toc = TOC(ELEMS1)
toc.insert(1, ('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
expected = list(ELEMS1)
expected.insert(1, ('li-la-lu', '/home/myself/li-la-su', 'SOMETHING'))
assert toc == expected
class Analysis(Target):
"""
Class does analysis of the user's main Python scripts.
An Analysis has five outputs, all TOCs (Table of Contents) accessed as
attributes of the analysis.
scripts
The scripts you gave Analysis as input, with any runtime hook scripts
prepended.
pure
The pure Python modules.
binaries
The extensionmodules and their dependencies. The secondary dependecies
are filtered. On Windows files from C:\\Windows are excluded by default.
On Linux/Unix only system libraries from /lib or /usr/lib are excluded.
datas
Data-file dependencies. These are data-file that are found to be needed
by modules. They can be anything: plugins, font files, images, translations,
etc.
zipfiles
The zipfiles dependencies (usually .egg files).
"""
_old_scripts = {
absnormpath(os.path.join(HOMEPATH, "support", "_mountzlib.py")),
absnormpath(os.path.join(HOMEPATH, "support", "useUnicode.py")),
absnormpath(os.path.join(HOMEPATH, "support", "useTK.py")),
absnormpath(os.path.join(HOMEPATH, "support", "unpackTK.py")),
absnormpath(os.path.join(HOMEPATH, "support", "removeTK.py"))
}
def __init__(self,
scripts,
pathex=None,
binaries=None,
datas=None,
hiddenimports=None,
hookspath=None,
hooksconfig=None,
excludes=None,
runtime_hooks=None,
cipher=None,
win_no_prefer_redirects=False,
win_private_assemblies=False,
noarchive=False):
"""
scripts
A list of scripts specified as file names.
pathex
An optional list of paths to be searched before sys.path.
binaries
An optional list of additional binaries (dlls, etc.) to include.
datas
An optional list of additional data files to include.
hiddenimport
An optional list of additional (hidden) modules to include.
hookspath
An optional list of additional paths to search for hooks.
(hook-modules).
hooksconfig
An optional dict of config settings for hooks.
(hook-modules).
excludes
An optional list of module or package names (their Python names,
not path names) that will be ignored (as though they were not found).
runtime_hooks
An optional list of scripts to use as users' runtime hooks. Specified
as file names.
cipher
Add optional instance of the pyz_crypto.PyiBlockCipher class
(with a provided key).
win_no_prefer_redirects
If True, prefers not to follow version redirects when searching for
Windows SxS Assemblies.
win_private_assemblies
If True, changes all bundled Windows SxS Assemblies into Private
Assemblies to enforce assembly versions.
noarchive
If True, don't place source files in a archive, but keep them as
individual files.
"""
super(Analysis, self).__init__()
from PyInstaller.config import CONF
self.inputs = []
spec_dir = os.path.dirname(CONF['spec'])
for script in scripts:
# If path is relative, it is relative to the location of .spec file.
if not os.path.isabs(script):
script = os.path.join(spec_dir, script)
if absnormpath(script) in self._old_scripts:
logger.warning('Ignoring obsolete auto-added script %s',
script)
continue
# Normalize script path.
script = os.path.normpath(script)
if not os.path.exists(script):
raise SystemExit("script '%s' not found" % script)
self.inputs.append(script)
# Django hook requires this variable to find the script manage.py.
CONF['main_script'] = self.inputs[0]
self.pathex = self._extend_pathex(pathex, self.inputs)
# Set global config variable 'pathex' to make it available for
# PyInstaller.utils.hooks and import hooks. Path extensions for module
# search.
CONF['pathex'] = self.pathex
# Extend sys.path so PyInstaller could find all necessary modules.
logger.info('Extending PYTHONPATH with paths\n' +
pprint.pformat(self.pathex))
sys.path.extend(self.pathex)
# Set global variable to hold assembly binding redirects
CONF['binding_redirects'] = []
self.hiddenimports = hiddenimports or []
# Include modules detected when parsing options, like 'codecs' and encodings.
self.hiddenimports.extend(CONF['hiddenimports'])
self.hookspath = []
# Append directories in `hookspath` (`--additional-hooks-dir`) to
# take precedence over those from the entry points.
if hookspath:
self.hookspath.extend(hookspath)
# Add hook directories from PyInstaller entry points.
self.hookspath += discover_hook_directories()
self.hooksconfig = {}
if hooksconfig:
self.hooksconfig.update(hooksconfig)
# Custom runtime hook files that should be included and started before
# any existing PyInstaller runtime hooks.
self.custom_runtime_hooks = runtime_hooks or []
if cipher:
logger.info('Will encrypt Python bytecode with key: %s',
cipher.key)
# Create a Python module which contains the decryption key which will
# be used at runtime by pyi_crypto.PyiBlockCipher.
pyi_crypto_key_path = os.path.join(CONF['workpath'],
'pyimod00_crypto_key.py')
with open(pyi_crypto_key_path, 'w', encoding='utf-8') as f:
f.write('# -*- coding: utf-8 -*-\n' 'key = %r\n' % cipher.key)
self.hiddenimports.append('tinyaes')
self.excludes = excludes or []
self.scripts = TOC()
self.pure = TOC()
self.binaries = TOC()
self.zipfiles = TOC()
self.zipped_data = TOC()
self.datas = TOC()
self.dependencies = TOC()
self.binding_redirects = CONF['binding_redirects'] = []
self.win_no_prefer_redirects = win_no_prefer_redirects
self.win_private_assemblies = win_private_assemblies
self._python_version = sys.version
self.noarchive = noarchive
self.__postinit__()
# TODO create function to convert datas/binaries from 'hook format' to TOC.
# Initialise 'binaries' and 'datas' with lists specified in .spec file.
if binaries:
logger.info("Appending 'binaries' from .spec")
for name, pth in format_binaries_and_datas(binaries,
workingdir=spec_dir):
self.binaries.append((name, pth, 'BINARY'))
if datas:
logger.info("Appending 'datas' from .spec")
for name, pth in format_binaries_and_datas(datas,
workingdir=spec_dir):
self.datas.append((name, pth, 'DATA'))
_GUTS = ( # input parameters
('inputs', _check_guts_eq), # parameter `scripts`
('pathex', _check_guts_eq),
('hiddenimports', _check_guts_eq),
('hookspath', _check_guts_eq),
('hooksconfig', _check_guts_eq),
('excludes', _check_guts_eq),
('custom_runtime_hooks', _check_guts_eq),
('win_no_prefer_redirects', _check_guts_eq),
('win_private_assemblies', _check_guts_eq),
('noarchive', _check_guts_eq),
#'cipher': no need to check as it is implied by an
# additional hidden import
#calculated/analysed values
('_python_version', _check_guts_eq),
('scripts', _check_guts_toc_mtime),
('pure', lambda *args: _check_guts_toc_mtime(*args, **{'pyc': 1})),
('binaries', _check_guts_toc_mtime),
('zipfiles', _check_guts_toc_mtime),
('zipped_data', None), # TODO check this, too
('datas', _check_guts_toc_mtime),
# TODO: Need to add "dependencies"?
# cached binding redirects - loaded into CONF for PYZ/COLLECT to find.
('binding_redirects', None),
)
def _extend_pathex(self, spec_pathex, scripts):
"""
Normalize additional paths where PyInstaller will look for modules and
add paths with scripts to the list of paths.
:param spec_pathex: Additional paths defined defined in .spec file.
:param scripts: Scripts to create executable from.
:return: list of updated paths
"""
# Based on main supplied script - add top-level modules directory to PYTHONPATH.
# Sometimes the main app script is not top-level module but submodule like 'mymodule.mainscript.py'.
# In that case PyInstaller will not be able find modules in the directory containing 'mymodule'.
# Add this directory to PYTHONPATH so PyInstaller could find it.
pathex = []
# Add scripts paths first.
for script in scripts:
logger.debug('script: %s' % script)
script_toplevel_dir = get_path_to_toplevel_modules(script)
if script_toplevel_dir:
pathex.append(script_toplevel_dir)
# Append paths from .spec.
if spec_pathex is not None:
pathex.extend(spec_pathex)
# Normalize paths in pathex and make them absolute.
return [absnormpath(p) for p in pathex]
def _check_guts(self, data, last_build):
if Target._check_guts(self, data, last_build):
return True
for fnm in self.inputs:
if mtime(fnm) > last_build:
logger.info("Building because %s changed", fnm)
return True
# Now we know that none of the input parameters and none of
# the input files has changed. So take the values calculated
# resp. analysed in the last run and store them in `self`.
self.scripts = TOC(data['scripts'])
self.pure = TOC(data['pure'])
self.binaries = TOC(data['binaries'])
self.zipfiles = TOC(data['zipfiles'])
self.zipped_data = TOC(data['zipped_data'])
self.datas = TOC(data['datas'])
# Store previously found binding redirects in CONF for later use by PKG/COLLECT
from PyInstaller.config import CONF
self.binding_redirects = CONF['binding_redirects'] = data[
'binding_redirects']
return False
def assemble(self):
"""
This method is the MAIN method for finding all necessary files to be bundled.
"""
from PyInstaller.config import CONF
for m in self.excludes:
logger.debug("Excluding module '%s'" % m)
self.graph = initialize_modgraph(excludes=self.excludes,
user_hook_dirs=self.hookspath)
# TODO Find a better place where to put 'base_library.zip' and when to created it.
# For Python 3 it is necessary to create file 'base_library.zip'
# containing core Python modules. In Python 3 some built-in modules
# are written in pure Python. base_library.zip is a way how to have
# those modules as "built-in".
libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip')
create_py3_base_library(libzip_filename, graph=self.graph)
# Bundle base_library.zip as data file.
# Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA')
self.datas.append(
(os.path.basename(libzip_filename), libzip_filename, 'DATA'))
# Expand sys.path of module graph.
# The attribute is the set of paths to use for imports: sys.path,
# plus our loader, plus other paths from e.g. --path option).
self.graph.path = self.pathex + self.graph.path
self.graph.set_setuptools_nspackages()
logger.info("running Analysis %s", self.tocbasename)
# Get paths to Python and, in Windows, the manifest.
python = compat.python_executable
if not is_win:
# Linux/MacOS: get a real, non-link path to the running Python executable.
while os.path.islink(python):
python = os.path.join(os.path.dirname(python),
os.readlink(python))
depmanifest = None
else:
# Windows: Create a manifest to embed into built .exe, containing the same
# dependencies as python.exe.
depmanifest = winmanifest.Manifest(
type_="win32",
name=CONF['specnm'],
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
depmanifest.filename = os.path.join(
CONF['workpath'], CONF['specnm'] + ".exe.manifest")
# We record "binaries" separately from the modulegraph, as there
# is no way to record those dependencies in the graph. These include
# the python executable and any binaries added by hooks later.
# "binaries" are not the same as "extensions" which are .so or .dylib
# that are found and recorded as extension nodes in the graph.
# Reset seen variable before running bindepend. We use bindepend only for
# the python executable.
bindepend.seen.clear()
# Add binary and assembly dependencies of Python.exe.
# This also ensures that its assembly depencies under Windows get added to the
# built .exe's manifest. Python 2.7 extension modules have no assembly
# dependencies, and rely on the app-global dependencies set by the .exe.
self.binaries.extend(
bindepend.Dependencies([('', python, '')],
manifest=depmanifest,
redirects=self.binding_redirects)[1:])
if is_win:
depmanifest.writeprettyxml()
### Module graph.
#
# Construct the module graph of import relationships between modules
# required by this user's application. For each entry point (top-level
# user-defined Python script), all imports originating from this entry
# point are recursively parsed into a subgraph of the module graph. This
# subgraph is then connected to this graph's root node, ensuring
# imported module nodes will be reachable from the root node -- which is
# is (arbitrarily) chosen to be the first entry point's node.
# List to hold graph nodes of scripts and runtime hooks in use order.
priority_scripts = []
# Assume that if the script does not exist, Modulegraph will raise error.
# Save the graph nodes of each in sequence.
for script in self.inputs:
logger.info("Analyzing %s", script)
priority_scripts.append(self.graph.add_script(script))
# Analyze the script's hidden imports (named on the command line)
self.graph.add_hiddenimports(self.hiddenimports)
### Post-graph hooks.
self.graph.process_post_graph_hooks(self)
# Update 'binaries' TOC and 'datas' TOC.
deps_proc = DependencyProcessor(self.graph,
self.graph._additional_files_cache)
self.binaries.extend(deps_proc.make_binaries_toc())
self.datas.extend(deps_proc.make_datas_toc())
self.zipped_data.extend(deps_proc.make_zipped_data_toc())
# Note: zipped eggs are collected below
### Look for dlls that are imported by Python 'ctypes' module.
# First get code objects of all modules that import 'ctypes'.
logger.info('Looking for ctypes DLLs')
# dict like: {'module1': code_obj, 'module2': code_obj}
ctypes_code_objs = self.graph.get_code_using("ctypes")
for name, co in ctypes_code_objs.items():
# Get dlls that might be needed by ctypes.
logger.debug('Scanning %s for shared libraries or dlls', name)
try:
ctypes_binaries = scan_code_for_ctypes(co)
self.binaries.extend(set(ctypes_binaries))
except Exception as ex:
raise RuntimeError(f"Failed to scan the module '{name}'. "
f"This is a bug. Please report it.") from ex
self.datas.extend((dest, source, "DATA")
for (dest, source) in format_binaries_and_datas(
self.graph.metadata_required()))
# Analyze run-time hooks.
# Run-time hooks has to be executed before user scripts. Add them
# to the beginning of 'priority_scripts'.
priority_scripts = self.graph.analyze_runtime_hooks(
self.custom_runtime_hooks) + priority_scripts
# 'priority_scripts' is now a list of the graph nodes of custom runtime
# hooks, then regular runtime hooks, then the PyI loader scripts.
# Further on, we will make sure they end up at the front of self.scripts
### Extract the nodes of the graph as TOCs for further processing.
# Initialize the scripts list with priority scripts in the proper order.
self.scripts = self.graph.nodes_to_toc(priority_scripts)
# Extend the binaries list with all the Extensions modulegraph has found.
self.binaries = self.graph.make_binaries_toc(self.binaries)
# Fill the "pure" list with pure Python modules.
assert len(self.pure) == 0
self.pure = self.graph.make_pure_toc()
# And get references to module code objects constructed by ModuleGraph
# to avoid writing .pyc/pyo files to hdd.
self.pure._code_cache = self.graph.get_code_objects()
# Add remaining binary dependencies - analyze Python C-extensions and what
# DLLs they depend on.
logger.info('Looking for dynamic libraries')
self.binaries.extend(
bindepend.Dependencies(self.binaries,
redirects=self.binding_redirects))
### Include zipped Python eggs.
logger.info('Looking for eggs')
self.zipfiles.extend(deps_proc.make_zipfiles_toc())
# Verify that Python dynamic library can be found.
# Without dynamic Python library PyInstaller cannot continue.
self._check_python_library(self.binaries)
if is_win:
# Remove duplicate redirects
self.binding_redirects[:] = list(set(self.binding_redirects))
logger.info("Found binding redirects: \n%s",
self.binding_redirects)
# Filter binaries to adjust path of extensions that come from
# python's lib-dynload directory. Prefix them with lib-dynload
# so that we'll collect them into subdirectory instead of
# directly into _MEIPASS
for idx, tpl in enumerate(self.binaries):
name, path, typecode = tpl
if typecode == 'EXTENSION' \
and not os.path.dirname(os.path.normpath(name)) \
and os.path.basename(os.path.dirname(path)) == 'lib-dynload':
name = os.path.join('lib-dynload', name)
self.binaries[idx] = (name, path, typecode)
# Place Python source in data files for the noarchive case.
if self.noarchive:
# Create a new TOC of ``(dest path for .pyc, source for .py, type)``.
new_toc = TOC()
for name, path, typecode in self.pure:
assert typecode == 'PYMODULE'
# Transform a python module name into a file name.
name = name.replace('.', os.sep)
# Special case: modules have an implied filename to add.
if os.path.splitext(os.path.basename(path))[0] == '__init__':
name += os.sep + '__init__'
# Append the extension for the compiled result.
# In python 3.5 (PEP-488) .pyo files were replaced by
# .opt-1.pyc and .opt-2.pyc. However, it seems that for
# bytecode-only module distribution, we always need to
# use the .pyc extension.
name += '.pyc'
new_toc.append((name, path, typecode))
# Put the result of byte-compiling this TOC in datas. Mark all entries as data.
for name, path, typecode in compile_py_files(
new_toc, CONF['workpath']):
self.datas.append((name, path, 'DATA'))
# Store no source in the archive.
self.pure = TOC()
# Write warnings about missing modules.
self._write_warnings()
# Write debug information about hte graph
self._write_graph_debug()
def _write_warnings(self):
"""
Write warnings about missing modules. Get them from the graph
and use the graph to figure out who tried to import them.
"""
def dependency_description(name, depInfo):
if not depInfo or depInfo == 'direct':
imptype = 0
else:
imptype = (depInfo.conditional + 2 * depInfo.function +
4 * depInfo.tryexcept)
return '%s (%s)' % (name, IMPORT_TYPES[imptype])
from PyInstaller.config import CONF
miss_toc = self.graph.make_missing_toc()
with open(CONF['warnfile'], 'w', encoding='utf-8') as wf:
wf.write(WARNFILE_HEADER)
for (n, p, status) in miss_toc:
importers = self.graph.get_importers(n)
print(status,
'module named',
n,
'- imported by',
', '.join(
dependency_description(name, data)
for name, data in importers),
file=wf)
logger.info("Warnings written to %s", CONF['warnfile'])
def _write_graph_debug(self):
"""Write a xref (in html) and with `--log-level DEBUG` a dot-drawing
of the graph.
"""
from PyInstaller.config import CONF
with open(CONF['xref-file'], 'w', encoding='utf-8') as fh:
self.graph.create_xref(fh)
logger.info("Graph cross-reference written to %s",
CONF['xref-file'])
if logger.getEffectiveLevel() > logging.DEBUG:
return
# The `DOT language's <https://www.graphviz.org/doc/info/lang.html>`_
# default character encoding (see the end of the linked page) is UTF-8.
with open(CONF['dot-file'], 'w', encoding='utf-8') as fh:
self.graph.graphreport(fh)
logger.info("Graph drawing written to %s", CONF['dot-file'])
def _check_python_library(self, binaries):
"""
Verify presence of the Python dynamic library in the binary dependencies.
Python library is an essential piece that has to be always included.
"""
# First check that libpython is in resolved binary dependencies.
for (nm, filename, typ) in binaries:
if typ == 'BINARY' and nm in PYDYLIB_NAMES:
# Just print its filename and return.
logger.info('Using Python library %s', filename)
# Checking was successful - end of function.
return
# Python lib not in dependencies - try to find it.
logger.info(
'Python library not in binary dependencies. Doing additional searching...'
)
python_lib = bindepend.get_python_library_path()
logger.debug('Adding Python library to binary dependencies')
binaries.append((os.path.basename(python_lib), python_lib, 'BINARY'))
logger.info('Using Python library %s', python_lib)
def exclude_system_libraries(self, list_of_exceptions=[]):
"""
This method may be optionally called from the spec file to exclude
any system libraries from the list of binaries other than those
containing the shell-style wildcards in list_of_exceptions.
Those that match '*python*' or are stored under 'lib-dynload' are
always treated as exceptions and not excluded.
"""
self.binaries = \
[i for i in self.binaries
if _should_include_system_binary(i, list_of_exceptions)]
def test_insert_existing():
toc = TOC(ELEMS1)
toc.insert(0, ELEMS1[-1])
toc.insert(1, ELEMS1[-1])
expected = list(ELEMS1)
assert toc == expected
def assemble(self):
"""
This method is the MAIN method for finding all necessary files to be bundled.
"""
from PyInstaller.config import CONF
for m in self.excludes:
logger.debug("Excluding module '%s'" % m)
self.graph = initialize_modgraph(excludes=self.excludes,
user_hook_dirs=self.hookspath)
# TODO Find a better place where to put 'base_library.zip' and when to created it.
# For Python 3 it is necessary to create file 'base_library.zip'
# containing core Python modules. In Python 3 some built-in modules
# are written in pure Python. base_library.zip is a way how to have
# those modules as "built-in".
libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip')
create_py3_base_library(libzip_filename, graph=self.graph)
# Bundle base_library.zip as data file.
# Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA')
self.datas.append(
(os.path.basename(libzip_filename), libzip_filename, 'DATA'))
# Expand sys.path of module graph.
# The attribute is the set of paths to use for imports: sys.path,
# plus our loader, plus other paths from e.g. --path option).
self.graph.path = self.pathex + self.graph.path
self.graph.set_setuptools_nspackages()
logger.info("running Analysis %s", self.tocbasename)
# Get paths to Python and, in Windows, the manifest.
python = compat.python_executable
if not is_win:
# Linux/MacOS: get a real, non-link path to the running Python executable.
while os.path.islink(python):
python = os.path.join(os.path.dirname(python),
os.readlink(python))
depmanifest = None
else:
# Windows: Create a manifest to embed into built .exe, containing the same
# dependencies as python.exe.
depmanifest = winmanifest.Manifest(
type_="win32",
name=CONF['specnm'],
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
depmanifest.filename = os.path.join(
CONF['workpath'], CONF['specnm'] + ".exe.manifest")
# We record "binaries" separately from the modulegraph, as there
# is no way to record those dependencies in the graph. These include
# the python executable and any binaries added by hooks later.
# "binaries" are not the same as "extensions" which are .so or .dylib
# that are found and recorded as extension nodes in the graph.
# Reset seen variable before running bindepend. We use bindepend only for
# the python executable.
bindepend.seen.clear()
# Add binary and assembly dependencies of Python.exe.
# This also ensures that its assembly depencies under Windows get added to the
# built .exe's manifest. Python 2.7 extension modules have no assembly
# dependencies, and rely on the app-global dependencies set by the .exe.
self.binaries.extend(
bindepend.Dependencies([('', python, '')],
manifest=depmanifest,
redirects=self.binding_redirects)[1:])
if is_win:
depmanifest.writeprettyxml()
### Module graph.
#
# Construct the module graph of import relationships between modules
# required by this user's application. For each entry point (top-level
# user-defined Python script), all imports originating from this entry
# point are recursively parsed into a subgraph of the module graph. This
# subgraph is then connected to this graph's root node, ensuring
# imported module nodes will be reachable from the root node -- which is
# is (arbitrarily) chosen to be the first entry point's node.
# List to hold graph nodes of scripts and runtime hooks in use order.
priority_scripts = []
# Assume that if the script does not exist, Modulegraph will raise error.
# Save the graph nodes of each in sequence.
for script in self.inputs:
logger.info("Analyzing %s", script)
priority_scripts.append(self.graph.add_script(script))
# Analyze the script's hidden imports (named on the command line)
self.graph.add_hiddenimports(self.hiddenimports)
### Post-graph hooks.
self.graph.process_post_graph_hooks(self)
# Update 'binaries' TOC and 'datas' TOC.
deps_proc = DependencyProcessor(self.graph,
self.graph._additional_files_cache)
self.binaries.extend(deps_proc.make_binaries_toc())
self.datas.extend(deps_proc.make_datas_toc())
self.zipped_data.extend(deps_proc.make_zipped_data_toc())
# Note: zipped eggs are collected below
### Look for dlls that are imported by Python 'ctypes' module.
# First get code objects of all modules that import 'ctypes'.
logger.info('Looking for ctypes DLLs')
# dict like: {'module1': code_obj, 'module2': code_obj}
ctypes_code_objs = self.graph.get_code_using("ctypes")
for name, co in ctypes_code_objs.items():
# Get dlls that might be needed by ctypes.
logger.debug('Scanning %s for shared libraries or dlls', name)
try:
ctypes_binaries = scan_code_for_ctypes(co)
self.binaries.extend(set(ctypes_binaries))
except Exception as ex:
raise RuntimeError(f"Failed to scan the module '{name}'. "
f"This is a bug. Please report it.") from ex
self.datas.extend((dest, source, "DATA")
for (dest, source) in format_binaries_and_datas(
self.graph.metadata_required()))
# Analyze run-time hooks.
# Run-time hooks has to be executed before user scripts. Add them
# to the beginning of 'priority_scripts'.
priority_scripts = self.graph.analyze_runtime_hooks(
self.custom_runtime_hooks) + priority_scripts
# 'priority_scripts' is now a list of the graph nodes of custom runtime
# hooks, then regular runtime hooks, then the PyI loader scripts.
# Further on, we will make sure they end up at the front of self.scripts
### Extract the nodes of the graph as TOCs for further processing.
# Initialize the scripts list with priority scripts in the proper order.
self.scripts = self.graph.nodes_to_toc(priority_scripts)
# Extend the binaries list with all the Extensions modulegraph has found.
self.binaries = self.graph.make_binaries_toc(self.binaries)
# Fill the "pure" list with pure Python modules.
assert len(self.pure) == 0
self.pure = self.graph.make_pure_toc()
# And get references to module code objects constructed by ModuleGraph
# to avoid writing .pyc/pyo files to hdd.
self.pure._code_cache = self.graph.get_code_objects()
# Add remaining binary dependencies - analyze Python C-extensions and what
# DLLs they depend on.
logger.info('Looking for dynamic libraries')
self.binaries.extend(
bindepend.Dependencies(self.binaries,
redirects=self.binding_redirects))
### Include zipped Python eggs.
logger.info('Looking for eggs')
self.zipfiles.extend(deps_proc.make_zipfiles_toc())
# Verify that Python dynamic library can be found.
# Without dynamic Python library PyInstaller cannot continue.
self._check_python_library(self.binaries)
if is_win:
# Remove duplicate redirects
self.binding_redirects[:] = list(set(self.binding_redirects))
logger.info("Found binding redirects: \n%s",
self.binding_redirects)
# Filter binaries to adjust path of extensions that come from
# python's lib-dynload directory. Prefix them with lib-dynload
# so that we'll collect them into subdirectory instead of
# directly into _MEIPASS
for idx, tpl in enumerate(self.binaries):
name, path, typecode = tpl
if typecode == 'EXTENSION' \
and not os.path.dirname(os.path.normpath(name)) \
and os.path.basename(os.path.dirname(path)) == 'lib-dynload':
name = os.path.join('lib-dynload', name)
self.binaries[idx] = (name, path, typecode)
# Place Python source in data files for the noarchive case.
if self.noarchive:
# Create a new TOC of ``(dest path for .pyc, source for .py, type)``.
new_toc = TOC()
for name, path, typecode in self.pure:
assert typecode == 'PYMODULE'
# Transform a python module name into a file name.
name = name.replace('.', os.sep)
# Special case: modules have an implied filename to add.
if os.path.splitext(os.path.basename(path))[0] == '__init__':
name += os.sep + '__init__'
# Append the extension for the compiled result.
# In python 3.5 (PEP-488) .pyo files were replaced by
# .opt-1.pyc and .opt-2.pyc. However, it seems that for
# bytecode-only module distribution, we always need to
# use the .pyc extension.
name += '.pyc'
new_toc.append((name, path, typecode))
# Put the result of byte-compiling this TOC in datas. Mark all entries as data.
for name, path, typecode in compile_py_files(
new_toc, CONF['workpath']):
self.datas.append((name, path, 'DATA'))
# Store no source in the archive.
self.pure = TOC()
# Write warnings about missing modules.
self._write_warnings()
# Write debug information about hte graph
self._write_graph_debug()
class PYZ(Target):
"""
Creates a ZlibArchive that contains all pure Python modules.
"""
typ = 'PYZ'
def __init__(self, *tocs, **kwargs):
"""
tocs
One or more TOCs (Tables of Contents), normally an
Analysis.pure.
If this TOC has an attribute `_code_cache`, this is
expected to be a dict of module code objects from
ModuleGraph.
kwargs
Possible keywork arguments:
name
A filename for the .pyz. Normally not needed, as the generated
name will do fine.
cipher
The block cipher that will be used to encrypt Python bytecode.
"""
from ..config import CONF
Target.__init__(self)
name = kwargs.get('name', None)
cipher = kwargs.get('cipher', None)
self.toc = TOC()
# If available, use code objects directly from ModuleGraph to
# speed up PyInstaller.
self.code_dict = {}
for t in tocs:
self.toc.extend(t)
self.code_dict.update(getattr(t, '_code_cache', {}))
self.name = name
if name is None:
self.name = os.path.splitext(self.tocfilename)[0] + '.pyz'
# PyInstaller bootstrapping modules.
self.dependencies = get_bootstrap_modules()
# Bundle the crypto key.
self.cipher = cipher
if cipher:
key_file = ('pyimod00_crypto_key',
os.path.join(CONF['workpath'], 'pyimod00_crypto_key.pyc'),
'PYMODULE')
# Insert the key as the first module in the list. The key module contains
# just variables and does not depend on other modules.
self.dependencies.insert(0, key_file)
# Compile the top-level modules so that they end up in the CArchive and can be
# imported by the bootstrap script.
self.dependencies = misc.compile_py_files(self.dependencies, CONF['workpath'])
self.__postinit__()
_GUTS = (# input parameters
('name', _check_guts_eq),
('toc', _check_guts_toc), # todo: pyc=1
# no calculated/analysed values
)
def _check_guts(self, data, last_build):
if Target._check_guts(self, data, last_build):
return True
return False
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc[:]:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
try:
self.code_dict[entry[0]] = get_code_object(entry[0], entry[1])
except SyntaxError:
# Exclude the module in case this is code meant for a newer Python version.
toc.remove(entry)
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher)
logger.info("Building PYZ (ZlibArchive) %s completed successfully.",
self.name)
def __init__(self, *args, **kwargs):
"""
args
One or more arguments that are either TOCs Targets.
kwargs
Possible keywork arguments:
bootloader_ignore_signals
Non-Windows only. If True, the bootloader process will ignore
all ignorable signals. If False (default), it will forward
all signals to the child process. Useful in situations where
e.g. a supervisor process signals both the bootloader and
child (e.g. via a process group) to avoid signalling the
child twice.
console
On Windows or OSX governs whether to use the console executable
or the windowed executable. Always True on Linux/Unix (always
console executable - it does not matter there).
debug
Setting to True gives you progress mesages from the executable
(for console=False there will be annoying MessageBoxes on Windows).
name
The filename for the executable. On Windows suffix '.exe' is
appended.
exclude_binaries
Forwarded to the PKG the EXE builds.
icon
Windows or OSX only. icon='myicon.ico' to use an icon file or
icon='notepad.exe,0' to grab an icon resource.
version
Windows only. version='myversion.txt'. Use grab_version.py to get
a version resource from an executable and then edit the output to
create your own. (The syntax of version resources is so arcane
that I wouldn't attempt to write one from scratch).
uac_admin
Windows only. Setting to True creates a Manifest with will request
elevation upon application restart
uac_uiaccess
Windows only. Setting to True allows an elevated application to
work with Remote Desktop
"""
from ..config import CONF
Target.__init__(self)
# Available options for EXE in .spec files.
self.exclude_binaries = kwargs.get('exclude_binaries', False)
self.bootloader_ignore_signals = kwargs.get(
'bootloader_ignore_signals', False)
self.console = kwargs.get('console', True)
self.debug = kwargs.get('debug', False)
self.name = kwargs.get('name', None)
self.icon = kwargs.get('icon', None)
self.versrsrc = kwargs.get('version', None)
self.manifest = kwargs.get('manifest', None)
self.resources = kwargs.get('resources', [])
self.strip = kwargs.get('strip', False)
self.runtime_tmpdir = kwargs.get('runtime_tmpdir', None)
# If ``append_pkg`` is false, the archive will not be appended
# to the exe, but copied beside it.
self.append_pkg = kwargs.get('append_pkg', True)
# On Windows allows the exe to request admin privileges.
self.uac_admin = kwargs.get('uac_admin', False)
self.uac_uiaccess = kwargs.get('uac_uiaccess', False)
if CONF['hasUPX']:
self.upx = kwargs.get('upx', False)
else:
self.upx = False
# Old .spec format included in 'name' the path where to put created
# app. New format includes only exename.
#
# Ignore fullpath in the 'name' and prepend DISTPATH or WORKPATH.
# DISTPATH - onefile
# WORKPATH - onedir
if self.exclude_binaries:
# onedir mode - create executable in WORKPATH.
self.name = os.path.join(CONF['workpath'],
os.path.basename(self.name))
else:
# onefile mode - create executable in DISTPATH.
self.name = os.path.join(CONF['distpath'],
os.path.basename(self.name))
# Old .spec format included on Windows in 'name' .exe suffix.
if is_win or is_cygwin:
# Append .exe suffix if it is not already there.
if not self.name.endswith('.exe'):
self.name += '.exe'
base_name = os.path.splitext(os.path.basename(self.name))[0]
else:
base_name = os.path.basename(self.name)
self.pkgname = base_name + '.pkg'
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append(
(os.path.basename(arg.name), arg.name, arg.typ))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
if self.runtime_tmpdir is not None:
self.toc.append(
("pyi-runtime-tmpdir " + self.runtime_tmpdir, "", "OPTION"))
if self.bootloader_ignore_signals:
# no value; presence means "true"
self.toc.append(("pyi-bootloader-ignore-signals", "", "OPTION"))
if is_win:
filename = os.path.join(CONF['workpath'],
CONF['specnm'] + ".exe.manifest")
self.manifest = winmanifest.create_manifest(
filename, self.manifest, self.console, self.uac_admin,
self.uac_uiaccess)
manifest_filename = os.path.basename(self.name) + ".manifest"
self.toc.append((manifest_filename, filename, 'BINARY'))
if not self.exclude_binaries:
# Onefile mode: manifest file is explicitly loaded.
# Store name of manifest file as bootloader option. Allows
# the exe to be renamed.
self.toc.append(
("pyi-windows-manifest-filename " + manifest_filename, "",
"OPTION"))
self.pkg = PKG(
self.toc,
cdict=kwargs.get('cdict', None),
exclude_binaries=self.exclude_binaries,
strip_binaries=self.strip,
upx_binaries=self.upx,
)
self.dependencies = self.pkg.dependencies
# Get the path of the bootloader and store it in a TOC, so it
# can be checked for being changed.
exe = self._bootloader_file('run',
'.exe' if is_win or is_cygwin else '')
self.exefiles = TOC([(os.path.basename(exe), exe, 'EXECUTABLE')])
self.__postinit__()
def test_insert_existing():
toc = TOC(ELEMS1)
toc.insert(0, ELEMS1[-1])
toc.insert(1, ELEMS1[-1])
expected = list(ELEMS1)
assert toc == expected
class COLLECT(Target):
"""
In one-dir mode creates the output folder with all necessary files.
"""
def __init__(self, *args, **kws):
"""
args
One or more arguments that are either TOCs Targets.
kws
Possible keywork arguments:
name
The name of the directory to be built.
"""
from ..config import CONF
Target.__init__(self)
self.strip_binaries = kws.get('strip', False)
self.console = True
if CONF['hasUPX']:
self.upx_binaries = kws.get('upx', False)
else:
self.upx_binaries = False
self.name = kws.get('name')
# Old .spec format included in 'name' the path where to collect files
# for the created app.
# app. New format includes only directory name.
#
# The 'name' directory is created in DISTPATH and necessary files are
# then collected to this directory.
self.name = os.path.join(CONF['distpath'], os.path.basename(self.name))
self.toc = TOC()
for arg in args:
if isinstance(arg, TOC):
self.toc.extend(arg)
elif isinstance(arg, Target):
self.toc.append(
(os.path.basename(arg.name), arg.name, arg.typ))
if isinstance(arg, EXE):
self.console = arg.console
for tocnm, fnm, typ in arg.toc:
if tocnm == os.path.basename(arg.name) + ".manifest":
self.toc.append((tocnm, fnm, typ))
if not arg.append_pkg:
self.toc.append((os.path.basename(arg.pkgname),
arg.pkgname, 'PKG'))
self.toc.extend(arg.dependencies)
else:
self.toc.extend(arg)
self.__postinit__()
_GUTS = (
# COLLECT always builds, just want the toc to be written out
('toc', None), )
def _check_guts(self, data, last_build):
# COLLECT always needs to be executed, since it will clean the output
# directory anyway to make sure there is no existing cruft accumulating
return 1
def assemble(self):
_make_clean_directory(self.name)
logger.info("Building COLLECT %s", self.tocbasename)
toc = add_suffix_to_extensions(self.toc)
for inm, fnm, typ in toc:
if not os.path.exists(
fnm) or not os.path.isfile(fnm) and is_path_to_egg(fnm):
# file is contained within python egg, it is added with the egg
continue
if os.pardir in os.path.normpath(inm).split(os.sep) \
or os.path.isabs(inm):
raise SystemExit('Security-Alert: try to store file outside '
'of dist-directory. Aborting. %r' % inm)
tofnm = os.path.join(self.name, inm)
todir = os.path.dirname(tofnm)
if not os.path.exists(todir):
os.makedirs(todir)
if typ in ('EXTENSION', 'BINARY'):
fnm = checkCache(fnm,
strip=self.strip_binaries,
upx=(self.upx_binaries
and (is_win or is_cygwin)),
dist_nm=inm)
if typ != 'DEPENDENCY':
if os.path.isdir(fnm):
# beacuse shutil.copy2() is the default copy function
# for shutil.copytree, this will also copy file metadata
shutil.copytree(fnm, tofnm)
else:
shutil.copy(fnm, tofnm)
try:
shutil.copystat(fnm, tofnm)
except OSError:
logger.warning("failed to copy flags of %s", fnm)
if typ in ('EXTENSION', 'BINARY'):
os.chmod(tofnm, 0o755)
logger.info("Building COLLECT %s completed successfully.",
self.tocbasename)
def test_insert_other_case():
# should not be added if the filenames are the same on a case-insensitive system.
toc = TOC(ELEMS1)
toc.insert(1, ('EnCodIngs', '/usr/lib/python2.7/encodings.py', 'BINARY'))
expected = list(ELEMS1)
assert toc == expected
