def test_format_binaries_and_datas_1(tmpdir):
def _(path):
return os.path.join(*path.split('/'))
datas = [(_('existing.txt'), '.'), (_('other.txt'), 'foo'),
(_('*.log'), 'logs'), (_('a/*.log'), 'lll'),
(_('a/here.tex'), '.'), (_('b/[abc].tex'), 'tex')]
expected = set()
for dest, src in (
('existing.txt', 'existing.txt'),
('foo/other.txt', 'other.txt'),
('logs/aaa.log', 'aaa.log'),
('logs/bbb.log', 'bbb.log'),
('lll/xxx.log', 'a/xxx.log'),
('lll/yyy.log', 'a/yyy.log'),
('here.tex', 'a/here.tex'),
('tex/a.tex', 'b/a.tex'),
('tex/b.tex', 'b/b.tex'),
):
src = tmpdir.join(_(src)).ensure()
expected.add((_(dest), str(src)))
# add some files which are not included
tmpdir.join(_('not.txt')).ensure()
tmpdir.join(_('a/not.txt')).ensure()
tmpdir.join(_('b/not.txt')).ensure()
res = utils.format_binaries_and_datas(datas, str(tmpdir))
assert res == expected
def test_format_binaries_and_datas_1(tmpdir):
def _(path): return os.path.join(*path.split('/'))
datas = [(_('existing.txt'), '.'),
(_('other.txt'), 'foo'),
(_('*.log'), 'logs'),
(_('a/*.log'), 'lll'),
(_('a/here.tex'), '.'),
(_('b/[abc].tex'), 'tex')]
expected = set()
for dest, src in (
('existing.txt', 'existing.txt'),
('foo/other.txt', 'other.txt'),
('logs/aaa.log', 'aaa.log'),
('logs/bbb.log', 'bbb.log'),
('lll/xxx.log', 'a/xxx.log'),
('lll/yyy.log', 'a/yyy.log'),
('here.tex', 'a/here.tex'),
('tex/a.tex', 'b/a.tex'),
('tex/b.tex', 'b/b.tex'),
):
src = tmpdir.join(_(src)).ensure()
expected.add((_(dest), str(src)))
# add some files which are not included
tmpdir.join(_('not.txt')).ensure()
tmpdir.join(_('a/not.txt')).ensure()
tmpdir.join(_('b/not.txt')).ensure()
res = utils.format_binaries_and_datas(datas, str(tmpdir))
assert res == expected
def add_datas(self, list_of_tuples):
"""
Add all external data files in the passed list of `(name, path)` 2-tuples as dependencies of the current
module. This is equivalent to adding to the global `datas` hook attribute.
For convenience, the `list_of_tuples` may also be a single TOC or TREE instance.
"""
if isinstance(list_of_tuples, TOC):
self._added_datas.extend(i[:2] for i in list_of_tuples)
else:
self._added_datas.extend(format_binaries_and_datas(list_of_tuples))
def test_format_binaries_and_datas_with_bracket(tmpdir):
# See issue #2314: the filename contains brackets which are
# interpreted by glob().
def _(path):
return os.path.join(*path.split('/'))
datas = [(_('b/[abc].tex'), 'tex')]
expected = set()
for dest, src in (('tex/[abc].tex', 'b/[abc].tex'), ):
src = tmpdir.join(_(src)).ensure()
expected.add((_(dest), str(src)))
# add some files which are not included
tmpdir.join(_('tex/not.txt')).ensure()
res = utils.format_binaries_and_datas(datas, str(tmpdir))
assert res == expected
def test_format_binaries_and_datas_with_bracket(tmpdir):
# See issue #2314: the filename contains brackets which are
# interpreted by glob().
def _(path): return os.path.join(*path.split('/'))
datas = [(_('b/[abc].tex'), 'tex')]
expected = set()
for dest, src in (
('tex/[abc].tex', 'b/[abc].tex'),
):
src = tmpdir.join(_(src)).ensure()
expected.add((_(dest), str(src)))
# add some files which are not included
tmpdir.join(_('tex/not.txt')).ensure()
res = utils.format_binaries_and_datas(datas, str(tmpdir))
assert res == expected
def assemble(self):
"""
This method is the MAIN method for finding all necessary files to be bundled.
"""
from PyInstaller.config import CONF
for m in self.excludes:
logger.debug("Excluding module '%s'" % m)
self.graph = initialize_modgraph(excludes=self.excludes,
user_hook_dirs=self.hookspath)
# TODO Find a better place where to put 'base_library.zip' and when to created it.
# For Python 3 it is necessary to create file 'base_library.zip'
# containing core Python modules. In Python 3 some built-in modules
# are written in pure Python. base_library.zip is a way how to have
# those modules as "built-in".
libzip_filename = os.path.join(CONF['workpath'], 'base_library.zip')
create_py3_base_library(libzip_filename, graph=self.graph)
# Bundle base_library.zip as data file.
# Data format of TOC item: ('relative_path_in_dist_dir', 'absolute_path_on_disk', 'DATA')
self.datas.append(
(os.path.basename(libzip_filename), libzip_filename, 'DATA'))
# Expand sys.path of module graph.
# The attribute is the set of paths to use for imports: sys.path,
# plus our loader, plus other paths from e.g. --path option).
self.graph.path = self.pathex + self.graph.path
self.graph.set_setuptools_nspackages()
logger.info("running Analysis %s", self.tocbasename)
# Get paths to Python and, in Windows, the manifest.
python = compat.python_executable
if not is_win:
# Linux/MacOS: get a real, non-link path to the running Python executable.
while os.path.islink(python):
python = os.path.join(os.path.dirname(python),
os.readlink(python))
depmanifest = None
else:
# Windows: Create a manifest to embed into built .exe, containing the same
# dependencies as python.exe.
depmanifest = winmanifest.Manifest(
type_="win32",
name=CONF['specnm'],
processorArchitecture=winmanifest.processor_architecture(),
version=(1, 0, 0, 0))
depmanifest.filename = os.path.join(
CONF['workpath'], CONF['specnm'] + ".exe.manifest")
# We record "binaries" separately from the modulegraph, as there
# is no way to record those dependencies in the graph. These include
# the python executable and any binaries added by hooks later.
# "binaries" are not the same as "extensions" which are .so or .dylib
# that are found and recorded as extension nodes in the graph.
# Reset seen variable before running bindepend. We use bindepend only for
# the python executable.
bindepend.seen.clear()
# Add binary and assembly dependencies of Python.exe.
# This also ensures that its assembly depencies under Windows get added to the
# built .exe's manifest. Python 2.7 extension modules have no assembly
# dependencies, and rely on the app-global dependencies set by the .exe.
self.binaries.extend(
bindepend.Dependencies([('', python, '')],
manifest=depmanifest,
redirects=self.binding_redirects)[1:])
if is_win:
depmanifest.writeprettyxml()
### Module graph.
#
# Construct the module graph of import relationships between modules
# required by this user's application. For each entry point (top-level
# user-defined Python script), all imports originating from this entry
# point are recursively parsed into a subgraph of the module graph. This
# subgraph is then connected to this graph's root node, ensuring
# imported module nodes will be reachable from the root node -- which is
# is (arbitrarily) chosen to be the first entry point's node.
# List to hold graph nodes of scripts and runtime hooks in use order.
priority_scripts = []
# Assume that if the script does not exist, Modulegraph will raise error.
# Save the graph nodes of each in sequence.
for script in self.inputs:
logger.info("Analyzing %s", script)
priority_scripts.append(self.graph.add_script(script))
# Analyze the script's hidden imports (named on the command line)
self.graph.add_hiddenimports(self.hiddenimports)
### Post-graph hooks.
self.graph.process_post_graph_hooks(self)
# Update 'binaries' TOC and 'datas' TOC.
deps_proc = DependencyProcessor(self.graph,
self.graph._additional_files_cache)
self.binaries.extend(deps_proc.make_binaries_toc())
self.datas.extend(deps_proc.make_datas_toc())
self.zipped_data.extend(deps_proc.make_zipped_data_toc())
# Note: zipped eggs are collected below
### Look for dlls that are imported by Python 'ctypes' module.
# First get code objects of all modules that import 'ctypes'.
logger.info('Looking for ctypes DLLs')
# dict like: {'module1': code_obj, 'module2': code_obj}
ctypes_code_objs = self.graph.get_code_using("ctypes")
for name, co in ctypes_code_objs.items():
# Get dlls that might be needed by ctypes.
logger.debug('Scanning %s for shared libraries or dlls', name)
try:
ctypes_binaries = scan_code_for_ctypes(co)
self.binaries.extend(set(ctypes_binaries))
except Exception as ex:
raise RuntimeError(f"Failed to scan the module '{name}'. "
f"This is a bug. Please report it.") from ex
self.datas.extend((dest, source, "DATA")
for (dest, source) in format_binaries_and_datas(
self.graph.metadata_required()))
# Analyze run-time hooks.
# Run-time hooks has to be executed before user scripts. Add them
# to the beginning of 'priority_scripts'.
priority_scripts = self.graph.analyze_runtime_hooks(
self.custom_runtime_hooks) + priority_scripts
# 'priority_scripts' is now a list of the graph nodes of custom runtime
# hooks, then regular runtime hooks, then the PyI loader scripts.
# Further on, we will make sure they end up at the front of self.scripts
### Extract the nodes of the graph as TOCs for further processing.
# Initialize the scripts list with priority scripts in the proper order.
self.scripts = self.graph.nodes_to_toc(priority_scripts)
# Extend the binaries list with all the Extensions modulegraph has found.
self.binaries = self.graph.make_binaries_toc(self.binaries)
# Fill the "pure" list with pure Python modules.
assert len(self.pure) == 0
self.pure = self.graph.make_pure_toc()
# And get references to module code objects constructed by ModuleGraph
# to avoid writing .pyc/pyo files to hdd.
self.pure._code_cache = self.graph.get_code_objects()
# Add remaining binary dependencies - analyze Python C-extensions and what
# DLLs they depend on.
logger.info('Looking for dynamic libraries')
self.binaries.extend(
bindepend.Dependencies(self.binaries,
redirects=self.binding_redirects))
### Include zipped Python eggs.
logger.info('Looking for eggs')
self.zipfiles.extend(deps_proc.make_zipfiles_toc())
# Verify that Python dynamic library can be found.
# Without dynamic Python library PyInstaller cannot continue.
self._check_python_library(self.binaries)
if is_win:
# Remove duplicate redirects
self.binding_redirects[:] = list(set(self.binding_redirects))
logger.info("Found binding redirects: \n%s",
self.binding_redirects)
# Filter binaries to adjust path of extensions that come from
# python's lib-dynload directory. Prefix them with lib-dynload
# so that we'll collect them into subdirectory instead of
# directly into _MEIPASS
for idx, tpl in enumerate(self.binaries):
name, path, typecode = tpl
if typecode == 'EXTENSION' \
and not os.path.dirname(os.path.normpath(name)) \
and os.path.basename(os.path.dirname(path)) == 'lib-dynload':
name = os.path.join('lib-dynload', name)
self.binaries[idx] = (name, path, typecode)
# Place Python source in data files for the noarchive case.
if self.noarchive:
# Create a new TOC of ``(dest path for .pyc, source for .py, type)``.
new_toc = TOC()
for name, path, typecode in self.pure:
assert typecode == 'PYMODULE'
# Transform a python module name into a file name.
name = name.replace('.', os.sep)
# Special case: modules have an implied filename to add.
if os.path.splitext(os.path.basename(path))[0] == '__init__':
name += os.sep + '__init__'
# Append the extension for the compiled result.
# In python 3.5 (PEP-488) .pyo files were replaced by
# .opt-1.pyc and .opt-2.pyc. However, it seems that for
# bytecode-only module distribution, we always need to
# use the .pyc extension.
name += '.pyc'
new_toc.append((name, path, typecode))
# Put the result of byte-compiling this TOC in datas. Mark all entries as data.
for name, path, typecode in compile_py_files(
new_toc, CONF['workpath']):
self.datas.append((name, path, 'DATA'))
# Store no source in the archive.
self.pure = TOC()
# Write warnings about missing modules.
self._write_warnings()
# Write debug information about hte graph
self._write_graph_debug()
def __init__(self,
scripts,
pathex=None,
binaries=None,
datas=None,
hiddenimports=None,
hookspath=None,
hooksconfig=None,
excludes=None,
runtime_hooks=None,
cipher=None,
win_no_prefer_redirects=False,
win_private_assemblies=False,
noarchive=False):
"""
scripts
A list of scripts specified as file names.
pathex
An optional list of paths to be searched before sys.path.
binaries
An optional list of additional binaries (dlls, etc.) to include.
datas
An optional list of additional data files to include.
hiddenimport
An optional list of additional (hidden) modules to include.
hookspath
An optional list of additional paths to search for hooks.
(hook-modules).
hooksconfig
An optional dict of config settings for hooks.
(hook-modules).
excludes
An optional list of module or package names (their Python names,
not path names) that will be ignored (as though they were not found).
runtime_hooks
An optional list of scripts to use as users' runtime hooks. Specified
as file names.
cipher
Add optional instance of the pyz_crypto.PyiBlockCipher class
(with a provided key).
win_no_prefer_redirects
If True, prefers not to follow version redirects when searching for
Windows SxS Assemblies.
win_private_assemblies
If True, changes all bundled Windows SxS Assemblies into Private
Assemblies to enforce assembly versions.
noarchive
If True, don't place source files in a archive, but keep them as
individual files.
"""
super(Analysis, self).__init__()
from PyInstaller.config import CONF
self.inputs = []
spec_dir = os.path.dirname(CONF['spec'])
for script in scripts:
# If path is relative, it is relative to the location of .spec file.
if not os.path.isabs(script):
script = os.path.join(spec_dir, script)
if absnormpath(script) in self._old_scripts:
logger.warning('Ignoring obsolete auto-added script %s',
script)
continue
# Normalize script path.
script = os.path.normpath(script)
if not os.path.exists(script):
raise SystemExit("script '%s' not found" % script)
self.inputs.append(script)
# Django hook requires this variable to find the script manage.py.
CONF['main_script'] = self.inputs[0]
self.pathex = self._extend_pathex(pathex, self.inputs)
# Set global config variable 'pathex' to make it available for
# PyInstaller.utils.hooks and import hooks. Path extensions for module
# search.
CONF['pathex'] = self.pathex
# Extend sys.path so PyInstaller could find all necessary modules.
logger.info('Extending PYTHONPATH with paths\n' +
pprint.pformat(self.pathex))
sys.path.extend(self.pathex)
# Set global variable to hold assembly binding redirects
CONF['binding_redirects'] = []
self.hiddenimports = hiddenimports or []
# Include modules detected when parsing options, like 'codecs' and encodings.
self.hiddenimports.extend(CONF['hiddenimports'])
self.hookspath = []
# Append directories in `hookspath` (`--additional-hooks-dir`) to
# take precedence over those from the entry points.
if hookspath:
self.hookspath.extend(hookspath)
# Add hook directories from PyInstaller entry points.
self.hookspath += discover_hook_directories()
self.hooksconfig = {}
if hooksconfig:
self.hooksconfig.update(hooksconfig)
# Custom runtime hook files that should be included and started before
# any existing PyInstaller runtime hooks.
self.custom_runtime_hooks = runtime_hooks or []
if cipher:
logger.info('Will encrypt Python bytecode with key: %s',
cipher.key)
# Create a Python module which contains the decryption key which will
# be used at runtime by pyi_crypto.PyiBlockCipher.
pyi_crypto_key_path = os.path.join(CONF['workpath'],
'pyimod00_crypto_key.py')
with open(pyi_crypto_key_path, 'w', encoding='utf-8') as f:
f.write('# -*- coding: utf-8 -*-\n' 'key = %r\n' % cipher.key)
self.hiddenimports.append('tinyaes')
self.excludes = excludes or []
self.scripts = TOC()
self.pure = TOC()
self.binaries = TOC()
self.zipfiles = TOC()
self.zipped_data = TOC()
self.datas = TOC()
self.dependencies = TOC()
self.binding_redirects = CONF['binding_redirects'] = []
self.win_no_prefer_redirects = win_no_prefer_redirects
self.win_private_assemblies = win_private_assemblies
self._python_version = sys.version
self.noarchive = noarchive
self.__postinit__()
# TODO create function to convert datas/binaries from 'hook format' to TOC.
# Initialise 'binaries' and 'datas' with lists specified in .spec file.
if binaries:
logger.info("Appending 'binaries' from .spec")
for name, pth in format_binaries_and_datas(binaries,
workingdir=spec_dir):
self.binaries.append((name, pth, 'BINARY'))
if datas:
logger.info("Appending 'datas' from .spec")
for name, pth in format_binaries_and_datas(datas,
workingdir=spec_dir):
self.datas.append((name, pth, 'DATA'))
def test_format_binaries_and_datas_not_found_raises_error(tmpdir):
datas = [('non-existing.txt', '.')]
tmpdir.join('existing.txt').ensure()
# TODO Tighten test when introducing PyInstaller.exceptions
with pytest.raises(SystemExit) as context:
utils.format_binaries_and_datas(datas, str(tmpdir))
def test_format_binaries_and_datas_not_found_raises_error(tmpdir):
datas = [('non-existing.txt', '.')]
tmpdir.join('existing.txt').ensure()
# TODO Tighten test when introducing PyInstaller.exceptions
with pytest.raises(SystemExit) as context:
utils.format_binaries_and_datas(datas, str(tmpdir))
