def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc[:]:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
try:
self.code_dict[entry[0]] = get_code_object(
entry[0], entry[1])
except SyntaxError:
# Exclude the module in case this is code meant for a newer Python version.
toc.remove(entry)
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name,
toc,
code_dict=self.code_dict,
cipher=self.cipher)
logger.info("Building PYZ (ZlibArchive) %s completed successfully.",
self.name)
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc[:]:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
try:
self.code_dict[entry[0]] = get_code_object(entry[0], entry[1])
except SyntaxError:
# Exclude the module in case this is code meant for a newer Python version.
toc.remove(entry)
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher)
logger.info("Building PYZ (ZlibArchive) %s completed successfully.",
self.name)
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
self.code_dict[entry[0]] = get_code_object(entry[0], entry[1])
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher)
def assemble(self):
logger.info("Building PYZ (ZlibArchive) %s", self.name)
# Do not bundle PyInstaller bootstrap modules into PYZ archive.
toc = self.toc - self.dependencies
for entry in toc:
if not entry[0] in self.code_dict and entry[2] == 'PYMODULE':
# For some reason the code-object, modulegraph created
# is not available. Recreate it
self.code_dict[entry[0]] = get_code_object(entry[0], entry[1])
# sort content alphabetically to support reproducible builds
toc.sort()
# Remove leading parts of paths in code objects
self.code_dict = {
key: strip_paths_in_code(code)
for key, code in self.code_dict.items()
}
pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher)
def add(self, entry):
"""
Add an ENTRY to the CArchive.
ENTRY must have:
entry[0] is name (under which it will be saved).
entry[1] is fullpathname of the file.
entry[2] is a flag for it's storage format (0==uncompressed,
1==compressed)
entry[3] is the entry's type code.
Version 5:
If the type code is 'o':
entry[0] is the runtime option
eg: v (meaning verbose imports)
u (menaing unbuffered)
W arg (warning option arg)
s (meaning do site.py processing.
"""
(nm, pathnm, flag, typcd) = entry[:4]
# FIXME Could we make the version 5 the default one?
# Version 5 - allow type 'o' = runtime option.
code_data = None
fh = None
try:
if typcd in ("o", "d"):
ulen = 0
flag = 0
elif typcd == "s":
# If it's a source code file, compile it to a code object and marshall
# the object so it can be unmarshalled by the bootloader.
code = get_code_object(nm, pathnm)
code = strip_paths_in_code(code)
code_data = marshal.dumps(code)
ulen = len(code_data)
else:
fh = open(pathnm, "rb")
ulen = os.fstat(fh.fileno()).st_size
except IOError:
print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd))
raise
where = self.lib.tell()
assert flag in range(3)
if not fh and not code_data:
# no need to write anything
pass
elif flag == 1:
comprobj = zlib.compressobj(self.LEVEL)
if code_data is not None:
self.lib.write(comprobj.compress(code_data))
else:
assert fh
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
self.lib.write(comprobj.compress(buf))
self.lib.write(comprobj.flush())
else:
if code_data is not None:
self.lib.write(code_data)
else:
assert fh
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
self.lib.write(buf)
dlen = self.lib.tell() - where
if typcd == "m":
if pathnm.find(".__init__.py") > -1:
typcd = "M"
# Record the entry in the CTOC
self.toc.add(where, dlen, ulen, flag, typcd, nm)
def add(self, entry):
"""
Add an ENTRY to the CArchive.
ENTRY must have:
entry[0] is name (under which it will be saved).
entry[1] is fullpathname of the file.
entry[2] is a flag for it's storage format (0==uncompressed,
1==compressed)
entry[3] is the entry's type code.
Version 5:
If the type code is 'o':
entry[0] is the runtime option
eg: v (meaning verbose imports)
u (menaing unbuffered)
W arg (warning option arg)
s (meaning do site.py processing.
"""
(nm, pathnm, flag, typcd) = entry[:4]
# FIXME Could we make the version 5 the default one?
# Version 5 - allow type 'o' = runtime option.
code_data = None
fh = None
try:
if typcd in ('o', 'd'):
ulen = 0
flag = 0
elif typcd == 's':
# If it's a source code file, compile it to a code object and marshall
# the object so it can be unmarshalled by the bootloader.
code = get_code_object(nm, pathnm)
code = strip_paths_in_code(code)
code_data = marshal.dumps(code)
ulen = len(code_data)
else:
fh = open(pathnm, 'rb')
ulen = os.fstat(fh.fileno()).st_size
except IOError:
print("Cannot find ('%s', '%s', %s, '%s')" %
(nm, pathnm, flag, typcd))
raise
where = self.lib.tell()
assert flag in range(3)
if not fh and not code_data:
# no need to write anything
pass
elif flag == 1:
comprobj = zlib.compressobj(self.LEVEL)
if code_data is not None:
self.lib.write(comprobj.compress(code_data))
else:
assert fh
# We only want to change it for pyc files
modify_header = typcd in ('M', 'm', 's')
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
if modify_header:
modify_header = False
buf = fake_pyc_timestamp(buf)
self.lib.write(comprobj.compress(buf))
self.lib.write(comprobj.flush())
else:
if code_data is not None:
self.lib.write(code_data)
else:
assert fh
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
self.lib.write(buf)
dlen = self.lib.tell() - where
if typcd == 'm':
if pathnm.find('.__init__.py') > -1:
typcd = 'M'
if fh:
fh.close()
# Record the entry in the CTOC
self.toc.add(where, dlen, ulen, flag, typcd, nm)
def create_py3_base_library(libzip_filename, graph):
"""
Package basic Python modules into .zip file. The .zip file with basic modules is necessary to have on PYTHONPATH
for initializing libpython3 in order to run the frozen executable with Python 3.
"""
# Import strip_paths_in_code locally to avoid cyclic import between building.utils and depend.utils (this module);
# building.utils imports depend.bindepend, which in turn imports depend.utils.
from PyInstaller.building.utils import strip_paths_in_code
# Construct regular expression for matching modules that should be bundled into base_library.zip. Excluded are plain
# 'modules' or 'submodules.ANY_NAME'. The match has to be exact - start and end of string not substring.
regex_modules = '|'.join([rf'(^{x}$)' for x in compat.PY3_BASE_MODULES])
regex_submod = '|'.join([rf'(^{x}\..*$)' for x in compat.PY3_BASE_MODULES])
regex_str = regex_modules + '|' + regex_submod
module_filter = re.compile(regex_str)
try:
# Remove .zip from previous run.
if os.path.exists(libzip_filename):
os.remove(libzip_filename)
logger.debug('Adding python files to base_library.zip')
# Class zipfile.PyZipFile is not suitable for PyInstaller needs.
with zipfile.ZipFile(libzip_filename, mode='w') as zf:
zf.debug = 3
# Sort the graph nodes by identifier to ensure repeatable builds
graph_nodes = list(graph.iter_graph())
graph_nodes.sort(key=lambda item: item.identifier)
for mod in graph_nodes:
if type(mod) in (modulegraph.SourceModule, modulegraph.Package,
modulegraph.CompiledModule):
# Bundling just required modules.
if module_filter.match(mod.identifier):
st = os.stat(mod.filename)
timestamp = int(st.st_mtime)
size = st.st_size & 0xFFFFFFFF
# Name inside the archive. The ZIP format specification requires forward slashes as directory
# separator.
if type(mod) is modulegraph.Package:
new_name = mod.identifier.replace(
'.', '/') + '/__init__.pyc'
else:
new_name = mod.identifier.replace('.',
'/') + '.pyc'
# Write code to a file. This code is similar to py_compile.compile().
with io.BytesIO() as fc:
# Prepare all data in byte stream file-like object.
fc.write(compat.BYTECODE_MAGIC)
if compat.is_py37:
# Additional bitfield according to PEP 552 0b01 means hash based but don't check the
# hash
fc.write(struct.pack('<I', 0b01))
with open(mod.filename, 'rb') as fs:
source_bytes = fs.read()
source_hash = importlib_source_hash(
source_bytes)
fc.write(source_hash)
else:
fc.write(struct.pack('<II', timestamp, size))
code = strip_paths_in_code(mod.code) # Strip paths
marshal.dump(code, fc)
# Use a ZipInfo to set timestamp for deterministic build.
info = zipfile.ZipInfo(new_name)
zf.writestr(info, fc.getvalue())
except Exception:
logger.error('base_library.zip could not be created!')
raise
def add(self, entry):
"""
Add an ENTRY to the CArchive.
ENTRY must have:
entry[0] is name (under which it will be saved).
entry[1] is fullpathname of the file.
entry[2] is a flag for it's storage format (0==uncompressed,
1==compressed)
entry[3] is the entry's type code.
Version 5:
If the type code is 'o':
entry[0] is the runtime option
eg: v (meaning verbose imports)
u (meaning unbuffered)
W arg (warning option arg)
s (meaning do site.py processing.
"""
(nm, pathnm, flag, typcd) = entry[:4]
# FIXME Could we make the version 5 the default one?
# Version 5 - allow type 'o' = runtime option.
code_data = None
fh = None
try:
if typcd in ('o', 'd'):
ulen = 0
flag = 0
elif typcd == 's':
# If it is a source code file, compile it to a code object and marshall the object, so it can be
# unmarshalled by the bootloader.
code = get_code_object(nm, pathnm)
code = strip_paths_in_code(code)
code_data = marshal.dumps(code)
ulen = len(code_data)
elif typcd == 'm':
fh = open(pathnm, 'rb')
ulen = os.fstat(fh.fileno()).st_size
# Check if it is a PYC file
header = fh.read(4)
fh.seek(0)
if header == BYTECODE_MAGIC:
# Read whole header and load code. According to PEP-552, in python versions prior to 3.7, the PYC
# header consists of three 32-bit words (magic, timestamp, and source file size).
# From python 3.7 on, the PYC header was extended to four 32-bit words (magic, flags, and, depending
# on the flags, either timestamp and source file size, or a 64-bit hash).
if is_py37:
header = fh.read(16)
else:
header = fh.read(12)
code = marshal.load(fh)
# Strip paths from code, marshal back into module form. The header fields (timestamp, size, hash,
# etc.) are all referring to the source file, so our modification of the code object does not affect
# them, and we can re-use the original header.
code = strip_paths_in_code(code)
data = header + marshal.dumps(code)
# Create file-like object for timestamp re-write in the subsequent steps.
fh = io.BytesIO(data)
ulen = len(data)
else:
fh = open(pathnm, 'rb')
ulen = os.fstat(fh.fileno()).st_size
except IOError:
print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd))
raise
where = self.lib.tell()
assert flag in range(3)
if not fh and not code_data:
# No need to write anything.
pass
elif flag == 1:
comprobj = zlib.compressobj(self.LEVEL)
if code_data is not None:
self.lib.write(comprobj.compress(code_data))
else:
assert fh
# We only want to change it for pyc files.
modify_header = typcd in ('M', 'm', 's')
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
if modify_header:
modify_header = False
buf = fake_pyc_timestamp(buf)
self.lib.write(comprobj.compress(buf))
self.lib.write(comprobj.flush())
else:
if code_data is not None:
self.lib.write(code_data)
else:
assert fh
while 1:
buf = fh.read(16 * 1024)
if not buf:
break
self.lib.write(buf)
dlen = self.lib.tell() - where
if typcd == 'm':
if pathnm.find('.__init__.py') > -1:
typcd = 'M'
if fh:
fh.close()
# Record the entry in the CTOC
self.toc.add(where, dlen, ulen, flag, typcd, nm)
