def assemble(self): logger.info("Building PYZ (ZlibArchive) %s", self.name) # Do not bundle PyInstaller bootstrap modules into PYZ archive. toc = self.toc - self.dependencies for entry in toc[:]: if not entry[0] in self.code_dict and entry[2] == 'PYMODULE': # For some reason the code-object, modulegraph created # is not available. Recreate it try: self.code_dict[entry[0]] = get_code_object( entry[0], entry[1]) except SyntaxError: # Exclude the module in case this is code meant for a newer Python version. toc.remove(entry) # sort content alphabetically to support reproducible builds toc.sort() # Remove leading parts of paths in code objects self.code_dict = { key: strip_paths_in_code(code) for key, code in self.code_dict.items() } pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher) logger.info("Building PYZ (ZlibArchive) %s completed successfully.", self.name)
def assemble(self): logger.info("Building PYZ (ZlibArchive) %s", self.name) # Do not bundle PyInstaller bootstrap modules into PYZ archive. toc = self.toc - self.dependencies for entry in toc[:]: if not entry[0] in self.code_dict and entry[2] == 'PYMODULE': # For some reason the code-object, modulegraph created # is not available. Recreate it try: self.code_dict[entry[0]] = get_code_object(entry[0], entry[1]) except SyntaxError: # Exclude the module in case this is code meant for a newer Python version. toc.remove(entry) # sort content alphabetically to support reproducible builds toc.sort() # Remove leading parts of paths in code objects self.code_dict = { key: strip_paths_in_code(code) for key, code in self.code_dict.items() } pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher) logger.info("Building PYZ (ZlibArchive) %s completed successfully.", self.name)
def assemble(self): logger.info("Building PYZ (ZlibArchive) %s", self.name) # Do not bundle PyInstaller bootstrap modules into PYZ archive. toc = self.toc - self.dependencies for entry in toc: if not entry[0] in self.code_dict and entry[2] == 'PYMODULE': # For some reason the code-object, modulegraph created # is not available. Recreate it self.code_dict[entry[0]] = get_code_object(entry[0], entry[1]) # sort content alphabetically to support reproducible builds toc.sort() # Remove leading parts of paths in code objects self.code_dict = { key: strip_paths_in_code(code) for key, code in self.code_dict.items() } pyz = ZlibArchiveWriter(self.name, toc, code_dict=self.code_dict, cipher=self.cipher)
def add(self, entry): """ Add an ENTRY to the CArchive. ENTRY must have: entry[0] is name (under which it will be saved). entry[1] is fullpathname of the file. entry[2] is a flag for it's storage format (0==uncompressed, 1==compressed) entry[3] is the entry's type code. Version 5: If the type code is 'o': entry[0] is the runtime option eg: v (meaning verbose imports) u (menaing unbuffered) W arg (warning option arg) s (meaning do site.py processing. """ (nm, pathnm, flag, typcd) = entry[:4] # FIXME Could we make the version 5 the default one? # Version 5 - allow type 'o' = runtime option. code_data = None fh = None try: if typcd in ("o", "d"): ulen = 0 flag = 0 elif typcd == "s": # If it's a source code file, compile it to a code object and marshall # the object so it can be unmarshalled by the bootloader. code = get_code_object(nm, pathnm) code = strip_paths_in_code(code) code_data = marshal.dumps(code) ulen = len(code_data) else: fh = open(pathnm, "rb") ulen = os.fstat(fh.fileno()).st_size except IOError: print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd)) raise where = self.lib.tell() assert flag in range(3) if not fh and not code_data: # no need to write anything pass elif flag == 1: comprobj = zlib.compressobj(self.LEVEL) if code_data is not None: self.lib.write(comprobj.compress(code_data)) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(comprobj.compress(buf)) self.lib.write(comprobj.flush()) else: if code_data is not None: self.lib.write(code_data) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(buf) dlen = self.lib.tell() - where if typcd == "m": if pathnm.find(".__init__.py") > -1: typcd = "M" # Record the entry in the CTOC self.toc.add(where, dlen, ulen, flag, typcd, nm)
def add(self, entry): """ Add an ENTRY to the CArchive. ENTRY must have: entry[0] is name (under which it will be saved). entry[1] is fullpathname of the file. entry[2] is a flag for it's storage format (0==uncompressed, 1==compressed) entry[3] is the entry's type code. Version 5: If the type code is 'o': entry[0] is the runtime option eg: v (meaning verbose imports) u (menaing unbuffered) W arg (warning option arg) s (meaning do site.py processing. """ (nm, pathnm, flag, typcd) = entry[:4] # FIXME Could we make the version 5 the default one? # Version 5 - allow type 'o' = runtime option. code_data = None fh = None try: if typcd in ('o', 'd'): ulen = 0 flag = 0 elif typcd == 's': # If it's a source code file, compile it to a code object and marshall # the object so it can be unmarshalled by the bootloader. code = get_code_object(nm, pathnm) code = strip_paths_in_code(code) code_data = marshal.dumps(code) ulen = len(code_data) else: fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size except IOError: print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd)) raise where = self.lib.tell() assert flag in range(3) if not fh and not code_data: # no need to write anything pass elif flag == 1: comprobj = zlib.compressobj(self.LEVEL) if code_data is not None: self.lib.write(comprobj.compress(code_data)) else: assert fh # We only want to change it for pyc files modify_header = typcd in ('M', 'm', 's') while 1: buf = fh.read(16 * 1024) if not buf: break if modify_header: modify_header = False buf = fake_pyc_timestamp(buf) self.lib.write(comprobj.compress(buf)) self.lib.write(comprobj.flush()) else: if code_data is not None: self.lib.write(code_data) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(buf) dlen = self.lib.tell() - where if typcd == 'm': if pathnm.find('.__init__.py') > -1: typcd = 'M' if fh: fh.close() # Record the entry in the CTOC self.toc.add(where, dlen, ulen, flag, typcd, nm)
def create_py3_base_library(libzip_filename, graph): """ Package basic Python modules into .zip file. The .zip file with basic modules is necessary to have on PYTHONPATH for initializing libpython3 in order to run the frozen executable with Python 3. """ # Import strip_paths_in_code locally to avoid cyclic import between building.utils and depend.utils (this module); # building.utils imports depend.bindepend, which in turn imports depend.utils. from PyInstaller.building.utils import strip_paths_in_code # Construct regular expression for matching modules that should be bundled into base_library.zip. Excluded are plain # 'modules' or 'submodules.ANY_NAME'. The match has to be exact - start and end of string not substring. regex_modules = '|'.join([rf'(^{x}$)' for x in compat.PY3_BASE_MODULES]) regex_submod = '|'.join([rf'(^{x}\..*$)' for x in compat.PY3_BASE_MODULES]) regex_str = regex_modules + '|' + regex_submod module_filter = re.compile(regex_str) try: # Remove .zip from previous run. if os.path.exists(libzip_filename): os.remove(libzip_filename) logger.debug('Adding python files to base_library.zip') # Class zipfile.PyZipFile is not suitable for PyInstaller needs. with zipfile.ZipFile(libzip_filename, mode='w') as zf: zf.debug = 3 # Sort the graph nodes by identifier to ensure repeatable builds graph_nodes = list(graph.iter_graph()) graph_nodes.sort(key=lambda item: item.identifier) for mod in graph_nodes: if type(mod) in (modulegraph.SourceModule, modulegraph.Package, modulegraph.CompiledModule): # Bundling just required modules. if module_filter.match(mod.identifier): st = os.stat(mod.filename) timestamp = int(st.st_mtime) size = st.st_size & 0xFFFFFFFF # Name inside the archive. The ZIP format specification requires forward slashes as directory # separator. if type(mod) is modulegraph.Package: new_name = mod.identifier.replace( '.', '/') + '/__init__.pyc' else: new_name = mod.identifier.replace('.', '/') + '.pyc' # Write code to a file. This code is similar to py_compile.compile(). with io.BytesIO() as fc: # Prepare all data in byte stream file-like object. fc.write(compat.BYTECODE_MAGIC) if compat.is_py37: # Additional bitfield according to PEP 552 0b01 means hash based but don't check the # hash fc.write(struct.pack('<I', 0b01)) with open(mod.filename, 'rb') as fs: source_bytes = fs.read() source_hash = importlib_source_hash( source_bytes) fc.write(source_hash) else: fc.write(struct.pack('<II', timestamp, size)) code = strip_paths_in_code(mod.code) # Strip paths marshal.dump(code, fc) # Use a ZipInfo to set timestamp for deterministic build. info = zipfile.ZipInfo(new_name) zf.writestr(info, fc.getvalue()) except Exception: logger.error('base_library.zip could not be created!') raise
def add(self, entry): """ Add an ENTRY to the CArchive. ENTRY must have: entry[0] is name (under which it will be saved). entry[1] is fullpathname of the file. entry[2] is a flag for it's storage format (0==uncompressed, 1==compressed) entry[3] is the entry's type code. Version 5: If the type code is 'o': entry[0] is the runtime option eg: v (meaning verbose imports) u (meaning unbuffered) W arg (warning option arg) s (meaning do site.py processing. """ (nm, pathnm, flag, typcd) = entry[:4] # FIXME Could we make the version 5 the default one? # Version 5 - allow type 'o' = runtime option. code_data = None fh = None try: if typcd in ('o', 'd'): ulen = 0 flag = 0 elif typcd == 's': # If it is a source code file, compile it to a code object and marshall the object, so it can be # unmarshalled by the bootloader. code = get_code_object(nm, pathnm) code = strip_paths_in_code(code) code_data = marshal.dumps(code) ulen = len(code_data) elif typcd == 'm': fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size # Check if it is a PYC file header = fh.read(4) fh.seek(0) if header == BYTECODE_MAGIC: # Read whole header and load code. According to PEP-552, in python versions prior to 3.7, the PYC # header consists of three 32-bit words (magic, timestamp, and source file size). # From python 3.7 on, the PYC header was extended to four 32-bit words (magic, flags, and, depending # on the flags, either timestamp and source file size, or a 64-bit hash). if is_py37: header = fh.read(16) else: header = fh.read(12) code = marshal.load(fh) # Strip paths from code, marshal back into module form. The header fields (timestamp, size, hash, # etc.) are all referring to the source file, so our modification of the code object does not affect # them, and we can re-use the original header. code = strip_paths_in_code(code) data = header + marshal.dumps(code) # Create file-like object for timestamp re-write in the subsequent steps. fh = io.BytesIO(data) ulen = len(data) else: fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size except IOError: print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd)) raise where = self.lib.tell() assert flag in range(3) if not fh and not code_data: # No need to write anything. pass elif flag == 1: comprobj = zlib.compressobj(self.LEVEL) if code_data is not None: self.lib.write(comprobj.compress(code_data)) else: assert fh # We only want to change it for pyc files. modify_header = typcd in ('M', 'm', 's') while 1: buf = fh.read(16 * 1024) if not buf: break if modify_header: modify_header = False buf = fake_pyc_timestamp(buf) self.lib.write(comprobj.compress(buf)) self.lib.write(comprobj.flush()) else: if code_data is not None: self.lib.write(code_data) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(buf) dlen = self.lib.tell() - where if typcd == 'm': if pathnm.find('.__init__.py') > -1: typcd = 'M' if fh: fh.close() # Record the entry in the CTOC self.toc.add(where, dlen, ulen, flag, typcd, nm)