def _collect_infos(dirname):
""" Utility function used by ExplodedZipFile to generate ZipInfo
entries for all of the files and directories under dirname """
for r, _ds, fs in walk(dirname):
if not islink(r) and r != dirname:
i = ZipInfo()
i.filename = join(relpath(r, dirname), "")
i.file_size = 0
i.compress_size = 0
i.CRC = 0
yield i.filename, i
for f in fs:
df = join(r, f)
relfn = relpath(join(r, f), dirname)
if islink(df):
pass
elif isfile(df):
i = ZipInfo()
i.filename = relfn
i.file_size = getsize(df)
i.compress_size = i.file_size
i.CRC = file_crc32(df)
yield i.filename, i
else:
# TODO: is there any more special treatment?
pass
def _collect_infos(dirname):
""" Utility function used by ExplodedZipFile to generate ZipInfo
entries for all of the files and directories under dirname """
for r, _ds, fs in walk(dirname):
if not islink(r) and r != dirname:
i = ZipInfo()
i.filename = join(relpath(r, dirname), "")
i.file_size = 0
i.compress_size = 0
i.CRC = 0
yield i.filename, i
for f in fs:
df = join(r, f)
relfn = relpath(join(r, f), dirname)
if islink(df):
pass
elif isfile(df):
i = ZipInfo()
i.filename = relfn
i.file_size = getsize(df)
i.compress_size = i.file_size
i.CRC = file_crc32(df)
yield i.filename, i
else:
# TODO: is there any more special treatment?
pass
def write_file(self, zinfo_or_arcname, file_object, compress_type=None):
if not isinstance(zinfo_or_arcname, ZipInfo):
zinfo = ZipInfo(filename=zinfo_or_arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = self.compression
if zinfo.filename[-1] == '/':
zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x
zinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
else:
zinfo = zinfo_or_arcname
zinfo.flag_bits |= 0x08 # we will write a data descriptor
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if compress_type is not None:
zinfo.compress_type = compress_type
else:
zinfo.compress_type = zipfile.ZIP_DEFLATED
zinfo.file_size = file_size = 0 # Uncompressed size
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
zinfo.CRC = crc = 0
zinfo.compress_size = compress_size = 0
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == zipfile.ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
else:
cmpr = None
while True:
buf = file_object.read(1024 * 1024 * 1)
if not buf:
break
file_size = file_size + len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = crc
zinfo.file_size = file_size
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def process_one(self, infile, filename):
""" process a non-zipped file. Is this needed? """
self.init_report()
resultfile = Tools.make_return_file(filename)
stat = os.stat(infile.fileno)
#Create a fake zipinfo
zi = ZipInfo(filename=filename)
zi.file_size = stat.st_size
data = self.process_file(infile)
resultfile.write(data)
self.finalize_report(resultfile)
return report
def process_one(self, infile, filename):
""" process a non-zipped file. Is this needed? """
self.init_report()
resultfile = Tools.make_return_file(filename)
stat = os.stat(infile.fileno)
#Create a fake zipinfo
zi = ZipInfo(filename=filename)
zi.file_size = stat.st_size
data = self.process_file(infile)
resultfile.write(data)
self.finalize_report(resultfile)
return report
def compress_buffer(self, arcname, buffer):
zinfo = ZipInfo(filename=arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = ZIP_DEFLATED
zinfo.compress_size = 0
zinfo.flag_bits = 0x08
zinfo.external_attr = 0o600 << 16
zinfo.file_size = 0
zinfo.header_offset = self._data_p
zinfo.CRC = crc = 0
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
yield self.__get_data(zinfo.FileHeader())
buffer.seek(0)
while 1:
buf = buffer.read(CHUNK_SIZE)
if not buf:
break
zinfo.file_size += len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
if cmpr:
buf = cmpr.flush()
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
else:
zinfo.compress_size = zinfo.file_size
zinfo.CRC = crc
zip64 = zinfo.file_size > ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT
fmt = '<4sLQQ' if zip64 else '<4sLLL'
data_descriptor = struct.pack(fmt, stringDataDescriptor, zinfo.CRC,
zinfo.compress_size, zinfo.file_size)
yield self.__get_data(data_descriptor)
self._filelist.append(zinfo)
def compress_string(self, arcname, data):
if isinstance(data, str):
data = data.encode("utf-8")
zinfo = ZipInfo(filename=arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = ZIP_DEFLATED
zinfo.external_attr = 0o600 << 16
zinfo.file_size = len(data)
zinfo.header_offset = self._data_p
zinfo.CRC = zlib.crc32(data) & 0xffffffff
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
if cmpr:
data = cmpr.compress(data) + cmpr.flush()
zinfo.compress_size = len(data)
else:
zinfo.compress_size = zinfo.file_size
zip64 = zinfo.file_size > ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT
yield self.__get_data(zinfo.FileHeader(zip64))
yield self.__get_data(data)
self._filelist.append(zinfo)
def compress_file(self, filename, arcname):
st = os.stat(filename)
zinfo = ZipInfo(arcname, time.localtime(time.time())[:6])
zinfo.external_attr = (st[0] & 0xFFFF) << 16
zinfo.compress_type = ZIP_DEFLATED
zinfo.flag_bits = 0x08
zinfo.header_offset = self._data_p
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
with open(filename, "rb") as fp:
zinfo.CRC = crc = 0
zinfo.compress_size = 0
zinfo.file_size = 0
yield self.__get_data(zinfo.FileHeader())
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
zinfo.file_size += len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
if cmpr:
buf = cmpr.flush()
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
else:
zinfo.compress_size = zinfo.file_size
zinfo.CRC = crc
zip64 = zinfo.file_size > ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT
fmt = '<4sLQQ' if zip64 else '<4sLLL'
data_descriptor = struct.pack(fmt, stringDataDescriptor, zinfo.CRC,
zinfo.compress_size, zinfo.file_size)
yield self.__get_data(data_descriptor)
self._filelist.append(zinfo)
def compress_stream(self, arcname, datagen):
zinfo = ZipInfo(arcname, time.localtime(time.time())[:6])
zinfo.external_attr = 0o600 << 16
zinfo.compress_type = ZIP_DEFLATED
zinfo.flag_bits = 0x08
zinfo.header_offset = self._data_p
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
zinfo.CRC = crc = 0
zinfo.compress_size = 0
zinfo.file_size = 0
yield self.__get_data(zinfo.FileHeader())
for buf in datagen:
if not buf:
continue
zinfo.file_size += len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
if cmpr:
buf = cmpr.flush()
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
else:
zinfo.compress_size = zinfo.file_size
zinfo.CRC = crc
zip64 = zinfo.file_size > ZIP64_LIMIT
fmt = '<4sLQQ' if zip64 else '<4sLLL'
data_descriptor = struct.pack(fmt, stringDataDescriptor, zinfo.CRC,
zinfo.compress_size, zinfo.file_size)
yield self.__get_data(data_descriptor)
self._filelist.append(zinfo)
def write(self, filename, arcname=None, compress_type=None):
"""
Fixed version of write supporting bitflag 0x08 to write crc and size
at end of file.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
self.start_dir = self.fp.tell()
return
zinfo.flag_bits |= 0x08
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
try:
# Python > 2.7.3
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
except TypeError:
# Python <= 2.7.3
zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Write CRC and file sizes after the file data
fmt = b'<LQQ' if zip64 else b'<LLL'
self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.start_dir = self.fp.tell()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname. The file is written in strictly sequential fashion - no seeking."""
# This code is a tweaked version of ZipFile.write ...
# TODO: add an alternative version that works with a stream rather than a filename.
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x08 # Use trailing data descriptor for file sizes and CRC
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
return
with open(filename, "rb") as fp:
# The CRC and sizes in the file header are zero ...
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
# Write the data descriptor after the file containing the true sizes and CRC
zinfo.CRC = CRC
zinfo.file_size = file_size
self.fp.write(struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname. The file is written in strictly sequential fashion - no seeking."""
# This code is a tweaked version of ZipFile.write ...
# TODO: add an alternative version that works with a stream rather than a filename.
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x08 # Use trailing data descriptor for file sizes and CRC
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
return
with open(filename, "rb") as fp:
# The CRC and sizes in the file header are zero ...
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
# Write the data descriptor after the file containing the true sizes and CRC
zinfo.CRC = CRC
zinfo.file_size = file_size
self.fp.write(
struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def my_zip_write(self, filename, arcname=None, compress_type=None,
date_time=None):
"""Copy of zipfile.ZipFile.write() with some modifications
- Allow overriding the timestamp for reproducible builds
- Calculate a SHA256 hash of the file as we write it and return the hash
object.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
if date_time is None:
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = zipfile.ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
self.fp.seek(getattr(self, 'start_dir', 0))
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader(False))
self.start_dir = self.fp.tell()
return
hashsum = hashlib.sha256()
cmpr = _get_compressor(zinfo.compress_type)
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
hashsum.update(buf)
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
self.start_dir = self.fp.tell() # Preserve current position in file
self.fp.seek(zinfo.header_offset, 0)
self.fp.write(zinfo.FileHeader(zip64))
self.fp.seek(self.start_dir, 0)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
return hashsum
def write(self, filename, arcname=None, compress_type=None):
"""
Fixed version of write supporting bitflag 0x08 to write crc and size
at end of file.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
self.start_dir = self.fp.tell()
return
zinfo.flag_bits |= 0x08
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
try:
# Python > 2.7.3
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
except TypeError:
# Python <= 2.7.3
zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Write CRC and file sizes after the file data
fmt = b'<LQQ' if zip64 else b'<LLL'
self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.start_dir = self.fp.tell()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def fix_zip(self):
if not self.broken:
return False
self.fp.seek(0, 2)
file_len = self.fp.tell()
mm = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)
offset = 0
file_list = {}
cd_list = {}
try:
# pass one, parse the zip file
while offset + 4 < file_len:
hdr_off = mm.find(b"PK", offset)
if hdr_off == -1:
break
hdr_type = mm[hdr_off:hdr_off + 4]
if hdr_type == stringFileHeader:
# local file header
if hdr_off + sizeFileHeader > file_len:
break
fheader = mm[hdr_off:hdr_off + sizeFileHeader]
fheader = struct.unpack(structFileHeader, fheader)
start = hdr_off
size = sizeFileHeader + fheader[_FH_COMPRESSED_SIZE] + fheader[_FH_FILENAME_LENGTH] + \
fheader[_FH_EXTRA_FIELD_LENGTH]
name = mm[hdr_off + sizeFileHeader:hdr_off +
sizeFileHeader + fheader[_FH_FILENAME_LENGTH]]
file_list[name] = [start, size, fheader]
offset = hdr_off + size
elif hdr_type == stringCentralDir:
if hdr_off + sizeCentralDir > file_len:
break
centdir = mm[hdr_off:hdr_off + sizeCentralDir]
centdir = struct.unpack(structCentralDir, centdir)
start = hdr_off
size = sizeCentralDir + centdir[_CD_FILENAME_LENGTH] + centdir[_CD_EXTRA_FIELD_LENGTH] + \
centdir[_CD_COMMENT_LENGTH]
name = mm[hdr_off + sizeCentralDir:hdr_off +
sizeCentralDir + centdir[_CD_FILENAME_LENGTH]]
cd_list[name] = [start, size, centdir]
offset = hdr_off + size
elif hdr_type == stringEndArchive:
offset = hdr_off + sizeEndCentDir
else:
offset = hdr_off + 1
# Guesses
last_cv = 20
last_ea = 0
last_cs = 0
last_dt = (0, 0)
# Pass two, repair
for filename, (start, end, centdir) in cd_list.items():
if filename not in file_list:
continue
if isinstance(filename, bytes):
x = ZipInfo(filename.decode('utf-8', 'backslashreplace'))
else:
x = ZipInfo(filename)
extra_off = start + sizeCentralDir
x.extra = mm[extra_off:extra_off +
centdir[_CD_EXTRA_FIELD_LENGTH]]
extra_off += centdir[_CD_EXTRA_FIELD_LENGTH]
x.comment = mm[extra_off:extra_off +
centdir[_CD_EXTRA_FIELD_LENGTH]]
x.header_offset = file_list[filename][0]
(x.create_version, x.create_system, x.extract_version,
x.reserved, x.flag_bits, x.compress_type, t, d, x.CRC,
x.compress_size, x.file_size) = centdir[1:12]
x.volume, x.internal_attr, x.external_attr = centdir[15:18]
# Convert date/time code to (year, month, day, hour, min, sec)
x._raw_time = t
x.date_time = ((d >> 9) + 1980, (d >> 5) & 0xF, d & 0x1F,
t >> 11, (t >> 5) & 0x3F, (t & 0x1F) * 2)
last_ea = x.external_attr
last_cs = x.create_system
last_cv = x.create_version
last_dt = (d, t)
# noinspection PyProtectedMember
x._decodeExtra()
# x.filename = x._decodeFilename()
self.filelist.append(x)
self.NameToInfo[x.filename] = x
for filename, (start, end, fheader) in file_list.items():
if filename in cd_list:
continue
x = ZipInfo(filename.decode('utf-8', 'backslashreplace'))
x.extra = ""
x.comment = ""
x.header_offset = file_list[filename][0]
x.create_version = last_cv
x.create_system = last_cs
x.extract_version = fheader[_FH_EXTRACT_VERSION]
x.reserved = 0
x.flag_bits = fheader[_FH_GENERAL_PURPOSE_FLAG_BITS]
x.compress_type = fheader[_FH_COMPRESSION_METHOD]
d, t = last_dt
x.CRC = fheader[_FH_CRC]
x.compress_size = fheader[_FH_COMPRESSED_SIZE]
x.file_size = fheader[_FH_UNCOMPRESSED_SIZE]
x.volume = 0
x.internal_attr = 0
x.external_attr = last_ea
# Convert date/time code to (year, month, day, hour, min, sec)
x._raw_time = t
x.date_time = ((d >> 9) + 1980, (d >> 5) & 0xF, d & 0x1F,
t >> 11, (t >> 5) & 0x3F, (t & 0x1F) * 2)
# noinspection PyProtectedMember
x._decodeExtra()
# x.filename = x._decodeFilename()
self.filelist.append(x)
self.NameToInfo[x.filename] = x
finally:
mm.close()
def write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader(False))
return
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(1024 * 1024)
if not buf:
break
file_size = file_size + len(buf)
CRC = zlib.crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
position = self.fp.tell() # Preserve current position in file
self.fp.seek(zinfo.header_offset, 0)
self.fp.write(zinfo.FileHeader(zip64))
self.fp.seek(position, 0)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def writecompressed(self,
zinfo_or_arcname,
compressed_input,
crc,
uncompressed_size,
compressed_size,
compress_type=ZIP_DEFLATED):
"""Write pre compressed data into the archive.
This method could be useful in a case when data needs to be stored in
a different order from what it is being produced. For example when
writing a GML-file, the bbox element needs to be written at the top of
the document. Naturally this can not be produced until all members of
the final document has been visited. The 'compressed_input' can be an
itarator of file like objects or strings. In the GML case the head of
the file would probably come from a StringIO object, bulk from temp file
and tail from StringIO object. Care has to be taken to properly compose
the different pre-compressed parts so that the concatenated value
becomes a valid deflate-stream. Also note that the combined crc32 has to
be calculated in correct order. This can be achieved using method
crc32_combine from zlib.
zinfo_or_arcname: Either a ZipInfo instance or the name of the file in
the archive.
compressed_input: The pre compressed content. This can be either a
string, a file object or an iterator. If input is an
iterator, each item will be checked if it's a string
or file object.
crc: The CRC32 checksum of the (combined) input.
uncompressed_size:
compressed_size:
compress_type:
"""
if not compress_type == self.compression:
raise RuntimeError(
"Pre compressed data has to be of same kind as this archive uses, got {}, expected {}"
.format(compress_type, self.compression))
if not isinstance(zinfo_or_arcname, ZipInfo):
zinfo = ZipInfo(filename=zinfo_or_arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = self.compression
if zinfo.filename[-1] == '/':
zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x
zinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
else:
zinfo = zinfo_or_arcname
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if compress_type is not None:
zinfo.compress_type = compress_type
zinfo.file_size = uncompressed_size # Uncompressed size
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
zinfo.CRC = crc & 0xffffffff # CRC-32 checksum
zinfo.compress_size = compressed_size
zip64 = zinfo.file_size > ZIP64_LIMIT or \
zinfo.compress_size > ZIP64_LIMIT
if zip64 and not self._allowZip64:
raise LargeZipFile("Filesize would require ZIP64 extensions")
self.fp.write(zinfo.FileHeader(zip64))
if isinstance(compressed_input, basestring):
self.fp.write(compressed_input)
elif hasattr(compressed_input, '__iter__'):
for o in compressed_input:
if isinstance(o, basestring):
self.fp.write(o)
else:
shutil.copyfileobj(o, self.fp)
else:
shutil.copyfileobj(compressed_input, self.fp)
if zinfo.flag_bits & 0x08:
# Write CRC and file sizes after the file data
fmt = '<LQQ' if zip64 else '<LLL'
self.fp.write(
struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.fp.flush()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
return
def write_iter(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
from zipfile import stat, time, ZipInfo, ZIP_DEFLATED, crc32, struct
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield zinfo.FileHeader()
#self.fp.write(zinfo.FileHeader())
return
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
zinfo.compress_size = file_size
zinfo.file_size = file_size
zinfo.CRC = CRC
#self.fp.write(zinfo.FileHeader())
yield zinfo.FileHeader()
fp.seek(0)
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
#self.fp.write(buf)
yield buf
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if self.compression == ZIP_AUTO:
ext = os.path.splitext(filename)[1].lower()
compression = ZIP_STORED if ext and ext[1:] in STORED_FORMATS \
else ZIP_DEFLATED
else:
compression = self.compression
if compress_type is None:
zinfo.compress_type = compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits |= 0x08
zinfo.header_offset = self.tell # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
header = zinfo.FileHeader()
yield header
self.tell += len(header)
return
fp = open(filename, "rb")
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
header = zinfo.FileHeader()
yield header
self.tell += len(header)
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield buf
fp.close()
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield buf
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
self.tell += zinfo.compress_size
zinfo.CRC = CRC
zinfo.file_size = file_size
# write the data descriptor
data_descriptor = struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size)
yield data_descriptor
self.tell += len(data_descriptor)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
