def full_check(self):
"""
fully check the fcode file, including all the checksum
raise error if any error occur
"""
try:
logger.info("Start Checking CRC %s" % str(self.data[:8]));
assert self.data[:8] == b"FCx0001\n"
logger.info("Passed Header Check");
self.script_size = uint_unpacker(self.data[8:12])
logger.info("Script size = " + str(self.script_size))
assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker(self.data[12 + self.script_size:16 + self.script_size])
logger.info("Passed CRC32 -1 ");
index = 16 + self.script_size
self.meta_size = uint_unpacker(self.data[index:index + 4])
index += 4
assert crc32(self.data[index:index + self.meta_size]) == uint_unpacker(self.data[index + self.meta_size:index + self.meta_size + 4])
logger.info("Passed CRC32 -2 ");
index = index + self.meta_size + 4
self.image_size = uint_unpacker(self.data[index:index + 4])
index += 4
logger.info("Passed full check");
return True
except AssertionError as e:
logger.info(str(e));
return False
def full_check(self):
"""
fully check the fcode file, including all the checksum
raise error if any error occur
"""
try:
logger.info("Start Checking CRC %s" % str(self.data[:8]))
assert self.data[:8] == b"FCx0001\n"
logger.info("Passed Header Check")
self.script_size = uint_unpacker(self.data[8:12])
logger.info("Script size = " + str(self.script_size))
assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker(
self.data[12 + self.script_size:16 + self.script_size])
logger.info("Passed CRC32 -1 ")
index = 16 + self.script_size
self.meta_size = uint_unpacker(self.data[index:index + 4])
index += 4
assert crc32(
self.data[index:index + self.meta_size]) == uint_unpacker(
self.data[index + self.meta_size:index + self.meta_size +
4])
logger.info("Passed CRC32 -2 ")
index = index + self.meta_size + 4
self.image_size = uint_unpacker(self.data[index:index + 4])
index += 4
logger.info("Passed full check")
return True
except AssertionError as e:
logger.info(str(e))
return False
def check_effaceable_header(plog):
z = xor_strings(plog[:16], plog[16:32])
if z[:4] != "ecaF":
return False
plog_generation = struct.unpack("<L", plog[0x38:0x3C])[0]
print "Effaceable generation" , plog_generation
plog_crc = crc32(plog[0x40:0x40 + 960], crc32(plog[0x20:0x3C], crc32(z))) & 0xffffffff
assert plog_crc == struct.unpack("<L", plog[0x3C:0x40])[0] , "Effaceable CRC"
print "Effaceable CRC OK"
return True
def check_effaceable_header(plog):
z = xor_strings(plog[:16], plog[16:32])
if z[:4] != "ecaF":
return False
plog_generation = struct.unpack("<L", plog[0x38:0x3C])[0]
print "Effaceable generation", plog_generation
plog_crc = crc32(plog[0x40:0x40 + 960], crc32(plog[0x20:0x3C],
crc32(z))) & 0xffffffff
assert plog_crc == struct.unpack("<L",
plog[0x3C:0x40])[0], "Effaceable CRC"
print "Effaceable CRC OK"
return True
def _load(self, task_file, crc_check):
t = task_file
# Check header
magic_num = t.read(8)
if magic_num != b"FCx0001\n":
if magic_num[:3] != b"FCx" or magic_num[3:].isdigit() is False:
raise RuntimeError(FILE_BROKEN)
else:
ver = int(magic_num[3:])
raise RuntimeError(FILE_BROKEN, NOT_SUPPORT, str(ver))
# Check script
script_size = UINT_PACKER.unpack(t.read(4))[0]
self.script_ptr = t.tell()
self.script_size = script_size
if crc_check:
script_crc32 = 0
f_ptr = 0
while f_ptr < script_size:
buf = t.read(min(script_size - f_ptr, 4096))
if buf:
f_ptr += len(buf)
script_crc32 = crc32(buf, script_crc32)
else:
raise RuntimeError(FILE_BROKEN, "LENGTH_ERROR")
req_script_crc32 = INT_PACKER.unpack(t.read(4))[0]
if req_script_crc32 != script_crc32:
raise RuntimeError(FILE_BROKEN, "SCRIPT_CRC32")
else:
t.seek(script_size + 4, 1)
# Check meta
meta_size = UINT_PACKER.unpack(t.read(4))[0]
meta_buf = t.read(meta_size)
req_metadata_crc32 = INT_PACKER.unpack(t.read(4))[0]
if crc_check and req_metadata_crc32 != crc32(meta_buf, 0):
raise RuntimeError(FILE_BROKEN, "METADATA_CRC32")
metadata = {}
for item in meta_buf.split("\x00"):
sitem = item.split("=", 1)
if len(sitem) == 2:
metadata[sitem[0]] = sitem[1]
self.metadata = metadata
t.seek(self.script_ptr)
def parse_lwvm(data, pageSize):
try:
hdr = LWVM_header.parse(data)
if hdr.type != LWVM_MAGIC:
print "LwVM magic mismatch"
return
tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000]
check = crc32(tocheck) & 0xffffffff
if check != hdr.crc32:
return None
print "LwVM header CRC OK"
partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions]
deviceSize=0
#XXX: HAAAAAAAX
for s in [8, 16, 32, 64, 128]:
if hdr.mediaSize < (s* 1024*1024*1024):
deviceSize = s
break
for i in xrange(len(hdr.chunks)):
if hdr.chunks[i] == 0x0:
lba0 = (i * deviceSize*1024*1024) / pageSize
partitions[0].first_lba = lba0
partitions[0].last_lba = lba0 + (partitions[0].end - partitions[0].begin) / pageSize
elif hdr.chunks[i] == 0x1000:
lbad = (i * deviceSize*1024*1024) / pageSize
partitions[1].first_lba = lbad
partitions[1].last_lba = lbad + (partitions[1].end - partitions[1].begin) / pageSize
return partitions
except:
return None
def parse_lwvm(data, pageSize):
try:
hdr = LWVM_header.parse(data)
if hdr.type != LWVM_MAGIC:
print "LwVM magic mismatch"
return
tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000]
check = crc32(tocheck) & 0xffffffff
if check != hdr.crc32:
return None
print "LwVM header CRC OK"
partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions]
deviceSize = 0
#XXX: HAAAAAAAX
for s in [8, 16, 32, 64, 128]:
if hdr.mediaSize < (s * 1024 * 1024 * 1024):
deviceSize = s
break
for i in xrange(len(hdr.chunks)):
if hdr.chunks[i] == 0x0:
lba0 = (i * deviceSize * 1024 * 1024) / pageSize
partitions[0].first_lba = lba0
partitions[0].last_lba = lba0 + (
partitions[0].end - partitions[0].begin) / pageSize
elif hdr.chunks[i] == 0x1000:
lbad = (i * deviceSize * 1024 * 1024) / pageSize
partitions[1].first_lba = lbad
partitions[1].last_lba = lbad + (
partitions[1].end - partitions[1].begin) / pageSize
return partitions
except:
return None
def BlurImages(ifile, ofile=None):
'''
If blurring is good enough, this function will take the document and
blur the image.
To change the amount of blurring, change Docx.blur
The Default value is: ImageFilter.GaussianBlur(50)
Adjust the Number inside GaussianBlur until the valus suits you
'''
# Thanks to mata for his/her help
# https://stackoverflow.com/questions/50963852/remove-all-images-from-docx-files
if ofile is None and osp.splitext(ifile)[1] == '.docx':
ofile = ifile.replace('.docx', '.redacted.docx')
elif ofile is None and osp.splitext(ifile)[1] == '.odt':
ofile = ifile.replace('.odt', '.redacted.odt')
elif ofile is None:
ofile = osp.splitext(ifile)[0] + '.redacted.' \
+ osp.splitext(ifile)[1]
izip = ZipFile(ifile)
ozip = ZipFile(ofile, 'w', zipfile.ZIP_DEFLATED)
for node in izip.infolist():
content = izip.read(node)
if osp.splitext(node.filename)[1] in Base.pictures:
# Blur Image and save it to a buffer
img = Image.open(io.BytesIO(content)) # Load Picture to memory
img = img.convert().filter(Base.blur) # Blur it
imgBuffer = io.BytesIO()
img.save(imgBuffer, osp.splitext(node.filename)[1][1:])
# Change the XML so that it points to the blurred image, not original
content = imgBuffer.getvalue()
node.file_size = len(content)
node.CRC = zipfile.crc32(content)
img.close()
ozip.writestr(node, content)
izip.close()
ozip.close()
def parse_lwvm(data, pageSize):
try:
hdr = LWVM_header.parse(data)
if hdr.type != LWVM_MAGIC:
print "LwVM magic mismatch"
return
tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000]
check = crc32(tocheck) & 0xffffffff
if check != hdr.crc32:
return None
print "LwVM header CRC OK"
partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions]
LwVM_rangeShiftValue = 32 - clz32((hdr.mediaSize - 1) >> 10 )
for i in xrange(len(hdr.chunks)):
if hdr.chunks[i] == 0x0:
lba0 = (i << LwVM_rangeShiftValue) / pageSize
partitions[0].first_lba = lba0
partitions[0].last_lba = lba0 + (partitions[0].end - partitions[0].begin) / pageSize
elif hdr.chunks[i] == 0x1000:
lbad = (i << LwVM_rangeShiftValue) / pageSize
partitions[1].first_lba = lbad
partitions[1].last_lba = lbad + (partitions[1].end - partitions[1].begin) / pageSize
return partitions
except:
return None
def _load(self, f):
if f.read(8) != b"FCx0001\n":
raise FCodeError("HEADER_ERROR")
script_size = UINT_PACKER.unpack(f.read(4))[0]
script_crc32 = 0
f_ptr = 0
self.script_ptr = f.tell()
self.script_size = script_size
while f_ptr < script_size:
buf = f.read(min(script_size - f_ptr, 4096))
if buf:
f_ptr += len(buf)
script_crc32 = crc32(buf, script_crc32)
else:
raise FCodeError("SIZE_ERROR", "SCRIPT")
req_script_crc32 = INT_PACKER.unpack(f.read(4))[0]
if req_script_crc32 != script_crc32:
raise FCodeError("CRC_ERROR", "SCRIPT")
# Check meta
meta_size = UINT_PACKER.unpack(f.read(4))[0]
meta_buf = f.read(meta_size)
req_metadata_crc32 = INT_PACKER.unpack(f.read(4))[0]
if req_metadata_crc32 != crc32(meta_buf, 0):
raise FCodeError("CRC_ERROR", "META")
metadata = {}
for item in meta_buf.split("\x00"):
sitem = item.split("=", 1)
if len(sitem) == 2:
metadata[sitem[0]] = sitem[1]
self.metadata = metadata
# Load image
self.image_buf = []
buf = f.read(4)
while len(buf) == 4:
image_size = UINT_PACKER.unpack(buf)[0]
if image_size > 0:
self.image_buf.append(f.read(image_size))
buf = f.read(4)
else:
break
def full_check(self):
"""
fully check the fcode file, including all the checksum
raise error if any error occur
"""
try:
assert self.data[:8] == b"FCx0001\n"
self.script_size = uint_unpacker(self.data[8:12])
assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker(self.data[12 + self.script_size:16 + self.script_size])
index = 16 + self.script_size
self.meta_size = uint_unpacker(self.data[index:index + 4])
index += 4
assert crc32(self.data[index:index + self.meta_size]) == uint_unpacker(self.data[index + self.meta_size:index + self.meta_size + 4])
index = index + self.meta_size + 4
self.image_size = uint_unpacker(self.data[index:index + 4])
index += 4
return True
except AssertionError as e:
raise RuntimeError(FILE_BROKEN, e.args[0] if e.args else "#")
def write(self, data):
if self.closed:
raise ValueError('I/O operation on closed file.')
nbytes = len(data)
self._file_size += nbytes
self._crc = crc32(data, self._crc)
if self._compressor:
data = self._compressor.compress(data)
self._compress_size += len(data)
self._fileobj.write(data)
return nbytes
def parse_gpt(data):
gpt = GPT_header.parse(data)
if gpt.signature != "EFI PART":
return None
print "Found GPT header current_lba=%d partition_entries_lba=%d" % (gpt.current_lba, gpt.partition_entries_lba)
assert gpt.partition_entries_lba > gpt.current_lba
check = gpt.crc
gpt.crc = 0
actual = crc32(GPT_header.build(gpt))
if actual != check:
print "GPT crc check fail %d vs %d" % (actual, check)
return None
return gpt
def WriteFileChunk(self, chunk):
"""Writes file chunk."""
if not self._stream:
raise ArchiveAlreadyClosedError(
"Attempting to write to a ZIP archive that was already closed.")
self.cur_file_size += len(chunk)
self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff
if self.cur_cmpr:
chunk = self.cur_cmpr.compress(chunk)
self.cur_compress_size += len(chunk)
self._stream.write(chunk)
return self._stream.GetValueAndReset()
def main(filename):
print 'Reading %s' % sys.argv[1]
f = open(filename, 'rb')
while True:
# Read and parse a file header
fheader = f.read(zipfile.sizeFileHeader)
if len(fheader) < zipfile.sizeFileHeader:
print 'Found end of file. Some entries missed.'
break
fheader = struct.unpack(zipfile.structFileHeader, fheader)
if fheader[zipfile._FH_SIGNATURE] == 'PK\x01\x02':
print 'Found start of central directory. All entries processed.'
break
fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH])
if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]:
f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH])
print 'Found %s' % fname
# Fake a zipinfo record
zi = zipfile.ZipInfo()
zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE]
zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD]
zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS]
zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE]
# Read the file contents
zef = zipfile.ZipExtFile(f, 'rb', zi)
data = zef.read()
# Sanity checks
if len(data) != fheader[zipfile._FH_UNCOMPRESSED_SIZE]:
raise Exception(
"Unzipped data doesn't match expected size! %d != %d, in %s" %
(len(data), fheader[zipfile._FH_UNCOMPRESSED_SIZE], fname))
calc_crc = zipfile.crc32(data) & 0xffffffff
if calc_crc != fheader[zipfile._FH_CRC]:
raise Exception('CRC mismatch! %d != %d, in %s' %
(calc_crc, fheader[zipfile._FH_CRC], fname))
# Write the file
write_data(fname, data)
f.close()
def __init__(self,
fileobj,
mode,
p4kinfo,
decrypter=None,
close_fileobj=False):
self._is_encrypted = p4kinfo.is_encrypted
self._decompressor = ZStdDecompressor()
self._fileobj = fileobj
self._decrypter = decrypter
self._close_fileobj = close_fileobj
self._compress_type = p4kinfo.compress_type
self._compress_left = p4kinfo.compress_size
self._left = p4kinfo.file_size
self._eof = False
self._readbuffer = b""
self._offset = 0
self.newlines = None
self.mode = mode
self.name = p4kinfo.filename
if hasattr(p4kinfo, 'CRC'):
self._expected_crc = p4kinfo.CRC
self._running_crc = zipfile.crc32(b'')
else:
self._expected_crc = None
# TODO: the CRCs don't match, but im getting the same outputs as unp4k - we should figure out what exactly is
# going into calculating the CRC for P4K entry
self._expected_crc = None
self._seekable = False
try:
if fileobj.seekable():
self._orig_compress_start = fileobj.tell()
self._orig_compress_size = p4kinfo.compress_size
self._orig_file_size = p4kinfo.file_size
self._orig_start_crc = self._running_crc
self._seekable = True
except AttributeError:
pass
def generator(self, source, compression=ZIP_STORED):
"""
Turn a streamed file source into a stream zipcrypted archive file.
Some lines are based upon Ivan Ergunov's work.
Some lines are based upon devthat's work on zipencrypt.
"""
if not self.zinfo:
raise RuntimeError(
"Ziphyr object not primed. "
"Please use either from_filepath() or from_metadata()."
)
self.stream = ZiphyrStream(self.password)
with self.ZipFile(
self.stream, mode='w', compression=compression
) as zfile:
clear_crc = 0
with zfile.open(self.zinfo, mode='w') as dest:
if self.password:
# the zipcrypto asks for twelve almost-random bytes
self.zinfo._raw_time = (
self.zinfo.date_time[3] << 11
| self.zinfo.date_time[4] << 5
| (self.zinfo.date_time[5] // 2))
check_byte = (self.zinfo._raw_time >> 8) & 0xff
twelve_angry = urandom(11) + struct.pack("B", check_byte)
dest.write(self.stream.cypher_chunk(twelve_angry))
for chunk in source:
# the internal crc is updated when dest.write()
# but we want it to be of the clear data
# hence doing our own in parallel
clear_crc = crc32(chunk, clear_crc)
if self.password:
chunk = self.stream.cypher_chunk(chunk)
dest.write(chunk)
yield self.stream.get()
if self.password:
self.zinfo.file_size -= 12 # withdrawing the 12 bytes
self.zinfo.CRC = clear_crc # overwriting with the clearbytes crc
yield self.stream.get()
def redact_images(filename, FilePath):
#outfile = filename.replace(".docx", "_redacted.docx")
with zipfile.ZipFile(filename) as inzip:
#with zipfile.ZipFile(outfile, "w") as outzip:
i = 0
for info in inzip.infolist():
name = info.filename
content = inzip.read(info)
if name.endswith((".png", ".jpeg", ".gif")):
fmt = name.split(".")[-1]
Name = name.split("/")[-1]
img = Image.open(io.BytesIO(content))
img.save(FilePath + str(Name))
outb = io.BytesIO()
img.save(outb, fmt)
content = outb.getvalue()
info.file_size = len(content)
info.CRC = zipfile.crc32(content)
i += 1
def main(filename):
print 'Reading %s' % sys.argv[1]
f = open(filename, 'rb')
while True:
# Read and parse a file header
fheader = f.read(zipfile.sizeFileHeader)
if len(fheader) < zipfile.sizeFileHeader:
print 'Found end of file. Some entries missed.'
break
fheader = struct.unpack(zipfile.structFileHeader, fheader)
if fheader[zipfile._FH_SIGNATURE] == 'PK\x01\x02':
print 'Found start of central directory. All entries processed.'
break
fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH])
if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]:
f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH])
print 'Found %s' % fname
# Fake a zipinfo record
zi = zipfile.ZipInfo()
zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE]
zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD]
zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS]
zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE]
# Read the file contents
zef = zipfile.ZipExtFile(f, 'rb', zi)
data = zef.read()
# Sanity checks
if len(data) != fheader[zipfile._FH_UNCOMPRESSED_SIZE]:
raise Exception("Unzipped data doesn't match expected size! %d != %d, in %s" % (len(data), fheader[zipfile._FH_UNCOMPRESSED_SIZE], fname))
calc_crc = zipfile.crc32(data) & 0xffffffff
if calc_crc != fheader[zipfile._FH_CRC]:
raise Exception('CRC mismatch! %d != %d, in %s' % (calc_crc, fheader[zipfile._FH_CRC], fname))
# Write the file
write_data(fname, data)
f.close()
def WriteFileChunk(self, chunk):
"""Writes file chunk."""
if not self._stream:
raise ArchiveAlreadyClosedError(
"Attempting to write to a ZIP archive that was already closed.")
self.cur_file_size += len(chunk)
# TODO(user):pytype: crc32 is not visible outside of zipfile.
# pytype: disable=module-attr
self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff
# pytype: enable=module-attr
if self.cur_cmpr:
chunk = self.cur_cmpr.compress(chunk)
self.cur_compress_size += len(chunk)
self._stream.write(chunk)
return self._stream.GetValueAndReset()
def WriteFileChunk(self, chunk):
"""Writes file chunk."""
if not self._stream:
raise ArchiveAlreadyClosedError(
"Attempting to write to a ZIP archive that was already closed.")
self.cur_file_size += len(chunk)
# TODO(user):pytype: crc32 is not visible outside of zipfile.
# pytype: disable=module-attr
self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff
# pytype: enable=module-attr
if self.cur_cmpr:
chunk = self.cur_cmpr.compress(chunk)
self.cur_compress_size += len(chunk)
self._stream.write(chunk)
return self._stream.GetValueAndReset()
def fast_read_meta(filename):
with open(filename, "rb") as f:
try:
# Check header
assert f.read(8) == b"FCx0001\n", "MAGIC_NUMBER_ERROR"
# Skip script size/crc32/body
script_size = UINT_PACKER.unpack(f.read(4))[0]
f.seek(script_size + 4, 1) # crc32 + size
# Load metadata
meta_size = UINT_PACKER.unpack(f.read(4))[0]
assert meta_size < 65536, "METADATA_TOO_LARGE"
meta_buf = f.read(meta_size)
meta_crc32 = INT_PACKER.unpack(f.read(4))[0]
assert meta_crc32 == crc32(meta_buf, 0), "METADATA_CRC32_ERROR"
metadata = {}
for item in meta_buf.split("\x00"):
sitem = item.split("=", 1)
if len(sitem) == 2:
metadata[sitem[0]] = sitem[1]
# Load image
images = []
size_buf = f.read(4)
while len(size_buf) == 4:
img_s = UINT_PACKER.unpack(size_buf)[0]
if img_s > 0:
images.append(f.read(img_s))
size_buf = f.read(4)
else:
break
return metadata, images
except struct.error as e:
raise RuntimeError(FILE_BROKEN)
except AssertionError as e:
raise RuntimeError(FILE_BROKEN, e.args[0] if e.args else "#")
def image_inverter(filename):
inverted_file = filename.replace('.docx', '_inverted.docx')
with zipfile.ZipFile(filename) as inzip:
with zipfile.ZipFile(inverted_file, 'w') as outzip:
for info in inzip.infolist():
name = info.filename
content = inzip.read(info)
if info.filename.endswith(('.png', '.jpeg', '.gif')):
print(name)
file_extension = name.split('.')[-1]
img = Image.open(io.BytesIO(content))
pixels = img.load()
if color_check(img, pixels):
pixels = inverter(img, pixels)
outb = io.BytesIO()
img.save(outb, file_extension)
content = outb.getvalue()
info.file_size = len(content)
info.CRC = zipfile.crc32(content)
outzip.writestr(info, content)
def main(filename):
print('Reading %s Central Directory' % filename)
# Get info from ZIP Central Directory
with zipfile.ZipFile(filename, 'r') as myzip:
files = myzip.namelist()
print('Found %d file(s) from Central Directory:' % (len(files)))
print('- ' + '\n- '.join(files))
print('Reading %s ZIP entry manually' % sys.argv[1])
f = open(filename, 'rb')
while True:
# Read and parse a file header
fheader = f.read(zipfile.sizeFileHeader)
if len(fheader) < zipfile.sizeFileHeader:
print('Found end of file. Some entries missed.')
break
fheader = struct.unpack(zipfile.structFileHeader, fheader)
if fheader[zipfile._FH_SIGNATURE] == zipfile.stringCentralDir:
print('Found start of central directory. All entries processed.')
break
if fheader[zipfile._FH_SIGNATURE] != zipfile.stringFileHeader:
raise Exception('Size mismatch! File Header expected, got "%s"' %
(fheader[zipfile._FH_SIGNATURE]))
if fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS] & 0x8:
data_descriptor = True
fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH])
if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]:
f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH])
print('Found %s' % fname.decode())
# Fake a zipinfo record
zi = zipfile.ZipInfo()
zi.filename = fname
zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE]
zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD]
zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS]
zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE]
zi.CRC = fheader[zipfile._FH_CRC]
if data_descriptor:
# Compress size is zero
# Get the real sizes with datadescriptor
ddescriptor = fdescriptor_reader(f, f.tell())
if ddescriptor is None:
break
zi.compress_size = ddescriptor[_DD_COMPRESSED_SIZE]
zi.file_size = ddescriptor[_DD_UNCOMPRESSED_SIZE]
zi.CRC = ddescriptor[_DD_CRC]
#print(zi)
# Read the file contents
zef = zipfile.ZipExtFile(f, 'rb', zi)
data = zef.read()
# Sanity checks
if len(data) != zi.file_size:
raise Exception(
"Unzipped data doesn't match expected size! %d != %d, in %s" %
(len(data), zi.file_size, fname))
calc_crc = zipfile.crc32(data) & 0xffffffff
if calc_crc != zi.CRC:
raise Exception('CRC mismatch! %d != %d, in %s' %
(calc_crc, zi.CRC, fname))
# Write the file
write_data(fname, data)
if data_descriptor:
# skip dataDescriptor before reading the next file
f.seek(f.tell() + sizeDataDescriptor)
f.close()
def my_zip_write(self, filename, arcname=None, compress_type=None,
date_time=None):
"""Copy of zipfile.ZipFile.write() with some modifications
- Allow overriding the timestamp for reproducible builds
- Calculate a SHA256 hash of the file as we write it and return the hash
object.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
if date_time is None:
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = zipfile.ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
self.fp.seek(getattr(self, 'start_dir', 0))
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader(False))
self.start_dir = self.fp.tell()
return
hashsum = hashlib.sha256()
cmpr = _get_compressor(zinfo.compress_type)
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
hashsum.update(buf)
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
self.start_dir = self.fp.tell() # Preserve current position in file
self.fp.seek(zinfo.header_offset, 0)
self.fp.write(zinfo.FileHeader(zip64))
self.fp.seek(self.start_dir, 0)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
return hashsum
def read(self, size):
buf = self.s.read(size)
self.length += size
self.crc32 = crc32(buf, self.crc32)
return buf
def _thread_py_id():
# return threading.get_ident()
return zipfile.crc32(int(threading.get_ident()).to_bytes(8, 'little'))
def write(self, filename, arcname=None, compress_type=None):
"""
Fixed version of write supporting bitflag 0x08 to write crc and size
at end of file.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
self.start_dir = self.fp.tell()
return
zinfo.flag_bits |= 0x08
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
try:
# Python > 2.7.3
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
except TypeError:
# Python <= 2.7.3
zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Write CRC and file sizes after the file data
fmt = b'<LQQ' if zip64 else b'<LLL'
self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.start_dir = self.fp.tell()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
async def _write(self,
filename=None,
iterable=None,
arcname=None,
compress_type=None):
"""
Put the bytes from filename into the archive under the name `arcname`.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if (filename is None
and iterable is None) or (filename is not None
and iterable is not None):
raise ValueError(
"either (exclusively) filename or iterable shall be not None")
if filename:
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
else:
st, isdir, date_time = None, False, time.localtime()[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
if st:
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
if st:
zinfo.file_size = st[6]
else:
zinfo.file_size = 0
zinfo.flag_bits = 0x00
zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == zipfile.ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield self.fp.write(zinfo.FileHeader(False))
return
cmpr = _get_compressor(zinfo.compress_type)
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT
yield self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
if filename:
async for buf in self.data_generator(filename):
file_size = file_size + len(buf)
CRC = zipfile.crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = await self._run_in_executor(cmpr.compress, buf)
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
else: # we have an iterable
for buf in iterable:
file_size = file_size + len(buf)
CRC = zipfile.crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = await self._run_in_executor(cmpr.compress, buf)
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > zipfile.ZIP64_LIMIT:
raise RuntimeError(
'File size has increased during compressing')
if compress_size > zipfile.ZIP64_LIMIT:
raise RuntimeError(
'Compressed size larger than uncompressed size')
yield self.fp.write(zinfo.DataDescriptor())
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def __write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield self.fp.write(zinfo.FileHeader(False))
return
cmpr = _get_compressor(zinfo.compress_type)
with open(filename, 'rb') as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
yield self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError(
'File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError(
'Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
# position = self.fp.tell() # Preserve current position in file
# self.fp.seek(zinfo.header_offset, 0)
# self.fp.write(zinfo.FileHeader(zip64))
# self.fp.seek(position, 0)
yield self.fp.write(zinfo.DataDescriptor())
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
async def _write(self, filename=None, iterable=None, arcname=None, compress_type=None):
"""
Put the bytes from filename into the archive under the name `arcname`.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if (filename is None and iterable is None) or (filename is not None and iterable is not None):
raise ValueError("either (exclusively) filename or iterable shall be not None")
if filename:
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
else:
st, isdir, date_time = None, False, time.localtime()[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
if st:
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
if st:
zinfo.file_size = st[6]
else:
zinfo.file_size = 0
zinfo.flag_bits = 0x00
zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == zipfile.ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
await yield_(self.fp.write(zinfo.FileHeader(False)))
return
cmpr = _get_compressor(zinfo.compress_type)
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT
await yield_(self.fp.write(zinfo.FileHeader(zip64)))
file_size = 0
if filename:
async for buf in self.data_generator(filename):
file_size = file_size + len(buf)
CRC = zipfile.crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = await self._run_in_executor(cmpr.compress, buf)
compress_size = compress_size + len(buf)
await yield_(self.fp.write(buf))
else: # we have an iterable
for buf in iterable:
file_size = file_size + len(buf)
CRC = zipfile.crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = await self._run_in_executor(cmpr.compress, buf)
compress_size = compress_size + len(buf)
await yield_(self.fp.write(buf))
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
await yield_(self.fp.write(buf))
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > zipfile.ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > zipfile.ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
await yield_(self.fp.write(zinfo.DataDescriptor()))
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None):
"""
Fixed version of write supporting bitflag 0x08 to write crc and size
at end of file.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
self.start_dir = self.fp.tell()
return
zinfo.flag_bits |= 0x08
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
try:
# Python > 2.7.3
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
except TypeError:
# Python <= 2.7.3
zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Write CRC and file sizes after the file data
fmt = b'<LQQ' if zip64 else b'<LLL'
self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.start_dir = self.fp.tell()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None, pwd=None):
"""Put the bytes from filename into the archive under the name
arcname."""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = zipfile.ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if isdir:
zinfo.compress_type = zipfile.ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader(False))
return
pwd = pwd or self.pwd
if pwd:
zinfo.flag_bits |= 0x8 | 0x1 # set stream and encrypted
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = crc = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
if zinfo.compress_type == zipfile.ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
if pwd:
ze = _ZipEncrypter(pwd)
encrypt = lambda x: "".join(map(ze, x))
zinfo._raw_time = (zinfo.date_time[3] << 11
| zinfo.date_time[4] << 5
| (zinfo.date_time[5] // 2))
check_byte = (zinfo._raw_time >> 8) & 0xff
enryption_header = os.urandom(11) + chr(check_byte)
self.fp.write(encrypt(enryption_header))
else:
encrypt = lambda x: x
file_size = 0
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
crc = zipfile.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(encrypt(buf))
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(encrypt(buf))
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = crc
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > zipfile.ZIP64_LIMIT:
raise RuntimeError(
'File size has increased during compressing')
if compress_size > zipfile.ZIP64_LIMIT:
raise RuntimeError(
'Compressed size larger than uncompressed size')
if pwd:
# Write CRC and file sizes after the file data
zinfo.compress_size += 12
fmt = '<LQQ' if zip64 else '<LLL'
self.fp.write(
struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.fp.flush()
else:
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
position = self.fp.tell() # Preserve current position in file
self.fp.seek(zinfo.header_offset, 0)
self.fp.write(zinfo.FileHeader(zip64))
self.fp.seek(position, 0)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def writestr(self, zinfo_or_arcname, _bytes, compress_type=None, pwd=None):
"""Write a file into the archive. The contents is the string
'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or
the name of the file in the archive."""
if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = self.compression
if zinfo.filename[-1] == '/':
zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x
zinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
else:
zinfo = zinfo_or_arcname
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if compress_type is not None:
zinfo.compress_type = compress_type
zinfo.file_size = len(_bytes) # Uncompressed size
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
zinfo.CRC = zipfile.crc32(_bytes) & 0xffffffff # CRC-32 checksum
if zinfo.compress_type == zipfile.ZIP_DEFLATED:
co = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED,
-15)
_bytes = co.compress(_bytes) + co.flush()
zinfo.compress_size = len(_bytes) # Compressed size
else:
zinfo.compress_size = zinfo.file_size
zip64 = zinfo.file_size > zipfile.ZIP64_LIMIT or zinfo.compress_size > zipfile.ZIP64_LIMIT
if zip64 and not self._allowZip64:
raise zipfile.LargeZipFile(
"Filesize would require ZIP64 extensions")
pwd = pwd or self.pwd
if pwd:
zinfo.flag_bits |= 0x01
zinfo.compress_size += 12 # 12 extra bytes for the header
if zinfo.flag_bits & 0x8:
zinfo._raw_time = (zinfo.date_time[3] << 11
| zinfo.date_time[4] << 5
| (zinfo.date_time[5] // 2))
check_byte = (zinfo._raw_time >> 8) & 0xff
else:
check_byte = (zinfo.CRC >> 24) & 0xff
enryption_header = os.urandom(11) + chr(check_byte)
ze = _ZipEncrypter(pwd)
_bytes = "".join(map(ze, enryption_header + _bytes))
self.fp.write(zinfo.FileHeader(zip64))
self.fp.write(_bytes)
if zinfo.flag_bits & 0x08:
# Write CRC and file sizes after the file data
fmt = '<LQQ' if zip64 else '<LLL'
self.fp.write(
struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.fp.flush()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def read(self, size):
buf = self.s.read(size)
self.length += size
self.crc32 = crc32(buf, self.crc32)
return buf
def WriteFromFD(self, src_fd, arcname=None, compress_type=None, st=None):
"""Write a zip member from a file like object.
Args:
src_fd: A file like object, must support seek(), tell(), read().
arcname: The name in the archive this should take.
compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED)
st: An optional stat object to be used for setting headers.
Raises:
RuntimeError: If the zip if already closed.
"""
zinfo = self.GenerateZipInfo(arcname=arcname, compress_type=compress_type,
st=st)
crc = 0
compress_size = 0
if not self.out_fd:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
zinfo.header_offset = self.out_fd.tell()
# Call _writeCheck(zinfo) to do sanity checking on zinfo structure that
# we've constructed.
self.zip_fd._writecheck(zinfo) # pylint: disable=protected-access
# Mark ZipFile as dirty. We have to keep self.zip_fd's internal state
# coherent so that it behaves correctly when close() is called.
self.zip_fd._didModify = True # pylint: disable=protected-access
# Write FileHeader now. It's incomplete, but CRC and uncompressed/compressed
# sized will be written later in data descriptor.
self.out_fd.write(zinfo.FileHeader())
if zinfo.compress_type == zipfile.ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = src_fd.read(1024 * 8)
if not buf:
break
file_size += len(buf)
crc = zipfile.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size += len(buf)
self.out_fd.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size += len(buf)
zinfo.compress_size = compress_size
self.out_fd.write(buf)
else:
zinfo.compress_size = file_size
zinfo.CRC = crc
zinfo.file_size = file_size
if file_size > zipfile.ZIP64_LIMIT or compress_size > zipfile.ZIP64_LIMIT:
# Writing data descriptor ZIP64-way:
# crc-32 8 bytes (little endian)
# compressed size 8 bytes (little endian)
# uncompressed size 8 bytes (little endian)
self.out_fd.write(struct.pack("<LLL", crc, compress_size, file_size))
else:
# Writing data descriptor non-ZIP64-way:
# crc-32 4 bytes (little endian)
# compressed size 4 bytes (little endian)
# uncompressed size 4 bytes (little endian)
self.out_fd.write(struct.pack("<III", crc, compress_size, file_size))
# Register the file in the zip file, so that central directory gets
# written correctly.
self.zip_fd.filelist.append(zinfo)
self.zip_fd.NameToInfo[zinfo.filename] = zinfo
def write_iter(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
from zipfile import stat, time, ZipInfo, ZIP_DEFLATED, crc32, struct
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield zinfo.FileHeader()
#self.fp.write(zinfo.FileHeader())
return
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
zinfo.compress_size = file_size
zinfo.file_size = file_size
zinfo.CRC = CRC
#self.fp.write(zinfo.FileHeader())
yield zinfo.FileHeader()
fp.seek(0)
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
#self.fp.write(buf)
yield buf
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def __write(self,
filename=None,
iterable=None,
arcname=None,
compress_type=None,
pwd=None):
"""Put the bytes from filename into the archive under the name
`arcname`."""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
if (filename is None
and iterable is None) or (filename is not None
and iterable is not None):
raise ValueError(
"either (exclusively) filename or iterable shall be not None")
if filename:
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
else:
st, isdir, date_time = None, False, time.localtime()[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
if st:
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
else:
zinfo.external_attr = 0o600 << 16 # ?rw-------
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
if st:
zinfo.file_size = st[6]
else:
zinfo.file_size = 0
zinfo.flag_bits = 0x00
pwd = pwd or self.pwd
if pwd:
zinfo.flag_bits |= 0x1 # set encrypted
zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield self.fp.write(zinfo.FileHeader(False))
return
cmpr = _get_compressor(zinfo.compress_type)
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
yield self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
if pwd:
encrypt = zip_encrypt(pwd)
zinfo._raw_time = (zinfo.date_time[3] << 11
| zinfo.date_time[4] << 5
| (zinfo.date_time[5] // 2))
check_byte = (zinfo._raw_time >> 8) & 0xff
encryption_header = randomFunc(11) + struct.pack(b"B", check_byte)
yield self.fp.write(encrypt(encryption_header))
else:
encrypt = lambda x: x
if filename:
with open(filename, 'rb') as fp:
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield self.fp.write(encrypt(buf))
else: # we have an iterable
for buf in iterable:
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield self.fp.write(encrypt(buf))
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield self.fp.write(encrypt(buf))
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError(
'File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError(
'Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
if pwd:
# Write CRC and file sizes after the file data
zinfo.compress_size += 12
fmt = b'<LQQ' if zip64 else b'<LLL'
yield self.fp.write(
struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
else:
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
# position = self.fp.tell() # Preserve current position in file
# self.fp.seek(zinfo.header_offset, 0)
# self.fp.write(zinfo.FileHeader(zip64))
# self.fp.seek(position, 0)
yield self.fp.write(zinfo.DataDescriptor())
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def update_event(self, inp=-1):
self.set_output_val(0, zipfile.crc32(self.input(0), self.input(1)))
def __write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
zinfo.header_offset = self.fp.tell() # Start of header bytes
if zinfo.compress_type == ZIP_LZMA:
# Compressed data includes an end-of-stream (EOS) marker
zinfo.flag_bits |= 0x02
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield self.fp.write(zinfo.FileHeader(False))
return
cmpr = _get_compressor(zinfo.compress_type)
with open(filename, 'rb') as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
yield self.fp.write(zinfo.FileHeader(zip64))
file_size = 0
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Seek backwards and write file header (which will now include
# correct CRC and file sizes)
# position = self.fp.tell() # Preserve current position in file
# self.fp.seek(zinfo.header_offset, 0)
# self.fp.write(zinfo.FileHeader(zip64))
# self.fp.seek(position, 0)
yield self.fp.write(zinfo.DataDescriptor())
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def WriteFromFD(self, src_fd, arcname=None, compress_type=None, st=None):
"""Write a zip member from a file like object.
Args:
src_fd: A file like object, must support seek(), tell(), read().
arcname: The name in the archive this should take.
compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED)
st: An optional stat object to be used for setting headers.
Raises:
ArchiveAlreadyClosedError: If the zip if already closed.
Yields:
Chunks of binary data.
"""
zinfo = self._GenerateZipInfo(arcname=arcname, compress_type=compress_type,
st=st)
crc = 0
compress_size = 0
if not self._stream:
raise ArchiveAlreadyClosedError(
"Attempting to write to a ZIP archive that was already closed.")
zinfo.header_offset = self._stream.tell()
# Call _writeCheck(zinfo) to do sanity checking on zinfo structure that
# we've constructed.
self._zip_fd._writecheck(zinfo) # pylint: disable=protected-access
# Mark ZipFile as dirty. We have to keep self._zip_fd's internal state
# coherent so that it behaves correctly when close() is called.
self._zip_fd._didModify = True # pylint: disable=protected-access
# Write FileHeader now. It's incomplete, but CRC and uncompressed/compressed
# sized will be written later in data descriptor.
self._stream.write(zinfo.FileHeader())
if zinfo.compress_type == zipfile.ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
chunk_size = 0
while 1:
buf = src_fd.read(1024 * 8)
if not buf:
break
file_size += len(buf)
chunk_size += len(buf)
crc = zipfile.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size += len(buf)
self._stream.write(buf)
if chunk_size > self.FILE_CHUNK_SIZE:
yield self._stream.GetValueAndReset()
chunk_size = 0
if cmpr:
buf = cmpr.flush()
compress_size += len(buf)
zinfo.compress_size = compress_size
self._stream.write(buf)
else:
zinfo.compress_size = file_size
zinfo.CRC = crc
zinfo.file_size = file_size
# Writing data descriptor ZIP64-way by default. We never know how large
# the archive may become as we're generating it dynamically.
#
# crc-32 8 bytes (little endian)
# compressed size 8 bytes (little endian)
# uncompressed size 8 bytes (little endian)
self._stream.write(struct.pack("<LLL", crc, compress_size, file_size))
# Register the file in the zip file, so that central directory gets
# written correctly.
self._zip_fd.filelist.append(zinfo)
self._zip_fd.NameToInfo[zinfo.filename] = zinfo
yield self._stream.GetValueAndReset()
def get_crc32(file_path):
with open(file_path, 'rb') as f:
return crc32(f.read())
