def full_check(self): """ fully check the fcode file, including all the checksum raise error if any error occur """ try: logger.info("Start Checking CRC %s" % str(self.data[:8])); assert self.data[:8] == b"FCx0001\n" logger.info("Passed Header Check"); self.script_size = uint_unpacker(self.data[8:12]) logger.info("Script size = " + str(self.script_size)) assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker(self.data[12 + self.script_size:16 + self.script_size]) logger.info("Passed CRC32 -1 "); index = 16 + self.script_size self.meta_size = uint_unpacker(self.data[index:index + 4]) index += 4 assert crc32(self.data[index:index + self.meta_size]) == uint_unpacker(self.data[index + self.meta_size:index + self.meta_size + 4]) logger.info("Passed CRC32 -2 "); index = index + self.meta_size + 4 self.image_size = uint_unpacker(self.data[index:index + 4]) index += 4 logger.info("Passed full check"); return True except AssertionError as e: logger.info(str(e)); return False
def full_check(self): """ fully check the fcode file, including all the checksum raise error if any error occur """ try: logger.info("Start Checking CRC %s" % str(self.data[:8])) assert self.data[:8] == b"FCx0001\n" logger.info("Passed Header Check") self.script_size = uint_unpacker(self.data[8:12]) logger.info("Script size = " + str(self.script_size)) assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker( self.data[12 + self.script_size:16 + self.script_size]) logger.info("Passed CRC32 -1 ") index = 16 + self.script_size self.meta_size = uint_unpacker(self.data[index:index + 4]) index += 4 assert crc32( self.data[index:index + self.meta_size]) == uint_unpacker( self.data[index + self.meta_size:index + self.meta_size + 4]) logger.info("Passed CRC32 -2 ") index = index + self.meta_size + 4 self.image_size = uint_unpacker(self.data[index:index + 4]) index += 4 logger.info("Passed full check") return True except AssertionError as e: logger.info(str(e)) return False
def check_effaceable_header(plog): z = xor_strings(plog[:16], plog[16:32]) if z[:4] != "ecaF": return False plog_generation = struct.unpack("<L", plog[0x38:0x3C])[0] print "Effaceable generation" , plog_generation plog_crc = crc32(plog[0x40:0x40 + 960], crc32(plog[0x20:0x3C], crc32(z))) & 0xffffffff assert plog_crc == struct.unpack("<L", plog[0x3C:0x40])[0] , "Effaceable CRC" print "Effaceable CRC OK" return True
def check_effaceable_header(plog): z = xor_strings(plog[:16], plog[16:32]) if z[:4] != "ecaF": return False plog_generation = struct.unpack("<L", plog[0x38:0x3C])[0] print "Effaceable generation", plog_generation plog_crc = crc32(plog[0x40:0x40 + 960], crc32(plog[0x20:0x3C], crc32(z))) & 0xffffffff assert plog_crc == struct.unpack("<L", plog[0x3C:0x40])[0], "Effaceable CRC" print "Effaceable CRC OK" return True
def _load(self, task_file, crc_check): t = task_file # Check header magic_num = t.read(8) if magic_num != b"FCx0001\n": if magic_num[:3] != b"FCx" or magic_num[3:].isdigit() is False: raise RuntimeError(FILE_BROKEN) else: ver = int(magic_num[3:]) raise RuntimeError(FILE_BROKEN, NOT_SUPPORT, str(ver)) # Check script script_size = UINT_PACKER.unpack(t.read(4))[0] self.script_ptr = t.tell() self.script_size = script_size if crc_check: script_crc32 = 0 f_ptr = 0 while f_ptr < script_size: buf = t.read(min(script_size - f_ptr, 4096)) if buf: f_ptr += len(buf) script_crc32 = crc32(buf, script_crc32) else: raise RuntimeError(FILE_BROKEN, "LENGTH_ERROR") req_script_crc32 = INT_PACKER.unpack(t.read(4))[0] if req_script_crc32 != script_crc32: raise RuntimeError(FILE_BROKEN, "SCRIPT_CRC32") else: t.seek(script_size + 4, 1) # Check meta meta_size = UINT_PACKER.unpack(t.read(4))[0] meta_buf = t.read(meta_size) req_metadata_crc32 = INT_PACKER.unpack(t.read(4))[0] if crc_check and req_metadata_crc32 != crc32(meta_buf, 0): raise RuntimeError(FILE_BROKEN, "METADATA_CRC32") metadata = {} for item in meta_buf.split("\x00"): sitem = item.split("=", 1) if len(sitem) == 2: metadata[sitem[0]] = sitem[1] self.metadata = metadata t.seek(self.script_ptr)
def parse_lwvm(data, pageSize): try: hdr = LWVM_header.parse(data) if hdr.type != LWVM_MAGIC: print "LwVM magic mismatch" return tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000] check = crc32(tocheck) & 0xffffffff if check != hdr.crc32: return None print "LwVM header CRC OK" partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions] deviceSize=0 #XXX: HAAAAAAAX for s in [8, 16, 32, 64, 128]: if hdr.mediaSize < (s* 1024*1024*1024): deviceSize = s break for i in xrange(len(hdr.chunks)): if hdr.chunks[i] == 0x0: lba0 = (i * deviceSize*1024*1024) / pageSize partitions[0].first_lba = lba0 partitions[0].last_lba = lba0 + (partitions[0].end - partitions[0].begin) / pageSize elif hdr.chunks[i] == 0x1000: lbad = (i * deviceSize*1024*1024) / pageSize partitions[1].first_lba = lbad partitions[1].last_lba = lbad + (partitions[1].end - partitions[1].begin) / pageSize return partitions except: return None
def parse_lwvm(data, pageSize): try: hdr = LWVM_header.parse(data) if hdr.type != LWVM_MAGIC: print "LwVM magic mismatch" return tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000] check = crc32(tocheck) & 0xffffffff if check != hdr.crc32: return None print "LwVM header CRC OK" partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions] deviceSize = 0 #XXX: HAAAAAAAX for s in [8, 16, 32, 64, 128]: if hdr.mediaSize < (s * 1024 * 1024 * 1024): deviceSize = s break for i in xrange(len(hdr.chunks)): if hdr.chunks[i] == 0x0: lba0 = (i * deviceSize * 1024 * 1024) / pageSize partitions[0].first_lba = lba0 partitions[0].last_lba = lba0 + ( partitions[0].end - partitions[0].begin) / pageSize elif hdr.chunks[i] == 0x1000: lbad = (i * deviceSize * 1024 * 1024) / pageSize partitions[1].first_lba = lbad partitions[1].last_lba = lbad + ( partitions[1].end - partitions[1].begin) / pageSize return partitions except: return None
def BlurImages(ifile, ofile=None): ''' If blurring is good enough, this function will take the document and blur the image. To change the amount of blurring, change Docx.blur The Default value is: ImageFilter.GaussianBlur(50) Adjust the Number inside GaussianBlur until the valus suits you ''' # Thanks to mata for his/her help # https://stackoverflow.com/questions/50963852/remove-all-images-from-docx-files if ofile is None and osp.splitext(ifile)[1] == '.docx': ofile = ifile.replace('.docx', '.redacted.docx') elif ofile is None and osp.splitext(ifile)[1] == '.odt': ofile = ifile.replace('.odt', '.redacted.odt') elif ofile is None: ofile = osp.splitext(ifile)[0] + '.redacted.' \ + osp.splitext(ifile)[1] izip = ZipFile(ifile) ozip = ZipFile(ofile, 'w', zipfile.ZIP_DEFLATED) for node in izip.infolist(): content = izip.read(node) if osp.splitext(node.filename)[1] in Base.pictures: # Blur Image and save it to a buffer img = Image.open(io.BytesIO(content)) # Load Picture to memory img = img.convert().filter(Base.blur) # Blur it imgBuffer = io.BytesIO() img.save(imgBuffer, osp.splitext(node.filename)[1][1:]) # Change the XML so that it points to the blurred image, not original content = imgBuffer.getvalue() node.file_size = len(content) node.CRC = zipfile.crc32(content) img.close() ozip.writestr(node, content) izip.close() ozip.close()
def parse_lwvm(data, pageSize): try: hdr = LWVM_header.parse(data) if hdr.type != LWVM_MAGIC: print "LwVM magic mismatch" return tocheck = data[:44] + "\x00\x00\x00\x00" + data[48:0x1000] check = crc32(tocheck) & 0xffffffff if check != hdr.crc32: return None print "LwVM header CRC OK" partitions = hdr.LWVM_partitionRecord[:hdr.numPartitions] LwVM_rangeShiftValue = 32 - clz32((hdr.mediaSize - 1) >> 10 ) for i in xrange(len(hdr.chunks)): if hdr.chunks[i] == 0x0: lba0 = (i << LwVM_rangeShiftValue) / pageSize partitions[0].first_lba = lba0 partitions[0].last_lba = lba0 + (partitions[0].end - partitions[0].begin) / pageSize elif hdr.chunks[i] == 0x1000: lbad = (i << LwVM_rangeShiftValue) / pageSize partitions[1].first_lba = lbad partitions[1].last_lba = lbad + (partitions[1].end - partitions[1].begin) / pageSize return partitions except: return None
def _load(self, f): if f.read(8) != b"FCx0001\n": raise FCodeError("HEADER_ERROR") script_size = UINT_PACKER.unpack(f.read(4))[0] script_crc32 = 0 f_ptr = 0 self.script_ptr = f.tell() self.script_size = script_size while f_ptr < script_size: buf = f.read(min(script_size - f_ptr, 4096)) if buf: f_ptr += len(buf) script_crc32 = crc32(buf, script_crc32) else: raise FCodeError("SIZE_ERROR", "SCRIPT") req_script_crc32 = INT_PACKER.unpack(f.read(4))[0] if req_script_crc32 != script_crc32: raise FCodeError("CRC_ERROR", "SCRIPT") # Check meta meta_size = UINT_PACKER.unpack(f.read(4))[0] meta_buf = f.read(meta_size) req_metadata_crc32 = INT_PACKER.unpack(f.read(4))[0] if req_metadata_crc32 != crc32(meta_buf, 0): raise FCodeError("CRC_ERROR", "META") metadata = {} for item in meta_buf.split("\x00"): sitem = item.split("=", 1) if len(sitem) == 2: metadata[sitem[0]] = sitem[1] self.metadata = metadata # Load image self.image_buf = [] buf = f.read(4) while len(buf) == 4: image_size = UINT_PACKER.unpack(buf)[0] if image_size > 0: self.image_buf.append(f.read(image_size)) buf = f.read(4) else: break
def full_check(self): """ fully check the fcode file, including all the checksum raise error if any error occur """ try: assert self.data[:8] == b"FCx0001\n" self.script_size = uint_unpacker(self.data[8:12]) assert crc32(self.data[12:12 + self.script_size]) == uint_unpacker(self.data[12 + self.script_size:16 + self.script_size]) index = 16 + self.script_size self.meta_size = uint_unpacker(self.data[index:index + 4]) index += 4 assert crc32(self.data[index:index + self.meta_size]) == uint_unpacker(self.data[index + self.meta_size:index + self.meta_size + 4]) index = index + self.meta_size + 4 self.image_size = uint_unpacker(self.data[index:index + 4]) index += 4 return True except AssertionError as e: raise RuntimeError(FILE_BROKEN, e.args[0] if e.args else "#")
def write(self, data): if self.closed: raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) if self._compressor: data = self._compressor.compress(data) self._compress_size += len(data) self._fileobj.write(data) return nbytes
def parse_gpt(data): gpt = GPT_header.parse(data) if gpt.signature != "EFI PART": return None print "Found GPT header current_lba=%d partition_entries_lba=%d" % (gpt.current_lba, gpt.partition_entries_lba) assert gpt.partition_entries_lba > gpt.current_lba check = gpt.crc gpt.crc = 0 actual = crc32(GPT_header.build(gpt)) if actual != check: print "GPT crc check fail %d vs %d" % (actual, check) return None return gpt
def WriteFileChunk(self, chunk): """Writes file chunk.""" if not self._stream: raise ArchiveAlreadyClosedError( "Attempting to write to a ZIP archive that was already closed.") self.cur_file_size += len(chunk) self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff if self.cur_cmpr: chunk = self.cur_cmpr.compress(chunk) self.cur_compress_size += len(chunk) self._stream.write(chunk) return self._stream.GetValueAndReset()
def main(filename): print 'Reading %s' % sys.argv[1] f = open(filename, 'rb') while True: # Read and parse a file header fheader = f.read(zipfile.sizeFileHeader) if len(fheader) < zipfile.sizeFileHeader: print 'Found end of file. Some entries missed.' break fheader = struct.unpack(zipfile.structFileHeader, fheader) if fheader[zipfile._FH_SIGNATURE] == 'PK\x01\x02': print 'Found start of central directory. All entries processed.' break fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH]) if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]: f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH]) print 'Found %s' % fname # Fake a zipinfo record zi = zipfile.ZipInfo() zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE] zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD] zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS] zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE] # Read the file contents zef = zipfile.ZipExtFile(f, 'rb', zi) data = zef.read() # Sanity checks if len(data) != fheader[zipfile._FH_UNCOMPRESSED_SIZE]: raise Exception( "Unzipped data doesn't match expected size! %d != %d, in %s" % (len(data), fheader[zipfile._FH_UNCOMPRESSED_SIZE], fname)) calc_crc = zipfile.crc32(data) & 0xffffffff if calc_crc != fheader[zipfile._FH_CRC]: raise Exception('CRC mismatch! %d != %d, in %s' % (calc_crc, fheader[zipfile._FH_CRC], fname)) # Write the file write_data(fname, data) f.close()
def __init__(self, fileobj, mode, p4kinfo, decrypter=None, close_fileobj=False): self._is_encrypted = p4kinfo.is_encrypted self._decompressor = ZStdDecompressor() self._fileobj = fileobj self._decrypter = decrypter self._close_fileobj = close_fileobj self._compress_type = p4kinfo.compress_type self._compress_left = p4kinfo.compress_size self._left = p4kinfo.file_size self._eof = False self._readbuffer = b"" self._offset = 0 self.newlines = None self.mode = mode self.name = p4kinfo.filename if hasattr(p4kinfo, 'CRC'): self._expected_crc = p4kinfo.CRC self._running_crc = zipfile.crc32(b'') else: self._expected_crc = None # TODO: the CRCs don't match, but im getting the same outputs as unp4k - we should figure out what exactly is # going into calculating the CRC for P4K entry self._expected_crc = None self._seekable = False try: if fileobj.seekable(): self._orig_compress_start = fileobj.tell() self._orig_compress_size = p4kinfo.compress_size self._orig_file_size = p4kinfo.file_size self._orig_start_crc = self._running_crc self._seekable = True except AttributeError: pass
def generator(self, source, compression=ZIP_STORED): """ Turn a streamed file source into a stream zipcrypted archive file. Some lines are based upon Ivan Ergunov's work. Some lines are based upon devthat's work on zipencrypt. """ if not self.zinfo: raise RuntimeError( "Ziphyr object not primed. " "Please use either from_filepath() or from_metadata()." ) self.stream = ZiphyrStream(self.password) with self.ZipFile( self.stream, mode='w', compression=compression ) as zfile: clear_crc = 0 with zfile.open(self.zinfo, mode='w') as dest: if self.password: # the zipcrypto asks for twelve almost-random bytes self.zinfo._raw_time = ( self.zinfo.date_time[3] << 11 | self.zinfo.date_time[4] << 5 | (self.zinfo.date_time[5] // 2)) check_byte = (self.zinfo._raw_time >> 8) & 0xff twelve_angry = urandom(11) + struct.pack("B", check_byte) dest.write(self.stream.cypher_chunk(twelve_angry)) for chunk in source: # the internal crc is updated when dest.write() # but we want it to be of the clear data # hence doing our own in parallel clear_crc = crc32(chunk, clear_crc) if self.password: chunk = self.stream.cypher_chunk(chunk) dest.write(chunk) yield self.stream.get() if self.password: self.zinfo.file_size -= 12 # withdrawing the 12 bytes self.zinfo.CRC = clear_crc # overwriting with the clearbytes crc yield self.stream.get()
def redact_images(filename, FilePath): #outfile = filename.replace(".docx", "_redacted.docx") with zipfile.ZipFile(filename) as inzip: #with zipfile.ZipFile(outfile, "w") as outzip: i = 0 for info in inzip.infolist(): name = info.filename content = inzip.read(info) if name.endswith((".png", ".jpeg", ".gif")): fmt = name.split(".")[-1] Name = name.split("/")[-1] img = Image.open(io.BytesIO(content)) img.save(FilePath + str(Name)) outb = io.BytesIO() img.save(outb, fmt) content = outb.getvalue() info.file_size = len(content) info.CRC = zipfile.crc32(content) i += 1
def main(filename): print 'Reading %s' % sys.argv[1] f = open(filename, 'rb') while True: # Read and parse a file header fheader = f.read(zipfile.sizeFileHeader) if len(fheader) < zipfile.sizeFileHeader: print 'Found end of file. Some entries missed.' break fheader = struct.unpack(zipfile.structFileHeader, fheader) if fheader[zipfile._FH_SIGNATURE] == 'PK\x01\x02': print 'Found start of central directory. All entries processed.' break fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH]) if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]: f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH]) print 'Found %s' % fname # Fake a zipinfo record zi = zipfile.ZipInfo() zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE] zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD] zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS] zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE] # Read the file contents zef = zipfile.ZipExtFile(f, 'rb', zi) data = zef.read() # Sanity checks if len(data) != fheader[zipfile._FH_UNCOMPRESSED_SIZE]: raise Exception("Unzipped data doesn't match expected size! %d != %d, in %s" % (len(data), fheader[zipfile._FH_UNCOMPRESSED_SIZE], fname)) calc_crc = zipfile.crc32(data) & 0xffffffff if calc_crc != fheader[zipfile._FH_CRC]: raise Exception('CRC mismatch! %d != %d, in %s' % (calc_crc, fheader[zipfile._FH_CRC], fname)) # Write the file write_data(fname, data) f.close()
def WriteFileChunk(self, chunk): """Writes file chunk.""" if not self._stream: raise ArchiveAlreadyClosedError( "Attempting to write to a ZIP archive that was already closed.") self.cur_file_size += len(chunk) # TODO(user):pytype: crc32 is not visible outside of zipfile. # pytype: disable=module-attr self.cur_crc = zipfile.crc32(chunk, self.cur_crc) & 0xffffffff # pytype: enable=module-attr if self.cur_cmpr: chunk = self.cur_cmpr.compress(chunk) self.cur_compress_size += len(chunk) self._stream.write(chunk) return self._stream.GetValueAndReset()
def fast_read_meta(filename): with open(filename, "rb") as f: try: # Check header assert f.read(8) == b"FCx0001\n", "MAGIC_NUMBER_ERROR" # Skip script size/crc32/body script_size = UINT_PACKER.unpack(f.read(4))[0] f.seek(script_size + 4, 1) # crc32 + size # Load metadata meta_size = UINT_PACKER.unpack(f.read(4))[0] assert meta_size < 65536, "METADATA_TOO_LARGE" meta_buf = f.read(meta_size) meta_crc32 = INT_PACKER.unpack(f.read(4))[0] assert meta_crc32 == crc32(meta_buf, 0), "METADATA_CRC32_ERROR" metadata = {} for item in meta_buf.split("\x00"): sitem = item.split("=", 1) if len(sitem) == 2: metadata[sitem[0]] = sitem[1] # Load image images = [] size_buf = f.read(4) while len(size_buf) == 4: img_s = UINT_PACKER.unpack(size_buf)[0] if img_s > 0: images.append(f.read(img_s)) size_buf = f.read(4) else: break return metadata, images except struct.error as e: raise RuntimeError(FILE_BROKEN) except AssertionError as e: raise RuntimeError(FILE_BROKEN, e.args[0] if e.args else "#")
def image_inverter(filename): inverted_file = filename.replace('.docx', '_inverted.docx') with zipfile.ZipFile(filename) as inzip: with zipfile.ZipFile(inverted_file, 'w') as outzip: for info in inzip.infolist(): name = info.filename content = inzip.read(info) if info.filename.endswith(('.png', '.jpeg', '.gif')): print(name) file_extension = name.split('.')[-1] img = Image.open(io.BytesIO(content)) pixels = img.load() if color_check(img, pixels): pixels = inverter(img, pixels) outb = io.BytesIO() img.save(outb, file_extension) content = outb.getvalue() info.file_size = len(content) info.CRC = zipfile.crc32(content) outzip.writestr(info, content)
def main(filename): print('Reading %s Central Directory' % filename) # Get info from ZIP Central Directory with zipfile.ZipFile(filename, 'r') as myzip: files = myzip.namelist() print('Found %d file(s) from Central Directory:' % (len(files))) print('- ' + '\n- '.join(files)) print('Reading %s ZIP entry manually' % sys.argv[1]) f = open(filename, 'rb') while True: # Read and parse a file header fheader = f.read(zipfile.sizeFileHeader) if len(fheader) < zipfile.sizeFileHeader: print('Found end of file. Some entries missed.') break fheader = struct.unpack(zipfile.structFileHeader, fheader) if fheader[zipfile._FH_SIGNATURE] == zipfile.stringCentralDir: print('Found start of central directory. All entries processed.') break if fheader[zipfile._FH_SIGNATURE] != zipfile.stringFileHeader: raise Exception('Size mismatch! File Header expected, got "%s"' % (fheader[zipfile._FH_SIGNATURE])) if fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS] & 0x8: data_descriptor = True fname = f.read(fheader[zipfile._FH_FILENAME_LENGTH]) if fheader[zipfile._FH_EXTRA_FIELD_LENGTH]: f.read(fheader[zipfile._FH_EXTRA_FIELD_LENGTH]) print('Found %s' % fname.decode()) # Fake a zipinfo record zi = zipfile.ZipInfo() zi.filename = fname zi.compress_size = fheader[zipfile._FH_COMPRESSED_SIZE] zi.compress_type = fheader[zipfile._FH_COMPRESSION_METHOD] zi.flag_bits = fheader[zipfile._FH_GENERAL_PURPOSE_FLAG_BITS] zi.file_size = fheader[zipfile._FH_UNCOMPRESSED_SIZE] zi.CRC = fheader[zipfile._FH_CRC] if data_descriptor: # Compress size is zero # Get the real sizes with datadescriptor ddescriptor = fdescriptor_reader(f, f.tell()) if ddescriptor is None: break zi.compress_size = ddescriptor[_DD_COMPRESSED_SIZE] zi.file_size = ddescriptor[_DD_UNCOMPRESSED_SIZE] zi.CRC = ddescriptor[_DD_CRC] #print(zi) # Read the file contents zef = zipfile.ZipExtFile(f, 'rb', zi) data = zef.read() # Sanity checks if len(data) != zi.file_size: raise Exception( "Unzipped data doesn't match expected size! %d != %d, in %s" % (len(data), zi.file_size, fname)) calc_crc = zipfile.crc32(data) & 0xffffffff if calc_crc != zi.CRC: raise Exception('CRC mismatch! %d != %d, in %s' % (calc_crc, zi.CRC, fname)) # Write the file write_data(fname, data) if data_descriptor: # skip dataDescriptor before reading the next file f.seek(f.tell() + sizeDataDescriptor) f.close()
def my_zip_write(self, filename, arcname=None, compress_type=None, date_time=None): """Copy of zipfile.ZipFile.write() with some modifications - Allow overriding the timestamp for reproducible builds - Calculate a SHA256 hash of the file as we write it and return the hash object. """ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) if date_time is None: mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes if isdir: zinfo.compress_type = zipfile.ZIP_STORED elif compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 self.fp.seek(getattr(self, 'start_dir', 0)) zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 zinfo.external_attr |= 0x10 # MS-DOS directory flag self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) self.start_dir = self.fp.tell() return hashsum = hashlib.sha256() cmpr = _get_compressor(zinfo.compress_type) with open(filename, "rb") as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff hashsum.update(buf) if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') # Seek backwards and write file header (which will now include # correct CRC and file sizes) self.start_dir = self.fp.tell() # Preserve current position in file self.fp.seek(zinfo.header_offset, 0) self.fp.write(zinfo.FileHeader(zip64)) self.fp.seek(self.start_dir, 0) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo return hashsum
def read(self, size): buf = self.s.read(size) self.length += size self.crc32 = crc32(buf, self.crc32) return buf
def _thread_py_id(): # return threading.get_ident() return zipfile.crc32(int(threading.get_ident()).to_bytes(8, 'little'))
def write(self, filename, arcname=None, compress_type=None): """ Fixed version of write supporting bitflag 0x08 to write crc and size at end of file. """ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes if isdir: zinfo.compress_type = ZIP_STORED elif compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.header_offset = self.fp.tell() # Start of header bytes self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 zinfo.external_attr |= 0x10 # MS-DOS directory flag self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader()) self.start_dir = self.fp.tell() return zinfo.flag_bits |= 0x08 with open(filename, "rb") as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 try: # Python > 2.7.3 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT self.fp.write(zinfo.FileHeader(zip64)) except TypeError: # Python <= 2.7.3 zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT self.fp.write(zinfo.FileHeader()) if zinfo.compress_type == ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None file_size = 0 while 1: buf = fp.read(CHUNK_SIZE) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') # Write CRC and file sizes after the file data fmt = b'<LQQ' if zip64 else b'<LLL' self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size, zinfo.file_size)) self.start_dir = self.fp.tell() self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
async def _write(self, filename=None, iterable=None, arcname=None, compress_type=None): """ Put the bytes from filename into the archive under the name `arcname`. """ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") if (filename is None and iterable is None) or (filename is not None and iterable is not None): raise ValueError( "either (exclusively) filename or iterable shall be not None") if filename: st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] else: st, isdir, date_time = None, False, time.localtime()[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) if st: zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes else: zinfo.external_attr = 0o600 << 16 # ?rw------- if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type if st: zinfo.file_size = st[6] else: zinfo.file_size = 0 zinfo.flag_bits = 0x00 zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == zipfile.ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield self.fp.write(zinfo.FileHeader(False)) return cmpr = _get_compressor(zinfo.compress_type) # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT yield self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 if filename: async for buf in self.data_generator(filename): file_size = file_size + len(buf) CRC = zipfile.crc32(buf, CRC) & 0xffffffff if cmpr: buf = await self._run_in_executor(cmpr.compress, buf) compress_size = compress_size + len(buf) yield self.fp.write(buf) else: # we have an iterable for buf in iterable: file_size = file_size + len(buf) CRC = zipfile.crc32(buf, CRC) & 0xffffffff if cmpr: buf = await self._run_in_executor(cmpr.compress, buf) compress_size = compress_size + len(buf) yield self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) yield self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > zipfile.ZIP64_LIMIT: raise RuntimeError( 'File size has increased during compressing') if compress_size > zipfile.ZIP64_LIMIT: raise RuntimeError( 'Compressed size larger than uncompressed size') yield self.fp.write(zinfo.DataDescriptor()) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def __write(self, filename, arcname=None, compress_type=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield self.fp.write(zinfo.FileHeader(False)) return cmpr = _get_compressor(zinfo.compress_type) with open(filename, 'rb') as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT yield self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) yield self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) yield self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError( 'File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError( 'Compressed size larger than uncompressed size') # Seek backwards and write file header (which will now include # correct CRC and file sizes) # position = self.fp.tell() # Preserve current position in file # self.fp.seek(zinfo.header_offset, 0) # self.fp.write(zinfo.FileHeader(zip64)) # self.fp.seek(position, 0) yield self.fp.write(zinfo.DataDescriptor()) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
async def _write(self, filename=None, iterable=None, arcname=None, compress_type=None): """ Put the bytes from filename into the archive under the name `arcname`. """ if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") if (filename is None and iterable is None) or (filename is not None and iterable is not None): raise ValueError("either (exclusively) filename or iterable shall be not None") if filename: st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] else: st, isdir, date_time = None, False, time.localtime()[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) if st: zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes else: zinfo.external_attr = 0o600 << 16 # ?rw------- if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type if st: zinfo.file_size = st[6] else: zinfo.file_size = 0 zinfo.flag_bits = 0x00 zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == zipfile.ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo await yield_(self.fp.write(zinfo.FileHeader(False))) return cmpr = _get_compressor(zinfo.compress_type) # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT await yield_(self.fp.write(zinfo.FileHeader(zip64))) file_size = 0 if filename: async for buf in self.data_generator(filename): file_size = file_size + len(buf) CRC = zipfile.crc32(buf, CRC) & 0xffffffff if cmpr: buf = await self._run_in_executor(cmpr.compress, buf) compress_size = compress_size + len(buf) await yield_(self.fp.write(buf)) else: # we have an iterable for buf in iterable: file_size = file_size + len(buf) CRC = zipfile.crc32(buf, CRC) & 0xffffffff if cmpr: buf = await self._run_in_executor(cmpr.compress, buf) compress_size = compress_size + len(buf) await yield_(self.fp.write(buf)) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) await yield_(self.fp.write(buf)) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > zipfile.ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > zipfile.ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') await yield_(self.fp.write(zinfo.DataDescriptor())) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def write(self, filename, arcname=None, compress_type=None, pwd=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = zipfile.ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes if isdir: zinfo.compress_type = zipfile.ZIP_STORED elif compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.header_offset = self.fp.tell() # Start of header bytes self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 zinfo.external_attr |= 0x10 # MS-DOS directory flag self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) return pwd = pwd or self.pwd if pwd: zinfo.flag_bits |= 0x8 | 0x1 # set stream and encrypted with open(filename, "rb") as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = crc = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and zinfo.file_size * 1.05 > zipfile.ZIP64_LIMIT self.fp.write(zinfo.FileHeader(zip64)) if zinfo.compress_type == zipfile.ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None if pwd: ze = _ZipEncrypter(pwd) encrypt = lambda x: "".join(map(ze, x)) zinfo._raw_time = (zinfo.date_time[3] << 11 | zinfo.date_time[4] << 5 | (zinfo.date_time[5] // 2)) check_byte = (zinfo._raw_time >> 8) & 0xff enryption_header = os.urandom(11) + chr(check_byte) self.fp.write(encrypt(enryption_header)) else: encrypt = lambda x: x file_size = 0 while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) crc = zipfile.crc32(buf, crc) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) self.fp.write(encrypt(buf)) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) self.fp.write(encrypt(buf)) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = crc zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > zipfile.ZIP64_LIMIT: raise RuntimeError( 'File size has increased during compressing') if compress_size > zipfile.ZIP64_LIMIT: raise RuntimeError( 'Compressed size larger than uncompressed size') if pwd: # Write CRC and file sizes after the file data zinfo.compress_size += 12 fmt = '<LQQ' if zip64 else '<LLL' self.fp.write( struct.pack(fmt, zinfo.CRC, zinfo.compress_size, zinfo.file_size)) self.fp.flush() else: # Seek backwards and write file header (which will now include # correct CRC and file sizes) position = self.fp.tell() # Preserve current position in file self.fp.seek(zinfo.header_offset, 0) self.fp.write(zinfo.FileHeader(zip64)) self.fp.seek(position, 0) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def writestr(self, zinfo_or_arcname, _bytes, compress_type=None, pwd=None): """Write a file into the archive. The contents is the string 'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or the name of the file in the archive.""" if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression if zinfo.filename[-1] == '/': zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag else: zinfo.external_attr = 0o600 << 16 # ?rw------- else: zinfo = zinfo_or_arcname if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") if compress_type is not None: zinfo.compress_type = compress_type zinfo.file_size = len(_bytes) # Uncompressed size zinfo.header_offset = self.fp.tell() # Start of header bytes self._writecheck(zinfo) self._didModify = True zinfo.CRC = zipfile.crc32(_bytes) & 0xffffffff # CRC-32 checksum if zinfo.compress_type == zipfile.ZIP_DEFLATED: co = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) _bytes = co.compress(_bytes) + co.flush() zinfo.compress_size = len(_bytes) # Compressed size else: zinfo.compress_size = zinfo.file_size zip64 = zinfo.file_size > zipfile.ZIP64_LIMIT or zinfo.compress_size > zipfile.ZIP64_LIMIT if zip64 and not self._allowZip64: raise zipfile.LargeZipFile( "Filesize would require ZIP64 extensions") pwd = pwd or self.pwd if pwd: zinfo.flag_bits |= 0x01 zinfo.compress_size += 12 # 12 extra bytes for the header if zinfo.flag_bits & 0x8: zinfo._raw_time = (zinfo.date_time[3] << 11 | zinfo.date_time[4] << 5 | (zinfo.date_time[5] // 2)) check_byte = (zinfo._raw_time >> 8) & 0xff else: check_byte = (zinfo.CRC >> 24) & 0xff enryption_header = os.urandom(11) + chr(check_byte) ze = _ZipEncrypter(pwd) _bytes = "".join(map(ze, enryption_header + _bytes)) self.fp.write(zinfo.FileHeader(zip64)) self.fp.write(_bytes) if zinfo.flag_bits & 0x08: # Write CRC and file sizes after the file data fmt = '<LQQ' if zip64 else '<LLL' self.fp.write( struct.pack(fmt, zinfo.CRC, zinfo.compress_size, zinfo.file_size)) self.fp.flush() self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def WriteFromFD(self, src_fd, arcname=None, compress_type=None, st=None): """Write a zip member from a file like object. Args: src_fd: A file like object, must support seek(), tell(), read(). arcname: The name in the archive this should take. compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED) st: An optional stat object to be used for setting headers. Raises: RuntimeError: If the zip if already closed. """ zinfo = self.GenerateZipInfo(arcname=arcname, compress_type=compress_type, st=st) crc = 0 compress_size = 0 if not self.out_fd: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") zinfo.header_offset = self.out_fd.tell() # Call _writeCheck(zinfo) to do sanity checking on zinfo structure that # we've constructed. self.zip_fd._writecheck(zinfo) # pylint: disable=protected-access # Mark ZipFile as dirty. We have to keep self.zip_fd's internal state # coherent so that it behaves correctly when close() is called. self.zip_fd._didModify = True # pylint: disable=protected-access # Write FileHeader now. It's incomplete, but CRC and uncompressed/compressed # sized will be written later in data descriptor. self.out_fd.write(zinfo.FileHeader()) if zinfo.compress_type == zipfile.ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None file_size = 0 while 1: buf = src_fd.read(1024 * 8) if not buf: break file_size += len(buf) crc = zipfile.crc32(buf, crc) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size += len(buf) self.out_fd.write(buf) if cmpr: buf = cmpr.flush() compress_size += len(buf) zinfo.compress_size = compress_size self.out_fd.write(buf) else: zinfo.compress_size = file_size zinfo.CRC = crc zinfo.file_size = file_size if file_size > zipfile.ZIP64_LIMIT or compress_size > zipfile.ZIP64_LIMIT: # Writing data descriptor ZIP64-way: # crc-32 8 bytes (little endian) # compressed size 8 bytes (little endian) # uncompressed size 8 bytes (little endian) self.out_fd.write(struct.pack("<LLL", crc, compress_size, file_size)) else: # Writing data descriptor non-ZIP64-way: # crc-32 4 bytes (little endian) # compressed size 4 bytes (little endian) # uncompressed size 4 bytes (little endian) self.out_fd.write(struct.pack("<III", crc, compress_size, file_size)) # Register the file in the zip file, so that central directory gets # written correctly. self.zip_fd.filelist.append(zinfo) self.zip_fd.NameToInfo[zinfo.filename] = zinfo
def write_iter(self, filename, arcname=None, compress_type=None): """Put the bytes from filename into the archive under the name arcname.""" from zipfile import stat, time, ZipInfo, ZIP_DEFLATED, crc32, struct if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.header_offset = self.fp.tell() # Start of header bytes self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield zinfo.FileHeader() #self.fp.write(zinfo.FileHeader()) return with open(filename, "rb") as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 zinfo.file_size = file_size = 0 cmpr = None while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff zinfo.compress_size = file_size zinfo.file_size = file_size zinfo.CRC = CRC #self.fp.write(zinfo.FileHeader()) yield zinfo.FileHeader() fp.seek(0) while 1: buf = fp.read(1024 * 8) if not buf: break #self.fp.write(buf) yield buf self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def __write(self, filename=None, iterable=None, arcname=None, compress_type=None, pwd=None): """Put the bytes from filename into the archive under the name `arcname`.""" if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") if (filename is None and iterable is None) or (filename is not None and iterable is not None): raise ValueError( "either (exclusively) filename or iterable shall be not None") if filename: st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] else: st, isdir, date_time = None, False, time.localtime()[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) if st: zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes else: zinfo.external_attr = 0o600 << 16 # ?rw------- if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type if st: zinfo.file_size = st[6] else: zinfo.file_size = 0 zinfo.flag_bits = 0x00 pwd = pwd or self.pwd if pwd: zinfo.flag_bits |= 0x1 # set encrypted zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield self.fp.write(zinfo.FileHeader(False)) return cmpr = _get_compressor(zinfo.compress_type) # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT yield self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 if pwd: encrypt = zip_encrypt(pwd) zinfo._raw_time = (zinfo.date_time[3] << 11 | zinfo.date_time[4] << 5 | (zinfo.date_time[5] // 2)) check_byte = (zinfo._raw_time >> 8) & 0xff encryption_header = randomFunc(11) + struct.pack(b"B", check_byte) yield self.fp.write(encrypt(encryption_header)) else: encrypt = lambda x: x if filename: with open(filename, 'rb') as fp: while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) yield self.fp.write(encrypt(buf)) else: # we have an iterable for buf in iterable: file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) yield self.fp.write(encrypt(buf)) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) yield self.fp.write(encrypt(buf)) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError( 'File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError( 'Compressed size larger than uncompressed size') # Seek backwards and write file header (which will now include # correct CRC and file sizes) if pwd: # Write CRC and file sizes after the file data zinfo.compress_size += 12 fmt = b'<LQQ' if zip64 else b'<LLL' yield self.fp.write( struct.pack(fmt, zinfo.CRC, zinfo.compress_size, zinfo.file_size)) else: # Seek backwards and write file header (which will now include # correct CRC and file sizes) # position = self.fp.tell() # Preserve current position in file # self.fp.seek(zinfo.header_offset, 0) # self.fp.write(zinfo.FileHeader(zip64)) # self.fp.seek(position, 0) yield self.fp.write(zinfo.DataDescriptor()) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def update_event(self, inp=-1): self.set_output_val(0, zipfile.crc32(self.input(0), self.input(1)))
def __write(self, filename, arcname=None, compress_type=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0x00 zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor zinfo.header_offset = self.fp.tell() # Start of header bytes if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= 0x02 self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo yield self.fp.write(zinfo.FileHeader(False)) return cmpr = _get_compressor(zinfo.compress_type) with open(filename, 'rb') as fp: # Must overwrite CRC and sizes with correct data later zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 # Compressed size can be larger than uncompressed size zip64 = self._allowZip64 and \ zinfo.file_size * 1.05 > ZIP64_LIMIT yield self.fp.write(zinfo.FileHeader(zip64)) file_size = 0 while 1: buf = fp.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) yield self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) yield self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') # Seek backwards and write file header (which will now include # correct CRC and file sizes) # position = self.fp.tell() # Preserve current position in file # self.fp.seek(zinfo.header_offset, 0) # self.fp.write(zinfo.FileHeader(zip64)) # self.fp.seek(position, 0) yield self.fp.write(zinfo.DataDescriptor()) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
def WriteFromFD(self, src_fd, arcname=None, compress_type=None, st=None): """Write a zip member from a file like object. Args: src_fd: A file like object, must support seek(), tell(), read(). arcname: The name in the archive this should take. compress_type: Compression type (zipfile.ZIP_DEFLATED, or ZIP_STORED) st: An optional stat object to be used for setting headers. Raises: ArchiveAlreadyClosedError: If the zip if already closed. Yields: Chunks of binary data. """ zinfo = self._GenerateZipInfo(arcname=arcname, compress_type=compress_type, st=st) crc = 0 compress_size = 0 if not self._stream: raise ArchiveAlreadyClosedError( "Attempting to write to a ZIP archive that was already closed.") zinfo.header_offset = self._stream.tell() # Call _writeCheck(zinfo) to do sanity checking on zinfo structure that # we've constructed. self._zip_fd._writecheck(zinfo) # pylint: disable=protected-access # Mark ZipFile as dirty. We have to keep self._zip_fd's internal state # coherent so that it behaves correctly when close() is called. self._zip_fd._didModify = True # pylint: disable=protected-access # Write FileHeader now. It's incomplete, but CRC and uncompressed/compressed # sized will be written later in data descriptor. self._stream.write(zinfo.FileHeader()) if zinfo.compress_type == zipfile.ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None file_size = 0 chunk_size = 0 while 1: buf = src_fd.read(1024 * 8) if not buf: break file_size += len(buf) chunk_size += len(buf) crc = zipfile.crc32(buf, crc) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size += len(buf) self._stream.write(buf) if chunk_size > self.FILE_CHUNK_SIZE: yield self._stream.GetValueAndReset() chunk_size = 0 if cmpr: buf = cmpr.flush() compress_size += len(buf) zinfo.compress_size = compress_size self._stream.write(buf) else: zinfo.compress_size = file_size zinfo.CRC = crc zinfo.file_size = file_size # Writing data descriptor ZIP64-way by default. We never know how large # the archive may become as we're generating it dynamically. # # crc-32 8 bytes (little endian) # compressed size 8 bytes (little endian) # uncompressed size 8 bytes (little endian) self._stream.write(struct.pack("<LLL", crc, compress_size, file_size)) # Register the file in the zip file, so that central directory gets # written correctly. self._zip_fd.filelist.append(zinfo) self._zip_fd.NameToInfo[zinfo.filename] = zinfo yield self._stream.GetValueAndReset()
def get_crc32(file_path): with open(file_path, 'rb') as f: return crc32(f.read())