def crc32(self): try: from zlib import crc32 try: if crc32('amuse')&0xffffffff == 0xc0cc9367: return crc32 except Exception: #python 3, crc32 needs bytes... def python3_crc32(x): x = crc32(bytes(x, 'ascii')) return x - ((x & 0x80000000) <<1) if python3_crc32('amuse')&0xffffffff == 0xc0cc9367: return python3_crc32 except Exception: pass try: from binascii import crc32 try: if crc32('amuse')&0xffffffff == 0xc0cc9367: return crc32 except Exception: #python 3, crc32 needs bytes... def python3_crc32(x): x = crc32(bytes(x, 'ascii')) return x - ((x & 0x80000000) <<1) if python3_crc32('amuse')&0xffffffff == 0xc0cc9367: return python3_crc32 except Exception: pass raise Exception("No working crc32 implementation found!")
def _to_png(data, width, height): # From MSS line = width * 3 png_filter = struct.pack('>B', 0) scanlines = b''.join( [png_filter + data[y * line:y * line + line] for y in range(height)] ) magic = struct.pack('>8B', 137, 80, 78, 71, 13, 10, 26, 10) # Header: size, marker, data, CRC32 ihdr = [b'', b'IHDR', b'', b''] ihdr[2] = struct.pack('>2I5B', width, height, 8, 2, 0, 0, 0) ihdr[3] = struct.pack('>I', crc32(b''.join(ihdr[1:3])) & 0xffffffff) ihdr[0] = struct.pack('>I', len(ihdr[2])) # Data: size, marker, data, CRC32 idat = [b'', b'IDAT', compress(scanlines), b''] idat[3] = struct.pack('>I', crc32(b''.join(idat[1:3])) & 0xffffffff) idat[0] = struct.pack('>I', len(idat[2])) # Footer: size, marker, None, CRC32 iend = [b'', b'IEND', b'', b''] iend[3] = struct.pack('>I', crc32(iend[1]) & 0xffffffff) iend[0] = struct.pack('>I', len(iend[2])) return b''.join([ magic, b''.join(ihdr), b''.join(idat), b''.join(iend) ])
def write_delete(self, id): fd = self.get_write_fd() header = self.header_no_crc_fmt.pack(self.put_header_fmt.size, TAG_DELETE) crc = self.crc_fmt.pack(crc32(id, crc32(header)) & 0xffffffff) fd.write(b''.join((crc, header, id))) self.offset += self.put_header_fmt.size return self.segment
def iter_objects(self, segment, include_data=False): fd = self.get_fd(segment) fd.seek(0) if fd.read(8) != MAGIC: raise IntegrityError('Invalid segment header') offset = 8 header = fd.read(self.header_fmt.size) while header: crc, size, tag = self.header_fmt.unpack(header) if size > MAX_OBJECT_SIZE: raise IntegrityError('Invalid segment object size') rest = fd.read(size - self.header_fmt.size) if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc: raise IntegrityError('Segment checksum mismatch') if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT): raise IntegrityError('Invalid segment entry header') key = None if tag in (TAG_PUT, TAG_DELETE): key = rest[:32] if include_data: yield tag, key, offset, rest[32:] else: yield tag, key, offset offset += size header = fd.read(self.header_fmt.size)
def crc32(*args): """ .. function:: crc32(args) -> int Returns the CRC32 of args. Numbers are converted to text before hashing is performed. Examples: >>> sql("select crc32(65)") crc32(65) ---------- 2658551721 >>> sql("select crc32(6,5)") crc32(6,5) ---------- 1565899724 >>> sql("select crc32(5)") crc32(5) ---------- 2226203566 >>> sql("select crc32('5')") crc32('5') ---------- 1201448970 """ if len(args) == 1: return zlib.crc32(repr(args[0])) & 0xFFFFFFFF else: return zlib.crc32(chr(30).join([repr(x) for x in args])) & 0xFFFFFFFF
def read_chunk(self): """ Read a PNG chunk from the input file, return tag name and data. """ # http://www.w3.org/TR/PNG/#5Chunk-layout try: data_bytes, tag = struct.unpack('!I4s', self.file.read(8)) except struct.error: raise ValueError('Chunk too short for header') data = self.file.read(data_bytes) if len(data) != data_bytes: raise ValueError('Chunk %s too short for required %i data octets' % (tag, data_bytes)) checksum = self.file.read(4) if len(checksum) != 4: raise ValueError('Chunk %s too short for checksum', tag) verify = zlib.crc32(tag) verify = zlib.crc32(data, verify) verify = struct.pack('!i', verify) if checksum != verify: # print repr(checksum) (a,) = struct.unpack('!I', checksum) (b,) = struct.unpack('!I', verify) raise ValueError("Checksum error in %s chunk: 0x%X != 0x%X" % (tag, a, b)) return tag, data
def _read(self, fd, fmt, header, segment, offset, acceptable_tags): # some code shared by read() and iter_objects() try: hdr_tuple = fmt.unpack(header) except struct.error as err: raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format( segment, offset, err)) if fmt is self.put_header_fmt: crc, size, tag, key = hdr_tuple elif fmt is self.header_fmt: crc, size, tag = hdr_tuple key = None else: raise TypeError("_read called with unsupported format") if size > MAX_OBJECT_SIZE or size < fmt.size: raise IntegrityError('Invalid segment entry size [segment {}, offset {}]'.format( segment, offset)) length = size - fmt.size data = fd.read(length) if len(data) != length: raise IntegrityError('Segment entry data short read [segment {}, offset {}]: expected {}, got {} bytes'.format( segment, offset, length, len(data))) if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc: raise IntegrityError('Segment entry checksum mismatch [segment {}, offset {}]'.format( segment, offset)) if tag not in acceptable_tags: raise IntegrityError('Invalid segment entry header, did not get acceptable tag [segment {}, offset {}]'.format( segment, offset)) if key is None and tag in (TAG_PUT, TAG_DELETE): key, data = data[:32], data[32:] return size, tag, key, data
def test_service_creation_with_offset(self): b = BlockBackend("/dev/null", "test-1") blocks = b.create_info_blocks({"test1": 300, "test2": 512, "test3": 1024*1024*50}, first_free=1000) self.assertEqual(3, len(blocks)) test1 = struct.pack("!4sQ64pQQQQ", self.SIGNATURE, 1001, "test1", 1003, 1, 0, 0) test1crc = struct.pack("!L", zlib.crc32(test1) & 0xffffffff) test2 = struct.pack("!4sQ64pQQQQ", self.SIGNATURE, 1002, "test2", 1004, 1, 0, 0) test2crc = struct.pack("!L", zlib.crc32(test2) & 0xffffffff) test3 = struct.pack("!4sQ64pQQQQ", self.SIGNATURE, 0, "test3", 1005, 102400, 0, 0) test3crc = struct.pack("!L", zlib.crc32(test3) & 0xffffffff) expected = [ test1 + test1crc, test2 + test2crc, test3 + test3crc ] self.assertEqual(expected, blocks)
def test_get_services(self): raw1 = struct.pack("!4sQ64pQQQQQQ", self.SIGNATURE, 1, "test", 1, 100, 102, 100, 0, 0) raw1crc = struct.pack("!L", zlib.crc32(raw1) & 0xffffffff) raw2 = struct.pack("!4sQ64pQQQQQQ", self.SIGNATURE, 0, "test2", 2, 200, 202, 200, 0, 0) raw2crc = struct.pack("!L", zlib.crc32(raw2) & 0xffffffff) b = BlockBackend("/dev/null", "test-1") blockdev = io.BytesIO() blockdev.write(raw1) blockdev.write(raw1crc) blockdev.seek(b.blocksize) blockdev.write(raw2) blockdev.write(raw2crc) blockdev.seek(0) expected = ({'test': [(1, 100), (102, 100)], 'test2': [(2, 200), (202, 200)]}, 1, 402) services = b.get_services(blockdev) self.assertEqual(expected, services)
def _write_block(self, block): #print("Saving %i bytes" % len(block)) start_offset = self._handle.tell() assert len(block) <= 65536 #Giving a negative window bits means no gzip/zlib headers, -15 used in samtools c = zlib.compressobj(self.compresslevel, zlib.DEFLATED, -15, zlib.DEF_MEM_LEVEL, 0) compressed = c.compress(block) + c.flush() del c assert len(compressed) < 65536, "TODO - Didn't compress enough, try less data in this block" crc = zlib.crc32(block) #Should cope with a mix of Python platforms... if crc < 0: crc = struct.pack("<i", crc) else: crc = struct.pack("<I", crc) bsize = struct.pack("<H", len(compressed)+25) # includes -1 crc = struct.pack("<I", zlib.crc32(block) & 0xffffffffL) uncompressed_length = struct.pack("<I", len(block)) #Fixed 16 bytes, # gzip magic bytes (4) mod time (4), # gzip flag (1), os (1), extra length which is six (2), # sub field which is BC (2), sub field length of two (2), #Variable data, #2 bytes: block length as BC sub field (2) #X bytes: the data #8 bytes: crc (4), uncompressed data length (4) data = _bgzf_header + bsize + compressed + crc + uncompressed_length self._handle.write(data)
def compress(body, compress_level): """Compress 'body' at the given compress_level.""" import zlib # See http://www.gzip.org/zlib/rfc-gzip.html yield ntob("\x1f\x8b") # ID1 and ID2: gzip marker yield ntob("\x08") # CM: compression method yield ntob("\x00") # FLG: none set # MTIME: 4 bytes yield struct.pack("<L", int(time.time()) & int("FFFFFFFF", 16)) yield ntob("\x02") # XFL: max compression, slowest algo yield ntob("\xff") # OS: unknown crc = zlib.crc32(ntob("")) size = 0 zobj = zlib.compressobj(compress_level, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) for line in body: size += len(line) crc = zlib.crc32(line, crc) yield zobj.compress(line) yield zobj.flush() # CRC32: 4 bytes yield struct.pack("<L", crc & int("FFFFFFFF", 16)) # ISIZE: 4 bytes yield struct.pack("<L", size & int("FFFFFFFF", 16))
def iter_objects(self, segment, include_data=False): fd = self.get_fd(segment) fd.seek(0) if fd.read(MAGIC_LEN) != MAGIC: raise IntegrityError('Invalid segment magic') offset = MAGIC_LEN header = fd.read(self.header_fmt.size) while header: try: crc, size, tag = self.header_fmt.unpack(header) except struct.error as err: raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err)) if size > MAX_OBJECT_SIZE: raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset)) length = size - self.header_fmt.size rest = fd.read(length) if len(rest) != length: raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format( offset, length, len(rest))) if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc: raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset)) if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT): raise IntegrityError('Invalid segment entry tag [offset {}]'.format(offset)) key = None if tag in (TAG_PUT, TAG_DELETE): key = rest[:32] if include_data: yield tag, key, offset, rest[32:] else: yield tag, key, offset offset += size header = fd.read(self.header_fmt.size)
def decrypt(cls, string, secret_key): data = super(SecureEncryptedCookie, cls).decrypt(string, secret_key) data, crc1 = data[:-4], data[-4:] crc2 = zlib.crc32(data, zlib.crc32(secret_key)) if crc1 != struct.pack('>I', crc2 & 0xffffffff): return b'' return data
def fileobj_to_generator(fileobj, bufsize=8192, gzipped=False): assert hasattr(fileobj, 'read') if not gzipped: while 1: data = fileobj.read(bufsize) if not data: fileobj.close() break else: yield data else: compressobj = zlib.compressobj(zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) crc = zlib.crc32('') size = 0 yield '\037\213\010\000' '\0\0\0\0' '\002\377' while 1: data = fileobj.read(bufsize) if not data: break crc = zlib.crc32(data, crc) size += len(data) zdata = compressobj.compress(data) if zdata: yield zdata zdata = compressobj.flush() if zdata: yield zdata yield struct.pack('<LL', crc&0xFFFFFFFFL, size&0xFFFFFFFFL)
def _read(self, readsize): """Bug was here with bad EOF signal""" data = self.fileobj.read(readsize) is_eof = True while True: if data == "": decompdata = self.decompobj.flush() else: decompdata = self.decompobj.decompress(data) decomplen = len(decompdata) self.buffer.append(decompdata) self.bufferlen += decomplen self.size += decomplen if decomplen: is_eof = False self.crcval = zlib.crc32(decompdata, self.crcval) if self.decompobj.unused_data: data = self._read_eof() self.decompobj = zlib.decompressobj(-zlib.MAX_WBITS) self.crcval = zlib.crc32("") self.size = 0 if data: continue break return is_eof
def decompress(input, type=None): if type == None: type = guesstype(input) if type == 'gzip': magic1, magic2, method, flags, mtime, xf, os = unpack('<BBBBIBB', input[:10]) if magic1 != 0x1F or magic2 != 0x8B: raise IOError('Not a gzipped file') if method != 8: raise IOError('Unknown compression method') if flags & 0xE0: raise IOError('Unknown flags') off = unpack('<H', input[10:12])[0] + 12 if flags & FEXTRA else 10 if flag & FNAME: off = input.index('\x00', off) + 1 if flag & FCOMMENT: off = input.index('\x00', off) + 1 if flags & FHCRC: if unpack('<H', input[off:off+2])[0] != (crc32(input[:off]) & 0xffff): raise IOError('Header corrupted') off += 2 crc32, isize = unpack('<II', input[-8:]) s = zdecompress(input[off:-8], -MAX_WBITS, isize) checksum = crc32(s) if crc32 != checksum: raise IOError("CRC32 check failed %08x != %08x" % (crc32, checksum)) if isize != (len(s) & 0xffffffffL): raise IOError("Incorrect length of data produced") return s elif type == 'zlib': header = unpack('>H', input[:2])[0] method = (header >> 8) & 0xF windowsize = ((header >> 12) & 0xF) + 8 fdict = (header & 0x20) != 0 if method != 8 or windowsize > MAX_WBITS or fdict: raise IOError('Unknown compression method') if header % 31 != 0: raise IOError('Header corrupted') s = zdecompress(input[2:-4], -windowsize) a32 = unpack('>I', input[-4:])[0] checksum = adler32(s) if a32 != checksum: raise IOError("Adler32 check failed %08x != %08x" % (a32, checksum)) return s elif type == 'deflate': return zdecompress(input) else: raise ValueError('Compression type must be one of deflate, zlib, gzip, or None')
def _write_box(self, name, data, length): if length > 0: self._writes.write(struct.pack('!L4s%dsl' % length, length, name, data, zlib.crc32(name+data))) else: self._writes.write(struct.pack('!L4sl', 0, name, zlib.crc32(name)))
def chunk(tag, data): return [ struct.pack("!I", len(data)), tag, data, struct.pack("!I", 0xFFFFFFFF & zlib.crc32(data, zlib.crc32(tag))) ]
def compress(chunks, compress_level, close=True): """ Compress 'chunks' at the given compress_level, where 'chunks' is an iterable over chunks of bytes. If close=True, then look for .close() method on chunks and call that when done iterating. """ try: # See http://www.gzip.org/zlib/rfc-gzip.html yield '\x1f\x8b' # ID1 and ID2: gzip marker yield '\x08' # CM: compression method yield '\x00' # FLG: none set # MTIME: 4 bytes yield struct.pack("<L", int(time.time()) & int('FFFFFFFF', 16)) yield '\x02' # XFL: max compression, slowest algo yield '\xff' # OS: unknown crc = zlib.crc32("") size = 0 zobj = zlib.compressobj(compress_level, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) for chunk in chunks: size += len(chunk) crc = zlib.crc32(chunk, crc) yield zobj.compress(chunk) yield zobj.flush() # CRC32: 4 bytes yield struct.pack("<L", crc & int('FFFFFFFF', 16)) # ISIZE: 4 bytes yield struct.pack("<L", size & int('FFFFFFFF', 16)) finally: if close and hasattr(chunks, 'close'): chunks.close()
def calcsum(objeto,tipoobjeto,algoritmo): valorhash = 0 if algoritmo == 'crc32': if tipoobjeto == 't': valorhash = zlib.crc32(objeto) else: fh = open(objeto,'rb') # Abre lectura en modo binario for linea in fh: valorhash = zlib.crc32(linea, valorhash) valorhash = "%X"%(valorhash & 0xFFFFFFFF) #Almacena el valor hash en hexadecimal else: if algoritmo == 'sha256': m = hashlib.sha256() elif algoritmo == 'sha1': m = hashlib.sha1() else: m = hashlib.md5() if tipoobjeto == 't': m.update(objeto) else: fh = open(objeto, 'rb') #Abre lectura en modo binario while True: data = fh.read(8192) #Lee el archivo en bloques de 8Kb if not data: break m.update(data) fh.close valorhash = m.hexdigest() return valorhash #Devuelve el valor hash en hexadecimal
def read_chunk(self): """ Read a PNG chunk from the input file, return tag name and data. """ # http://www.w3.org/TR/PNG/#5Chunk-layout try: data_bytes, tag = struct.unpack('!I4s', self.file.read(8)) except struct.error: raise ValueError('Chunk too short for header') data = self.file.read(data_bytes) if len(data) != data_bytes: raise ValueError('Chunk %s too short for required %i data octets' % (tag, data_bytes)) checksum = self.file.read(4) if len(checksum) != 4: raise ValueError('Chunk %s too short for checksum', tag) verify = zlib.crc32(tag) verify = zlib.crc32(data, verify) # Whether the output from zlib.crc32 is signed or not varies # according to hideous implementation details, see # http://bugs.python.org/issue1202 . # We coerce it to be positive here (in a way which works on # Python 2.3 and older). verify &= 2**32 - 1 verify = struct.pack('!I', verify) if checksum != verify: # print repr(checksum) (a,) = struct.unpack('!I', checksum) (b,) = struct.unpack('!I', verify) raise ValueError("Checksum error in %s chunk: 0x%X != 0x%X" % (tag, a, b)) return tag, data
def fix_png(data): """ Fix the signature and checksums on a fuzzed PNG image. """ out = [b"\x89PNG\r\n\x1A\n"] data = bytes(data[8:]) chunk = 0 while len(data) >= 8: chunklen = data[:4] out.append(chunklen) chunklen = struct.unpack("!I", chunklen)[0] if chunk == 0: chunkname = b"IHDR" # make sure the first tag is correct else: chunkname = data[4:8] #chunkname = bytes(_coerce_ascii(c) for c in data[4:8]) out.append(chunkname) data = data[8:] if len(data) < chunklen: break else: chunkdata = data[:chunklen] chunkcrc = zlib.crc32(chunkname) & 0xFFFFFFFF chunkcrc = zlib.crc32(chunkdata, chunkcrc) & 0xFFFFFFFF out.append(chunkdata) out.append(struct.pack("!I", chunkcrc)) data = data[chunklen+4:] # skip the old crc chunk += 1 out.append(data) return b"".join(out)
def patch(orig_fname, diff_fname, dest_fname, crc_offset): # crcoffset = 0x044E80 # Iphone 3Gs - 5.0.1 crcoffset = 0x0409A9 - 4 # Ipad 1 - 5.0.1 # crcoffset = 0x042170 - 4 # Galaxy tab - ?? # read original firmare. origdata = '' with open(orig_fname, "rb") as f: origdata = f.read() if pack("<l", crc32(origdata[:crc_offset]))[:4] != origdata[crc_offset:crc_offset+4]: raise Exception("checksum mismatch!") print "checksum ok!" # read list of changes to do. changes = readDiff(diff_fname) # apply changes. newdata = origdata[:] for offset, before, after in changes: if origdata[offset] != chr(before): raise Exception("data mismatch at %x expecting %x found %x" % (offset, before, origdata[offset])) newdata = newdata[:offset] + chr(after) + newdata[offset+1:] # fix checksum newchecksum = pack("<l", crc32(newdata[:crc_offset]))[:4] newdata = newdata[:crc_offset] + newchecksum + newdata[crc_offset+4:] # write new file with open(dest_fname, "wb") as f: f.write(newdata)
def compress(body, compress_level): """Compress 'body' at the given compress_level.""" # Header yield b"\037\213\010\0" \ + struct.pack("<L", int(time.time())) \ + b"\002\377" size = 0 crc = zlib.crc32(b"") zobj = zlib.compressobj( compress_level, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0, ) for chunk in body: if not isinstance(chunk, bytes): chunk = chunk.encode("utf-8") size += len(chunk) crc = zlib.crc32(chunk, crc) yield zobj.compress(chunk) yield zobj.flush() \ + struct.pack("<l", crc) \ + struct.pack("<L", size & 0xFFFFFFFF)
def crc32(data): if DEBUG: print '++++++ CRC32 ++++++' print 'input: ' + data print 'crc32: ' + hex(zlib.crc32(data) & 0xffffffff) print '+++++++++++++++++++' return hex(zlib.crc32(data) & 0xffffffff) # crc32 returns a signed value, &-ing it will match py3k
def main(arg): with open("C:/Users/user/Documents/Visual Studio 2013/Projects/pCTF2015/forensics/150/test_crc_binary_second_byte.bin", "rb") as fi: data = fi.read() b = bytearray(data) for x in range(0, 256): # data[0] = 0x00 b[len(data) - 1] = x d = binascii.a2b_hex(binascii.hexlify(b[4:0x20004])) # calculate tag crc tag_crc = binascii.crc32(b[0:4]) item = ctypes.c_ulong(zlib.crc32(d, tag_crc)) item2 = ctypes.c_ulong(zlib.crc32(d)) # print "0x%s"%binascii.hexlify(d) final_crc = "%x" % item.value # if arg in final_crc: print "%s" % binascii.hexlify(d[len(d) - 4:len(d)]) print "final crc is 0x%x" % (item.value) # print "final crc without tag is 0x%x"%(item2.value) # print "%s"%binascii.hexlify(d[0x1fffc:0x20000]) # b.append(data) # print "0x%s"%binascii.hexlify(b) print "data length is 0x%x" % (len(data) - 4) pass
def decode(filName): # read file as binary filObj = open(filName, 'rb') filCon = filObj.read() filObj.close() # get file size to forecast output array statinfo = os.stat(filName) # process file header setWidth = struct.unpack('B', filCon[:1]) setNames = [] for ii in range(0, setWidth[0]): setNames.append(filCon[ii*5+1:ii*5+6]) print("[CRC] of file header:", end="") crcVal = crc32(filCon[0:setWidth[0]*5+1+4]) & 0xffffffff crcErrors = 0 if ( crcVal == 0xffffffff): print("\tOK\t["+hex(crcVal)+"]") else: print("\tERROR\t["+hex(crcVal)+"]") crcErrors += 1 offset = setWidth[0]*5+5 # process data sets setCon = np.zeros(statinfo.st_size // 4) idx = 0 fmtStr = "" setBytes = 0 for setName in setNames: fmtStr += chr(setName[0]) setBytes += fmtChars[chr(setName[0])] while(offset < len(filCon)): setNumber = struct.unpack('B', filCon[offset:offset+1]) offset += 1 for ii in range(setNumber[0]): setCon[idx:idx+setWidth[0]] = np.array(struct.unpack(fmtStr, filCon[offset:setBytes+offset])) offset += setBytes idx += setWidth[0] crcVal = crc32(filCon[offset-setBytes*setNumber[0]-1:offset+4]) & 0xffffffff print("[CRC] of data set:", end="") if ( crcVal == 0xffffffff): print("\tOK\t["+hex(crcVal)+"]") else: print("\tERROR\t["+hex(crcVal)+"]") crcErrors += 1 offset += 4 if (not crcErrors): print("[CRC] no errors occurred:\tOK") else: print("[CRC] {0} errors occurred:\tERROR".format(crcErrors)) # remove not required elements and reshape as matrix setCon = np.reshape(setCon[0:idx], (setWidth[0], idx//setWidth[0]), 'f') # create output dictionary output = {} for ii in range(setWidth[0]): output[setNames[ii][1:].decode("utf-8").strip()] = setCon[ii] return output
def get_crc32(data): '''Returns the crc32 value of the input string. ''' if PY_VER_2: return crc32(data) strbytes = bytes(data, encoding='UTF-8') return crc32(strbytes)
def patch(orig_fname, diff_fname, dest_fname, crc_offset): """TODO""" # Read original firmare. origdata = '' with open(orig_fname, "rb") as f: origdata = f.read() checksum = pack("<l", crc32(origdata[:crc_offset]))[:4] if checksum != origdata[crc_offset:crc_offset+4]: raise Exception("Checksum mismatch!") print "Checksum ok!" # Read list of changes to do. changes = readDiff(diff_fname) # Apply changes. newdata = origdata[:] for offset, before, after in changes: if origdata[offset] != chr(before): byte = origdata[offset] msg = "Data mismatch at %X expecting %X found %X" % (offset, before, byte) raise Exception(msg) newdata = newdata[:offset] + chr(after) + newdata[offset+1:] # Fix checksum newchecksum = pack("<l", crc32(newdata[:crc_offset]))[:4] newdata = newdata[:crc_offset] + newchecksum + newdata[crc_offset+4:] # Write new file with open(dest_fname, "wb") as f: f.write(newdata)
def pack(self): b1 = struct.pack('>BIIIIIH', self.flag, self.window, self.seq, self.ack, self.sndtime, self.acktime, len(self.content)) crc = zlib.crc32(b1) if len(self.content) > 0: crc = zlib.crc32(self.content, crc) b2 = struct.pack('>H', crc & 0xffff) return b1 + b2 + self.content
def get_category_info_version(): return zlib.crc32(get_category_info_string().encode('utf-8')) & 0x7ffffff
def delServer(self,ip,port): str = ip + port index = zlib.crc32(str) del self.servers[index]
ucode_blob_sz = ucode_size & ~0xfff ucode_jt_sz = ucode_size & 0xfff ucode_blob = ucode[:ucode_blob_sz] ucode_jt = ucode[ucode_blob_sz:] print "have: 0x%x / 0x%x / 0x%x (total, blob, jt)" % ( ucode_size, ucode_blob_sz, ucode_jt_sz) if ucode_blob_sz < want_blob_sz: ucode_blob += "\x00" * (want_blob_sz - ucode_blob_sz) elif ucode_blob_sz > want_blob_sz: ucode_blob = ucode_blob[:want_blob_sz] if ucode_jt_sz < want_jt_sz: ucode_jt += ucode_jt[-4:] * ((want_jt_sz - ucode_jt_sz) / 4) elif ucode_jt_sz > want_jt_sz: ucode_jt = ucode_jt[:want_jt_sz] sub_hdr = struct.pack("<III212x", ucode_feature_version, want_blob_sz / 4, want_jt_sz / 4) payload = sub_hdr + ucode_blob + ucode_jt assert len(payload) == want_size + 0xe0 crc = zlib.crc32(payload) hdr = struct.pack("<I16sIII", len(payload) + 0x20, p1, want_size, 0x100, crc & 0xffffffff) with open(sys.argv[3], "wb") as fd: fd.write(hdr) fd.write(payload)
def advisory_lock(lock_id, shared=False, wait=True, using=None): import six from django.db import DEFAULT_DB_ALIAS, connections, transaction if using is None: using = DEFAULT_DB_ALIAS # Assemble the function name based on the options. function_name = 'pg_' if not wait: function_name += 'try_' function_name += 'advisory_lock' if shared: function_name += '_shared' release_function_name = 'pg_advisory_unlock' if shared: release_function_name += '_shared' # Format up the parameters. tuple_format = False if isinstance(lock_id, ( list, tuple, )): if len(lock_id) != 2: raise ValueError( "Tuples and lists as lock IDs must have exactly two entries.") if not isinstance(lock_id[0], six.integer_types) or not isinstance( lock_id[1], six.integer_types): raise ValueError( "Both members of a tuple/list lock ID must be integers") tuple_format = True elif isinstance(lock_id, six.string_types): # Generates an id within postgres integer range (-2^31 to 2^31 - 1). # crc32 generates an unsigned integer in Py3, we convert it into # a signed integer using 2's complement (this is a noop in Py2) pos = crc32(lock_id.encode("utf-8")) lock_id = (2**31 - 1) & pos if pos & 2**31: lock_id -= 2**31 elif not isinstance(lock_id, six.integer_types): raise ValueError("Cannot use %s as a lock id" % lock_id) if tuple_format: base = "SELECT %s(%d, %d)" params = ( lock_id[0], lock_id[1], ) else: base = "SELECT %s(%d)" params = (lock_id, ) acquire_params = (function_name, ) + params command = base % acquire_params cursor = connections[using].cursor() cursor.execute(command) if not wait: acquired = cursor.fetchone()[0] else: acquired = True try: yield acquired finally: if acquired: release_params = (release_function_name, ) + params command = base % release_params cursor.execute(command) cursor.close()
print(aux) if (aux): col += 1 print('\nCONTRASEÑAS ROTAS HASTA AHORA = %d' % col) cr += aux else: h = p for i in range(pf): if (i % 2 == 0): r = str(h % 1000000) else: r = r2(h) h = crc32(bytes(r, 'ascii')) if ((h in hashes) and (h not in cr)): print('\nContraseña rota al %d intento en el bucle 1' % (i + 1)) col += 1 aux = [v for v in hashes if v == h] cr += aux print('\nCONTRASEÑAS ROTAS HASTA AHORA = %d' % col) break if (not aux): h = p for i in range(pf): if (i % 2 == 0): r = r2(h) else:
def simple_hash(s): # & 0xffffffff avoids python2/python3 compatibility return zlib.crc32(s.encode()) & 0xffffffff
def process(self, rec_data, result): result[self.mName] = crc32(bytes(result[self.mBaseName], 'utf-8')) % (1 << 32)
import zlib # TODO(davedash): liberate this def manual_order(qs, pks, pk_name='id'): """ Given a query set and a list of primary keys, return a set of objects from the query set in that exact order. """ if not pks: return qs.none() objects = qs.filter(id__in=pks).extra(select={ '_manual': 'FIELD(%s, %s)' % (pk_name, ','.join(map(str, pks))) }, order_by=['_manual']) return objects crc32 = lambda x: zlib.crc32(x) & 0xffffffff
def file_checksum(self, file_path): """ Generate checksum from file. """ with open(file_path, "rb") as in_file: return zlib.crc32(in_file.read())
def _get_server(self, key): # Naive, low-performance implementation assert self._ips, "_find_server should not have been called with self._ips == []" return self._ips[zlib.crc32(key) % len(self._ips)]
def png_pack(png_tag, data): chunk_head = png_tag + data return struct.pack("!I", len(data)) + chunk_head + struct.pack( "!I", 0xFFFFFFFF & zlib.crc32(chunk_head))
def _ranged_scan_and_read(infile, start, end, buffer=collections.deque()): assert start < end, "must span at least one byte!" infile.seek(start, os.SEEK_SET) # read the bgzip header # see https://samtools.github.io/hts-specs/SAMv1.pdf pattern = "<BBBBIBBHBBHH" patternsize = struct.calcsize(pattern) got_content = False block_start = start - len(buffer) while block_start < end: block_start_next = block_start if len(buffer) < patternsize: bytesread = infile.read(patternsize - len(buffer)) buffer.extend(bytesread) if len(buffer) < patternsize: logger.warning(f"Unable to read up to {patternsize} at {block_start}") break headerbuffer = bytes((buffer[i] for i in range(patternsize))) header = struct.unpack(pattern, headerbuffer) logger.debug( f"Header at {block_start} is {header} or in raw bytes {headerbuffer}" ) if check_bytes_header(header): logger.debug(f"Matched header at {block_start} {header}") # this is a valid location for a block blocksize = header[11] - header[7] - 19 logger.debug(f"Block at {block_start} has cdata size = {blocksize}") cdata = infile.read(blocksize) assert ( len(cdata) == blocksize ), f"Unable to read up to {blocksize} of cdata at {block_start}" buffer.clear() # now do the actual decompression decompressor = zlib.decompressobj( wbits=-15 ) # we've alread read the header, so ignore it decompressed = decompressor.decompress(cdata) assert ( not decompressor.unconsumed_tail ), f"unconsumed tail of {len(decompressor.unconsumed_tail)} at {block_start}" assert ( not decompressor.unused_data ), f"unused data present of {len(decompressor.unused_data)} at {block_start}" # read isize and crc check tailpattern = "<II" tailpatternsize = struct.calcsize(tailpattern) tailbytes = infile.read(tailpatternsize) if len(tailbytes) != tailpatternsize: raise ValueError( f"Unable to read {tailpatternsize} bytes for tail at {block_start}" ) tail_crc, tail_isize = struct.unpack(tailpattern, tailbytes) # check decompressed size is expected assert len(decompressed) == tail_isize # check crc check is expected assert zlib.crc32(decompressed) == tail_crc # last block has no compressed content if decompressed: got_content = True yield decompressed else: logger.warning(f"Found empty block at {block_start}") break logger.debug(f"Read block from {block_start} to {infile.tell()}") block_start_next += patternsize + blocksize + tailpatternsize else: # move ahead a byte buffer.popleft() block_start_next += 1 block_start = block_start_next logger.debug(f"End of scan from {start} to {end} at {block_start}-{infile.tell()}") if not got_content: logger.warning(f"Got to end of {start}-{end} without finding a block")
def query_performance_express_report(code, start_date=None, end_date=None): """公司业绩快报。 @param code: 证券代码,不可为空 @param start_date: 开始日期,默认2015-01-01;发布日期或更新日期在这个范围内。 @param end_date: 结束日期,默认系统当前日期;发布日期或更新日期在这个范围内。 """ data = rs.ResultData() if code is None or code == "": print("股票代码不能为空,请检查。") data.error_msg = "股票代码不能为空,请检查。" data.error_code = cons.BSERR_PARAM_ERR return data if len(code) != cons.STOCK_CODE_LENGTH: print("股票代码应为" + str(cons.STOCK_CODE_LENGTH) + "位,请检查。格式示例:sh.600000。") data.error_msg = "股票代码应为" + str( cons.STOCK_CODE_LENGTH) + "位,请检查。格式示例:sh.600000。" data.error_code = cons.BSERR_PARAM_ERR return data code = code.lower() if (code.endswith("sh") or code.endswith("sz")): code = code[7:9].lower() + "." + code[0:6] if start_date is None or start_date == "": start_date = cons.DEFAULT_START_DATE if end_date is None or end_date == "": end_date = time.strftime("%Y-%m-%d", time.localtime()) if start_date != "" and start_date is not None and end_date != "" and end_date is not None: if strUtil.is_valid_date(start_date) and strUtil.is_valid_date( end_date): start_date_time = datetime.datetime.strptime( start_date, '%Y-%m-%d') end_date_time = datetime.datetime.strptime(end_date, '%Y-%m-%d') if end_date_time < start_date_time: print("起始日期大于终止日期,请修改。") data.error_code = cons.BSERR_START_BIGTHAN_END data.error_msg = "起始日期大于终止日期,请修改。" return data else: print("日期格式不正确,请修改。") return user_id = "" try: user_id = getattr(conx, "user_id") except Exception: print("you don't login.") data.error_code = cons.BSERR_NO_LOGIN data.error_msg = "you don't login." return data param = "%s,%s,%s,%s,%s,%s,%s" % ( "query_performance_express_report", user_id, "1", cons.BAOSTOCK_PER_PAGE_COUNT, code, start_date, end_date) msg_body = strUtil.organize_msg_body(param) msg_header = msgheader.to_message_header( cons.MESSAGE_TYPE_QUERYPERFORMANCEEXPRESSREPORT_REQUEST, len(msg_body)) data.msg_type = cons.MESSAGE_TYPE_QUERYPERFORMANCEEXPRESSREPORT_REQUEST data.msg_body = msg_body head_body = msg_header + msg_body crc32str = zlib.crc32(bytes(head_body, encoding='utf-8')) receive_data = sock.send_msg(head_body + cons.MESSAGE_SPLIT + str(crc32str)) if receive_data is None or receive_data.strip() == "": data.error_code = cons.BSERR_RECVSOCK_FAIL data.error_msg = "网络接收错误。" return data msg_header = receive_data[0:cons.MESSAGE_HEADER_LENGTH] msg_body = receive_data[cons.MESSAGE_HEADER_LENGTH:-1] header_arr = msg_header.split(cons.MESSAGE_SPLIT) body_arr = msg_body.split(cons.MESSAGE_SPLIT) data.msg_body_length = header_arr[2] data.error_code = body_arr[0] data.error_msg = body_arr[1] if cons.BSERR_SUCCESS == data.error_code: data.method = body_arr[2] data.user_id = body_arr[3] data.cur_page_num = body_arr[4] data.per_page_count = body_arr[5] data.setData(body_arr[6]) data.code = body_arr[7] data.start_date = body_arr[8] data.end_date = body_arr[9] data.setFields(body_arr[10]) return data
def CreateHashValue(self, hashString): return zlib.crc32(hashString.encode()) # % (1<<32)
def prepared(self): finished = base64.b64decode(self.stitched()) prev = 0 prev = zlib.crc32(finished, prev) text_crc = "%x"%(prev & 0xFFFFFFFF) return finished, bool(self.crc == text_crc)
def getCRC32(fileName): prev = 0 for eachLine in open(fileName,"rb"): prev = zlib.crc32(eachLine, prev) return "%X"%(prev & 0xFFFFFFFF)
def _init_read(self): self._crc = zlib.crc32('') self._stream_size = 0
def main(): def file_rel(fullpath: str) -> str: ''' Returns relative path ''' return os.path.relpath(fullpath, FOLDER) # Get path from command-line or clipboard: try: FOLDER = argv[1] except IndexError: FOLDER = paste() except: raise # Check if folder looks like path: if FOLDER.find(':\\') != 1: input(f'Wrong folder:\r\n\r\n{FOLDER}\n\nPress Enter to exit') exit() print(f'Searching for duplicates in {FOLDER}\n') alg = int( input('Algorythm:\n' + '0 - Cancel\n' + '1 - By size\n' + f'2 - By hash of first {HASH_SIZE_PERCENT} percent of file\n' + '3 - By hash of full file\n\n' + 'Choice: ')) if alg == 0: return dups = [] dups_final = [] files = {} hashes = [] pathlist = Path(FOLDER).rglob('*') widgets = ['Processed: ', Counter(), ' files (', Timer(), ')'] pbar = ProgressBar(widgets=widgets) for fi in pbar(pathlist): # Skip folders: if fi.is_file(): filesize = fi.stat().st_size if filesize in files: # print(file_rel(str(fi))) # Collect all files with known size: dups.append(str(fi)) else: # Collect unique file size and file: files.update({filesize: str(fi)}) # Now we have list and dictionary: # {files} - files with unique size # [dups] - files with duplicate sizes print('\nDuplicates by size:') print(*list(map(file_rel, dups)), sep='\n') print(f'Total: {len(dups)}') if alg == 1: # We already have duplicates dups_final = dups elif alg == 2: print('Now compare hashes') # First HASH_SIZE_PERCENT of files widgets = ['Processed: ', Counter(), ' files (', Timer(), ')'] pbar = ProgressBar(widgets=widgets) for fi in pbar(dups): file_size = os.stat(fi).st_size read_limit = int(file_size / 100 * HASH_SIZE_PERCENT) with open(fi, 'rb') as f: hash_dup = crc32(f.read(read_limit)) & 0xFFFFFFFF if hash_dup in hashes: dups_final.append(fi) else: # Calculate hash of file with same size from {files} with open(files[file_size], 'rb') as f: hash_prev = crc32(f.read(read_limit)) & 0xFFFFFFFF hashes.append(hash_prev) if hash_dup == hash_prev: # It's the same file: dups_final.append(fi) else: # Same size but different file: hashes.append(hash_dup) elif alg == 3: # By hash of full file: widgets = ['Processed: ', Counter(), ' files (', Timer(), ')'] pbar = ProgressBar(widgets=widgets) for fi in pbar(dups): file_size = os.stat(fi).st_size with open(fi, 'rb') as f: hash_dup = crc32(f.read()) & 0xFFFFFFFF if hash_dup in hashes: dups_final.append(fi) else: # Calculate hash of file with same size from {files} with open(files[file_size], 'rb') as f: hash_prev = crc32(f.read()) & 0xFFFFFFFF hashes.append(hash_prev) if hash_dup == hash_prev: # It's the same file: dups_final.append(fi) else: # Same size but different file: hashes.append(hash_dup) else: print('WTF?') # Final list of duplicates: print('\nFinal list of duplicates:') print(*list(map(file_rel, dups_final)), sep='\n') print(f'Final total: {len(dups_final)}') MessageBox(None, 'Search for duplicates is complete', MSG_TITLE, 64 + 4096) if len(dups_final) > 0: if input('\nDelete duplicates? (Y/n): ') == 'Y': for dup in dups_final: try: os.remove(dup) except: print(f'Can\'t delete file: {dup}') raise else: exit()
from zlib import crc32 import numpy as np if __name__ == '__main__': mail = input('enter phystech e-mail\n') to_hash = mail.strip().split('@')[0] h = crc32(to_hash.encode('utf-8')) + 127 seed = h % (2**32 - 1) rs = np.random.RandomState(seed) task1 = rs.randint(low=1, high=5) task2 = rs.randint(low=1, high=5) print('Your tasks are 1.{}, 2.{}'.format(task1, task2))
def _add_read_data(self, data): self._crc = zlib.crc32(data, self._crc) self._stream_size = self._stream_size + len(data)
#!/usr/bin/env python # Demo program for zlib; it compresses or decompresses files, but *doesn't* # delete the original. This doesn't support all of gzip's options. FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 def write32(output, value): output.write(chr(value & 255)) value = value / 256 output.write(chr(value & 255)) value = value / 256 output.write(chr(value & 255)) value = value / 256 output.write(chr(value & 255)) def read32(input): v = ord(input.read(1)) v = v + (ord(input.read(1)) << 8) v = v + (ord(input.read(1)) << 16) v = v + (ord(input.read(1)) << 24) return v import zlib, sys if len(sys.argv) != 2: print 'Usage: minigzip.py <filename>' print ' The file will be compressed or decompressed.' sys.exit(0) filename = sys.argv[1] compressing = 1 outputname = filename + '.gz' if filename[-3:] == '.gz':
# Not meaningful but does what is expected. # Date compression. import zlib # gzip lzma bz2 tarfile s = b'witch which has which witches wrist watch' len(s) # Now compress compressed = zlib.compress(s) len(compressed) zlib.decompress(compressed) # Perform a cyclic redundancy check. crc32 zlib.crc32(s) # Compute speed from timeit import Timer Timer('t=a; a=b; b=t', 'a=1; b=2').timeit() # use pstats to find time critical sections. # Quality Control. # Write tests, test them with doctest def avg(*values): '''Returns the mean of a list of values
def _monitor_recv(self): clk = RisingEdge(self.clock) self._pkt = "" while True: yield clk ctrl, bytes = self._get_bytes() if ctrl[0] and bytes[0] == _XGMII_START: ctrl, bytes = ctrl[1:], bytes[1:] while self._add_payload(ctrl, bytes): yield clk ctrl, bytes = self._get_bytes() elif self.bytes == 8: if ctrl[4] and bytes[4] == _XGMII_START: ctrl, bytes = ctrl[5:], bytes[5:] while self._add_payload(ctrl, bytes): yield clk ctrl, bytes = self._get_bytes() if self._pkt: self.log.debug("Received:\n%s" % (hexdump(self._pkt))) if len(self._pkt) < 64 + 7: self.log.error("Received a runt frame!") if len(self._pkt) < 12: self.log.error("No data to extract") self._pkt = "" continue preamble_sfd = self._pkt[0:7] crc32 = self._pkt[-4:] payload = self._pkt[7:-4] if preamble_sfd != _PREAMBLE_SFD: self.log.error("Got a frame with unknown preamble/SFD") self.log.error(hexdump(preamble_sfd)) self._pkt = "" continue expected_crc = struct.pack("<I", (zlib.crc32(payload) & 0xFFFFFFFF)) if crc32 != expected_crc: self.log.error("Incorrect CRC on received packet") self.log.info("Expected: %s" % (hexdump(expected_crc))) self.log.info("Received: %s" % (hexdump(crc32))) # Use scapy to decode the packet if _have_scapy: p = Ether(payload) self.log.debug("Received decoded packet:\n%s" % p.show2()) else: p = payload self._recv(p) self._pkt = ""
def test_set_check(identifier, test_ratio): return crc32(np.int64(identifier)) & 0xffffffff < test_ratio * 2**32
def read_png2(stream): if not stream.read(8) == b'\x89PNG\r\n\x1a\n': stream.seek(0, 0) # 重置数据流 return # 读取需要的数据 _width = _height = _depth = _color_type = _interlace = _pixel_bytes = \ _palette = _background = _extra_alpha = None _data = [] while True: try: _length, _mask = struct.unpack('!I4s', stream.read(8)) _chunk = stream.read(_length) _crc32 = struct.unpack('!I', stream.read(4))[0] if zlib.crc32(_mask + _chunk) != _crc32: break if _mask == _HEADER: (_width, _height, _depth, _color_type, _compress_method, _filter_method, _interlace) = struct.unpack('!2I5B', _chunk) _planes = (1, -1, 3, 1, 2, -1, 4)[_color_type] _pixel_bytes = (_depth * _planes + 7) // 8 elif _mask == _PALETTE: _palette = tuple(_chunk[_i:_i + 3] for _i in range(0, _length, 3)) elif _mask == _DATA: _data.append(_chunk) elif _mask == _END: break elif _mask == b'tRNS': # 透明信息 if _color_type == _COLOR_TYPE_GRAY: _extra_alpha = _chunk[1] # 只取一半 elif _color_type == _COLOR_TYPE_RGB: # 只取一半 _extra_alpha = _chunk[1], _chunk[3], _chunk[5] if _color_type == _COLOR_TYPE_PALETTE: # 这时候是一个 alpha table _length = len(_chunk) _extra_alpha = tuple(_chunk[_i] if _i < _length else 255 for _i in range(256)) elif _mask == b'bKGD': if _color_type in (_COLOR_TYPE_GRAY, _COLOR_TYPE_GRAY_ALPHA): _background = _chunk[1] elif _color_type in (_COLOR_TYPE_RGB, _COLOR_TYPE_RGB_ALPHA): _background = _chunk[1], _chunk[3], _chunk[5] elif _color_type == _COLOR_TYPE_PALETTE: _background = ord(_chunk) # 指向 _palette 不过这里写做代替 except struct.error: break # LZ77 解压 _decompress_obj = zlib.decompressobj() _unzip_data = itertools.chain( *(_decompress_obj.decompress(_chunk) for _chunk in _data), _decompress_obj.flush()) # 按行分割,_adam 表示 pass 提取算法得到的缩小图的每行像素个数 _adam = (_adam7iter if _interlace else _adam1iter)(_width, _height) _reduced_images = tuple([] for _ in range(7 if _interlace else 1)) # 容器 _current_image = 0 # 不要这个变量也能正常运行 while True: try: _line_bytes = next(_adam) if _line_bytes > 0: # 大于 0 表示读取长度 _reduced_images[_current_image].append( bytearray( # 这是一个从迭代器中依次读取数量个数的方法 # 下面的意思是从 _unzip_data 里读取一定数量的字节 # 这个字节长度是缩小图每行的字节长度 +1,多出来的是滤波标记 _read_iter(_unzip_data, _line_bytes * _pixel_bytes + 1))) else: # 小于等于 0 表示切换图片,注意缩小图是 7 张 _current_image = abs(_line_bytes) except StopIteration: break # 滤波重构 _un_filter_images = ( # 把多个图片的打包在一起 # 把一个图片的行打包在一起 tuple(_un_filter_image_lines(_image_lines, _pixel_bytes)) for _image_lines in _reduced_images) # 数据回填 if _interlace: _result = _un_interlace(_un_filter_images, _width, _height, _pixel_bytes) else: _result = itertools.chain(*next(_un_filter_images)) # 这时只有一个图片 if _depth == 16: # 要记得放缩深度哦 _result = (_j for _i, _j in enumerate(_result) if (_i % 2)) if _color_type == _COLOR_TYPE_GRAY: if _extra_alpha: # 如果有额外的 alpha 通道 if _background: # 把额外通道合并到灰度通道上,这比较简单 _mode = 'gray' _result = (_background + (i - _background) * _extra_alpha // 255 for i in _result) else: _mode = 'full' # 为了把透明度全部体现 _result = _t_gray_to_rgb_alpha(_result, _extra_alpha) else: # 不做任何处理 _mode = 'gray' elif _color_type == _COLOR_TYPE_GRAY_ALPHA: # 因为已经有了 alpha 通道,所以不关心 _extra_alpha _mode = 'full' _result = _t_gray_alpha_to_rgb_alpha(_result, _background or 0) elif _color_type == _COLOR_TYPE_PALETTE: if _extra_alpha: if _background: _mode = 'rgb' _result = _t_palette_to_rgb(_result, _palette, _extra_alpha, _background) else: _mode = 'full' _result = _t_palette_to_rgb_alpha(_result, _palette, _extra_alpha) else: _mode = 'rgb' _result = _t_palette_to_rgb2(_result, _palette) elif _color_type == _COLOR_TYPE_RGB: if _extra_alpha: if _background: _mode = 'rgb' _result = _t_rgb_to_rgb(_result, _extra_alpha, _background) else: _mode = 'full' _result = _t_rgb_to_rgb_alpha(_result, _extra_alpha) else: # 不做任何处理 _mode = 'rgb' elif _color_type == _COLOR_TYPE_RGB_ALPHA: if _background: _mode = 'rgb' _result = _t_rgb_alpha_to_rgb(_result, _background) else: _mode = 'full' else: raise return _width, _height, _mode, bytes(_result)
# 报头信息。 head = { 'filepath': r'D:\PythonProject\IPv6_file_trans', 'filename': r'file.zip', 'filesize': None, 'CRC32': None } file_path = os.path.join(head['filepath'], head['filename']) # 计算文件的大小 file_size = os.path.getsize(os.path.join(head['filepath'], head['filename'])) head['filesize'] = file_size # 计算CRC32值并且和文件头里面的做比较。 with open(file_path, 'rb') as f: head['CRC32'] = zlib.crc32(f.read()) json_head = json.dumps(head) # 利用json将字典转成字符串 bytes_head = json_head.encode('utf-8') # 字符串转bytes # 计算head长度 head_len = len(bytes_head) # 报头的长度 # 利用struct将int类型的数据打包成4个字节的byte,所以服务器端接受这个长度的时候可以固定缓冲区大小为4 pack_len = struct.pack('i', head_len) # 先将报头长度发出去 sender.send(pack_len) # 再发送bytes类型的报头 sender.send(bytes_head) listener.listen() conn, addr = listener.accept()
def get_crc32(file_bytes): """Returns the CRC32.""" return hex(zlib.crc32(file_bytes))
def _crc32(data, seed=0): return zlib.crc32(data, seed) & 0xffffffff
def parse_folder(target_folder, output_file): """ read each file and produce a hash value. """ # list folders and files to exclude banned_folders = ("/AUTO/", "/CPAK/", "/Documentation/", "/ED64/", "/EDFC/", "/EDGB/", "/EDMD/", "/Extended SSF Dev Demo Sample - Krikzz/src/", "/Firmware Backup/", "/GBASYS/", "/Images/", "/MEGA/", "/Manuals/", "/PALETTE/", "/PATTERN/", "/SAVE/", "/SNAP/", "/SOUNDS/", "/SPED/", "/SYSTEM/", "/System Test Images/", "/System Volume Information/", "/TBED/", "/TEXT/", "/_PREVIEW/", "/menu/", "/ntm_firmware_ver", "/sd2snes Themes/", "/sd2snes/") banned_suffixes = (".001", ".002", ".003", ".004", ".005", ".006", ".007", ".008", ".009", ".aps", ".asm", ".bak", ".bat", ".bsa", ".bps", ".BPS", ".bst", ".c", ".cht", ".dat", ".db", ".docx", ".exe", ".ips", ".jpg", ".json", ".mso", ".ods", ".odt", ".pc", ".pdf", ".png", ".sav", ".srm", ".sto", ".txt", ".tmp", ".xdelta", ".xls", "OS.PCE", "Thumbs.db", "menu.bin", "desktop.ini", ".DS_Store") with open(output_file, "w") as output_file: i = 0 # make sure subfolders are alphanumerically sorted sorted_files = sorted(os.walk(target_folder)) for dirpath, dirnames, filenames in sorted_files: if filenames: # make sure files are alphanumerically sorted filenames.sort() for f in filenames: filename = os.path.join(os.path.normpath(dirpath), f) absolute_filename = os.path.abspath(filename) os.path.isfile(absolute_filename) # convert to Unix format by default filename = filename.replace("\\", "/") # Report filenames with non-ASCII characters try: filename.encode('ascii') except UnicodeEncodeError: print("Error (non-ASCII character):", filename, file=sys.stdout) time.sleep(10) # alternatively: sys.exit(1) sha256 = hashlib.sha256() sha1 = hashlib.sha1() md5 = hashlib.md5() crc = 0 # exclude certain folders and files if not (any(s in filename for s in banned_folders) or filename.endswith(banned_suffixes)): try: with open(absolute_filename, "rb", buffering=0) as f: # use a small buffer to compute hash # values to avoid storing large files # in memory (changing buffer size does # not change parsing speed much) for b in iter(lambda: f.read(128 * 1024), b''): sha256.update(b) sha1.update(b) md5.update(b) crc = zlib.crc32(b, crc) except FileNotFoundError: # Windows default API is limited to paths of # 260 characters absolute_filename = u'\\\\?\\' + absolute_filename with open(absolute_filename, "rb", buffering=0) as f: for b in iter(lambda: f.read(128 * 1024), b''): sha256.update(b) sha1.update(b) md5.update(b) crc = zlib.crc32(b, crc) print(sha256.hexdigest(), filename, sha1.hexdigest(), md5.hexdigest(), '{0:08x}'.format(crc & 0xffffffff), sep="\t", file=output_file) i += 1 print_progress(i, END_LINE) else: if not args.new_line: print_progress(i, "\n") return None
def read_png(stream): if not stream.read(8) == b'\x89PNG\r\n\x1a\n': stream.seek(0, 0) # 重置数据流 return # 读取需要的数据 _width = _height = _depth = _color_type = _interlace = _pixel_bytes = \ _palette = None _data = [] while True: try: _length, _mask = struct.unpack('!I4s', stream.read(8)) _chunk = stream.read(_length) _crc32 = struct.unpack('!I', stream.read(4))[0] if zlib.crc32(_mask + _chunk) != _crc32: break if _mask == _HEADER: (_width, _height, _depth, _color_type, _compress_method, _filter_method, _interlace) = struct.unpack('!2I5B', _chunk) _planes = (1, -1, 3, 1, 2, -1, 4)[_color_type] _pixel_bytes = (_depth * _planes + 7) // 8 elif _mask == _PALETTE: _palette = tuple(_chunk[_i:_i + 3] for _i in range(0, _length, 3)) elif _mask == _DATA: _data.append(_chunk) elif _mask == _END: break except struct.error: break # LZ77 解压 _decompress_obj = zlib.decompressobj() _unzip_data = itertools.chain( *(_decompress_obj.decompress(_chunk) for _chunk in _data), _decompress_obj.flush()) # 按行分割,_adam 表示 pass 提取算法得到的缩小图的每行像素个数 _adam = (_adam7iter if _interlace else _adam1iter)(_width, _height) _reduced_images = tuple([] for _ in range(7 if _interlace else 1)) # 容器 _current_image = 0 # 不要这个变量也能正常运行 while True: try: _line_bytes = next(_adam) if _line_bytes > 0: # 大于 0 表示读取长度 _reduced_images[_current_image].append( bytearray( # 这是一个从迭代器中依次读取数量个数的方法 # 下面的意思是从 _unzip_data 里读取一定数量的字节 # 这个字节长度是缩小图每行的字节长度 +1,多出来的是滤波标记 _read_iter(_unzip_data, _line_bytes * _pixel_bytes + 1))) else: # 小于等于 0 表示切换图片,注意缩小图是 7 张 _current_image = abs(_line_bytes) except StopIteration: break # 滤波重构 _un_filter_images = ( # 把多个图片的打包在一起 # 把一个图片的行打包在一起 tuple(_un_filter_image_lines(_image_lines, _pixel_bytes)) for _image_lines in _reduced_images) # 数据回填 if _interlace: _result = _un_interlace(_un_filter_images, _width, _height, _pixel_bytes) else: _result = itertools.chain(*next(_un_filter_images)) # 这时只有一个图片 if _depth == 16: # 要记得放缩深度哦 _result = (_j for _i, _j in enumerate(_result) if (_i % 2)) elif _palette: _result = itertools.chain(*(_palette[_i] for _i in _result)) return _width, _height, _color_type, bytes(_result)