def scan_hint(self, hint_path): with open(hint_path, 'rb') as f: hint = f.read() if hint_path.endswith('.qlz'): try: hint = quicklz.decompress(hint) except ValueError as e: msg = str(e) if msg.startswith('compressed length not match'): hint = hint[:int(msg.split('!=')[1])] hint = quicklz.decompress(hint) key_filter = self.key_filter or (lambda x: True) with open(self.path, 'rb') as dataf: p = 0 while p < len(hint): pos, ver, _ = struct.unpack("IiH", hint[p:p + 10]) p += 10 ksz = pos & 0xff key = hint[p:p + ksz] if key_filter(key): dataf.seek(pos & 0xffffff00) r, err = read_record(dataf) if err is not None: logger.error("read failed from %s at %d", self.path, pos & 0xffffff00) else: rsize, key, value = r value, err = self.restore(value) if not err: yield key, value p += ksz + 1 # \x00
def scan_hint(self, hint_path): with open(hint_path, 'rb') as f: hint = f.read() if hint_path.endswith('.qlz'): try: hint = quicklz.decompress(hint) except ValueError as e: msg = str(e) if msg.startswith('compressed length not match'): hint = hint[:int(msg.split('!=')[1])] hint = quicklz.decompress(hint) key_filter = self.key_filter or (lambda x: True) with open(self.path, 'rb') as dataf: p = 0 while p < len(hint): pos, ver, _ = struct.unpack("IiH", hint[p:p + 10]) p += 10 ksz = pos & 0xff key = hint[p: p + ksz] if key_filter(key): dataf.seek(pos & 0xffffff00) r, err = read_record(dataf) if err is not None: logger.error("read failed from %s at %d", self.path, pos & 0xffffff00) else: rsize, key, value = r value, err = self.restore(value) if not err: yield key, value p += ksz + 1 # \x00
def scan_hint(self, hint_path): hint = open(hint_path).read() if hint_path.endswith(".qlz"): try: hint = quicklz.decompress(hint) except ValueError, e: msg = str(e.message) if msg.startswith("compressed length not match"): hint = hint[: int(msg.split("!=")[1])] hint = quicklz.decompress(hint)
def scan_hint(self, hint_path): hint = open(hint_path).read() if hint_path.endswith('.qlz'): try: hint = quicklz.decompress(hint) except ValueError, e: msg = str(e.message) if msg.startswith('compressed length not match'): hint = hint[:int(msg.split('!=')[1])] hint = quicklz.decompress(hint)
def __init__(self, path, is_new=None, stop_on_bad=False, check_khash=False): # print check_khash self.path = path self.stop_on_bad = stop_on_bad self.check_khash = check_khash path_low = path.lower() if is_new in [True, False]: self.is_new = is_new else: if path_low.endswith('.tmp'): path_low = path_low[:-4] if path_low.endswith('.idx.s'): self.is_new = True elif path_low.endswith('.hint.qlz'): self.is_new = False else: raise Exception("%s has unexpected suffix" % path) with open(self.path, 'r') as f: hint_data = f.read() if self.is_new: index_off_s, count, datasize = parse_new_hint_header(hint_data) print "index_off_s, count, datasize: ", index_off_s, count, datasize hint_len = len(hint_data) if index_off_s == 0 else index_off_s self.g = parse_new_hint_body( hint_data[FILE_HEADER_SIZE_NEW:hint_len], check_khash ) else: hint_data = quicklz.decompress(hint_data) self.g = parse_old_hint(hint_data)
def test_try_compress(self): cases = [ (b'hello', False, 'short string'), (os.urandom(1024 * 100), False, 'random str'), (marshal.dumps(list(range(1024 * 4))), True, 'marshal of range()'), ] for text, c, desc in cases: ok, d = quicklz.try_compress(text) assert ok == c, desc if ok: assert text == quicklz.decompress(d), 'data not match'
def restore_value(flag, val): if flag & FLAG_COMPRESS: val = quicklz.decompress(val) if flag & FLAG_COMPRESS1: val = zlib.decompress(val) if flag & FLAG_BOOL: val = bool(int(val)) elif flag & FLAG_INTEGER: val = int(val) elif flag & FLAG_MARSHAL: val = marshal.loads(val) elif flag & FLAG_PICKLE: val = cPickle.loads(val) return val
def restore_value(flag, val): # will ignore pickled flag = int(flag) if flag & FLAG_COMPRESS: val = quicklz.decompress(val) if flag & FLAG_COMPRESS1: val = zlib.decompress(val) if flag & FLAG_BOOL: val = bool(int(val)) elif flag & FLAG_INTEGER: val = int(val) elif flag & FLAG_MARSHAL: val = marshal.loads(val) return val
def _check_hint_with_key(file_path, key): with open(file_path, 'r') as f: hint_data = f.read() if file_path.endswith('.qlz'): hint_data = quicklz.decompress(hint_data) hint_len = len(hint_data) off_s = 0 while off_s < hint_len: header = hint_data[off_s:off_s + 10] if not header: raise ValueError('%s error' % (file_path)) pos, ver, hash_ = struct.unpack('IiH', header) off_s += 10 ksz = pos & 0xff key_ = hint_data[off_s:off_s + ksz] if key_ == key: return pos & 0xffffff00, ver, hash_ off_s += ksz + 1 return None
def check_data_with_key(file_path, key, ver_=None, hash_=None, pos=None): """ if pos is None, iterate data file to match key and ver_, otherwise seek to pos and check key and ver_ and hash_ """ with open(file_path, 'r') as f: while True: if pos is not None: f.seek(pos, 0) block = f.read(PADDING) if not block: if pos is not None: raise Exception("no data at pos %s" % (pos)) return False crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24]) if not (0 < ksz < 255 and 0 <= vsz < (50<<20)): raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (file_path, ksz, vsz, f.tell())) rsize = 24 + ksz + vsz if rsize & 0xff: rsize = ((rsize >> 8) + 1) << 8 if rsize > PADDING: block += f.read(rsize-PADDING) crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff if crc != crc32: raise CRCError('%s crc wrong with key %s' % (file_path, key)) key_ = block[24:24+ksz] if pos is not None: eq_(key, key_) if ver_ is not None and ver_ != ver: raise ValueError('%s key %s expect ver %s != %s', file_path, key, ver_, ver) else: if key != key_: continue if ver_ is not None and ver_ != ver: continue value = block[24+ksz:24+ksz+vsz] if flag & FLAG_COMPRESS: value = quicklz.decompress(value) _hash = get_data_hash(value) if hash_ is not None and _hash != hash_: raise ValueError("%s key %s expect hash 0x%x != 0x%x" % (file_path, key, hash_, _hash)) return True return False
def _check_data_with_hint(data_file, hint_file): hint_keys = _build_key_list_from_hint(hint_file) j = 0 pos = 0 with open(data_file, 'r') as f: while True: block = f.read(PADDING) _pos = PADDING if not block: if j < len(hint_keys): raise Exception("data is less than hint: %s" % (data_file)) return crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24]) if not (0 < ksz < 255 and 0 <= vsz < (50<<20)): raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (data_file, ksz, vsz, f.tell())) rsize = 24 + ksz + vsz if rsize & 0xff: rsize = ((rsize >> 8) + 1) << 8 if rsize > PADDING: block += f.read(rsize-PADDING) _pos += rsize - PADDING crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff if crc != crc32: raise ValueError('%s crc wrong, pos=%s' % (data_file, pos)) key = block[24:24+ksz] value = block[24+ksz:24+ksz+vsz] hint_key = hint_keys[j] if pos < hint_key[0]: pos += _pos continue elif pos > hint_key[0]: raise Exception('%s pos %s > hint pos %s' % (data_file, pos, hint_key[0])) eq_(hint_key[1], key, "%s: %s" % (data_file, key)) eq_(hint_key[2], ver, "%s: %s" % (data_file, key)) if flag & FLAG_COMPRESS: value = quicklz.decompress(value) _hash = get_data_hash(value) eq_(hint_key[3], _hash, "%s: %s, hash 0x%x != 0x%x" % (data_file, key, _hash, hint_key[3])) pos += _pos j += 1
def _build_key_list_from_hint(file_path): with open(file_path, 'r') as f: hint_data = f.read() if file_path.endswith('.qlz'): hint_data = quicklz.decompress(hint_data) key_list = list() hint_len = len(hint_data) off_s = 0 while off_s < hint_len: header = hint_data[off_s:off_s + 10] if not header: raise ValueError('%s error' % (file_path)) pos, ver, hash_ = struct.unpack('IiH', header) off_s += 10 ksz = pos & 0xff key = hint_data[off_s:off_s + ksz] key_list.append((pos & 0xffffff00, key, ver, hash_)) off_s += ksz + 1 key_list.sort(cmp=lambda a, b: cmp(a[0], b[0])) return key_list
def read_record(f, decompress_value=True, check_crc=True): '''read a rec from f''' block = f.read(PADDING) if not block: return crc, tstamp, flag, ver, ksz, vsz = parse_header(block) if not (0 < ksz <= MAX_KEY_LEN) or not 0 <= vsz <= MAX_VALUE_SIZE: raise SizeError("size %d %d" % (ksz, vsz)) rsize = 24 + ksz + vsz if rsize & 0xff: rsize = ((rsize >> 8) + 1) << 8 if rsize > PADDING: block += f.read(rsize - PADDING) if check_crc: crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff if crc != crc32: raise CRCError("crc") key = block[24:24 + ksz] value = block[24 + ksz:24 + ksz + vsz] if decompress_value and (flag & FLAG_COMPRESS): value = quicklz.decompress(value) flag -= FLAG_COMPRESS return (key, vsz, value, flag, tstamp, ver)
def decompress(val): return quicklz.decompress(val)
def test_compress(self): text = b'hello' assert text == quicklz.decompress(quicklz.compress(text)) text = os.urandom(10000) assert text == quicklz.decompress(quicklz.compress(text))
def __packetLoop(self): try: while not self.__halt: _r, _w, _e = select([self.__sock], [], [self.__sock], 1) if self.__sock in _e: return elif self.__sock not in _r: continue res = self.__parse_packet(self.__sock.recvfrom(512)[0]) q = self.__packetQueueTmp[res['tp']] if res['tp'] == 6: continue elif res['tp'] == 5: self.__lastPong = time.time() continue if res['tp'] < 8 and res['pid'] > 0: if self.__packetCounterRecv[ res['tp']] != res['pid'] and res['tp'] != 4: continue else: if self.__packetCounterRecv[res['tp']] != res['pid']: self.__packetCounterRecv[res['tp']] = res['pid'] self.__packetCounterRecv[res['tp']] += 1 if res['tp'] == 4 or res['tp'] == 2: self.__send_packet( None, self.__packetCounter[6 if res['tp'] == 2 else 5], self.__clientID, 6 if res['tp'] == 2 else 5, 1 if res['tp'] == 4 else 0, 0, 1 if res['tp'] == 2 else 0, 0, struct.pack('>H', res['pid'] & 0xFFFF)) self.__packetCounter[6 if res['tp'] == 2 else 5] += 1 if res['tp'] == 3: self.__send_packet(None, self.__packetCounter[7], self.__clientID, 7, 0, 0, 0, 0, struct.pack('>H', res['pid'] & 0xFFFF)) self.__packetCounter[7] += 1 if res['tp'] == 4: continue if len(q) > 0: q.append(res) if res['flag_f'] == 1: payload = decompress(b''.join([ i['payload'] for i in q ])) if q[0]['flag_c'] == 1 else b''.join( [i['payload'] for i in q]) res = q[0] res['payload'] = payload self.__packetQueue.put(res) del q[:] else: if res['flag_f'] == 1: q.append(res) else: if res['flag_c'] == 1: res['payload'] = decompress(res['payload']) self.__packetQueue.put(res) except: raise return print('Returned normally')
def test_empty_string(self): text = b'' assert quicklz.compress(text) == b'' assert text == quicklz.decompress(quicklz.compress(text))