Exemplo n.º 1
0
    def scan_hint(self, hint_path):
        with open(hint_path, 'rb') as f:
            hint = f.read()

        if hint_path.endswith('.qlz'):
            try:
                hint = quicklz.decompress(hint)
            except ValueError as e:
                msg = str(e)
                if msg.startswith('compressed length not match'):
                    hint = hint[:int(msg.split('!=')[1])]
                    hint = quicklz.decompress(hint)

        key_filter = self.key_filter or (lambda x: True)
        with open(self.path, 'rb') as dataf:
            p = 0
            while p < len(hint):
                pos, ver, _ = struct.unpack("IiH", hint[p:p + 10])
                p += 10
                ksz = pos & 0xff
                key = hint[p:p + ksz]
                if key_filter(key):
                    dataf.seek(pos & 0xffffff00)
                    r, err = read_record(dataf)
                    if err is not None:
                        logger.error("read failed from %s at %d", self.path,
                                     pos & 0xffffff00)
                    else:
                        rsize, key, value = r
                        value, err = self.restore(value)
                        if not err:
                            yield key, value
                p += ksz + 1  # \x00
Exemplo n.º 2
0
    def scan_hint(self, hint_path):
        with open(hint_path, 'rb') as f:
            hint = f.read()

        if hint_path.endswith('.qlz'):
            try:
                hint = quicklz.decompress(hint)
            except ValueError as e:
                msg = str(e)
                if msg.startswith('compressed length not match'):
                    hint = hint[:int(msg.split('!=')[1])]
                    hint = quicklz.decompress(hint)

        key_filter = self.key_filter or (lambda x: True)
        with open(self.path, 'rb') as dataf:
            p = 0
            while p < len(hint):
                pos, ver, _ = struct.unpack("IiH", hint[p:p + 10])
                p += 10
                ksz = pos & 0xff
                key = hint[p: p + ksz]
                if key_filter(key):
                    dataf.seek(pos & 0xffffff00)
                    r, err = read_record(dataf)
                    if err is not None:
                        logger.error("read failed from %s at %d",
                                     self.path, pos & 0xffffff00)
                    else:
                        rsize, key, value = r
                        value, err = self.restore(value)
                        if not err:
                            yield key, value
                p += ksz + 1  # \x00
Exemplo n.º 3
0
 def scan_hint(self, hint_path):
     hint = open(hint_path).read()
     if hint_path.endswith(".qlz"):
         try:
             hint = quicklz.decompress(hint)
         except ValueError, e:
             msg = str(e.message)
             if msg.startswith("compressed length not match"):
                 hint = hint[: int(msg.split("!=")[1])]
                 hint = quicklz.decompress(hint)
Exemplo n.º 4
0
 def scan_hint(self, hint_path):
     hint = open(hint_path).read()
     if hint_path.endswith('.qlz'):
         try:
             hint = quicklz.decompress(hint)
         except ValueError, e:
             msg = str(e.message)
             if msg.startswith('compressed length not match'):
                 hint = hint[:int(msg.split('!=')[1])]
                 hint = quicklz.decompress(hint)
Exemplo n.º 5
0
    def __init__(self, path, is_new=None, stop_on_bad=False, check_khash=False):
        # print check_khash
        self.path = path
        self.stop_on_bad = stop_on_bad
        self.check_khash = check_khash

        path_low = path.lower()
        if is_new in [True, False]:
            self.is_new = is_new
        else:
            if path_low.endswith('.tmp'):
                path_low = path_low[:-4]
            if path_low.endswith('.idx.s'):
                self.is_new = True
            elif path_low.endswith('.hint.qlz'):
                self.is_new = False
            else:
                raise Exception("%s has unexpected suffix" % path)

        with open(self.path, 'r') as f:
            hint_data = f.read()
        if self.is_new:
            index_off_s, count, datasize = parse_new_hint_header(hint_data)
            print "index_off_s, count, datasize: ", index_off_s, count, datasize
            hint_len = len(hint_data) if index_off_s == 0 else index_off_s
            self.g = parse_new_hint_body(
                hint_data[FILE_HEADER_SIZE_NEW:hint_len],
                check_khash
            )
        else:
            hint_data = quicklz.decompress(hint_data)
            self.g = parse_old_hint(hint_data)
Exemplo n.º 6
0
 def test_try_compress(self):
     cases = [
         (b'hello', False, 'short string'),
         (os.urandom(1024 * 100), False, 'random str'),
         (marshal.dumps(list(range(1024 * 4))), True, 'marshal of range()'),
     ]
     for text, c, desc in cases:
         ok, d = quicklz.try_compress(text)
         assert ok == c, desc
         if ok:
             assert text == quicklz.decompress(d), 'data not match'
Exemplo n.º 7
0
def restore_value(flag, val):
    if flag & FLAG_COMPRESS:
        val = quicklz.decompress(val)
    if flag & FLAG_COMPRESS1:
        val = zlib.decompress(val)

    if flag & FLAG_BOOL:
        val = bool(int(val))
    elif flag & FLAG_INTEGER:
        val = int(val)
    elif flag & FLAG_MARSHAL:
        val = marshal.loads(val)
    elif flag & FLAG_PICKLE:
        val = cPickle.loads(val)
    return val
Exemplo n.º 8
0
def restore_value(flag, val):
    if flag & FLAG_COMPRESS:
        val = quicklz.decompress(val)
    if flag & FLAG_COMPRESS1:
        val = zlib.decompress(val)

    if flag & FLAG_BOOL:
        val = bool(int(val))
    elif flag & FLAG_INTEGER:
        val = int(val)
    elif flag & FLAG_MARSHAL:
        val = marshal.loads(val)
    elif flag & FLAG_PICKLE:
        val = cPickle.loads(val)
    return val
Exemplo n.º 9
0
def restore_value(flag, val):
    # will ignore pickled
    flag = int(flag)
    if flag & FLAG_COMPRESS:
        val = quicklz.decompress(val)
    if flag & FLAG_COMPRESS1:
        val = zlib.decompress(val)

    if flag & FLAG_BOOL:
        val = bool(int(val))
    elif flag & FLAG_INTEGER:
        val = int(val)
    elif flag & FLAG_MARSHAL:
        val = marshal.loads(val)
    return val
Exemplo n.º 10
0
def _check_hint_with_key(file_path, key):
    with open(file_path, 'r') as f:
        hint_data = f.read()
    if file_path.endswith('.qlz'):
        hint_data = quicklz.decompress(hint_data)
    hint_len = len(hint_data)
    off_s = 0
    while off_s < hint_len:
        header = hint_data[off_s:off_s + 10]
        if not header:
            raise ValueError('%s error' % (file_path))
        pos, ver, hash_ = struct.unpack('IiH', header)
        off_s += 10
        ksz = pos & 0xff
        key_ = hint_data[off_s:off_s + ksz]
        if key_ == key:
            return pos & 0xffffff00, ver, hash_
        off_s += ksz + 1
    return None
Exemplo n.º 11
0
def check_data_with_key(file_path, key, ver_=None, hash_=None, pos=None):
    """ if pos is None, iterate data file to match key and ver_,
        otherwise seek to pos and check key and ver_ and hash_
    """
    with open(file_path, 'r') as f:
        while True:
            if pos is not None:
                f.seek(pos, 0)
            block = f.read(PADDING)
            if not block:
                if pos is not None:
                    raise Exception("no data at pos %s" % (pos))
                return False
            crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24])
            if not (0 < ksz < 255 and 0 <= vsz < (50<<20)):
                raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (file_path, ksz, vsz, f.tell()))
            rsize = 24 + ksz + vsz
            if rsize & 0xff:
                rsize = ((rsize >> 8) + 1) << 8
            if rsize > PADDING:
                block += f.read(rsize-PADDING)
            crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff
            if crc != crc32:
                raise CRCError('%s crc wrong with key %s' % (file_path, key))
            key_ = block[24:24+ksz]
            if pos is not None:
                eq_(key, key_)
                if ver_ is not None and ver_ != ver:
                    raise ValueError('%s key %s expect ver %s != %s', file_path, key, ver_, ver)
            else:
                if key != key_:
                    continue
                if ver_ is not None and ver_ != ver:
                    continue

            value = block[24+ksz:24+ksz+vsz]
            if flag & FLAG_COMPRESS:
                value = quicklz.decompress(value)
            _hash = get_data_hash(value)
            if hash_ is not None and _hash != hash_:
                raise ValueError("%s key %s expect hash 0x%x != 0x%x" % (file_path, key, hash_, _hash))
            return True
    return False
Exemplo n.º 12
0
def _check_data_with_hint(data_file, hint_file):
    hint_keys = _build_key_list_from_hint(hint_file)
    j = 0
    pos = 0
    with open(data_file, 'r') as f:
        while True:
            block = f.read(PADDING)
            _pos = PADDING
            if not block:
                if j < len(hint_keys):
                    raise Exception("data is less than hint: %s" % (data_file))
                return
            crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24])
            if not (0 < ksz < 255 and 0 <= vsz < (50<<20)):
                raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (data_file, ksz, vsz, f.tell()))

            rsize = 24 + ksz + vsz
            if rsize & 0xff:
                rsize = ((rsize >> 8) + 1) << 8
            if rsize > PADDING:
                block += f.read(rsize-PADDING)
                _pos += rsize - PADDING
            crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff
            if crc != crc32:
                raise ValueError('%s crc wrong, pos=%s' % (data_file, pos))
            key = block[24:24+ksz]
            value = block[24+ksz:24+ksz+vsz]
            hint_key = hint_keys[j]
            if pos < hint_key[0]:
                pos += _pos
                continue
            elif pos > hint_key[0]:
                raise Exception('%s pos %s > hint pos %s' % (data_file, pos, hint_key[0]))
            eq_(hint_key[1], key, "%s: %s" % (data_file, key))
            eq_(hint_key[2], ver, "%s: %s" % (data_file, key))

            if flag & FLAG_COMPRESS:
                value = quicklz.decompress(value)
            _hash = get_data_hash(value)
            eq_(hint_key[3], _hash, "%s: %s, hash 0x%x != 0x%x" % (data_file, key, _hash, hint_key[3]))
            pos += _pos
            j += 1
Exemplo n.º 13
0
def _build_key_list_from_hint(file_path):
    with open(file_path, 'r') as f:
        hint_data = f.read()
    if file_path.endswith('.qlz'):
        hint_data = quicklz.decompress(hint_data)
    key_list = list()
    hint_len = len(hint_data)
    off_s = 0
    while off_s < hint_len:
        header = hint_data[off_s:off_s + 10]
        if not header:
            raise ValueError('%s error' % (file_path))
        pos, ver, hash_ = struct.unpack('IiH', header)
        off_s += 10
        ksz = pos & 0xff
        key = hint_data[off_s:off_s + ksz]
        key_list.append((pos & 0xffffff00, key, ver, hash_))
        off_s += ksz + 1
    key_list.sort(cmp=lambda a, b: cmp(a[0], b[0]))
    return key_list
Exemplo n.º 14
0
def read_record(f, decompress_value=True, check_crc=True):
    '''read a rec from f'''
    block = f.read(PADDING)
    if not block:
        return
    crc, tstamp, flag, ver, ksz, vsz = parse_header(block)
    if not (0 < ksz <= MAX_KEY_LEN) or not 0 <= vsz <= MAX_VALUE_SIZE:
        raise SizeError("size %d %d" % (ksz, vsz))
    rsize = 24 + ksz + vsz
    if rsize & 0xff:
        rsize = ((rsize >> 8) + 1) << 8
    if rsize > PADDING:
        block += f.read(rsize - PADDING)
    if check_crc:
        crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff
        if crc != crc32:
            raise CRCError("crc")
    key = block[24:24 + ksz]
    value = block[24 + ksz:24 + ksz + vsz]
    if decompress_value and (flag & FLAG_COMPRESS):
        value = quicklz.decompress(value)
        flag -= FLAG_COMPRESS
    return (key, vsz, value, flag, tstamp, ver)
Exemplo n.º 15
0
def decompress(val):
    return quicklz.decompress(val)
Exemplo n.º 16
0
 def test_compress(self):
     text = b'hello'
     assert text == quicklz.decompress(quicklz.compress(text))
     text = os.urandom(10000)
     assert text == quicklz.decompress(quicklz.compress(text))
Exemplo n.º 17
0
    def __packetLoop(self):
        try:
            while not self.__halt:
                _r, _w, _e = select([self.__sock], [], [self.__sock], 1)

                if self.__sock in _e:
                    return
                elif self.__sock not in _r:
                    continue

                res = self.__parse_packet(self.__sock.recvfrom(512)[0])
                q = self.__packetQueueTmp[res['tp']]

                if res['tp'] == 6:
                    continue
                elif res['tp'] == 5:
                    self.__lastPong = time.time()
                    continue

                if res['tp'] < 8 and res['pid'] > 0:
                    if self.__packetCounterRecv[
                            res['tp']] != res['pid'] and res['tp'] != 4:
                        continue
                    else:
                        if self.__packetCounterRecv[res['tp']] != res['pid']:
                            self.__packetCounterRecv[res['tp']] = res['pid']

                        self.__packetCounterRecv[res['tp']] += 1

                        if res['tp'] == 4 or res['tp'] == 2:
                            self.__send_packet(
                                None, self.__packetCounter[6 if res['tp'] ==
                                                           2 else 5],
                                self.__clientID, 6 if res['tp'] == 2 else 5,
                                1 if res['tp'] == 4 else 0, 0,
                                1 if res['tp'] == 2 else 0, 0,
                                struct.pack('>H', res['pid'] & 0xFFFF))
                            self.__packetCounter[6 if res['tp'] ==
                                                 2 else 5] += 1

                if res['tp'] == 3:
                    self.__send_packet(None, self.__packetCounter[7],
                                       self.__clientID, 7, 0, 0, 0, 0,
                                       struct.pack('>H', res['pid'] & 0xFFFF))
                    self.__packetCounter[7] += 1

                if res['tp'] == 4:
                    continue

                if len(q) > 0:
                    q.append(res)

                    if res['flag_f'] == 1:
                        payload = decompress(b''.join([
                            i['payload'] for i in q
                        ])) if q[0]['flag_c'] == 1 else b''.join(
                            [i['payload'] for i in q])
                        res = q[0]
                        res['payload'] = payload
                        self.__packetQueue.put(res)

                        del q[:]
                else:
                    if res['flag_f'] == 1:
                        q.append(res)
                    else:
                        if res['flag_c'] == 1:
                            res['payload'] = decompress(res['payload'])

                        self.__packetQueue.put(res)

        except:
            raise
            return

        print('Returned normally')
Exemplo n.º 18
0
 def test_empty_string(self):
     text = b''
     assert quicklz.compress(text) == b''
     assert text == quicklz.decompress(quicklz.compress(text))