Example #1
0
    def scan_hint(self, hint_path):
        with open(hint_path, 'rb') as f:
            hint = f.read()

        if hint_path.endswith('.qlz'):
            try:
                hint = quicklz.decompress(hint)
            except ValueError as e:
                msg = str(e)
                if msg.startswith('compressed length not match'):
                    hint = hint[:int(msg.split('!=')[1])]
                    hint = quicklz.decompress(hint)

        key_filter = self.key_filter or (lambda x: True)
        with open(self.path, 'rb') as dataf:
            p = 0
            while p < len(hint):
                pos, ver, _ = struct.unpack("IiH", hint[p:p + 10])
                p += 10
                ksz = pos & 0xff
                key = hint[p: p + ksz]
                if key_filter(key):
                    dataf.seek(pos & 0xffffff00)
                    r, err = read_record(dataf)
                    if err is not None:
                        logger.error("read failed from %s at %d",
                                     self.path, pos & 0xffffff00)
                    else:
                        rsize, key, value = r
                        value, err = self.restore(value)
                        if not err:
                            yield key, value
                p += ksz + 1  # \x00
Example #2
0
 def scan_hint(self, hint_path):
     hint = open(hint_path).read()
     if hint_path.endswith(".qlz"):
         try:
             hint = quicklz.decompress(hint)
         except ValueError, e:
             msg = str(e.message)
             if msg.startswith("compressed length not match"):
                 hint = hint[: int(msg.split("!=")[1])]
                 hint = quicklz.decompress(hint)
Example #3
0
def restore_value(flag, val):
    if flag & FLAG_COMPRESS:
        val = quicklz.decompress(val)
    if flag & FLAG_COMPRESS1:
        val = zlib.decompress(val)

    if flag & FLAG_BOOL:
        val = bool(int(val))
    elif flag & FLAG_INTEGER:
        val = int(val)
    elif flag & FLAG_MARSHAL:
        val = marshal.loads(val)
    elif flag & FLAG_PICKLE:
        val = cPickle.loads(val)
    return val
Example #4
0
def _check_hint_with_key(file_path, key):
    with open(file_path, 'r') as f:
        hint_data = f.read()
    if file_path.endswith('.qlz'):
        hint_data = quicklz.decompress(hint_data)
    hint_len = len(hint_data)
    off_s = 0
    while off_s < hint_len:
        header = hint_data[off_s:off_s + 10]
        if not header:
            raise ValueError('%s error' % (file_path))
        pos, ver, hash_ = struct.unpack('IiH', header)
        off_s += 10
        ksz = pos & 0xff
        key_ = hint_data[off_s:off_s + ksz]
        if key_ == key:
            return pos & 0xffffff00, ver, hash_
        off_s += ksz + 1
    return None
Example #5
0
def check_data_with_key(file_path, key, ver_=None, hash_=None, pos=None):
    """ if pos is None, iterate data file to match key and ver_,
        otherwise seek to pos and check key and ver_ and hash_
    """
    with open(file_path, 'r') as f:
        while True:
            if pos is not None:
                f.seek(pos, 0)
            block = f.read(PADDING)
            if not block:
                if pos is not None:
                    raise Exception("no data at pos %s" % (pos))
                return False
            crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24])
            if not (0 < ksz < 255 and 0 <= vsz < (50<<20)):
                raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (file_path, ksz, vsz, f.tell()))
            rsize = 24 + ksz + vsz
            if rsize & 0xff:
                rsize = ((rsize >> 8) + 1) << 8
            if rsize > PADDING:
                block += f.read(rsize-PADDING)
            crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff
            if crc != crc32:
                raise CRCError('%s crc wrong with key %s' % (file_path, key))
            key_ = block[24:24+ksz]
            if pos is not None:
                eq_(key, key_)
                if ver_ is not None and ver_ != ver:
                    raise ValueError('%s key %s expect ver %s != %s', file_path, key, ver_, ver)
            else:
                if key != key_:
                    continue
                if ver_ is not None and ver_ != ver:
                    continue

            value = block[24+ksz:24+ksz+vsz]
            if flag & FLAG_COMPRESS:
                value = quicklz.decompress(value)
            _hash = get_data_hash(value)
            if hash_ is not None and _hash != hash_:
                raise ValueError("%s key %s expect hash 0x%x != 0x%x" % (file_path, key, hash_, _hash))
            return True
    return False
Example #6
0
def _check_data_with_hint(data_file, hint_file):
    hint_keys = _build_key_list_from_hint(hint_file)
    j = 0
    pos = 0
    with open(data_file, 'r') as f:
        while True:
            block = f.read(PADDING)
            _pos = PADDING
            if not block:
                if j < len(hint_keys):
                    raise Exception("data is less than hint: %s" % (data_file))
                return
            crc, tstamp, flag, ver, ksz, vsz = struct.unpack("IiiiII", block[:24])
            if not (0 < ksz < 255 and 0 <= vsz < (50<<20)):
                raise ValueError('%s header out of bound, ksz %s, vsz %s, offset %s' % (data_file, ksz, vsz, f.tell()))

            rsize = 24 + ksz + vsz
            if rsize & 0xff:
                rsize = ((rsize >> 8) + 1) << 8
            if rsize > PADDING:
                block += f.read(rsize-PADDING)
                _pos += rsize - PADDING
            crc32 = binascii.crc32(block[4:24 + ksz + vsz]) & 0xffffffff
            if crc != crc32:
                raise ValueError('%s crc wrong, pos=%s' % (data_file, pos))
            key = block[24:24+ksz]
            value = block[24+ksz:24+ksz+vsz]
            hint_key = hint_keys[j]
            if pos < hint_key[0]:
                pos += _pos
                continue
            elif pos > hint_key[0]:
                raise Exception('%s pos %s > hint pos %s' % (data_file, pos, hint_key[0]))
            eq_(hint_key[1], key, "%s: %s" % (data_file, key))
            eq_(hint_key[2], ver, "%s: %s" % (data_file, key))

            if flag & FLAG_COMPRESS:
                value = quicklz.decompress(value)
            _hash = get_data_hash(value)
            eq_(hint_key[3], _hash, "%s: %s, hash 0x%x != 0x%x" % (data_file, key, _hash, hint_key[3]))
            pos += _pos
            j += 1
Example #7
0
def _build_key_list_from_hint(file_path):
    with open(file_path, 'r') as f:
        hint_data = f.read()
    if file_path.endswith('.qlz'):
        hint_data = quicklz.decompress(hint_data)
    key_list = list()
    hint_len = len(hint_data)
    off_s = 0
    while off_s < hint_len:
        header = hint_data[off_s:off_s + 10]
        if not header:
            raise ValueError('%s error' % (file_path))
        pos, ver, hash_ = struct.unpack('IiH', header)
        off_s += 10
        ksz = pos & 0xff
        key = hint_data[off_s:off_s + ksz]
        key_list.append((pos & 0xffffff00, key, ver, hash_))
        off_s += ksz + 1
    key_list.sort(cmp=lambda a, b: cmp(a[0], b[0]))
    return key_list