def prepare_value(val, compress): flag = 0 if isinstance(val, str): pass elif isinstance(val, (bool)): flag = FLAG_BOOL val = str(int(val)) elif isinstance(val, (int, long)): flag = FLAG_INTEGER val = str(val) elif isinstance(val, unicode): flag = FLAG_MARSHAL val = marshal.dumps(val, 2) else: try: val = marshal.dumps(val, 2) flag = FLAG_MARSHAL except ValueError: val = cPickle.dumps(val, -1) flag = FLAG_PICKLE if compress and len(val) > 1024: flag |= FLAG_COMPRESS val = quicklz.compress(val) return flag, val
def prepare_value(val, compress): flag = 0 if isinstance(val, six.binary_type): pass elif isinstance(val, bool): flag = FLAG_BOOL val = str(int(val)).encode('utf-8') elif isinstance(val, six.integer_types): flag = FLAG_INTEGER val = str(val).encode('utf-8') elif isinstance(val, six.text_type): flag = FLAG_MARSHAL val = marshal.dumps(val, 2) else: try: val = marshal.dumps(val, 2) flag = FLAG_MARSHAL except ValueError: val = cPickle.dumps(val, -1) flag = FLAG_PICKLE if compress and len(val) > 1024: flag |= FLAG_COMPRESS val = quicklz.compress(val) return flag, val
def write_bucket(self, it, index): """ 0 <= index < 256 yield from it write to "*/%03d.data" % index """ N = 16**self.depth if self.depth > 0: fmt = '%%0%dx' % self.depth ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)] else: ds = [self.path] pname = '%03d.data' % index tname = '.%03d.data.%s.tmp' % (index, socket.gethostname()) p = [os.path.join(d, pname) for d in ds] tp = [os.path.join(d, tname) for d in ds] pos = [0] * N f = [open(t, 'wb', 1 << 20) for t in tp] now = int(time.time()) hint = [[] for d in ds] bits = 32 - self.depth * 4 for key, value in it: if not isinstance(key, (six.string_types, six.binary_type)): key = str(key) if isinstance(key, six.text_type): key = key.encode('utf-8') if not is_valid_key(key): logger.warning("ignored invalid key: %s", [key]) continue i = fnv1a(key) >> bits hint[i].append( struct.pack("IIH", pos[i] + len(key), 1, 0) + key + b'\x00') pos[i] += self.write_record(f[i], key, value, now) if pos[i] > (4000 << 20): raise Exception("beansdb data file is larger than 4000M") for i in f: i.close() for i in range(N): if hint[i] and not os.path.exists(p[i]): os.rename(tp[i], p[i]) hintdata = b''.join(hint[i]) hint_path = os.path.join(os.path.dirname(p[i]), '%03d.hint' % index) if self.compress: hintdata = quicklz.compress(hintdata) hint_path += '.qlz' with open(hint_path, 'wb') as f: f.write(hintdata) else: os.remove(tp[i]) return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def write_bucket(self, it, index): """ 0 <= index < 256 yield from it write to "*/%03d.data" % index """ N = 16 ** self.depth if self.depth > 0: fmt = '%%0%dx' % self.depth ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)] else: ds = [self.path] pname = '%03d.data' % index tname = '.%03d.data.%s.tmp' % (index, socket.gethostname()) p = [os.path.join(d, pname) for d in ds] tp = [os.path.join(d, tname) for d in ds] pos = [0] * N f = [open(t, 'wb', 1 << 20) for t in tp] now = int(time.time()) hint = [[] for d in ds] bits = 32 - self.depth * 4 for key, value in it: if not isinstance(key, (six.string_types, six.binary_type)): key = str(key) if isinstance(key, six.text_type): key = key.encode('utf-8') if not is_valid_key(key): logger.warning("ignored invalid key: %s", [key]) continue i = fnv1a(key) >> bits hint[i].append(struct.pack("IIH", pos[i] + len(key), 1, 0) + key + b'\x00') pos[i] += self.write_record(f[i], key, value, now) for i in f: i.close() for i in range(N): if hint[i] and not os.path.exists(p[i]): os.rename(tp[i], p[i]) hintdata = b''.join(hint[i]) hint_path = os.path.join(os.path.dirname(p[i]), '%03d.hint' % index) if self.compress: hintdata = quicklz.compress(hintdata) hint_path += '.qlz' with open(hint_path, 'wb') as f: f.write(hintdata) else: os.remove(tp[i]) return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def write_bucket(self, it, index): """ 0 <= index < 256 yield from it write to "*/%03d.data" % index """ N = 16 ** self.depth if self.depth > 0: fmt = "%%0%dx" % self.depth ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)] else: ds = [self.path] pname = "%03d.data" % index tname = ".%03d.data.%s.tmp" % (index, socket.gethostname()) p = [os.path.join(d, pname) for d in ds] tp = [os.path.join(d, tname) for d in ds] pos = [0] * N f = [open(t, "w", 1 << 20) for t in tp] now = int(time.time()) hint = [[] for d in ds] bits = 32 - self.depth * 4 for key, value in it: key = str(key) if not is_valid_key(key): logger.warning("ignored invalid key: %s", [key]) continue i = fnv1a(key) >> bits hint[i].append(struct.pack("IIH", pos[i] + len(key), 1, 0) + key + "\x00") pos[i] += self.write_record(f[i], key, value, now) if pos[i] > (4000 << 20): raise Exception("beansdb data file is larger than 4000M") for i in f: i.close() for i in range(N): if hint[i] and not os.path.exists(p[i]): os.rename(tp[i], p[i]) hintdata = "".join(hint[i]) hint_path = os.path.join(os.path.dirname(p[i]), "%03d.hint" % index) if self.compress: hintdata = quicklz.compress(hintdata) hint_path += ".qlz" open(hint_path, "w").write(hintdata) else: os.remove(tp[i]) return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def test_compress(self): text = b'hello' assert text == quicklz.decompress(quicklz.compress(text)) text = os.urandom(10000) assert text == quicklz.decompress(quicklz.compress(text))
def test_empty_string(self): text = b'' assert quicklz.compress(text) == b'' assert text == quicklz.decompress(quicklz.compress(text))