def prepare_value(val, compress):
flag = 0
if isinstance(val, str):
pass
elif isinstance(val, (bool)):
flag = FLAG_BOOL
val = str(int(val))
elif isinstance(val, (int, long)):
flag = FLAG_INTEGER
val = str(val)
elif isinstance(val, unicode):
flag = FLAG_MARSHAL
val = marshal.dumps(val, 2)
else:
try:
val = marshal.dumps(val, 2)
flag = FLAG_MARSHAL
except ValueError:
val = cPickle.dumps(val, -1)
flag = FLAG_PICKLE
if compress and len(val) > 1024:
flag |= FLAG_COMPRESS
val = quicklz.compress(val)
return flag, val
def prepare_value(val, compress):
flag = 0
if isinstance(val, six.binary_type):
pass
elif isinstance(val, bool):
flag = FLAG_BOOL
val = str(int(val)).encode('utf-8')
elif isinstance(val, six.integer_types):
flag = FLAG_INTEGER
val = str(val).encode('utf-8')
elif isinstance(val, six.text_type):
flag = FLAG_MARSHAL
val = marshal.dumps(val, 2)
else:
try:
val = marshal.dumps(val, 2)
flag = FLAG_MARSHAL
except ValueError:
val = cPickle.dumps(val, -1)
flag = FLAG_PICKLE
if compress and len(val) > 1024:
flag |= FLAG_COMPRESS
val = quicklz.compress(val)
return flag, val
def prepare_value(val, compress):
flag = 0
if isinstance(val, six.binary_type):
pass
elif isinstance(val, bool):
flag = FLAG_BOOL
val = str(int(val)).encode('utf-8')
elif isinstance(val, six.integer_types):
flag = FLAG_INTEGER
val = str(val).encode('utf-8')
elif isinstance(val, six.text_type):
flag = FLAG_MARSHAL
val = marshal.dumps(val, 2)
else:
try:
val = marshal.dumps(val, 2)
flag = FLAG_MARSHAL
except ValueError:
val = cPickle.dumps(val, -1)
flag = FLAG_PICKLE
if compress and len(val) > 1024:
flag |= FLAG_COMPRESS
val = quicklz.compress(val)
return flag, val
def prepare_value(val, compress):
flag = 0
if isinstance(val, str):
pass
elif isinstance(val, (bool)):
flag = FLAG_BOOL
val = str(int(val))
elif isinstance(val, (int, long)):
flag = FLAG_INTEGER
val = str(val)
elif isinstance(val, unicode):
flag = FLAG_MARSHAL
val = marshal.dumps(val, 2)
else:
try:
val = marshal.dumps(val, 2)
flag = FLAG_MARSHAL
except ValueError:
val = cPickle.dumps(val, -1)
flag = FLAG_PICKLE
if compress and len(val) > 1024:
flag |= FLAG_COMPRESS
val = quicklz.compress(val)
return flag, val
def write_bucket(self, it, index):
""" 0 <= index < 256
yield from it
write to "*/%03d.data" % index
"""
N = 16**self.depth
if self.depth > 0:
fmt = '%%0%dx' % self.depth
ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)]
else:
ds = [self.path]
pname = '%03d.data' % index
tname = '.%03d.data.%s.tmp' % (index, socket.gethostname())
p = [os.path.join(d, pname) for d in ds]
tp = [os.path.join(d, tname) for d in ds]
pos = [0] * N
f = [open(t, 'wb', 1 << 20) for t in tp]
now = int(time.time())
hint = [[] for d in ds]
bits = 32 - self.depth * 4
for key, value in it:
if not isinstance(key, (six.string_types, six.binary_type)):
key = str(key)
if isinstance(key, six.text_type):
key = key.encode('utf-8')
if not is_valid_key(key):
logger.warning("ignored invalid key: %s", [key])
continue
i = fnv1a(key) >> bits
hint[i].append(
struct.pack("IIH", pos[i] + len(key), 1, 0) + key + b'\x00')
pos[i] += self.write_record(f[i], key, value, now)
if pos[i] > (4000 << 20):
raise Exception("beansdb data file is larger than 4000M")
for i in f:
i.close()
for i in range(N):
if hint[i] and not os.path.exists(p[i]):
os.rename(tp[i], p[i])
hintdata = b''.join(hint[i])
hint_path = os.path.join(os.path.dirname(p[i]),
'%03d.hint' % index)
if self.compress:
hintdata = quicklz.compress(hintdata)
hint_path += '.qlz'
with open(hint_path, 'wb') as f:
f.write(hintdata)
else:
os.remove(tp[i])
return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def write_bucket(self, it, index):
""" 0 <= index < 256
yield from it
write to "*/%03d.data" % index
"""
N = 16 ** self.depth
if self.depth > 0:
fmt = '%%0%dx' % self.depth
ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)]
else:
ds = [self.path]
pname = '%03d.data' % index
tname = '.%03d.data.%s.tmp' % (index, socket.gethostname())
p = [os.path.join(d, pname) for d in ds]
tp = [os.path.join(d, tname) for d in ds]
pos = [0] * N
f = [open(t, 'wb', 1 << 20) for t in tp]
now = int(time.time())
hint = [[] for d in ds]
bits = 32 - self.depth * 4
for key, value in it:
if not isinstance(key, (six.string_types, six.binary_type)):
key = str(key)
if isinstance(key, six.text_type):
key = key.encode('utf-8')
if not is_valid_key(key):
logger.warning("ignored invalid key: %s", [key])
continue
i = fnv1a(key) >> bits
hint[i].append(struct.pack("IIH", pos[i] + len(key), 1, 0) + key + b'\x00')
pos[i] += self.write_record(f[i], key, value, now)
for i in f:
i.close()
for i in range(N):
if hint[i] and not os.path.exists(p[i]):
os.rename(tp[i], p[i])
hintdata = b''.join(hint[i])
hint_path = os.path.join(os.path.dirname(p[i]), '%03d.hint' % index)
if self.compress:
hintdata = quicklz.compress(hintdata)
hint_path += '.qlz'
with open(hint_path, 'wb') as f:
f.write(hintdata)
else:
os.remove(tp[i])
return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def write_bucket(self, it, index):
""" 0 <= index < 256
yield from it
write to "*/%03d.data" % index
"""
N = 16 ** self.depth
if self.depth > 0:
fmt = "%%0%dx" % self.depth
ds = [os.path.join(self.path, *list(fmt % i)) for i in range(N)]
else:
ds = [self.path]
pname = "%03d.data" % index
tname = ".%03d.data.%s.tmp" % (index, socket.gethostname())
p = [os.path.join(d, pname) for d in ds]
tp = [os.path.join(d, tname) for d in ds]
pos = [0] * N
f = [open(t, "w", 1 << 20) for t in tp]
now = int(time.time())
hint = [[] for d in ds]
bits = 32 - self.depth * 4
for key, value in it:
key = str(key)
if not is_valid_key(key):
logger.warning("ignored invalid key: %s", [key])
continue
i = fnv1a(key) >> bits
hint[i].append(struct.pack("IIH", pos[i] + len(key), 1, 0) + key + "\x00")
pos[i] += self.write_record(f[i], key, value, now)
if pos[i] > (4000 << 20):
raise Exception("beansdb data file is larger than 4000M")
for i in f:
i.close()
for i in range(N):
if hint[i] and not os.path.exists(p[i]):
os.rename(tp[i], p[i])
hintdata = "".join(hint[i])
hint_path = os.path.join(os.path.dirname(p[i]), "%03d.hint" % index)
if self.compress:
hintdata = quicklz.compress(hintdata)
hint_path += ".qlz"
open(hint_path, "w").write(hintdata)
else:
os.remove(tp[i])
return sum([([p[i]] if hint[i] else []) for i in range(N)], [])
def test_compress(self):
text = b'hello'
assert text == quicklz.decompress(quicklz.compress(text))
text = os.urandom(10000)
assert text == quicklz.decompress(quicklz.compress(text))
def test_empty_string(self):
text = b''
assert quicklz.compress(text) == b''
assert text == quicklz.decompress(quicklz.compress(text))
