def do_test(self, k):
packed = keylib.packs(k)
try:
unpacked = keylib.unpack(packed)
eq(k, keylib.unpack(keylib.packs(k)))
except:
print 'failing enc was: %r' % (packed,)
raise
def do_test(self, k):
packed = keylib.packs('', k)
try:
unpacked = keylib.unpack('', packed)
eq(k, keylib.unpack('', keylib.packs('', k)))
except:
print 'failing enc was: %r' % (packed,)
raise
def _index_keys(self, key, obj):
"""Generate a list of encoded keys representing index entries for `obj`
existing under `key`."""
res = self.func(obj)
if type(res) is list:
return [keylib.packs([ik, key], self.prefix) for ik in res]
elif res is not None:
return [keylib.packs([res, key], self.prefix)]
def test1(self):
for i in EncodeIntTest.INTS:
native = _keylib.packs(i)
python = keylib.packs(i)
try:
eq(native, python)
except:
print 'failing int was ' + str(i)
raise
def test1(self):
for i in self.INTS:
s = keylib.packs('', i)
try:
j, = keylib.unpack('', s)
eq(j, i)
except:
print [i, s]
raise
def test1(self):
for i in EncodeIntTest.INTS:
native = _keylib.packs('', i)
python = keylib.packs('', i)
try:
eq(native, python)
except:
print 'failing int was ' + str(i)
raise
def test1(self):
for i in self.INTS:
s = keylib.packs(i)
try:
j, = keylib.unpack(s)
eq(j, i)
except:
print [i, s]
raise
def test1(self):
for seq in self.SEQS:
packed = map(lambda s: keylib.packs(s), seq)
rnge = range(len(packed))
done = sorted(rnge, key=packed.__getitem__)
try:
eq(done, rnge)
except:
print 'failed:', seq
raise
def test_pos(self):
tz = dateutil.tz.gettz('Etc/GMT+1')
dt = datetime.now(tz)
sn = _keylib.packs(dt)
sp = keylib.packs(dt)
eq(sn, sp)
dn = _keylib.unpacks(sn)
dp = keylib.unpacks(sp)
eq(dn, dp)
def test1(self):
for seq in self.SEQS:
packed = map(lambda s: keylib.packs('', s), seq)
rnge = range(len(packed))
done = sorted(rnge, key=packed.__getitem__)
try:
eq(done, rnge)
except:
print 'failed:', seq
raise
def test_pos(self):
tz = dateutil.tz.gettz('Etc/GMT+1')
dt = datetime.now(tz)
sn = _keylib.packs('', dt)
sp = keylib.packs('', dt)
eq(sn, sp)
dn = _keylib.unpacks('', sn)
dp = keylib.unpacks('', sp)
eq(dn, dp)
def test_utc(self):
tz = dateutil.tz.gettz('Etc/UTC')
dt = datetime.now(tz)
sn = _keylib.packs(dt)
sp = keylib.packs(dt)
eq(sn, sp)
dn = _keylib.unpacks(sn)
dp = keylib.unpacks(sp)
eq(dn, dp)
def _index_keys(self, key, obj):
"""Generate a list of encoded keys representing index entries for `obj`
existing under `key`."""
idx_keys = []
for idx in self.indices.itervalues():
lst = idx.func(obj)
if lst:
if type(lst) is not list:
lst = [lst]
for idx_key in lst:
idx_keys.append(keylib.packs(idx.prefix, [idx_key, key]))
return idx_keys
def _prepare_batch(self, items):
"""Encode a list of records into their batch representation, returning
a tuple of the encoded physical key and value."""
keytups = [key for key, _ in reversed(items)]
phys = keylib.packs(keytups, self.prefix)
if len(items) == 1:
out = items[0][1]
else:
out = bytearray()
keylib.write_int(len(items), out.append, 0)
for _, data in items:
keylib.write_int(len(data), out.append, 0)
concat = ''.join(data for _, data in items)
out.extend(self.compressor.pack(concat))
return phys, bytes(out)
def delete(self, key):
"""Delete any existing record filed under `key`.
"""
it = self._iter(key, None, None, None, None, True, None)
txn = self.store._txn_context.get()
for batch, key_, data in it:
if key != key_:
break
obj = self.encoder.unpack(key, data)
if self.indices:
for key in self._index_keys(key, obj):
txn.delete(key)
if batch:
self._split_batch(key)
else:
txn.delete(keylib.packs(self.prefix, key))
def _prepare_batch(self, items, packer):
packer_prefix = self.store._encoder_prefix.get(packer)
if not packer_prefix:
packer_prefix = self.store.add_encoder(packer)
keytups = [key for key, _ in reversed(items)]
phys = keylib.packs(self.prefix, keytups)
out = bytearray()
if len(items) == 1:
out.extend(packer_prefix)
out.extend(packer.pack(items[0][1]))
else:
keylib.write_int(len(items), out.append, 0)
for _, data in items:
keylib.write_int(len(data), out.append, 0)
out.extend(packer_prefix)
concat = ''.join(data for _, data in items)
out.extend(packer.pack(concat))
return phys, str(out)
def _prepare_batch(self, items):
if len(items) == 1:
phys = keylib.Key(items[0][0]).to_raw(self.prefix)
return phys, items[0][1]
high_key_s = items[-1][0].to_raw()
low_key_s = items[0][0].to_raw()
cp_len = iterators.common_prefix_len(high_key_s, low_key_s)
raw = bytearray(2 + (2 * (1 + len(items))))
raw[0:2] = struct.pack('>HH', len(items), len(raw))
for idx, (key, value) in enumerate(items):
opos = 2 + (2 * idx)
raw[opos:opos + 2] = struct.pack('>H', len(raw))
suffix = key.to_raw()[cp_len:]
raw.append(len(suffix))
raw.extend(suffix)
raw.extend(value)
opos = 2 + (2 * len(items))
raw[opos:opos + 2] = struct.pack('>H', len(raw))
phys = keylib.packs([items[-1][0], items[0][0]], self.prefix)
return phys, self.compressor.pack(bytes(raw))
def testUuid(self):
t = ('a', uuid.uuid4(), 'b')
s = keylib.packs(t)
eq(t, keylib.unpack(s))
def test_single_sort_lower(self):
for val in self.SINGLE_VALS:
e1 = keylib.packs('', (val,))
e2 = keylib.packs('', [(val, val),])
lt(e1, e2, 'eek %r' % (val,))
def test_single(self):
for val in self.SINGLE_VALS:
encoded = keylib.packs('', (val,))
decoded = keylib.unpacks('', encoded)
eq([(val,)], decoded, 'input was %r' % (val,))
def test3(self):
t = [('index:Item:first_last', 11, 'item')]
p = keylib.packs(t)
eq(keylib.unpacks(p), t)
def test3(self):
t = [('index:Item:first_last', 11, 'item')]
p = keylib.packs('', t)
eq(keylib.unpacks('', p), t)
def test1(self):
t = [('', 11, 'item')]
p = keylib.packs('', t)
eq(keylib.unpacks('', p), t)
def testStringSorting(self):
strs = [(x,) for x in ('dave', 'dave\x00', 'dave\x01', 'davee\x01')]
encs = map(lambda o: keylib.packs('', o), strs)
encs.sort()
eq(strs, [keylib.unpack('', x) for x in encs])
def _encode(self, s):
return keylib.packs(s, self.prefix)
def test2(self):
t = [('index:I', 11, 'item')]
p = keylib.packs(t)
eq(keylib.unpacks(p), t)
def test_utc(self):
tz = dateutil.tz.gettz('Etc/UTC')
dt = self._now_truncate(tz)
s = keylib.packs('', dt)
dt2, = keylib.unpack('', s)
eq(dt, dt2)
def test_minusone(self):
tz = dateutil.tz.gettz('Etc/GMT-1')
dt = self._now_truncate(tz)
s = keylib.packs(dt)
dt2, = keylib.unpack(s)
eq(dt, dt2)
def test_utc(self):
tz = dateutil.tz.gettz('Etc/UTC')
dt = self._now_truncate(tz)
s = keylib.packs(dt)
dt2, = keylib.unpack(s)
eq(dt, dt2)
def test_naive(self):
dt = self._now_truncate()
s = keylib.packs(dt)
dt2, = keylib.unpack(s)
eq(dt.utctimetuple(), dt2.utctimetuple())
def batch(self,
lo=None,
hi=None,
prefix=None,
max_recs=None,
max_bytes=None,
max_keylen=None,
preserve=True,
max_phys=None,
grouper=None):
"""
Search the key range *lo..hi* for individual records, combining them
into a batches.
Returns `(found, made, last_key)` indicating the number of records
combined, the number of batches produced, and the last key visited
before `max_phys` was exceeded.
Batch size is controlled via `max_recs` and `max_bytes`; at least one
must not be ``None``. Larger sizes may cause pathological behaviour in
the storage engine (for example, space inefficiency). Since batches are
fully decompressed before any member may be accessed via
:py:meth:`get() <Collection.get>` or :py:meth:`iteritems()
<Collection.iteritems>`, larger sizes may slow decompression, waste IO
bandwidth, and temporarily use more RAM.
`lo`:
Lowest search key.
`hi`:
Highest search key.
`max_recs`:
Maximum number of records contained by any single batch. When
this count is reached, the current batch is saved and a new one
is created.
`max_bytes`:
Maximum size in bytes of the batch record's value after
compression, or ``None`` for no maximum size. When not
``None``, values are recompressed after each member is
appended, in order to test if `max_bytes` has been reached. This
is inefficient, but provides the best guarantee of final record
size. Single records are skipped if they exceed this size when
compressed individually.
`preserve`:
If ``True``, then existing batch records in the database are
left untouched. When one is found within `lo..hi`, the
currently building batch is finished and the found batch is
skipped over.
If ``False``, found batches are exploded and their members
contribute to the currently building batch.
`max_phys`:
Maximum number of physical keys to visit in any particular
call. A collection may be incrementally batched by repeatedly
invoking :py:meth:`Collection.batch` with `max` set, and `lo`
set to `last_key` of the previous run, until `found` returns
``0``. This allows batching to complete over several
transactions without blocking other users.
`grouper`:
Specifies a grouping function used to decide when to avoid
compressing unrelated records. The function is passed a
record's value. A new batch is triggered each time the
function's return value changes.
"""
assert max_bytes or max_recs, 'max_bytes and/or max_recs is required.'
txn = self.store._txn_context.get()
it = self.ITERATOR_CLASS(txn, self.prefix, self.compressor)
groupval = object()
items = []
for r in iterators.from_args(it, None, lo, hi, prefix, False, None,
True, max_phys):
if preserve and len(r.keys) > 1:
self._write_batch(txn, items, self.compressor)
else:
txn.delete(keylib.packs(r.key, self.prefix))
items.append((r.key, r.data))
if max_bytes:
_, encoded = self._prepare_batch(items)
if len(encoded) > max_bytes:
items.pop()
self._write_batch(txn, items)
items.append((r.key, r.data))
done = max_recs and len(items) == max_recs
if (not done) and grouper:
val = grouper(self.encoder.unpack(r.key, r.data))
done = val != groupval
groupval = val
if done:
self._write_batch(txn, items)
self._write_batch(txn, items)
def test1(self):
t = [('', 11, 'item')]
p = keylib.packs(t)
eq(keylib.unpacks(p), t)
def testUuid(self):
t = ('a', uuid.uuid4(), 'b')
s = keylib.packs('', t)
eq(t, keylib.unpack('', s))
def testStringSorting(self):
strs = [(x,) for x in ('dave', 'dave\x00', 'dave\x01', 'davee\x01')]
encs = map(lambda o: keylib.packs(o), strs)
encs.sort()
eq(strs, [keylib.unpack(x) for x in encs])
def test2(self):
t = [('index:I', 11, 'item')]
p = keylib.packs('', t)
eq(keylib.unpacks('', p), t)
def test_list(self):
lst = [(1,), (2,)]
eq(lst, keylib.unpacks(keylib.packs(lst)))
def test_naive(self):
dt = datetime.now()
sn = _keylib.packs('', dt)
sp = keylib.packs('', dt)
eq(sn, sp)
def test_naive(self):
dt = datetime.now()
sn = _keylib.packs(dt)
sp = keylib.packs(dt)
eq(sn, sp)
def test_naive(self):
dt = self._now_truncate()
s = keylib.packs('', dt)
dt2, = keylib.unpack('', s)
eq(dt.utctimetuple(), dt2.utctimetuple())
def test_single_sort_lower(self):
for val in self.SINGLE_VALS:
e1 = keylib.packs((val,))
e2 = keylib.packs([(val, val),])
lt(e1, e2, 'eek %r' % (val,))
def test_minusone(self):
tz = dateutil.tz.gettz('Etc/GMT-1')
dt = self._now_truncate(tz)
s = keylib.packs('', dt)
dt2, = keylib.unpack('', s)
eq(dt, dt2)
def test1(self):
s = keylib.packs((1, 2, 3, 4))
assert keylib.unpacks(s, s) == _keylib.unpacks(s, s)
def _enc(self, *args, **kwargs):
return keylib.packs('', *args, **kwargs)
def test_counter(self):
s = keylib.packs(('dave', 1))
eq([('dave', 1)], keylib.unpacks(s))
def _encode(self, s):
return keylib.packs(s, self.prefix)
def test_single(self):
for val in self.SINGLE_VALS:
encoded = keylib.packs((val,))
decoded = keylib.unpacks(encoded)
eq([(val,)], decoded, 'input was %r' % (val,))