Ejemplo n.º 1
0
 def do_test(self, k):
     packed = keylib.packs(k)
     try:
         unpacked = keylib.unpack(packed)
         eq(k, keylib.unpack(keylib.packs(k)))
     except:
         print 'failing enc was: %r' % (packed,)
         raise
Ejemplo n.º 2
0
 def do_test(self, k):
     packed = keylib.packs('', k)
     try:
         unpacked = keylib.unpack('', packed)
         eq(k, keylib.unpack('', keylib.packs('', k)))
     except:
         print 'failing enc was: %r' % (packed,)
         raise
Ejemplo n.º 3
0
 def _index_keys(self, key, obj):
     """Generate a list of encoded keys representing index entries for `obj`
     existing under `key`."""
     res = self.func(obj)
     if type(res) is list:
         return [keylib.packs([ik, key], self.prefix) for ik in res]
     elif res is not None:
         return [keylib.packs([res, key], self.prefix)]
Ejemplo n.º 4
0
 def test1(self):
     for i in EncodeIntTest.INTS:
         native = _keylib.packs(i)
         python = keylib.packs(i)
         try:
             eq(native, python)
         except:
             print 'failing int was ' + str(i)
             raise
Ejemplo n.º 5
0
 def test1(self):
     for i in self.INTS:
         s = keylib.packs('', i)
         try:
             j, = keylib.unpack('', s)
             eq(j, i)
         except:
             print [i, s]
             raise
Ejemplo n.º 6
0
 def test1(self):
     for i in EncodeIntTest.INTS:
         native = _keylib.packs('', i)
         python = keylib.packs('', i)
         try:
             eq(native, python)
         except:
             print 'failing int was ' + str(i)
             raise
Ejemplo n.º 7
0
 def test1(self):
     for i in self.INTS:
         s = keylib.packs(i)
         try:
             j, = keylib.unpack(s)
             eq(j, i)
         except:
             print [i, s]
             raise
Ejemplo n.º 8
0
 def test1(self):
     for seq in self.SEQS:
         packed = map(lambda s: keylib.packs(s), seq)
         rnge = range(len(packed))
         done = sorted(rnge, key=packed.__getitem__)
         try:
             eq(done, rnge)
         except:
             print 'failed:', seq
             raise
Ejemplo n.º 9
0
    def test_pos(self):
        tz = dateutil.tz.gettz('Etc/GMT+1')
        dt = datetime.now(tz)
        sn = _keylib.packs(dt)
        sp = keylib.packs(dt)
        eq(sn, sp)

        dn = _keylib.unpacks(sn)
        dp = keylib.unpacks(sp)
        eq(dn, dp)
Ejemplo n.º 10
0
 def test1(self):
     for seq in self.SEQS:
         packed = map(lambda s: keylib.packs('', s), seq)
         rnge = range(len(packed))
         done = sorted(rnge, key=packed.__getitem__)
         try:
             eq(done, rnge)
         except:
             print 'failed:', seq
             raise
Ejemplo n.º 11
0
    def test_pos(self):
        tz = dateutil.tz.gettz('Etc/GMT+1')
        dt = datetime.now(tz)
        sn = _keylib.packs('', dt)
        sp = keylib.packs('', dt)
        eq(sn, sp)

        dn = _keylib.unpacks('', sn)
        dp = keylib.unpacks('', sp)
        eq(dn, dp)
Ejemplo n.º 12
0
    def test_utc(self):
        tz = dateutil.tz.gettz('Etc/UTC')
        dt = datetime.now(tz)
        sn = _keylib.packs(dt)
        sp = keylib.packs(dt)
        eq(sn, sp)

        dn = _keylib.unpacks(sn)
        dp = keylib.unpacks(sp)
        eq(dn, dp)
Ejemplo n.º 13
0
 def _index_keys(self, key, obj):
     """Generate a list of encoded keys representing index entries for `obj`
     existing under `key`."""
     idx_keys = []
     for idx in self.indices.itervalues():
         lst = idx.func(obj)
         if lst:
             if type(lst) is not list:
                 lst = [lst]
             for idx_key in lst:
                 idx_keys.append(keylib.packs(idx.prefix, [idx_key, key]))
     return idx_keys
Ejemplo n.º 14
0
    def _prepare_batch(self, items):
        """Encode a list of records into their batch representation, returning
        a tuple of the encoded physical key and value."""
        keytups = [key for key, _ in reversed(items)]
        phys = keylib.packs(keytups, self.prefix)

        if len(items) == 1:
            out = items[0][1]
        else:
            out = bytearray()
            keylib.write_int(len(items), out.append, 0)
            for _, data in items:
                keylib.write_int(len(data), out.append, 0)
            concat = ''.join(data for _, data in items)
            out.extend(self.compressor.pack(concat))
        return phys, bytes(out)
Ejemplo n.º 15
0
 def delete(self, key):
     """Delete any existing record filed under `key`.
     """
     it = self._iter(key, None, None, None, None, True, None)
     txn = self.store._txn_context.get()
     for batch, key_, data in it:
         if key != key_:
             break
         obj = self.encoder.unpack(key, data)
         if self.indices:
             for key in self._index_keys(key, obj):
                 txn.delete(key)
         if batch:
             self._split_batch(key)
         else:
             txn.delete(keylib.packs(self.prefix, key))
Ejemplo n.º 16
0
    def _prepare_batch(self, items, packer):
        packer_prefix = self.store._encoder_prefix.get(packer)
        if not packer_prefix:
            packer_prefix = self.store.add_encoder(packer)
        keytups = [key for key, _ in reversed(items)]
        phys = keylib.packs(self.prefix, keytups)
        out = bytearray()

        if len(items) == 1:
            out.extend(packer_prefix)
            out.extend(packer.pack(items[0][1]))
        else:
            keylib.write_int(len(items), out.append, 0)
            for _, data in items:
                keylib.write_int(len(data), out.append, 0)
            out.extend(packer_prefix)
            concat = ''.join(data for _, data in items)
            out.extend(packer.pack(concat))
        return phys, str(out)
Ejemplo n.º 17
0
    def _prepare_batch(self, items):
        if len(items) == 1:
            phys = keylib.Key(items[0][0]).to_raw(self.prefix)
            return phys, items[0][1]

        high_key_s = items[-1][0].to_raw()
        low_key_s = items[0][0].to_raw()
        cp_len = iterators.common_prefix_len(high_key_s, low_key_s)

        raw = bytearray(2 + (2 * (1 + len(items))))
        raw[0:2] = struct.pack('>HH', len(items), len(raw))
        for idx, (key, value) in enumerate(items):
            opos = 2 + (2 * idx)
            raw[opos:opos + 2] = struct.pack('>H', len(raw))
            suffix = key.to_raw()[cp_len:]
            raw.append(len(suffix))
            raw.extend(suffix)
            raw.extend(value)

        opos = 2 + (2 * len(items))
        raw[opos:opos + 2] = struct.pack('>H', len(raw))
        phys = keylib.packs([items[-1][0], items[0][0]], self.prefix)
        return phys, self.compressor.pack(bytes(raw))
Ejemplo n.º 18
0
 def testUuid(self):
     t = ('a', uuid.uuid4(), 'b')
     s = keylib.packs(t)
     eq(t, keylib.unpack(s))
Ejemplo n.º 19
0
 def test_single_sort_lower(self):
     for val in self.SINGLE_VALS:
         e1 = keylib.packs('', (val,))
         e2 = keylib.packs('', [(val, val),])
         lt(e1, e2, 'eek %r' % (val,))
Ejemplo n.º 20
0
 def test_single(self):
     for val in self.SINGLE_VALS:
         encoded = keylib.packs('', (val,))
         decoded = keylib.unpacks('', encoded)
         eq([(val,)], decoded, 'input was %r' % (val,))
Ejemplo n.º 21
0
 def test3(self):
     t = [('index:Item:first_last', 11, 'item')]
     p = keylib.packs(t)
     eq(keylib.unpacks(p), t)
Ejemplo n.º 22
0
 def test3(self):
     t = [('index:Item:first_last', 11, 'item')]
     p = keylib.packs('', t)
     eq(keylib.unpacks('', p), t)
Ejemplo n.º 23
0
 def test1(self):
     t = [('', 11, 'item')]
     p = keylib.packs('', t)
     eq(keylib.unpacks('', p), t)
Ejemplo n.º 24
0
 def testStringSorting(self):
     strs = [(x,) for x in ('dave', 'dave\x00', 'dave\x01', 'davee\x01')]
     encs = map(lambda o: keylib.packs('', o), strs)
     encs.sort()
     eq(strs, [keylib.unpack('', x) for x in encs])
Ejemplo n.º 25
0
 def _encode(self, s):
     return keylib.packs(s, self.prefix)
Ejemplo n.º 26
0
 def test2(self):
     t = [('index:I', 11, 'item')]
     p = keylib.packs(t)
     eq(keylib.unpacks(p), t)
Ejemplo n.º 27
0
 def test_utc(self):
     tz = dateutil.tz.gettz('Etc/UTC')
     dt = self._now_truncate(tz)
     s = keylib.packs('', dt)
     dt2, = keylib.unpack('', s)
     eq(dt, dt2)
Ejemplo n.º 28
0
 def test_minusone(self):
     tz = dateutil.tz.gettz('Etc/GMT-1')
     dt = self._now_truncate(tz)
     s = keylib.packs(dt)
     dt2, = keylib.unpack(s)
     eq(dt, dt2)
Ejemplo n.º 29
0
 def test_utc(self):
     tz = dateutil.tz.gettz('Etc/UTC')
     dt = self._now_truncate(tz)
     s = keylib.packs(dt)
     dt2, = keylib.unpack(s)
     eq(dt, dt2)
Ejemplo n.º 30
0
 def test_naive(self):
     dt = self._now_truncate()
     s = keylib.packs(dt)
     dt2, = keylib.unpack(s)
     eq(dt.utctimetuple(), dt2.utctimetuple())
Ejemplo n.º 31
0
    def batch(self,
              lo=None,
              hi=None,
              prefix=None,
              max_recs=None,
              max_bytes=None,
              max_keylen=None,
              preserve=True,
              max_phys=None,
              grouper=None):
        """
        Search the key range *lo..hi* for individual records, combining them
        into a batches.

        Returns `(found, made, last_key)` indicating the number of records
        combined, the number of batches produced, and the last key visited
        before `max_phys` was exceeded.

        Batch size is controlled via `max_recs` and `max_bytes`; at least one
        must not be ``None``. Larger sizes may cause pathological behaviour in
        the storage engine (for example, space inefficiency). Since batches are
        fully decompressed before any member may be accessed via
        :py:meth:`get() <Collection.get>` or :py:meth:`iteritems()
        <Collection.iteritems>`, larger sizes may slow decompression, waste IO
        bandwidth, and temporarily use more RAM.

            `lo`:
                Lowest search key.

            `hi`:
                Highest search key.

            `max_recs`:
                Maximum number of records contained by any single batch. When
                this count is reached, the current batch is saved and a new one
                is created.

            `max_bytes`:
                Maximum size in bytes of the batch record's value after
                compression, or ``None`` for no maximum size. When not
                ``None``, values are recompressed after each member is
                appended, in order to test if `max_bytes` has been reached. This
                is inefficient, but provides the best guarantee of final record
                size. Single records are skipped if they exceed this size when
                compressed individually.

            `preserve`:
                If ``True``, then existing batch records in the database are
                left untouched. When one is found within `lo..hi`, the
                currently building batch is finished and the found batch is
                skipped over.

                If ``False``, found batches are exploded and their members
                contribute to the currently building batch.

            `max_phys`:
                Maximum number of physical keys to visit in any particular
                call. A collection may be incrementally batched by repeatedly
                invoking :py:meth:`Collection.batch` with `max` set, and `lo`
                set to `last_key` of the previous run, until `found` returns
                ``0``. This allows batching to complete over several
                transactions without blocking other users.

            `grouper`:
                Specifies a grouping function used to decide when to avoid
                compressing unrelated records. The function is passed a
                record's value. A new batch is triggered each time the
                function's return value changes.

        """
        assert max_bytes or max_recs, 'max_bytes and/or max_recs is required.'

        txn = self.store._txn_context.get()
        it = self.ITERATOR_CLASS(txn, self.prefix, self.compressor)
        groupval = object()
        items = []

        for r in iterators.from_args(it, None, lo, hi, prefix, False, None,
                                     True, max_phys):
            if preserve and len(r.keys) > 1:
                self._write_batch(txn, items, self.compressor)
            else:
                txn.delete(keylib.packs(r.key, self.prefix))
                items.append((r.key, r.data))
                if max_bytes:
                    _, encoded = self._prepare_batch(items)
                    if len(encoded) > max_bytes:
                        items.pop()
                        self._write_batch(txn, items)
                        items.append((r.key, r.data))
                done = max_recs and len(items) == max_recs
                if (not done) and grouper:
                    val = grouper(self.encoder.unpack(r.key, r.data))
                    done = val != groupval
                    groupval = val
                if done:
                    self._write_batch(txn, items)
        self._write_batch(txn, items)
Ejemplo n.º 32
0
 def test1(self):
     t = [('', 11, 'item')]
     p = keylib.packs(t)
     eq(keylib.unpacks(p), t)
Ejemplo n.º 33
0
 def testUuid(self):
     t = ('a', uuid.uuid4(), 'b')
     s = keylib.packs('', t)
     eq(t, keylib.unpack('', s))
Ejemplo n.º 34
0
 def testStringSorting(self):
     strs = [(x,) for x in ('dave', 'dave\x00', 'dave\x01', 'davee\x01')]
     encs = map(lambda o: keylib.packs(o), strs)
     encs.sort()
     eq(strs, [keylib.unpack(x) for x in encs])
Ejemplo n.º 35
0
 def test2(self):
     t = [('index:I', 11, 'item')]
     p = keylib.packs('', t)
     eq(keylib.unpacks('', p), t)
Ejemplo n.º 36
0
 def test_list(self):
     lst = [(1,), (2,)]
     eq(lst, keylib.unpacks(keylib.packs(lst)))
Ejemplo n.º 37
0
 def test_naive(self):
     dt = datetime.now()
     sn = _keylib.packs('', dt)
     sp = keylib.packs('', dt)
     eq(sn, sp)
Ejemplo n.º 38
0
 def test_naive(self):
     dt = datetime.now()
     sn = _keylib.packs(dt)
     sp = keylib.packs(dt)
     eq(sn, sp)
Ejemplo n.º 39
0
 def test_naive(self):
     dt = self._now_truncate()
     s = keylib.packs('', dt)
     dt2, = keylib.unpack('', s)
     eq(dt.utctimetuple(), dt2.utctimetuple())
Ejemplo n.º 40
0
 def test_single_sort_lower(self):
     for val in self.SINGLE_VALS:
         e1 = keylib.packs((val,))
         e2 = keylib.packs([(val, val),])
         lt(e1, e2, 'eek %r' % (val,))
Ejemplo n.º 41
0
 def test_minusone(self):
     tz = dateutil.tz.gettz('Etc/GMT-1')
     dt = self._now_truncate(tz)
     s = keylib.packs('', dt)
     dt2, = keylib.unpack('', s)
     eq(dt, dt2)
Ejemplo n.º 42
0
 def test1(self):
     s = keylib.packs((1, 2, 3, 4))
     assert keylib.unpacks(s, s) == _keylib.unpacks(s, s)
Ejemplo n.º 43
0
 def _enc(self, *args, **kwargs):
     return keylib.packs('', *args, **kwargs)
Ejemplo n.º 44
0
 def test_counter(self):
     s = keylib.packs(('dave', 1))
     eq([('dave', 1)], keylib.unpacks(s))
Ejemplo n.º 45
0
Archivo: core_test.py Proyecto: dw/acid
 def _encode(self, s):
     return keylib.packs(s, self.prefix)
Ejemplo n.º 46
0
 def test_single(self):
     for val in self.SINGLE_VALS:
         encoded = keylib.packs((val,))
         decoded = keylib.unpacks(encoded)
         eq([(val,)], decoded, 'input was %r' % (val,))