async def test_lmdbslab_hotcount(self): with self.getTestDir() as dirn: path = os.path.join(dirn, 'test.lmdb') async with await s_lmdbslab.Slab.anit(path, map_size=1000000, lockmemory=True) as slab, \ await s_lmdbslab.HotCount.anit(slab, 'counts') as ctr: self.eq(0, ctr.get('foo')) self.eq({}, ctr.pack()) ctr.inc('foo') self.eq({'foo': 1}, ctr.pack()) self.eq(1, ctr.get('foo')) ctr.set('bar', 42) self.eq({'foo': 1, 'bar': 42}, ctr.pack()) ctr.sync() self.eq({'foo': 1, 'bar': 42}, ctr.pack()) ctr.inc('foo') ctr.inc('foo') ctr.set('bar', 37) ctr.sync() cache = [] for lkey, lval in slab.scanByFull(db='counts'): cache.append((lkey, s_common.int64un(lval))) self.len(1, [k for k, v in cache if k == b'foo']) self.len(1, [k for k, v in cache if k == b'bar'])
def rows(self, offs): ''' Iterate over raw indx, bytes tuples from a given offset. ''' lkey = s_common.int64en(offs) for lkey, byts in self.slab.scanByRange(lkey, db=self.db): indx = s_common.int64un(lkey) yield indx, byts
def _initFormCounts(self): self.counts = {} self.formcountdb = self.slab.initdb('form:counts') for lkey, lval in self.slab.scanByFull(db=self.formcountdb): form = lkey.decode('utf8') valu = s_common.int64un(lval) self.counts[form] = valu
async def getLyrErrs(self, lyriden): lpref = s_common.uhex(lyriden) errs = [] for lkey, errb in self.slab.scanByPref(lpref, db=self.errors): offset = s_common.int64un(lkey[16:]) err = s_msgpack.un(errb) errs.append((offset, err)) return errs
def get(self, iden): buid = s_common.uhex(iden) byts = self.lenv.get(buid, db=self.db) if byts is None: return 0 return s_common.int64un(byts)
def last(self): last = self.slab.last(db=self.db) if last is None: return None lkey, lval = last indx = s_common.int64un(lkey) return indx, s_msgpack.un(lval)
def __init__(self, slab, name): self.slab = slab self.name2abrv = slab.initdb(f'{name}:byts2abrv') self.abrv2name = slab.initdb(f'{name}:abrv2byts') self.offs = 0 item = self.slab.last(db=self.abrv2name) if item is not None: self.offs = s_common.int64un(item[0]) + 1
async def iterUserNotifs(self, useriden, size=None): # iterate user notifications backward userbyts = s_common.uhex(useriden) count = 0 for _, indxbyts in self.slab.scanByPrefBack(userbyts, db=self.notif_indx_usertype): indx = s_common.int64un(indxbyts) mesg = self.notifseqn.getraw(indxbyts) yield (indx, mesg) count += 1 if size is not None and count >= size: break
def nextindx(self): ''' Determine the next insert offset according to storage. Returns: int: The next insert offset. ''' byts = self.slab.lastkey(db=self.db) if byts is None: return 0 return s_common.int64un(byts) + 1
def nextindx(self): ''' Determine the next insert offset according to storage. Returns: int: The next insert offset. ''' indx = 0 with s_lmdbslab.Scan(self.slab, self.db) as curs: last_key = curs.last_key() if last_key is not None: indx = s_common.int64un(last_key) + 1 return indx
async def __anit__(self, slab, name): await s_base.Base.__anit__(self) self.slab = slab self.cache = collections.defaultdict(int) self.dirty = set() self.countdb = self.slab.initdb(name, integerkey=True) for lkey, lval in self.slab.scanByFull(db=self.countdb): self.cache[lkey] = s_common.int64un(lval) slab.on('commit', self._onSlabCommit) self.onfini(self.sync)
async def aiter(self, offs, wait=False, timeout=None): ''' Iterate over items in a sequence from a given offset. Args: offs (int): The offset to begin iterating from. wait (boolean): Once caught up, yield new results in realtime. timeout (int): Max time to wait for a new item. Yields: (indx, valu): The index and valu of the item. ''' startkey = s_common.int64en(offs) for lkey, lval in self.slab.scanByRange(startkey, db=self.db): offs = s_common.int64un(lkey) valu = s_msgpack.un(lval) yield offs, valu # no awaiting between here and evnt.timewait() if wait: evnt = s_coro.Event() try: self.addevents.append(evnt) while True: evnt.clear() if not await evnt.timewait(timeout=timeout): return startkey = s_common.int64en(offs + 1) for lkey, lval in self.slab.scanByRange(startkey, db=self.db): offs = s_common.int64un(lkey) valu = s_msgpack.un(lval) yield offs, valu finally: self.addevents.remove(evnt)
def iter(self, offs): ''' Iterate over items in a sequence from a given offset. Args: offs (int): The offset to begin iterating from. Yields: (indx, valu): The index and valu of the item. ''' startkey = s_common.int64en(offs) for lkey, lval in self.slab.scanByRange(startkey, db=self.db): offs = s_common.int64un(lkey) valu = s_msgpack.un(lval) yield offs, valu
def nextindx(self): ''' Determine the next insert offset according to storage. Args: xact (lmdb.Transaction): An LMDB transaction. Returns: int: The next insert offset. ''' indx = 0 with s_lmdbslab.Scan(self.lenv, self.db) as curs: last_key = curs.last_key() if last_key is not None: indx = s_common.int64un(last_key) + 1 return indx
async def gets(self, name, offs, size=None, cull=False, wait=False): ''' Yield (offs, item) tuples from the message queue. ''' if self.queues.get(name) is None: mesg = f'No queue named {name}.' raise s_exc.NoSuchName(mesg=mesg, name=name) if cull and offs > 0: await self.cull(name, offs - 1) abrv = self.abrv.nameToAbrv(name) count = 0 while not self.slab.isfini: indx = s_common.int64en(offs) for lkey, lval in self.slab.scanByRange(abrv + indx, abrv + int64max, db=self.qdata): offs = s_common.int64un(lkey[8:]) yield offs, s_msgpack.un(lval) offs += 1 # in case of wait, we're at next offset count += 1 if size is not None and count >= size: return if not wait: return evnt = self.waiters[name] evnt.clear() await evnt.wait()
def _getFirstIndx(slab) -> Optional[int]: db = slab.initdb('info') bytz = slab.get(b'firstindx', db=db) if bytz is None: return 0 return s_common.int64un(bytz)
async def getLyrErrs(self, lyriden): lpref = s_common.uhex(lyriden) for lkey, errb in self.slab.scanByPref(lpref, db=self.errors): offset = s_common.int64un(lkey[16:]) err = s_msgpack.un(errb) yield offset, err
def first(self): for lkey, lval in self.slab.scanByFull(db=self.db): return s_common.int64un(lkey), s_msgpack.un(lval) return None