async def test_lmdbslab_hotcount(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
async with await s_lmdbslab.Slab.anit(path, map_size=1000000, lockmemory=True) as slab, \
await s_lmdbslab.HotCount.anit(slab, 'counts') as ctr:
self.eq(0, ctr.get('foo'))
self.eq({}, ctr.pack())
ctr.inc('foo')
self.eq({'foo': 1}, ctr.pack())
self.eq(1, ctr.get('foo'))
ctr.set('bar', 42)
self.eq({'foo': 1, 'bar': 42}, ctr.pack())
ctr.sync()
self.eq({'foo': 1, 'bar': 42}, ctr.pack())
ctr.inc('foo')
ctr.inc('foo')
ctr.set('bar', 37)
ctr.sync()
cache = []
for lkey, lval in slab.scanByFull(db='counts'):
cache.append((lkey, s_common.int64un(lval)))
self.len(1, [k for k, v in cache if k == b'foo'])
self.len(1, [k for k, v in cache if k == b'bar'])
def rows(self, offs):
'''
Iterate over raw indx, bytes tuples from a given offset.
'''
lkey = s_common.int64en(offs)
for lkey, byts in self.slab.scanByRange(lkey, db=self.db):
indx = s_common.int64un(lkey)
yield indx, byts
def _initFormCounts(self):
self.counts = {}
self.formcountdb = self.slab.initdb('form:counts')
for lkey, lval in self.slab.scanByFull(db=self.formcountdb):
form = lkey.decode('utf8')
valu = s_common.int64un(lval)
self.counts[form] = valu
async def getLyrErrs(self, lyriden):
lpref = s_common.uhex(lyriden)
errs = []
for lkey, errb in self.slab.scanByPref(lpref, db=self.errors):
offset = s_common.int64un(lkey[16:])
err = s_msgpack.un(errb)
errs.append((offset, err))
return errs
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.lenv.get(buid, db=self.db)
if byts is None:
return 0
return s_common.int64un(byts)
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.lenv.get(buid, db=self.db)
if byts is None:
return 0
return s_common.int64un(byts)
def last(self):
last = self.slab.last(db=self.db)
if last is None:
return None
lkey, lval = last
indx = s_common.int64un(lkey)
return indx, s_msgpack.un(lval)
def __init__(self, slab, name):
self.slab = slab
self.name2abrv = slab.initdb(f'{name}:byts2abrv')
self.abrv2name = slab.initdb(f'{name}:abrv2byts')
self.offs = 0
item = self.slab.last(db=self.abrv2name)
if item is not None:
self.offs = s_common.int64un(item[0]) + 1
async def iterUserNotifs(self, useriden, size=None):
# iterate user notifications backward
userbyts = s_common.uhex(useriden)
count = 0
for _, indxbyts in self.slab.scanByPrefBack(userbyts, db=self.notif_indx_usertype):
indx = s_common.int64un(indxbyts)
mesg = self.notifseqn.getraw(indxbyts)
yield (indx, mesg)
count += 1
if size is not None and count >= size:
break
def nextindx(self):
'''
Determine the next insert offset according to storage.
Returns:
int: The next insert offset.
'''
byts = self.slab.lastkey(db=self.db)
if byts is None:
return 0
return s_common.int64un(byts) + 1
def nextindx(self):
'''
Determine the next insert offset according to storage.
Returns:
int: The next insert offset.
'''
indx = 0
with s_lmdbslab.Scan(self.slab, self.db) as curs:
last_key = curs.last_key()
if last_key is not None:
indx = s_common.int64un(last_key) + 1
return indx
async def __anit__(self, slab, name):
await s_base.Base.__anit__(self)
self.slab = slab
self.cache = collections.defaultdict(int)
self.dirty = set()
self.countdb = self.slab.initdb(name, integerkey=True)
for lkey, lval in self.slab.scanByFull(db=self.countdb):
self.cache[lkey] = s_common.int64un(lval)
slab.on('commit', self._onSlabCommit)
self.onfini(self.sync)
async def aiter(self, offs, wait=False, timeout=None):
'''
Iterate over items in a sequence from a given offset.
Args:
offs (int): The offset to begin iterating from.
wait (boolean): Once caught up, yield new results in realtime.
timeout (int): Max time to wait for a new item.
Yields:
(indx, valu): The index and valu of the item.
'''
startkey = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(startkey, db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
# no awaiting between here and evnt.timewait()
if wait:
evnt = s_coro.Event()
try:
self.addevents.append(evnt)
while True:
evnt.clear()
if not await evnt.timewait(timeout=timeout):
return
startkey = s_common.int64en(offs + 1)
for lkey, lval in self.slab.scanByRange(startkey,
db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
finally:
self.addevents.remove(evnt)
def iter(self, offs):
'''
Iterate over items in a sequence from a given offset.
Args:
offs (int): The offset to begin iterating from.
Yields:
(indx, valu): The index and valu of the item.
'''
startkey = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(startkey, db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
def nextindx(self):
'''
Determine the next insert offset according to storage.
Args:
xact (lmdb.Transaction): An LMDB transaction.
Returns:
int: The next insert offset.
'''
indx = 0
with s_lmdbslab.Scan(self.lenv, self.db) as curs:
last_key = curs.last_key()
if last_key is not None:
indx = s_common.int64un(last_key) + 1
return indx
async def gets(self, name, offs, size=None, cull=False, wait=False):
'''
Yield (offs, item) tuples from the message queue.
'''
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
if cull and offs > 0:
await self.cull(name, offs - 1)
abrv = self.abrv.nameToAbrv(name)
count = 0
while not self.slab.isfini:
indx = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(abrv + indx,
abrv + int64max,
db=self.qdata):
offs = s_common.int64un(lkey[8:])
yield offs, s_msgpack.un(lval)
offs += 1 # in case of wait, we're at next offset
count += 1
if size is not None and count >= size:
return
if not wait:
return
evnt = self.waiters[name]
evnt.clear()
await evnt.wait()
def _getFirstIndx(slab) -> Optional[int]:
db = slab.initdb('info')
bytz = slab.get(b'firstindx', db=db)
if bytz is None:
return 0
return s_common.int64un(bytz)
async def getLyrErrs(self, lyriden):
lpref = s_common.uhex(lyriden)
for lkey, errb in self.slab.scanByPref(lpref, db=self.errors):
offset = s_common.int64un(lkey[16:])
err = s_msgpack.un(errb)
yield offset, err
def first(self):
for lkey, lval in self.slab.scanByFull(db=self.db):
return s_common.int64un(lkey), s_msgpack.un(lval)
return None
