def add(self, item, indx=None):
'''
Add a single item to the sequence.
'''
if indx is not None:
if indx >= self.indx:
self.slab.put(s_common.int64en(indx),
s_msgpack.en(item),
append=True,
db=self.db)
self.indx = indx + 1
self.size += 1
self._wake_waiters()
return indx
oldv = self.slab.replace(s_common.int64en(indx),
s_msgpack.en(item),
db=self.db)
if oldv is None:
self.size += 1
return indx
indx = self.indx
retn = self.slab.put(s_common.int64en(indx),
s_msgpack.en(item),
append=True,
db=self.db)
assert retn, "Not adding the largest index"
self.indx += 1
self.size += 1
self._wake_waiters()
return indx
async def delUserNotif(self, indx):
envl = self.notifseqn.pop(indx)
if envl is None:
return
mesg = envl[1]
useriden, mesgtime, mesgtype, mesgdata = mesg
indxbyts = s_common.int64en(indx)
userbyts = s_common.uhex(useriden)
timebyts = s_common.int64en(mesgtime)
typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode())
self.slab.delete(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime)
self.slab.delete(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype)
async def _addUserNotif(self, mesg, nexsitem):
indx = self.notifseqn.add(mesg, indx=nexsitem[0])
indxbyts = s_common.int64en(indx)
useriden, mesgtime, mesgtype, mesgdata = mesg
userbyts = s_common.uhex(useriden)
timebyts = s_common.int64en(mesgtime)
typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode())
self.slab.put(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime, dupdata=True)
self.slab.put(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype, dupdata=True)
return indx
async def puts(self, name, items):
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
abrv = self.abrv.nameToAbrv(name)
offs = retn = self.offsets.get(name, 0)
for item in items:
self.slab.put(abrv + s_common.int64en(offs),
s_msgpack.en(item),
db=self.qdata)
self.sizes.inc(name, 1)
offs = self.offsets.inc(name, 1)
# wake the sleepers
evnt = self.waiters.get(name)
if evnt is not None:
evnt.set()
return retn
async def puts(self, name, items, reqid=None):
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
abrv = self.abrv.nameToAbrv(name)
offs = retn = self.offsets.get(name, 0)
if reqid is not None:
if reqid == self.lastreqid.get(name):
return retn
self.lastreqid.set(name, reqid)
for item in items:
putv = self.slab.put(abrv + s_common.int64en(offs), s_msgpack.en(item), db=self.qdata)
assert putv, 'Put failed'
self.sizes.inc(name, 1)
offs = self.offsets.inc(name, 1)
# wake the sleepers
evnt = self.waiters.get(name)
if evnt is not None:
evnt.set()
return retn
def save(self, items):
'''
Save a series of items to a sequence.
Args:
items (tuple): The series of items to save into the sequence.
Returns:
None
'''
rows = []
indx = self.indx
for item in items:
byts = s_msgpack.en(item)
lkey = s_common.int64en(indx)
indx += 1
rows.append((lkey, byts))
self.lenv.putmulti(rows, append=True, db=self.db)
self.indx = indx
async def _calcFormCounts(self):
'''
Recalculate form counts from scratch.
'''
logger.info('Calculating form counts from scratch.')
self.counts.clear()
nameforms = list(self.model.forms.items())
fairiter = 5
tcount = 0
for i, (name, form) in enumerate(nameforms, 1):
logger.info('Calculating form counts for [%s] [%s/%s]',
name, i, len(nameforms))
count = 0
async for buid, valu in self.layer.iterFormRows(name):
count += 1
tcount += 1
if count % fairiter == 0:
await asyncio.sleep(0)
# identity check for small integer
if fairiter is 5 and tcount > 100000:
fairiter = 1000
self.counts[name] = count
for name, valu in self.counts.items():
byts = s_common.int64en(valu)
self.slab.put(name.encode('utf8'), byts, db=self.formcountdb)
logger.info('Done calculating form counts.')
def save(self, items):
'''
Save a series of items to a sequence.
Args:
items (tuple): The series of items to save into the sequence.
Returns:
The index of the first item
'''
rows = []
indx = self.indx
size = 0
tick = s_common.now()
for item in items:
byts = s_msgpack.en(item)
size += len(byts)
lkey = s_common.int64en(indx)
indx += 1
rows.append((lkey, byts))
self.slab.putmulti(rows, append=True, db=self.db)
took = s_common.now() - tick
origindx = self.indx
self.indx = indx
return {'indx': indx, 'size': size, 'count': len(items), 'time': tick, 'took': took, 'orig': origindx}
def pop(self, offs):
'''
Pop a single entry at the given offset.
'''
byts = self.slab.pop(s_common.int64en(offs), db=self.db)
if byts is not None:
return (offs, s_msgpack.un(byts))
def sync(self):
tups = [(p, s_common.int64en(self.cache[p])) for p in self.dirty]
if not tups:
return
self.slab.putmulti(tups, db=self.countdb)
self.dirty.clear()
def rows(self, offs):
'''
Iterate over raw indx, bytes tuples from a given offset.
'''
lkey = s_common.int64en(offs)
for lkey, byts in self.slab.scanByRange(lkey, db=self.db):
indx = s_common.int64un(lkey)
yield indx, byts
async def pop(self, name, offs):
'''
Pop a single entry from the named queue by offset.
'''
abrv = self.abrv.nameToAbrv(name)
byts = self.slab.pop(abrv + s_common.int64en(offs), db=self.qdata)
if byts is not None:
return (offs, s_msgpack.un(byts))
def get(self, offs):
'''
Retrieve a single row by offset
'''
lkey = s_common.int64en(offs)
valu = self.slab.get(lkey, db=self.db)
if valu is not None:
return s_msgpack.un(valu)
def pokeFormCount(self, form, valu):
curv = self.counts.get(form, 0)
newv = curv + valu
self.counts[form] = newv
byts = s_common.int64en(newv)
self.slab.put(form.encode('utf8'), byts, db=self.formcountdb)
async def _setLyrErr(self, lyriden, offset, err):
self.errcnts.inc(lyriden)
lkey = s_common.uhex(lyriden) + s_common.int64en(offset)
errb = s_msgpack.en(err)
return self.slab.put(lkey,
errb,
dupdata=False,
overwrite=True,
db=self.errors)
def add(self, item):
'''
Add a single item to the sequence.
'''
byts = s_msgpack.en(item)
lkey = s_common.int64en(self.indx)
self.slab.put(lkey, byts, db=self.db)
self.indx += 1
async def dele(self, name, minoffs, maxoffs):
'''
Remove queue entries from minoffs, up-to (and including) the queue entry at maxoffs.
'''
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
if minoffs < 0 or maxoffs < 0 or maxoffs < minoffs:
return
minindx = s_common.int64en(minoffs)
maxindx = s_common.int64en(maxoffs)
abrv = self.abrv.nameToAbrv(name)
for lkey, _ in self.slab.scanByRange(abrv + minindx, abrv + maxindx, db=self.qdata):
self.slab.delete(lkey, db=self.qdata)
self.sizes.set(name, self.sizes.get(name) - 1)
await asyncio.sleep(0)
def append(self, item):
byts = s_msgpack.en(item)
indx = self.indx
lkey = s_common.int64en(indx)
self.slab.put(lkey, byts, db=self.db)
self.indx += 1
return indx
async def cull(self, offs):
'''
Remove entries up to (and including) the given offset.
'''
for itemoffs, valu in self.iter(0):
if itemoffs > offs:
return
self.slab.delete(s_common.int64en(itemoffs), db=self.db)
await asyncio.sleep(0)
def add(self, item):
'''
Add a single item to the sequence.
'''
indx = self.indx
self.slab.put(s_common.int64en(indx), s_msgpack.en(item), db=self.db)
self.indx += 1
self._wake_waiters()
return indx
def setBytsToAbrv(self, byts):
try:
return self.bytsToAbrv(byts)
except s_exc.NoSuchAbrv:
pass
abrv = s_common.int64en(self.offs)
self.offs += 1
self.slab.put(byts, abrv, db=self.name2abrv)
self.slab.put(abrv, byts, db=self.abrv2name)
return abrv
def bytsToAbrv(self, byts):
abrv = self.slab.get(byts, db=self.name2abrv)
if abrv is not None:
return abrv
abrv = s_common.int64en(self.offs)
self.offs += 1
self.slab.put(byts, abrv, db=self.name2abrv)
self.slab.put(abrv, byts, db=self.abrv2name)
return abrv
def trim(self, offs):
'''
Delete entries starting at offset and moving forward.
'''
retn = False
startkey = s_common.int64en(offs)
for lkey, _ in self.slab.scanByRange(startkey, db=self.db):
retn = True
self.slab.delete(lkey, db=self.db)
if retn:
self.indx = self.nextindx()
return retn
def nameToAbrv(self, name):
lkey = name.encode()
abrv = self.slab.get(lkey, db=self.name2abrv)
if abrv is not None:
return abrv
abrv = s_common.int64en(self.offs)
self.offs += 1
self.slab.put(lkey, abrv, db=self.name2abrv)
self.slab.put(abrv, lkey, db=self.abrv2name)
return abrv
async def aiter(self, offs, wait=False, timeout=None):
'''
Iterate over items in a sequence from a given offset.
Args:
offs (int): The offset to begin iterating from.
wait (boolean): Once caught up, yield new results in realtime.
timeout (int): Max time to wait for a new item.
Yields:
(indx, valu): The index and valu of the item.
'''
startkey = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(startkey, db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
# no awaiting between here and evnt.timewait()
if wait:
evnt = s_coro.Event()
try:
self.addevents.append(evnt)
while True:
evnt.clear()
if not await evnt.timewait(timeout=timeout):
return
startkey = s_common.int64en(offs + 1)
for lkey, lval in self.slab.scanByRange(startkey,
db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
finally:
self.addevents.remove(evnt)
def iter(self, offs):
'''
Iterate over items in a sequence from a given offset.
Args:
offs (int): The offset to begin iterating from.
Yields:
(indx, valu): The index and valu of the item.
'''
startkey = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(startkey, db=self.db):
offs = s_common.int64un(lkey)
valu = s_msgpack.un(lval)
yield offs, valu
async def cull(self, name, offs):
'''
Remove up-to (and including) the queue entry at offs.
'''
if offs < 0:
return
indx = s_common.int64en(offs)
abrv = self.abrv.nameToAbrv(name)
for lkey, _ in self.slab.scanByRange(abrv + int64min,
abrv + indx,
db=self.qdata):
self.slab.delete(lkey, db=self.qdata)
self.sizes.set(name, self.sizes.get(name) - 1)
await asyncio.sleep(0)
async def cull(self, name, offs):
'''
Remove up-to (and including) the queue entry at offs.
'''
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
if offs < 0:
return
indx = s_common.int64en(offs)
abrv = self.abrv.nameToAbrv(name)
for lkey, _ in self.slab.scanByRange(abrv + int64min, abrv + indx, db=self.qdata):
self.slab.delete(lkey, db=self.qdata)
self.sizes.set(name, self.sizes.get(name) - 1)
await asyncio.sleep(0)
def getNameAbrv(self, name):
'''
Create or return a layer specific abbreviation for the given name.
'''
utf8 = name.encode()
abrv = self.layrslab.get(utf8, db=self.name2abrv)
if abrv is not None:
return abrv
nexi = self.metadict.get('nameabrv', 0)
self.metadict.set('nameabrv', nexi + 1)
abrv = s_common.int64en(nexi)
self.layrslab.put(utf8, abrv, db=self.name2abrv)
self.layrslab.put(abrv, utf8, db=self.abrv2name)
return abrv
async def gets(self, name, offs, size=None, cull=False, wait=False):
'''
Yield (offs, item) tuples from the message queue.
'''
if self.queues.get(name) is None:
mesg = f'No queue named {name}.'
raise s_exc.NoSuchName(mesg=mesg, name=name)
if cull and offs > 0:
await self.cull(name, offs - 1)
abrv = self.abrv.nameToAbrv(name)
count = 0
while not self.slab.isfini:
indx = s_common.int64en(offs)
for lkey, lval in self.slab.scanByRange(abrv + indx,
abrv + int64max,
db=self.qdata):
offs = s_common.int64un(lkey[8:])
yield offs, s_msgpack.un(lval)
offs += 1 # in case of wait, we're at next offset
count += 1
if size is not None and count >= size:
return
if not wait:
return
evnt = self.waiters[name]
evnt.clear()
await evnt.wait()
def set(self, iden, offs):
buid = s_common.uhex(iden)
byts = s_common.int64en(offs)
self.lenv.put(buid, byts, db=self.db)
