async def test_slab_seqn(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.eq(seqn.nextindx(), 0)
items = ('foo', 10, 20)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.raises(TypeError, seqn.save, ({'set'},))
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.eq(seqn.nextindx(), 3)
await slab.fini()
# Reopen the seqn and continue where we left off
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.eq(seqn.index(), 3)
self.eq(seqn.nextindx(), 3)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20),
(3, 'foo'), (4, 10), (5, 20)))
self.eq(seqn.nextindx(), 6)
# We can also start in the middle of the sequence
retn = tuple(seqn.iter(4))
self.eq(retn, ((4, 10), (5, 20)))
# iterating past the end yields nothing
retn = tuple(seqn.iter(100))
self.eq(retn, ())
seqn.save(items)
retn = tuple(seqn.iter(0))
self.len(9, retn)
self.eq('foo', seqn.getByIndxByts(b'\x00' * 8))
await slab.fini()
async def __anit__(self, dirn, conf=None): # type: ignore
await s_cell.Cell.__anit__(self, dirn, conf=conf)
# share ourself via the cell dmon as "axon"
# for potential default remote use
self.dmon.share('axon', self)
path = s_common.gendir(self.dirn, 'axon.lmdb')
self.axonslab = await s_lmdbslab.Slab.anit(path)
self.sizes = self.axonslab.initdb('sizes')
self.onfini(self.axonslab.fini)
self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history')
self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn')
node = await self.hive.open(('axon', 'metrics'))
self.axonmetrics = await node.dict()
self.axonmetrics.setdefault('size:bytes', 0)
self.axonmetrics.setdefault('file:count', 0)
self.addHealthFunc(self._axonHealth)
# modularize blob storage
await self._initBlobStor()
self._initAxonHttpApi()
async def __anit__(self, dirn, conf=None):
await s_cell.Cell.__anit__(self, dirn)
# share ourself via the cell dmon as "axon"
# for potential default remote use
self.dmon.share('axon', self)
path = s_common.gendir(self.dirn, 'axon.lmdb')
self.axonslab = await s_lmdbslab.Slab.anit(path)
self.sizes = self.axonslab.initdb('sizes')
self.onfini(self.axonslab.fini)
self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history')
path = s_common.gendir(self.dirn, 'blob.lmdb')
self.blobslab = await s_lmdbslab.Slab.anit(path)
self.blobs = self.blobslab.initdb('blobs')
self.onfini(self.blobslab.fini)
self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn')
node = await self.hive.open(('axon', 'metrics'))
self.axonmetrics = await node.dict()
self.axonmetrics.setdefault('size:bytes', 0)
self.axonmetrics.setdefault('file:count', 0)
async def __anit__(self,
dirn: str,
dologging: bool = True): # type: ignore
await s_base.Base.__anit__(self)
import synapse.lib.lmdbslab as s_lmdbslab # avoid import cycle
import synapse.lib.slabseqn as s_slabseqn # avoid import cycle
self.dirn = dirn
self.readonly = False
self._nexskids: Dict[str, 'Pusher'] = {}
self.mirrors: List[ChangeDist] = []
self.dologging = dologging
if self.dologging:
path = s_common.genpath(self.dirn, 'slabs', 'nexus.lmdb')
self.nexusslab = await s_lmdbslab.Slab.anit(path, map_async=False)
self.nexuslog = s_slabseqn.SlabSeqn(self.nexusslab, 'nexuslog')
async def fini():
if self.dologging:
await self.nexusslab.fini()
[(await dist.fini()) for dist in self.mirrors]
self.onfini(fini)
async def __anit__(self, core, node):
await s_layer.Layer.__anit__(self, core, node)
path = os.path.join(self.dirn, 'layer.lmdb')
splicepath = os.path.join(self.dirn, 'splices.lmdb')
self.fresh = not os.path.exists(path)
mapsize = self.conf.get('lmdb:mapsize')
readahead = self.conf.get('lmdb:readahead')
maxsize = self.conf.get('lmdb:maxsize')
growsize = self.conf.get('lmdb:growsize')
# First check hive configuration. If not set, use passed-in parameter (that defaults to False)
self.lockmemory = self.conf.get('lmdb:lockmemory')
if self.lockmemory is None:
self.lockmemory = core.conf.get('dedicated')
self.layrslab = await s_lmdbslab.Slab.anit(path,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead,
lockmemory=self.lockmemory)
self.onfini(self.layrslab.fini)
self.spliceslab = await s_lmdbslab.Slab.anit(splicepath,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead)
self.onfini(self.spliceslab.fini)
metadb = self.layrslab.initdb('meta')
self.metadict = s_lmdbslab.SlabDict(self.layrslab, metadb)
self._migrate_splices_pre010()
self.dbs = {}
self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu>
self.byprop = await self.initdb('byprop', dupsort=True
) # <form>00<prop>00<indx>=<buid>
self.byuniv = await self.initdb('byuniv',
dupsort=True) # <prop>00<indx>=<buid>
offsdb = await self.initdb('offsets')
self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb)
self.splicelog = s_slabseqn.SlabSeqn(self.spliceslab, 'splices')
self.indxfunc = {
'eq': self._rowsByEq,
'pref': self._rowsByPref,
'range': self._rowsByRange,
}
async def __anit__(self, dirn):
await s_base.Base.__anit__(self)
path = str(pathlib.Path(dirn) / 'slabs' / self.PROV_FN)
self.slab = await s_lmdbslab.Slab.anit(path,
map_size=self.PROV_MAP_SIZE)
self.onfini(self.slab.fini)
self.db = self.slab.initdb('prov')
self.provseq = s_slabseqn.SlabSeqn(self.slab, 'provs')
async def __anit__(self, dirn, conf=None):
await s_base.Base.__anit__(self)
if conf is None:
conf = {}
self.conf = conf
self.dirn = s_common.gendir(dirn)
self._iden = self._getTankIden()
path = s_common.gendir(self.dirn, 'tank.lmdb')
self.slab = await s_lmdbslab.Slab.anit(path, map_async=True, **conf)
self.offs = s_slaboffs.SlabOffs(self.slab, 'offsets')
self._items = s_slabseqn.SlabSeqn(self.slab, 'items')
self._metrics = s_slabseqn.SlabSeqn(self.slab, 'metrics')
self.onfini(self.slab.fini)
async def __anit__(self, dirn: str, donexslog: bool = True): # type: ignore
await s_base.Base.__anit__(self)
import synapse.lib.lmdbslab as s_lmdbslab # avoid import cycle
import synapse.lib.slabseqn as s_slabseqn # avoid import cycle
self.dirn = dirn
self._nexskids: Dict[str, 'Pusher'] = {}
self._mirrors: List[ChangeDist] = []
self.donexslog = donexslog
self._state_lock = asyncio.Lock()
self._state_funcs: List[Callable] = [] # External Callbacks for state changes
# These are used when this cell is a mirror.
self._ldrurl: Optional[str] = None
self._ldr: Optional[s_telepath.Proxy] = None # only set by looptask
self._looptask: Optional[asyncio.Task] = None
self._ldrready = asyncio.Event()
# Used to match pending follower write requests with the responses arriving on the log
self._futures: Dict[str, asyncio.Future] = {}
if self.donexslog:
path = s_common.genpath(self.dirn, 'slabs', 'nexus.lmdb')
self._nexusslab = await s_lmdbslab.Slab.anit(path, map_async=False)
self._nexuslog = s_slabseqn.SlabSeqn(self._nexusslab, 'nexuslog')
async def fini():
if self._looptask:
self._looptask.cancel()
try:
await self._looptask
except Exception:
pass
for futu in self._futures.values():
futu.cancel()
if self._ldr:
self._ldrready.clear()
await self._ldr.fini()
if self.donexslog:
await self._nexusslab.fini()
[(await dist.fini()) for dist in self._mirrors]
self.onfini(fini)
async def __anit__(self, dirn, proven=True):
await s_base.Base.__anit__(self)
global ProvenanceEnabled
ProvenanceEnabled = proven
self.enabled = proven
if self.enabled:
path = str(pathlib.Path(dirn) / 'slabs' / self.PROV_FN)
self.slab = await s_lmdbslab.Slab.anit(path,
map_size=self.PROV_MAP_SIZE)
self.onfini(self.slab.fini)
self.db = self.slab.initdb('prov')
self.provseq = s_slabseqn.SlabSeqn(self.slab, 'provs')
async def __anit__(self, dirn, readonly=False):
await s_layer.Layer.__anit__(self, dirn, readonly=readonly)
path = os.path.join(self.dirn, 'layer.lmdb')
self.fresh = not os.path.exists(path)
mapsize = self.conf.get('lmdb:mapsize')
readahead = self.conf.get('lmdb:readahead')
maxsize = self.conf.get('lmdb:maxsize')
growsize = self.conf.get('lmdb:growsize')
self.layrslab = await s_lmdbslab.Slab.anit(path,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead,
readonly=readonly)
self.onfini(self.layrslab.fini)
self.dbs = {}
self.utf8 = s_layer.Utf8er()
self.encoder = s_layer.Encoder()
self.tid = s_threads.iden()
self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu>
self.byprop = await self.initdb('byprop', dupsort=True
) # <form>00<prop>00<indx>=<buid>
self.byuniv = await self.initdb('byuniv',
dupsort=True) # <prop>00<indx>=<buid>
offsdb = await self.initdb('offsets')
self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb)
self.splicedb = await self.initdb('splices')
self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')
async def delayNdefProps(self):
'''
Hold this during a series of renames to delay ndef
secondary property processing until the end....
'''
async with self.getTempSlab() as slab:
seqn = s_slabseqn.SlabSeqn(slab, 'ndef')
self.ndefdelay = seqn
yield
self.ndefdelay = None
logger.info(f'Processing {seqn.index()} delayed values.')
# process them all now...
for i, (oldv, newv) in seqn.iter(0):
await self.editNdefProps(oldv, newv)
if i and i % _progress == 0:
logger.info(f'Processed {i} delayed values.')
def getSeqn(self, name):
return s_slabseqn.SlabSeqn(self, name)
async def test_slab_seqn(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.chk_size(seqn)
self.eq(seqn.nextindx(), 0)
items = ('foo', 10, 20)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.chk_size(seqn)
self.raises(s_exc.NotMsgpackSafe, seqn.save, ({'set'}, ))
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.eq(seqn.nextindx(), 3)
await slab.fini()
# Reopen the seqn and continue where we left off
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.eq(seqn.index(), 3)
self.chk_size(seqn)
self.eq(seqn.nextindx(), 3)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20), (3, 'foo'), (4, 10),
(5, 20)))
self.eq(seqn.nextindx(), 6)
# We can also start in the middle of the sequence
retn = tuple(seqn.iter(4))
self.eq(retn, ((4, 10), (5, 20)))
# iterating past the end yields nothing
retn = tuple(seqn.iter(100))
self.eq(retn, ())
evnt = seqn.getOffsetEvent(4)
self.true(evnt.is_set())
evnt1 = seqn.getOffsetEvent(8)
evnt2 = seqn.getOffsetEvent(9)
evnt3 = seqn.getOffsetEvent(8)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.len(9, retn)
self.chk_size(seqn)
self.eq('foo', seqn.getByIndxByts(b'\x00' * 8))
self.true(evnt1.is_set())
self.true(await seqn.waitForOffset(8, timeout=0.5))
self.false(evnt2.is_set())
self.false(await seqn.waitForOffset(9, timeout=0.1))
self.true(evnt3.is_set())
state = None
started = asyncio.Event()
async def taskloop():
nonlocal state
state = 'started'
started.set()
state = await seqn.waitForOffset(9, timeout=5)
task = asyncio.get_running_loop().create_task(taskloop())
self.true(await s_coro.event_wait(started, 2))
self.eq(state, 'started')
seqn.add('bar')
self.true(evnt2.is_set())
self.true(state)
await task
self.eq((0, 'foo'), seqn.pop(0))
self.none(seqn.pop(0))
self.chk_size(seqn)
async def getter():
retn = []
async for item in seqn.gets(8):
if item[1] is None:
return retn
retn.append(item)
return retn
task = slab.schedCoro(getter())
await asyncio.sleep(0)
seqn.add(None)
self.eq(((8, 20), (9, 'bar')), await asyncio.wait_for(task,
timeout=3))
await seqn.cull(8)
self.chk_size(seqn)
self.eq(((9, 'bar'), (10, None)),
[x async for x in seqn.gets(8, wait=False)])
# overwrite existing
seqn.add('baz', indx=9)
self.chk_size(seqn)
# no oldv for smaller indx
seqn.add('bam', indx=8)
self.chk_size(seqn)
# append indx
seqn.add('faz', indx=15)
self.chk_size(seqn)
await seqn.cull(14)
self.chk_size(seqn)
seqn.trim(0)
self.chk_size(seqn)
await slab.fini()
async def test_slab_seqn(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.eq(seqn.nextindx(), 0)
items = ('foo', 10, 20)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.raises(s_exc.NotMsgpackSafe, seqn.save, ({'set'},))
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20)))
self.eq(seqn.nextindx(), 3)
await slab.fini()
# Reopen the seqn and continue where we left off
slab = await s_lmdbslab.Slab.anit(path, map_size=1000000)
seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
self.eq(seqn.index(), 3)
self.eq(seqn.nextindx(), 3)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.eq(retn, ((0, 'foo'), (1, 10), (2, 20),
(3, 'foo'), (4, 10), (5, 20)))
self.eq(seqn.nextindx(), 6)
# We can also start in the middle of the sequence
retn = tuple(seqn.iter(4))
self.eq(retn, ((4, 10), (5, 20)))
# iterating past the end yields nothing
retn = tuple(seqn.iter(100))
self.eq(retn, ())
evnt = seqn.getOffsetEvent(4)
self.true(evnt.is_set())
evnt1 = seqn.getOffsetEvent(8)
evnt2 = seqn.getOffsetEvent(9)
evnt3 = seqn.getOffsetEvent(8)
seqn.save(items)
retn = tuple(seqn.iter(0))
self.len(9, retn)
self.eq('foo', seqn.getByIndxByts(b'\x00' * 8))
self.true(evnt1.is_set())
self.true(await seqn.waitForOffset(8, timeout=0.5))
self.false(evnt2.is_set())
self.false(await seqn.waitForOffset(9, timeout=0.1))
self.true(evnt3.is_set())
state = None
started = asyncio.Event()
async def taskloop():
nonlocal state
state = 'started'
started.set()
state = await seqn.waitForOffset(9, timeout=5)
task = asyncio.get_running_loop().create_task(taskloop())
self.true(await s_coro.event_wait(started, 2))
self.eq(state, 'started')
seqn.add('bar')
self.true(evnt2.is_set())
self.true(state)
await task
await slab.fini()
