async def test_slab_seqn(self): with self.getTestDir() as dirn: path = os.path.join(dirn, 'test.lmdb') slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.eq(seqn.nextindx(), 0) items = ('foo', 10, 20) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.raises(TypeError, seqn.save, ({'set'},)) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.eq(seqn.nextindx(), 3) await slab.fini() # Reopen the seqn and continue where we left off slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.eq(seqn.index(), 3) self.eq(seqn.nextindx(), 3) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20), (3, 'foo'), (4, 10), (5, 20))) self.eq(seqn.nextindx(), 6) # We can also start in the middle of the sequence retn = tuple(seqn.iter(4)) self.eq(retn, ((4, 10), (5, 20))) # iterating past the end yields nothing retn = tuple(seqn.iter(100)) self.eq(retn, ()) seqn.save(items) retn = tuple(seqn.iter(0)) self.len(9, retn) self.eq('foo', seqn.getByIndxByts(b'\x00' * 8)) await slab.fini()
async def __anit__(self, dirn, conf=None): # type: ignore await s_cell.Cell.__anit__(self, dirn, conf=conf) # share ourself via the cell dmon as "axon" # for potential default remote use self.dmon.share('axon', self) path = s_common.gendir(self.dirn, 'axon.lmdb') self.axonslab = await s_lmdbslab.Slab.anit(path) self.sizes = self.axonslab.initdb('sizes') self.onfini(self.axonslab.fini) self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history') self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn') node = await self.hive.open(('axon', 'metrics')) self.axonmetrics = await node.dict() self.axonmetrics.setdefault('size:bytes', 0) self.axonmetrics.setdefault('file:count', 0) self.addHealthFunc(self._axonHealth) # modularize blob storage await self._initBlobStor() self._initAxonHttpApi()
async def __anit__(self, dirn, conf=None): await s_cell.Cell.__anit__(self, dirn) # share ourself via the cell dmon as "axon" # for potential default remote use self.dmon.share('axon', self) path = s_common.gendir(self.dirn, 'axon.lmdb') self.axonslab = await s_lmdbslab.Slab.anit(path) self.sizes = self.axonslab.initdb('sizes') self.onfini(self.axonslab.fini) self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history') path = s_common.gendir(self.dirn, 'blob.lmdb') self.blobslab = await s_lmdbslab.Slab.anit(path) self.blobs = self.blobslab.initdb('blobs') self.onfini(self.blobslab.fini) self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn') node = await self.hive.open(('axon', 'metrics')) self.axonmetrics = await node.dict() self.axonmetrics.setdefault('size:bytes', 0) self.axonmetrics.setdefault('file:count', 0)
async def __anit__(self, dirn: str, dologging: bool = True): # type: ignore await s_base.Base.__anit__(self) import synapse.lib.lmdbslab as s_lmdbslab # avoid import cycle import synapse.lib.slabseqn as s_slabseqn # avoid import cycle self.dirn = dirn self.readonly = False self._nexskids: Dict[str, 'Pusher'] = {} self.mirrors: List[ChangeDist] = [] self.dologging = dologging if self.dologging: path = s_common.genpath(self.dirn, 'slabs', 'nexus.lmdb') self.nexusslab = await s_lmdbslab.Slab.anit(path, map_async=False) self.nexuslog = s_slabseqn.SlabSeqn(self.nexusslab, 'nexuslog') async def fini(): if self.dologging: await self.nexusslab.fini() [(await dist.fini()) for dist in self.mirrors] self.onfini(fini)
async def __anit__(self, core, node): await s_layer.Layer.__anit__(self, core, node) path = os.path.join(self.dirn, 'layer.lmdb') splicepath = os.path.join(self.dirn, 'splices.lmdb') self.fresh = not os.path.exists(path) mapsize = self.conf.get('lmdb:mapsize') readahead = self.conf.get('lmdb:readahead') maxsize = self.conf.get('lmdb:maxsize') growsize = self.conf.get('lmdb:growsize') # First check hive configuration. If not set, use passed-in parameter (that defaults to False) self.lockmemory = self.conf.get('lmdb:lockmemory') if self.lockmemory is None: self.lockmemory = core.conf.get('dedicated') self.layrslab = await s_lmdbslab.Slab.anit(path, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead, lockmemory=self.lockmemory) self.onfini(self.layrslab.fini) self.spliceslab = await s_lmdbslab.Slab.anit(splicepath, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead) self.onfini(self.spliceslab.fini) metadb = self.layrslab.initdb('meta') self.metadict = s_lmdbslab.SlabDict(self.layrslab, metadb) self._migrate_splices_pre010() self.dbs = {} self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu> self.byprop = await self.initdb('byprop', dupsort=True ) # <form>00<prop>00<indx>=<buid> self.byuniv = await self.initdb('byuniv', dupsort=True) # <prop>00<indx>=<buid> offsdb = await self.initdb('offsets') self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb) self.splicelog = s_slabseqn.SlabSeqn(self.spliceslab, 'splices') self.indxfunc = { 'eq': self._rowsByEq, 'pref': self._rowsByPref, 'range': self._rowsByRange, }
async def __anit__(self, dirn): await s_base.Base.__anit__(self) path = str(pathlib.Path(dirn) / 'slabs' / self.PROV_FN) self.slab = await s_lmdbslab.Slab.anit(path, map_size=self.PROV_MAP_SIZE) self.onfini(self.slab.fini) self.db = self.slab.initdb('prov') self.provseq = s_slabseqn.SlabSeqn(self.slab, 'provs')
async def __anit__(self, dirn, conf=None): await s_base.Base.__anit__(self) if conf is None: conf = {} self.conf = conf self.dirn = s_common.gendir(dirn) self._iden = self._getTankIden() path = s_common.gendir(self.dirn, 'tank.lmdb') self.slab = await s_lmdbslab.Slab.anit(path, map_async=True, **conf) self.offs = s_slaboffs.SlabOffs(self.slab, 'offsets') self._items = s_slabseqn.SlabSeqn(self.slab, 'items') self._metrics = s_slabseqn.SlabSeqn(self.slab, 'metrics') self.onfini(self.slab.fini)
async def __anit__(self, dirn: str, donexslog: bool = True): # type: ignore await s_base.Base.__anit__(self) import synapse.lib.lmdbslab as s_lmdbslab # avoid import cycle import synapse.lib.slabseqn as s_slabseqn # avoid import cycle self.dirn = dirn self._nexskids: Dict[str, 'Pusher'] = {} self._mirrors: List[ChangeDist] = [] self.donexslog = donexslog self._state_lock = asyncio.Lock() self._state_funcs: List[Callable] = [] # External Callbacks for state changes # These are used when this cell is a mirror. self._ldrurl: Optional[str] = None self._ldr: Optional[s_telepath.Proxy] = None # only set by looptask self._looptask: Optional[asyncio.Task] = None self._ldrready = asyncio.Event() # Used to match pending follower write requests with the responses arriving on the log self._futures: Dict[str, asyncio.Future] = {} if self.donexslog: path = s_common.genpath(self.dirn, 'slabs', 'nexus.lmdb') self._nexusslab = await s_lmdbslab.Slab.anit(path, map_async=False) self._nexuslog = s_slabseqn.SlabSeqn(self._nexusslab, 'nexuslog') async def fini(): if self._looptask: self._looptask.cancel() try: await self._looptask except Exception: pass for futu in self._futures.values(): futu.cancel() if self._ldr: self._ldrready.clear() await self._ldr.fini() if self.donexslog: await self._nexusslab.fini() [(await dist.fini()) for dist in self._mirrors] self.onfini(fini)
async def __anit__(self, dirn, proven=True): await s_base.Base.__anit__(self) global ProvenanceEnabled ProvenanceEnabled = proven self.enabled = proven if self.enabled: path = str(pathlib.Path(dirn) / 'slabs' / self.PROV_FN) self.slab = await s_lmdbslab.Slab.anit(path, map_size=self.PROV_MAP_SIZE) self.onfini(self.slab.fini) self.db = self.slab.initdb('prov') self.provseq = s_slabseqn.SlabSeqn(self.slab, 'provs')
async def __anit__(self, dirn, readonly=False): await s_layer.Layer.__anit__(self, dirn, readonly=readonly) path = os.path.join(self.dirn, 'layer.lmdb') self.fresh = not os.path.exists(path) mapsize = self.conf.get('lmdb:mapsize') readahead = self.conf.get('lmdb:readahead') maxsize = self.conf.get('lmdb:maxsize') growsize = self.conf.get('lmdb:growsize') self.layrslab = await s_lmdbslab.Slab.anit(path, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead, readonly=readonly) self.onfini(self.layrslab.fini) self.dbs = {} self.utf8 = s_layer.Utf8er() self.encoder = s_layer.Encoder() self.tid = s_threads.iden() self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu> self.byprop = await self.initdb('byprop', dupsort=True ) # <form>00<prop>00<indx>=<buid> self.byuniv = await self.initdb('byuniv', dupsort=True) # <prop>00<indx>=<buid> offsdb = await self.initdb('offsets') self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb) self.splicedb = await self.initdb('splices') self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')
async def delayNdefProps(self): ''' Hold this during a series of renames to delay ndef secondary property processing until the end.... ''' async with self.getTempSlab() as slab: seqn = s_slabseqn.SlabSeqn(slab, 'ndef') self.ndefdelay = seqn yield self.ndefdelay = None logger.info(f'Processing {seqn.index()} delayed values.') # process them all now... for i, (oldv, newv) in seqn.iter(0): await self.editNdefProps(oldv, newv) if i and i % _progress == 0: logger.info(f'Processed {i} delayed values.')
def getSeqn(self, name): return s_slabseqn.SlabSeqn(self, name)
async def test_slab_seqn(self): with self.getTestDir() as dirn: path = os.path.join(dirn, 'test.lmdb') slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.chk_size(seqn) self.eq(seqn.nextindx(), 0) items = ('foo', 10, 20) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.chk_size(seqn) self.raises(s_exc.NotMsgpackSafe, seqn.save, ({'set'}, )) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.eq(seqn.nextindx(), 3) await slab.fini() # Reopen the seqn and continue where we left off slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.eq(seqn.index(), 3) self.chk_size(seqn) self.eq(seqn.nextindx(), 3) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20), (3, 'foo'), (4, 10), (5, 20))) self.eq(seqn.nextindx(), 6) # We can also start in the middle of the sequence retn = tuple(seqn.iter(4)) self.eq(retn, ((4, 10), (5, 20))) # iterating past the end yields nothing retn = tuple(seqn.iter(100)) self.eq(retn, ()) evnt = seqn.getOffsetEvent(4) self.true(evnt.is_set()) evnt1 = seqn.getOffsetEvent(8) evnt2 = seqn.getOffsetEvent(9) evnt3 = seqn.getOffsetEvent(8) seqn.save(items) retn = tuple(seqn.iter(0)) self.len(9, retn) self.chk_size(seqn) self.eq('foo', seqn.getByIndxByts(b'\x00' * 8)) self.true(evnt1.is_set()) self.true(await seqn.waitForOffset(8, timeout=0.5)) self.false(evnt2.is_set()) self.false(await seqn.waitForOffset(9, timeout=0.1)) self.true(evnt3.is_set()) state = None started = asyncio.Event() async def taskloop(): nonlocal state state = 'started' started.set() state = await seqn.waitForOffset(9, timeout=5) task = asyncio.get_running_loop().create_task(taskloop()) self.true(await s_coro.event_wait(started, 2)) self.eq(state, 'started') seqn.add('bar') self.true(evnt2.is_set()) self.true(state) await task self.eq((0, 'foo'), seqn.pop(0)) self.none(seqn.pop(0)) self.chk_size(seqn) async def getter(): retn = [] async for item in seqn.gets(8): if item[1] is None: return retn retn.append(item) return retn task = slab.schedCoro(getter()) await asyncio.sleep(0) seqn.add(None) self.eq(((8, 20), (9, 'bar')), await asyncio.wait_for(task, timeout=3)) await seqn.cull(8) self.chk_size(seqn) self.eq(((9, 'bar'), (10, None)), [x async for x in seqn.gets(8, wait=False)]) # overwrite existing seqn.add('baz', indx=9) self.chk_size(seqn) # no oldv for smaller indx seqn.add('bam', indx=8) self.chk_size(seqn) # append indx seqn.add('faz', indx=15) self.chk_size(seqn) await seqn.cull(14) self.chk_size(seqn) seqn.trim(0) self.chk_size(seqn) await slab.fini()
async def test_slab_seqn(self): with self.getTestDir() as dirn: path = os.path.join(dirn, 'test.lmdb') slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.eq(seqn.nextindx(), 0) items = ('foo', 10, 20) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.raises(s_exc.NotMsgpackSafe, seqn.save, ({'set'},)) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20))) self.eq(seqn.nextindx(), 3) await slab.fini() # Reopen the seqn and continue where we left off slab = await s_lmdbslab.Slab.anit(path, map_size=1000000) seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test') self.eq(seqn.index(), 3) self.eq(seqn.nextindx(), 3) seqn.save(items) retn = tuple(seqn.iter(0)) self.eq(retn, ((0, 'foo'), (1, 10), (2, 20), (3, 'foo'), (4, 10), (5, 20))) self.eq(seqn.nextindx(), 6) # We can also start in the middle of the sequence retn = tuple(seqn.iter(4)) self.eq(retn, ((4, 10), (5, 20))) # iterating past the end yields nothing retn = tuple(seqn.iter(100)) self.eq(retn, ()) evnt = seqn.getOffsetEvent(4) self.true(evnt.is_set()) evnt1 = seqn.getOffsetEvent(8) evnt2 = seqn.getOffsetEvent(9) evnt3 = seqn.getOffsetEvent(8) seqn.save(items) retn = tuple(seqn.iter(0)) self.len(9, retn) self.eq('foo', seqn.getByIndxByts(b'\x00' * 8)) self.true(evnt1.is_set()) self.true(await seqn.waitForOffset(8, timeout=0.5)) self.false(evnt2.is_set()) self.false(await seqn.waitForOffset(9, timeout=0.1)) self.true(evnt3.is_set()) state = None started = asyncio.Event() async def taskloop(): nonlocal state state = 'started' started.set() state = await seqn.waitForOffset(9, timeout=5) task = asyncio.get_running_loop().create_task(taskloop()) self.true(await s_coro.event_wait(started, 2)) self.eq(state, 'started') seqn.add('bar') self.true(evnt2.is_set()) self.true(state) await task await slab.fini()