async def __anit__(self, dirn, conf=None): await s_cell.Cell.__anit__(self, dirn, conf=conf) self.src = self.conf.get('src') self.dest = self.conf.get('dest') self.offsfile = self.conf.get('offsfile') self.poll_s = self.conf.get('poll_s') self.err_lim = self.conf.get('err_lim') self.q_size = self.conf.get('queue_size') self.offs_logging = self.conf.get('offs_logging') # Check that form or prop is not from an older model rev # which we know will fail - only loaded for historical splice migration self.form_chk = {} self.prop_chk = {} self.pull_fair_iter = 10 self.push_fair_iter = 100 self.q_cap = int(self.q_size * 0.9) self.layers = await self.hive.dict(('sync:layer', )) # lyridens self.migrmode = await self.hive.open( ('sync:migrmode', ) ) # Override migration mode defaults with True/False self.migrmode_evnt = asyncio.Event() # if migrmode was overridden true, try to re-enable on startup if self.migrmode.valu: await self.enableMigrationMode() path = s_common.gendir(dirn, 'slabs', 'migrsync.lmdb') self.slab = await s_lmdbslab.Slab.anit(path, map_async=True) self.onfini(self.slab.fini) self.pull_offs = s_slaboffs.SlabOffs(self.slab, 'pull_offs') # key=lyriden self.push_offs = s_slaboffs.SlabOffs(self.slab, 'push_offs') # key=lyriden self.errors = self.slab.initdb( 'errors', dupsort=False) # key=<lyriden><offset>, val=err self.errcnts = await self.slab.getHotCount('count:errors') self.model = {} self._pull_status = {} # lyriden: status self.pull_tasks = {} # lyriden: Task self.push_tasks = {} self.pull_last_start = {} # lyriden: epoch self.push_last_start = {} self._pull_evnts = {} # lyriden: event (set when up-to-date) self._push_evnts = {} # lyriden: event (set when queue is empty) self._queues = {} # lyriden: queue of splices
async def __anit__(self, core, node): await s_layer.Layer.__anit__(self, core, node) path = os.path.join(self.dirn, 'layer.lmdb') splicepath = os.path.join(self.dirn, 'splices.lmdb') self.fresh = not os.path.exists(path) mapsize = self.conf.get('lmdb:mapsize') readahead = self.conf.get('lmdb:readahead') maxsize = self.conf.get('lmdb:maxsize') growsize = self.conf.get('lmdb:growsize') # First check hive configuration. If not set, use passed-in parameter (that defaults to False) self.lockmemory = self.conf.get('lmdb:lockmemory') if self.lockmemory is None: self.lockmemory = core.conf.get('dedicated') self.layrslab = await s_lmdbslab.Slab.anit(path, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead, lockmemory=self.lockmemory) self.onfini(self.layrslab.fini) self.spliceslab = await s_lmdbslab.Slab.anit(splicepath, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead) self.onfini(self.spliceslab.fini) metadb = self.layrslab.initdb('meta') self.metadict = s_lmdbslab.SlabDict(self.layrslab, metadb) self._migrate_splices_pre010() self.dbs = {} self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu> self.byprop = await self.initdb('byprop', dupsort=True ) # <form>00<prop>00<indx>=<buid> self.byuniv = await self.initdb('byuniv', dupsort=True) # <prop>00<indx>=<buid> offsdb = await self.initdb('offsets') self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb) self.splicelog = s_slabseqn.SlabSeqn(self.spliceslab, 'splices') self.indxfunc = { 'eq': self._rowsByEq, 'pref': self._rowsByPref, 'range': self._rowsByRange, }
async def __anit__(self, dirn, conf=None): await s_cell.Cell.__anit__(self, dirn, conf=conf) self.src = self.conf.get('src') self.dest = self.conf.get('dest') self.offsfile = self.conf.get('offsfile') self.poll_s = self.conf.get('poll_s') self.err_lim = self.conf.get('err_lim') self.q_size = self.conf.get('queue_size') self.pull_fair_iter = 10 self.push_fair_iter = 100 self.q_cap = int(self.q_size * 0.9) self.layers = await self.hive.dict(('sync:layer', )) # lyridens path = s_common.gendir(dirn, 'slabs', 'migrsync.lmdb') self.slab = await s_lmdbslab.Slab.anit(path, map_async=True) self.onfini(self.slab.fini) self.pull_offs = s_slaboffs.SlabOffs(self.slab, 'pull_offs') # key=lyriden self.push_offs = s_slaboffs.SlabOffs(self.slab, 'push_offs') # key=lyriden self.errors = self.slab.initdb('errors', dupsort=False) # key=<lyriden><offset>, val=err self.model = {} self._pull_status = {} # lyriden: status self._pull_tasks = {} # lyriden: Task self._push_tasks = {} self.pull_last_start = {} # lyriden: epoch self.push_last_start = {} self._pull_evnts = {} # lyriden: event (set when up-to-date) self._push_evnts = {} # lyriden: event (set when queue is empty) self._queues = {} # lyriden: queue of splices
async def test_lmdbslab_base(self): with self.getTestDir() as dirn: path = os.path.join(dirn, 'test.lmdb') async with await s_lmdbslab.Slab.anit(path) as slab: offsdb = slab.initdb('offsets') offs = s_slaboffs.SlabOffs(slab, offsdb) guid = s_common.guid() self.eq(0, offs.get(guid)) offs.set(guid, 42) self.eq(42, offs.get(guid)) offs.delete(guid) self.eq(0, offs.get(guid))
async def __anit__(self, dirn, conf=None): await s_base.Base.__anit__(self) if conf is None: conf = {} self.conf = conf self.dirn = s_common.gendir(dirn) self._iden = self._getTankIden() path = s_common.gendir(self.dirn, 'tank.lmdb') self.slab = await s_lmdbslab.Slab.anit(path, map_async=True, **conf) self.offs = s_slaboffs.SlabOffs(self.slab, 'offsets') self._items = s_slabseqn.SlabSeqn(self.slab, 'items') self._metrics = s_slabseqn.SlabSeqn(self.slab, 'metrics') self.onfini(self.slab.fini)
async def __anit__(self, dirn, readonly=False): await s_layer.Layer.__anit__(self, dirn, readonly=readonly) path = os.path.join(self.dirn, 'layer.lmdb') self.fresh = not os.path.exists(path) mapsize = self.conf.get('lmdb:mapsize') readahead = self.conf.get('lmdb:readahead') maxsize = self.conf.get('lmdb:maxsize') growsize = self.conf.get('lmdb:growsize') self.layrslab = await s_lmdbslab.Slab.anit(path, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead, readonly=readonly) self.onfini(self.layrslab.fini) self.dbs = {} self.utf8 = s_layer.Utf8er() self.encoder = s_layer.Encoder() self.tid = s_threads.iden() self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu> self.byprop = await self.initdb('byprop', dupsort=True ) # <form>00<prop>00<indx>=<buid> self.byuniv = await self.initdb('byuniv', dupsort=True) # <prop>00<indx>=<buid> offsdb = await self.initdb('offsets') self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb) self.splicedb = await self.initdb('splices') self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')