async def __anit__(self, dirn, conf=None):
await s_cell.Cell.__anit__(self, dirn, conf=conf)
self.src = self.conf.get('src')
self.dest = self.conf.get('dest')
self.offsfile = self.conf.get('offsfile')
self.poll_s = self.conf.get('poll_s')
self.err_lim = self.conf.get('err_lim')
self.q_size = self.conf.get('queue_size')
self.offs_logging = self.conf.get('offs_logging')
# Check that form or prop is not from an older model rev
# which we know will fail - only loaded for historical splice migration
self.form_chk = {}
self.prop_chk = {}
self.pull_fair_iter = 10
self.push_fair_iter = 100
self.q_cap = int(self.q_size * 0.9)
self.layers = await self.hive.dict(('sync:layer', )) # lyridens
self.migrmode = await self.hive.open(
('sync:migrmode', )
) # Override migration mode defaults with True/False
self.migrmode_evnt = asyncio.Event()
# if migrmode was overridden true, try to re-enable on startup
if self.migrmode.valu:
await self.enableMigrationMode()
path = s_common.gendir(dirn, 'slabs', 'migrsync.lmdb')
self.slab = await s_lmdbslab.Slab.anit(path, map_async=True)
self.onfini(self.slab.fini)
self.pull_offs = s_slaboffs.SlabOffs(self.slab,
'pull_offs') # key=lyriden
self.push_offs = s_slaboffs.SlabOffs(self.slab,
'push_offs') # key=lyriden
self.errors = self.slab.initdb(
'errors', dupsort=False) # key=<lyriden><offset>, val=err
self.errcnts = await self.slab.getHotCount('count:errors')
self.model = {}
self._pull_status = {} # lyriden: status
self.pull_tasks = {} # lyriden: Task
self.push_tasks = {}
self.pull_last_start = {} # lyriden: epoch
self.push_last_start = {}
self._pull_evnts = {} # lyriden: event (set when up-to-date)
self._push_evnts = {} # lyriden: event (set when queue is empty)
self._queues = {} # lyriden: queue of splices
async def __anit__(self, core, node):
await s_layer.Layer.__anit__(self, core, node)
path = os.path.join(self.dirn, 'layer.lmdb')
splicepath = os.path.join(self.dirn, 'splices.lmdb')
self.fresh = not os.path.exists(path)
mapsize = self.conf.get('lmdb:mapsize')
readahead = self.conf.get('lmdb:readahead')
maxsize = self.conf.get('lmdb:maxsize')
growsize = self.conf.get('lmdb:growsize')
# First check hive configuration. If not set, use passed-in parameter (that defaults to False)
self.lockmemory = self.conf.get('lmdb:lockmemory')
if self.lockmemory is None:
self.lockmemory = core.conf.get('dedicated')
self.layrslab = await s_lmdbslab.Slab.anit(path,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead,
lockmemory=self.lockmemory)
self.onfini(self.layrslab.fini)
self.spliceslab = await s_lmdbslab.Slab.anit(splicepath,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead)
self.onfini(self.spliceslab.fini)
metadb = self.layrslab.initdb('meta')
self.metadict = s_lmdbslab.SlabDict(self.layrslab, metadb)
self._migrate_splices_pre010()
self.dbs = {}
self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu>
self.byprop = await self.initdb('byprop', dupsort=True
) # <form>00<prop>00<indx>=<buid>
self.byuniv = await self.initdb('byuniv',
dupsort=True) # <prop>00<indx>=<buid>
offsdb = await self.initdb('offsets')
self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb)
self.splicelog = s_slabseqn.SlabSeqn(self.spliceslab, 'splices')
self.indxfunc = {
'eq': self._rowsByEq,
'pref': self._rowsByPref,
'range': self._rowsByRange,
}
async def __anit__(self, dirn, conf=None):
await s_cell.Cell.__anit__(self, dirn, conf=conf)
self.src = self.conf.get('src')
self.dest = self.conf.get('dest')
self.offsfile = self.conf.get('offsfile')
self.poll_s = self.conf.get('poll_s')
self.err_lim = self.conf.get('err_lim')
self.q_size = self.conf.get('queue_size')
self.pull_fair_iter = 10
self.push_fair_iter = 100
self.q_cap = int(self.q_size * 0.9)
self.layers = await self.hive.dict(('sync:layer', )) # lyridens
path = s_common.gendir(dirn, 'slabs', 'migrsync.lmdb')
self.slab = await s_lmdbslab.Slab.anit(path, map_async=True)
self.onfini(self.slab.fini)
self.pull_offs = s_slaboffs.SlabOffs(self.slab, 'pull_offs') # key=lyriden
self.push_offs = s_slaboffs.SlabOffs(self.slab, 'push_offs') # key=lyriden
self.errors = self.slab.initdb('errors', dupsort=False) # key=<lyriden><offset>, val=err
self.model = {}
self._pull_status = {} # lyriden: status
self._pull_tasks = {} # lyriden: Task
self._push_tasks = {}
self.pull_last_start = {} # lyriden: epoch
self.push_last_start = {}
self._pull_evnts = {} # lyriden: event (set when up-to-date)
self._push_evnts = {} # lyriden: event (set when queue is empty)
self._queues = {} # lyriden: queue of splices
async def test_lmdbslab_base(self):
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'test.lmdb')
async with await s_lmdbslab.Slab.anit(path) as slab:
offsdb = slab.initdb('offsets')
offs = s_slaboffs.SlabOffs(slab, offsdb)
guid = s_common.guid()
self.eq(0, offs.get(guid))
offs.set(guid, 42)
self.eq(42, offs.get(guid))
offs.delete(guid)
self.eq(0, offs.get(guid))
async def __anit__(self, dirn, conf=None):
await s_base.Base.__anit__(self)
if conf is None:
conf = {}
self.conf = conf
self.dirn = s_common.gendir(dirn)
self._iden = self._getTankIden()
path = s_common.gendir(self.dirn, 'tank.lmdb')
self.slab = await s_lmdbslab.Slab.anit(path, map_async=True, **conf)
self.offs = s_slaboffs.SlabOffs(self.slab, 'offsets')
self._items = s_slabseqn.SlabSeqn(self.slab, 'items')
self._metrics = s_slabseqn.SlabSeqn(self.slab, 'metrics')
self.onfini(self.slab.fini)
async def __anit__(self, dirn, readonly=False):
await s_layer.Layer.__anit__(self, dirn, readonly=readonly)
path = os.path.join(self.dirn, 'layer.lmdb')
self.fresh = not os.path.exists(path)
mapsize = self.conf.get('lmdb:mapsize')
readahead = self.conf.get('lmdb:readahead')
maxsize = self.conf.get('lmdb:maxsize')
growsize = self.conf.get('lmdb:growsize')
self.layrslab = await s_lmdbslab.Slab.anit(path,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead,
readonly=readonly)
self.onfini(self.layrslab.fini)
self.dbs = {}
self.utf8 = s_layer.Utf8er()
self.encoder = s_layer.Encoder()
self.tid = s_threads.iden()
self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu>
self.byprop = await self.initdb('byprop', dupsort=True
) # <form>00<prop>00<indx>=<buid>
self.byuniv = await self.initdb('byuniv',
dupsort=True) # <prop>00<indx>=<buid>
offsdb = await self.initdb('offsets')
self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb)
self.splicedb = await self.initdb('splices')
self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')