def test_lmdb_metrics(self):
with self.getTestDir() as dirn:
lenv = lmdb.open(dirn, writemap=True, max_dbs=128)
metr = s_lmdb.Metrics(lenv)
self.eq(metr.stat(), {})
self.eq(0, metr.indx)
with lenv.begin(write=True) as xact:
metr.inc(xact, 'woot', 20)
metr.inc(xact, 'woot', 20)
metr.record(xact, {'hehe': 10, 'haha': 20})
metr.record(xact, {'hehe': 20, 'haha': 30})
self.eq(2, metr.indx)
lenv.sync()
lenv.close()
lenv = lmdb.open(dirn, writemap=True, max_dbs=128)
metr = s_lmdb.Metrics(lenv)
with lenv.begin(write=True) as xact:
self.eq(metr.info.get('woot'), 40)
retn = list(metr.iter(xact, 1))
self.len(1, retn)
self.eq(retn[0][0], 1)
self.eq(retn[0][1].get('hehe'), 20)
self.len(0, list(metr.iter(xact, 1234567890)))
self.eq(2, metr.indx)
with lenv.begin(write=True) as xact:
metr.record(xact, {'hehe': 30, 'haha': 20})
metr.record(xact, {'hehe': 40, 'haha': 30})
self.eq(4, metr.indx)
with lenv.begin(write=False) as xact:
retn = list(metr.iter(xact, 0))
self.eq([off for off, item in retn], [0, 1, 2, 3])
self.eq(metr.stat(), {'woot': 40})
async def __anit__(self, dirn: str, conf=None) -> None: # type: ignore
await s_cell.Cell.__anit__(self, dirn)
path = s_common.gendir(self.dirn, 'axon.lmdb')
mapsize = self.conf.get('mapsize')
self.lenv = lmdb.open(path, writemap=True, max_dbs=128)
self.lenv.set_mapsize(mapsize)
self.bloblocs = self.lenv.open_db(
b'axon:blob:locs', dupsort=True,
dupfixed=True) # <sha256>=blobstor_bsid
self.offsets = self.lenv.open_db(
b'axon:blob:offsets') # <sha256>=blobstor_bsid
# Persistent settings
self.settings = self.lenv.open_db(b'axon:settings', dupsort=True)
self._metrics = s_lmdb.Metrics(self.lenv)
self._proxykeeper = await _ProxyKeeper.anit()
# Clear the global proxykeeper bsid->telepath path map (really just for unit tests)
_ProxyKeeper.bsidpathmap = {}
paths = self._get_stored_blobstorpaths()
self.blobstorwatchers = {} # type: ignore
async def _connect_to_blobstors():
# Wait a few seconds for the daemon to register all of its shared objects
DAEMON_DELAY = 3
await asyncio.sleep(DAEMON_DELAY)
for path in paths:
try:
await self._start_watching_blobstor(path)
except asyncio.CancelledError: # pragma: no cover
break
except Exception:
logger.error(
'At axon startup, failed to connect to stored blobstor path %s',
_path_sanitize(path))
conn_future = self.schedCoro(_connect_to_blobstors())
self._workq = await s_queue.AsyncQueue.anit(50)
self.xact = IncrementalTransaction(self.lenv)
self._worker = self._workloop()
self._worker.name = 'Axon Writer'
async def fini():
conn_future.cancel()
for stop_event in self.blobstorwatchers.values():
stop_event.set()
await self._workq.fini()
await self._proxykeeper.fini()
self._worker.join()
self.onfini(fini)
def __init__(self, dirn, mapsize=s_const.tebibyte):
s_eventbus.EventBus.__init__(self)
path = s_common.gendir(dirn, 'blobs.lmdb')
self.lenv = lmdb.open(path, writemap=True, max_dbs=128)
self.lenv.set_mapsize(mapsize)
self._blob_info = self.lenv.open_db(b'info')
self._blob_bytes = self.lenv.open_db(b'bytes') # <fidn><foff>=<byts>
self._blob_clone = s_lmdb.Seqn(self.lenv, b'clone')
self._blob_metrics = s_lmdb.Metrics(self.lenv)
def fini():
self.lenv.sync()
self.lenv.close()
self.onfini(fini)
async def __anit__(self, dirn: str, conf=None) -> None: # type: ignore
await s_cell.Cell.__anit__(self, dirn)
self.clonetask = None
if conf is not None:
self.conf.update(conf)
path = s_common.gendir(self.dirn, 'blobs.lmdb')
mapsize = self.conf.get('mapsize')
self.lenv = lmdb.open(path,
writemap=True,
max_dbs=128,
map_size=mapsize)
self._blob_info = self.lenv.open_db(b'info')
self._blob_bytes = self.lenv.open_db(b'bytes') # <sha256>=<byts>
self._clone_seqn = s_lmdb.Seqn(self.lenv, b'clone')
self._metrics = s_lmdb.Metrics(self.lenv)
self._newdataevent = asyncio.Event(loop=self.loop)
def delevent():
del self._newdataevent
self.onfini(delevent)
self._recover_partial()
self._partial_hash = None # hash currently being written
self.writer = await _BlobStorWriter.anit(self)
self.cloneof = self.conf.get('cloneof')
if self.cloneof is not None:
self.clonetask = self.schedCoro(self._cloneeLoop(self.cloneof))
self.onfini(self.writer.fini)
def postCell(self):
if self.cellpool is None:
raise s_common.BadConfValu(
mesg='AxonCell requires a neuron and CellPool')
mapsize = self.getConfOpt('axon:mapsize')
path = self.getCellDir('axon.lmdb')
self.lenv = lmdb.open(path, writemap=True, max_dbs=128)
self.lenv.set_mapsize(mapsize)
self.blobhas = self.lenv.open_db(b'axon:blob:has') # <sha256>=<size>
self.bloblocs = self.lenv.open_db(
b'axon:blob:locs') # <sha256><loc>=<buid>
self.metrics = s_lmdb.Metrics(self.lenv)
self.blobs = s_cell.CellPool(self.cellauth, self.neuraddr)
self.blobs.neurwait(timeout=10)
for name in self.getConfOpt('axon:blobs'):
self.blobs.add(name)