def test_cryo_tank(self):
with self.getTestDir() as dirn:
with s_cryotank.CryoTank(
dirn) as tank: # type: s_cryotank.CryoTank
# Default configable option
self.eq(tank.getConfOpt('mapsize'), s_const.tebibyte)
info = tank.info()
self.eq(0, info.get('indx'))
self.eq(0, info.get('metrics'))
self.nn(info.get('stat').get('entries'))
tank.puts(cryodata)
info = tank.info()
self.eq(2, info.get('indx'))
self.eq(1, info.get('metrics'))
self.eq('baz', tank.last()[1][0])
retn = tuple(tank.slice(0, 1))
self.eq(retn, ((0, cryodata[0]), ))
retn = tuple(tank.rows(0, 1)) # coverage related
self.len(1, retn)
retn = tuple(tank.rows(0, 2))
self.len(2, retn)
data = retn[-1]
self.eq(data, (1, b'\x92\xa3baz\x81\xa3faz\x14'))
info = tank.info()
self.eq(2, info.get('indx'))
self.eq(1, info.get('metrics'))
retn = tuple(tank.metrics(0, 1))[0]
self.nn(retn[1].get('time'))
self.eq(retn[1].get('size'), 22)
self.eq(retn[1].get('count'), 2)
# Slices and rows can start at offsets
retn = tuple(tank.rows(1, 4))
self.len(1, retn)
retn = tuple(tank.slice(1, 4))
self.len(1, retn)
retn = tuple(tank.rows(4, 4))
self.len(0, retn)
retn = tuple(tank.slice(4, 4))
self.len(0, retn)
# Test empty puts
tank.puts(tuple())
info = tank.info()
self.eq(2, info.get('indx'))
self.eq(2, info.get('metrics'))
def test_cryotank_index_nest(self):
with self.getTestDir() as dirn, s_cryotank.CryoTank(
dirn, {'mapsize': s_iq.TEST_MAP_SIZE}) as tank:
idxr = tank.indexer
item = {'hehe': {'haha': {'key': 'valu'}}}
waiter = self.initWaiter(tank, 'addIndex')
idxr.addIndex('key', 'str:lwr', ['hehe/haha/key'])
self.wait(waiter)
waiter = self.initWaiter(tank, 'None')
tank.puts([item])
self.wait(waiter)
idxs = idxr.getIndices()
self.eq(idxs[0]['ngood'], 1)
def test_cryo_mapsize(self):
self.skipTest('LMDB Failure modes need research')
mapsize = s_const.mebibyte * 1
v = s_const.kibibyte * b'''In that flickering pallor it had the effect of a large and clumsy black insect, an insect the size of an ironclad cruiser, crawling obliquely to the first line of trenches and firing shots out of portholes in its side. And on its carcass the bullets must have been battering with more than the passionate violence of hail on a roof of tin.'''
with self.getTestDir() as dirn:
with s_cryotank.CryoTank(
dirn,
{'mapsize': mapsize}) as tank: # type: s_cryotank.CryoTank
tank.puts([(0, {'key': v})])
tank.puts([(1, {'key': v})])
# Now we fail
with self.getLoggerStream('synapse.cryotank') as stream:
self.none(tank.puts([(2, {'key': v})]))
stream.seek(0)
mesgs = stream.read()
self.isin('Error appending items to the cryotank', mesgs)
# We can still put a small item in though!
self.eq(tank.puts([(2, {'key': 'tinyitem'})]))
def test_cryotank_index(self):
with self.getTestDir() as dirn, s_cryotank.CryoTank(
dirn, {'mapsize': s_iq.TEST_MAP_SIZE}) as tank:
idxr = tank.indexer
data1 = {'foo': 1234, 'bar': 'stringval'}
data2 = {'foo': 2345, 'baz': 4567, 'bar': 'strinstrin'}
data3 = {'foo': 388383, 'bar': ('strinstrin' * 20)}
data4 = {'foo2': 9999, 'baz': 4567}
baddata = {'foo': 'bad'}
# Simple index add/remove
self.eq([], idxr.getIndices())
idxr.addIndex('first', 'int', ['foo'])
self.raises(DupIndx, idxr.addIndex, 'first', 'int', ['foo'])
idxs = idxr.getIndices()
self.eq(idxs[0]['propname'], 'first')
idxr.delIndex('first')
self.raises(NoSuchIndx, idxr.delIndex, 'first')
self.eq([], idxr.getIndices())
self.genraises(NoSuchIndx, idxr.queryNormValu, 'notanindex')
# Check simple 1 record, 1 index index and retrieval
waiter = self.initWaiter(tank, 'addIndex')
tank.puts([data1])
idxr.addIndex('first', 'int', ['foo'])
self.wait(waiter)
waiter = self.initWaiter(tank, 'getIndices')
idxs = idxr.getIndices()
self.wait(waiter)
self.eq(1, idxs[0]['nextoffset'])
self.eq(1, idxs[0]['ngood'])
retn = list(idxr.queryNormRecords('first'))
self.eq(1, len(retn))
t = retn[0]
self.eq(2, len(t))
self.eq(t[0], 0)
self.eq(t[1], {'first': 1234})
waiter = self.initWaiter(tank, 'None')
tank.puts([data2])
self.wait(waiter)
idxs = idxr.getIndices()
self.eq(2, idxs[0]['nextoffset'])
self.eq(2, idxs[0]['ngood'])
# exact query
retn = list(idxr.queryRows('first', valu=2345, exact=True))
self.eq(1, len(retn))
t = retn[0]
self.eq(2, len(t))
self.eq(t[0], 1)
self.eq(s_msgpack.un(t[1]), data2)
# second index
waiter = self.initWaiter(tank, 'addIndex')
idxr.addIndex('second', 'str', ['bar'])
self.wait(waiter)
# prefix search
retn = list(idxr.queryNormValu('second', valu='strin'))
self.eq(retn, [(0, 'stringval'), (1, 'strinstrin')])
# long value, exact
waiter = self.initWaiter(tank, 'None')
tank.puts([data3])
self.wait(waiter)
retn = list(
idxr.queryRows('second', valu='strinstrin' * 20, exact=True))
self.eq(1, len(retn))
self.eq(s_msgpack.un(retn[0][1]), data3)
# long value with prefix
retn = list(idxr.queryRows('second', valu='str'))
self.eq(3, len(retn))
# long value with long prefix
self.genraises(s_exc.BadOperArg,
idxr.queryNormRecords,
'second',
valu='strinstrin' * 15)
# Bad data
waiter = self.initWaiter(tank, 'None')
tank.puts([baddata])
self.wait(waiter)
idxs = idxr.getIndices()
idx = next(i for i in idxs if i['propname'] == 'first')
self.eq(4, idx['nextoffset'])
self.eq(3, idx['ngood'])
self.eq(1, idx['nnormfail'])
waiter = self.initWaiter(tank, 'delIndex')
idxr.delIndex('second')
self.wait(waiter)
# Multiple datapaths
waiter = self.initWaiter(tank, 'delIndex')
idxr.delIndex('first')
self.wait(waiter)
waiter = self.initWaiter(tank, 'addIndex')
idxr.addIndex('first', 'int', ('foo', 'foo2'))
self.wait(waiter)
waiter = self.initWaiter(tank, 'None')
tank.puts([data4])
self.wait(waiter)
retn = list(idxr.queryNormValu('first'))
self.eq(retn, [(0, 1234), (1, 2345), (4, 9999), (2, 388383)])
waiter = self.initWaiter(tank, 'pauseIndex')
idxr.pauseIndex('first')
self.wait(waiter)
waiter = self.initWaiter(tank, 'resumeIndex')
idxr.resumeIndex('first')
self.wait(waiter)
waiter = self.initWaiter(tank, 'getIndices')
before_idxs = idxr.getIndices()
before_idx = next(i for i in before_idxs
if i['propname'] == 'first')
self.wait(waiter)
waiter = self.initWaiter(tank, 'None')
tank.puts([data1, data2, data3, data4] * 1000)
self.wait(waiter)
after_idxs = idxr.getIndices()
after_idx = next(i for i in after_idxs if i['propname'] == 'first')
self.lt(before_idx['ngood'], after_idx['ngood'])