def test_cryocat(self):
with self.getTestDir() as dirn:
conf = {'host': '127.0.0.1'}
celldir = os.path.join(dirn, 'cell')
authfp = os.path.join(dirn, 'user.auth')
with s_cryotank.CryoCell(celldir, conf) as cell:
port = cell.getCellPort()
auth = cell.genUserAuth('*****@*****.**')
self.cell_populate(port, auth)
with genfile(authfp) as fd:
fd.write(s_msgpack.en(auth))
addr = 'cell://127.0.0.1:%s/%s' % (port, 'test:hehe')
outp = self.getTestOutp()
argv = ['--list', addr]
self.eq(s_cryocat.main(argv, outp), 1)
self.true(outp.expect('Currently requires --authfile'))
outp = self.getTestOutp()
argv = ['--list', '--authfile', authfp, addr]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect('test:hehe'))
self.true(outp.expect('test:haha'))
outp = self.getTestOutp()
argv = [
'--offset', '0', '--size', '1', '--authfile', authfp, addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect("(0, (None, {'key': 0}))"))
outp = self.getTestOutp()
argv = [
'--offset', '0', '--size', '20', '--authfile', authfp, addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect("(0, (None, {'key': 0}))"))
self.true(outp.expect("(9, (None, {'key': 9}))"))
self.false(
outp.expect("(10, (None, {'key': 10}))", throw=False))
outp = self.getTestOutp()
argv = [
'--offset', '10', '--size', '20', '--authfile', authfp,
addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.false(outp.expect("(9, (None, {'key': 9}))", throw=False))
self.false(
outp.expect("(10, (None, {'key': 10}))", throw=False))
def test_cryo_cell_indexing(self):
conf = {
'bind': '127.0.0.1',
'host': 'localhost',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}
with self.getTestDir() as dirn, s_cryotank.CryoCell(dirn,
conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
# Test index operations
self.raises(s_exc.RetnErr, user.getIndices, 'notpresent')
self.eq((), user.getIndices('woot:woot'))
self.raises(s_exc.BadOperArg, user.addIndex, 'woot:woot',
'prop1', 'str', [])
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.delIndex('woot:woot', 'prop1')
self.raises(s_exc.RetnErr, user.delIndex, 'woot:woot',
'noexist')
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.pauseIndex('woot:woot', 'prop1')
user.pauseIndex('woot:woot')
user.resumeIndex('woot:woot')
self.eq([(1, 'baz'), (0, 'foo')],
list(user.queryNormValu('woot:woot', 'prop1')))
self.eq([(1, 'baz')],
list(user.queryNormValu('woot:woot', 'prop1',
valu='b')))
self.eq([],
list(
user.queryNormValu('woot:woot',
'prop1',
valu='bz',
timeout=10)))
self.eq([(1, {
'prop1': 'baz'
})], (list(
user.queryNormRecords('woot:woot', 'prop1', valu='b'))))
self.eq([(1, s_msgpack.en(('baz', {
'faz': 20
})))], list(user.queryRows('woot:woot', 'prop1', valu='b')))
user.init('woot:boring', {'noindex': True})
self.raises(s_exc.RetnErr, user.getIndices, 'woot:boring')
def cryo(dirn, conf=None):
return s_cryotank.CryoCell(dirn, conf=conf)
def test_cryocat(self):
with self.getTestDir() as dirn:
conf = {
'bind': '127.0.0.1',
'host': 'localhost',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}
celldir = os.path.join(dirn, 'cell')
authfp = os.path.join(dirn, 'user.auth')
with s_cryotank.CryoCell(celldir, conf) as cell:
addr = cell.getCellAddr()
port = addr[1]
auth = cell.genUserAuth('*****@*****.**')
self.cell_populate(port, auth)
with genfile(authfp) as fd:
fd.write(s_msgpack.en(auth))
addr = 'cell://127.0.0.1:%s/%s' % (port, 'test:hehe')
outp = self.getTestOutp()
argv = ['--list', addr]
with self.getLoggerStream('synapse.tools.cryo.cat') as stream:
self.eq(s_cryocat.main(argv, outp), 1)
stream.seek(0)
log_msgs = stream.read()
self.isin('Currently requires --authfile', log_msgs)
outp = self.getTestOutp()
argv = ['-v', '--list', '--authfile', authfp, addr]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect('test:hehe'))
self.true(outp.expect('test:haha'))
# Make sure that --ingest without a format dies
outp = self.getTestOutp()
argv = ['--ingest', '--authfile', authfp, addr]
self.eq(s_cryocat.main(argv, outp), 1)
# Happy path jsonl ingest
outp = self.getTestOutp()
argv = ['--ingest', '--jsonl', '--authfile', authfp, addr]
inp = io.StringIO('{"foo": "bar"}\n[]\n')
with self.redirectStdin(inp):
self.eq(s_cryocat.main(argv, outp), 0)
# Sad path jsonl ingest
outp = self.getTestOutp()
argv = ['--ingest', '--jsonl', '--authfile', authfp, addr]
inp = io.StringIO('{"foo: "bar"}\n[]\n')
msg = 'Failure parsing line'
with self.redirectStdin(inp):
with self.getLoggerStream('synapse.lib.net',
msg) as stream:
self.eq(s_cryocat.main(argv, outp), 0)
self.true(stream.wait(10))
stream.seek(0)
log_msgs = stream.read()
self.isin(msg, log_msgs)
# Happy path msgpack ingest
outp = self.getTestOutp()
argv = ['--ingest', '--msgpack', '--authfile', authfp, addr]
to_ingest1 = s_msgpack.en({'foo': 'bar'})
to_ingest2 = s_msgpack.en(['lol', 'brb'])
inp = Mock()
inp.buffer = io.BytesIO(to_ingest1 + to_ingest2)
with self.redirectStdin(inp):
self.eq(s_cryocat.main(argv, outp), 0)
# Sad path msgpack ingest
outp = self.getTestOutp()
argv = ['--ingest', '--msgpack', '--authfile', authfp, addr]
good_encoding = s_msgpack.en({'foo': 'bar'})
bad_encoding = bytearray(good_encoding)
bad_encoding[2] = 0xff
inp = Mock()
inp.buffer = io.BytesIO(bad_encoding)
msg = 'UnpackValueError'
with self.redirectStdin(inp):
with self.getLoggerStream('synapse.lib.net',
msg) as stream:
self.eq(s_cryocat.main(argv, outp), 0)
self.true(stream.wait(10))
stream.seek(0)
log_msgs = stream.read()
self.isin(msg, log_msgs)
self.isin('Error parsing item', log_msgs)
outp = self.getTestOutp()
argv = [
'--offset', '0', '--size', '1', '--authfile', authfp, addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect("(0, (None, {'key': 0}))"))
outp = self.getTestOutp()
argv = [
'--offset', '0', '--jsonl', '--size', '2', '--omit-offset',
'--authfile', authfp, addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(
outp.expect('[null, {"key": 0}]\n[null, {"key": 1}]\n'))
outp = self.getTestOutp()
argv = [
'--offset', '0', '--size', '20', '--authfile', authfp, addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.true(outp.expect("(0, (None, {'key': 0}))"))
self.true(outp.expect("(9, (None, {'key': 9}))"))
# Verify the ingested data
self.true(outp.expect("(10, {'foo': 'bar'})"))
self.true(outp.expect("(11, ())"))
self.true(outp.expect("(12, {'foo': 'bar'})"))
self.true(outp.expect("(13, ('lol', 'brb'))"))
self.false(
outp.expect("(14, (None, {'key': 10}))", throw=False))
outp = self.getTestOutp()
argv = [
'--offset', '10', '--size', '20', '--authfile', authfp,
addr
]
self.eq(s_cryocat.main(argv, outp), 0)
self.false(outp.expect("(9, (None, {'key': 9}))", throw=False))
self.false(
outp.expect("(10, (None, {'key': 10}))", throw=False))
def test_cryo_cell(self):
with self.getTestDir() as dirn:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
metr = list(user.metrics('woot:woot', 0, 100, timeout=2))
self.len(2, metr)
self.eq(metr[0][1]['count'], 1)
user.puts('woot:woot', cryodata, timeout=2)
retn = list(user.slice('woot:woot', 2, 2, timeout=3))
self.len(2, retn)
self.eq(2, retn[0][0])
retn = list(user.rows('woot:woot', 0, 2, timeout=3))
self.len(2, retn)
self.eq(retn[0], (0, s_msgpack.en(cryodata[0])))
self.eq(retn[1], (1, s_msgpack.en(cryodata[1])))
# Reset chunksize
user._chunksize = s_cryotank.CryoClient._chunksize
user.puts('woot:hehe', cryodata, timeout=5)
user.puts('woot:hehe', cryodata, timeout=5)
retn = list(user.slice('woot:hehe', 1, 2))
retn = [val for indx, val in retn[::-1]]
self.eq(tuple(retn), cryodata)
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
listd = dict(user.list(timeout=3))
self.isin('woot:hehe', listd)
self.eq(user.last('woot:hehe', timeout=3),
(3, cryodata[1]))
# delete woot.hehe and then call apis on it
self.true(user.delete('woot:hehe'))
self.false(user.delete('woot:hehe'))
self.none(cell.tanks.get('woot:hehe'))
self.none(cell.names.get('woot:hehe'))
self.genraises(s_exc.RetnErr,
user.slice,
'woot:hehe',
1,
2,
timeout=3)
self.genraises(s_exc.RetnErr,
user.rows,
'woot:hehe',
1,
2,
timeout=3)
listd = dict(user.list(timeout=3))
self.notin('woot:hehe', listd)
self.none(user.last('woot:hehe', timeout=3))
self.genraises(s_exc.RetnErr,
user.metrics,
'woot:hehe',
0,
100,
timeout=3)
# Adding data re-adds the tank
user._chunksize = 1000
user.puts('woot:hehe', cryodata, timeout=5)
metr = list(user.metrics('woot:hehe', 0))
self.len(1, metr)
# We can initialize a new tank directly with a custom map size
self.true(
user.init('weee:imthebest', {'mapsize': 5558675309}))
self.false(user.init('woot:hehe'))
# error when we specify an invalid config option
self.raises(s_exc.RetnErr, user.init, 'weee:danktank',
{'newp': 'hehe'})
# Turn it back on
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
listd = dict(user.list(timeout=3))
self.len(3, listd)
self.isin('weee:imthebest', listd)
self.isin('woot:woot', listd)
self.isin('woot:hehe', listd)
self.istufo(user.last('woot:woot')[1])
self.istufo(user.last('woot:hehe')[1])
self.none(user.last('weee:imthebest'))
# Test empty puts
user.puts('woot:hehe', tuple())
listd = dict(user.list(timeout=3))
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
self.nn(user.last('woot:hehe'))
def test_cryo_cell(self):
with self.getTestDir() as dirn:
conf = {'host': '127.0.0.1'}
with s_cryotank.CryoCell(dirn, conf) as cell:
port = cell.getCellPort()
auth = cell.genUserAuth('*****@*****.**')
addr = ('127.0.0.1', port)
user = s_cryotank.CryoUser(auth, addr, timeout=2)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list()
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
metr = user.metrics('woot:woot', 0, 100, timeout=2)
self.len(2, metr)
self.eq(metr[0][1]['count'], 1)
user.puts('woot:woot', cryodata, timeout=2)
retn = list(user.slice('woot:woot', 2, 2))
self.len(2, retn)
self.eq(2, retn[0][0])
retn = list(user.rows('woot:woot', 0, 2))
self.len(2, retn)
self.eq(retn[0], (0, s_msgpack.en(cryodata[0])))
self.eq(retn[1], (1, s_msgpack.en(cryodata[1])))
# Reset chunksize
user._chunksize = s_cryotank.CryoUser._chunksize
user.puts('woot:hehe', cryodata, timeout=5)
user.puts('woot:hehe', cryodata, timeout=5)
retn = list(user.slice('woot:hehe', 1, 2))
retn = [val for indx, val in retn[::-1]]
self.eq(tuple(retn), cryodata)
self.len(2, (user.metrics('woot:hehe', 0, 100)))
listd = dict(user.list())
self.isin('woot:hehe', listd)
self.eq(user.last('woot:hehe'), (3, cryodata[1]))
# delete woot.hehe and then call apis on it
self.true(user.delete('woot:hehe'))
self.false(user.delete('woot:hehe'))
self.none(cell.tanks.get('woot:hehe'))
self.none(cell.names.get('woot:hehe'))
self.eq(list(user.slice('woot:hehe', 1, 2)), [])
self.eq(list(user.rows('woot:hehe', 1, 2)), [])
listd = dict(user.list())
self.notin('woot:hehe', listd)
self.none(user.metrics('woot:hehe', 0, 100))
self.none(user.last('woot:hehe'))
# Adding data re-adds the tank
user.puts('woot:hehe', cryodata, timeout=5)
self.len(1, (user.metrics('woot:hehe', 0, 100)))
# We can initialize a new tank directly with a custom map size
self.true(user.init('weee:imthebest', {'mapsize': 5558675309}))
self.false(user.init('woot:hehe'))
with self.getLoggerStream('synapse.cryotank') as stream:
self.false(user.init('weee:danktank', {'newp': 'hehe'}))
stream.seek(0)
mesgs = stream.read()
self.isin('Error making CryoTank', mesgs)
# Turn it back on
with s_cryotank.CryoCell(dirn, conf) as cell:
# auth and port persist
user = s_cryotank.CryoUser(auth, addr, timeout=2)
listd = dict(user.list())
self.len(3, listd)
self.isin('weee:imthebest', listd)
self.isin('woot:woot', listd)
self.isin('woot:hehe', listd)
self.istufo(user.last('woot:woot')[1])
self.istufo(user.last('woot:hehe')[1])
self.none(user.last('weee:imthebest'))
# Test empty puts
user.puts('woot:hehe', tuple())
listd = dict(user.list())
self.len(2, (user.metrics('woot:hehe', 0, 100)))
self.nn(user.metrics('woot:hehe', 0, 100))
self.nn(user.last('woot:hehe'))