def test_cryo_cell_indexing(self):
conf = {
'bind': '127.0.0.1',
'host': 'localhost',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}
with self.getTestDir() as dirn, s_cryotank.CryoCell(dirn,
conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
# Test index operations
self.raises(s_exc.RetnErr, user.getIndices, 'notpresent')
self.eq((), user.getIndices('woot:woot'))
self.raises(s_exc.BadOperArg, user.addIndex, 'woot:woot',
'prop1', 'str', [])
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.delIndex('woot:woot', 'prop1')
self.raises(s_exc.RetnErr, user.delIndex, 'woot:woot',
'noexist')
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.pauseIndex('woot:woot', 'prop1')
user.pauseIndex('woot:woot')
user.resumeIndex('woot:woot')
self.eq([(1, 'baz'), (0, 'foo')],
list(user.queryNormValu('woot:woot', 'prop1')))
self.eq([(1, 'baz')],
list(user.queryNormValu('woot:woot', 'prop1',
valu='b')))
self.eq([],
list(
user.queryNormValu('woot:woot',
'prop1',
valu='bz',
timeout=10)))
self.eq([(1, {
'prop1': 'baz'
})], (list(
user.queryNormRecords('woot:woot', 'prop1', valu='b'))))
self.eq([(1, s_msgpack.en(('baz', {
'faz': 20
})))], list(user.queryRows('woot:woot', 'prop1', valu='b')))
user.init('woot:boring', {'noindex': True})
self.raises(s_exc.RetnErr, user.getIndices, 'woot:boring')
def cell_populate(self, port, auth):
# Populate the cell with data
addr = ('127.0.0.1', port)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
nodes = [(None, {'key': i}) for i in range(10)]
user.puts('test:hehe', nodes, 4)
self.len(10, list(user.slice('test:hehe', 0, 100)))
user.puts('test:haha', nodes, 4)
self.len(2, user.list())
def test_cryo_cell_daemon(self):
with self.getTestDir() as dirn:
celldir = os.path.join(dirn, 'cell')
port = random.randint(10000, 50000)
conf = {
'cells': [
(celldir, {
'ctor': 'synapse.cryotank.CryoCell',
'port': port,
'host': 'localhost',
'bind': '127.0.0.1',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}),
],
}
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
with genfile(celldir, 'cell.lock') as fd:
self.true(checkLock(fd, 30))
authfp = os.path.join(celldir, 'cell.auth')
auth = s_msgpack.loadfile(authfp)
addr = ('127.0.0.1', port)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
retn = user.list(timeout=3)
self.eq(retn, ())
user.puts('woot:woot', cryodata, timeout=2)
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
# ensure dmon cell processes are fini'd
for celldir, proc in dmon.cellprocs.items():
self.false(proc.is_alive())
def test_cryo_cell(self):
with self.getTestDir() as dirn:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
metr = list(user.metrics('woot:woot', 0, 100, timeout=2))
self.len(2, metr)
self.eq(metr[0][1]['count'], 1)
user.puts('woot:woot', cryodata, timeout=2)
retn = list(user.slice('woot:woot', 2, 2, timeout=3))
self.len(2, retn)
self.eq(2, retn[0][0])
retn = list(user.rows('woot:woot', 0, 2, timeout=3))
self.len(2, retn)
self.eq(retn[0], (0, s_msgpack.en(cryodata[0])))
self.eq(retn[1], (1, s_msgpack.en(cryodata[1])))
# Reset chunksize
user._chunksize = s_cryotank.CryoClient._chunksize
user.puts('woot:hehe', cryodata, timeout=5)
user.puts('woot:hehe', cryodata, timeout=5)
retn = list(user.slice('woot:hehe', 1, 2))
retn = [val for indx, val in retn[::-1]]
self.eq(tuple(retn), cryodata)
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
listd = dict(user.list(timeout=3))
self.isin('woot:hehe', listd)
self.eq(user.last('woot:hehe', timeout=3),
(3, cryodata[1]))
# delete woot.hehe and then call apis on it
self.true(user.delete('woot:hehe'))
self.false(user.delete('woot:hehe'))
self.none(cell.tanks.get('woot:hehe'))
self.none(cell.names.get('woot:hehe'))
self.genraises(s_exc.RetnErr,
user.slice,
'woot:hehe',
1,
2,
timeout=3)
self.genraises(s_exc.RetnErr,
user.rows,
'woot:hehe',
1,
2,
timeout=3)
listd = dict(user.list(timeout=3))
self.notin('woot:hehe', listd)
self.none(user.last('woot:hehe', timeout=3))
self.genraises(s_exc.RetnErr,
user.metrics,
'woot:hehe',
0,
100,
timeout=3)
# Adding data re-adds the tank
user._chunksize = 1000
user.puts('woot:hehe', cryodata, timeout=5)
metr = list(user.metrics('woot:hehe', 0))
self.len(1, metr)
# We can initialize a new tank directly with a custom map size
self.true(
user.init('weee:imthebest', {'mapsize': 5558675309}))
self.false(user.init('woot:hehe'))
# error when we specify an invalid config option
self.raises(s_exc.RetnErr, user.init, 'weee:danktank',
{'newp': 'hehe'})
# Turn it back on
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
listd = dict(user.list(timeout=3))
self.len(3, listd)
self.isin('weee:imthebest', listd)
self.isin('woot:woot', listd)
self.isin('woot:hehe', listd)
self.istufo(user.last('woot:woot')[1])
self.istufo(user.last('woot:hehe')[1])
self.none(user.last('weee:imthebest'))
# Test empty puts
user.puts('woot:hehe', tuple())
listd = dict(user.list(timeout=3))
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
self.nn(user.last('woot:hehe'))
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument(
'cryocell',
help=
'The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list',
default=False,
action='store_true',
help='List tanks in the remote cell and return')
pars.add_argument('--offset',
default=0,
type=int,
help='Begin at offset index')
pars.add_argument('--size',
default=10,
type=int,
help='How many items to display')
pars.add_argument('--timeout',
default=10,
type=int,
help='The network timeout setting')
pars.add_argument('--authfile',
help='Path to your auth file for the remote cell')
group = pars.add_mutually_exclusive_group()
group.add_argument('--jsonl',
action='store_true',
help='Input/Output items in jsonl format')
group.add_argument('--msgpack',
action='store_true',
help='Input/Output items in msgpack format')
pars.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='Verbose output')
pars.add_argument(
'--ingest',
'-i',
default=False,
action='store_true',
help=
'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format'
)
pars.add_argument(
'--omit-offset',
default=False,
action='store_true',
help=
"Don't output offsets of objects. This is recommended to be used when jsonl/msgpack"
" output is used.")
opts = pars.parse_args(argv)
if opts.verbose:
logger.setLevel(logging.INFO)
if not opts.authfile:
logger.error(
'Currently requires --authfile until neuron protocol is supported')
return 1
if opts.ingest and not opts.jsonl and not opts.msgpack:
logger.error(
'Must specify exactly one of --jsonl or --msgpack if --ingest is specified'
)
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
logger.info('connecting to: %r', addr)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=opts.timeout) as sess:
cryo = s_cryotank.CryoClient(sess)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
if opts.ingest:
if opts.msgpack:
fd = sys.stdin.buffer
item_it = _except_wrap(s_msgpack.iterfd(fd),
lambda x: 'Error parsing item %d' % x)
else:
fd = sys.stdin
item_it = _except_wrap(
(json.loads(s) for s in fd), lambda x:
('Failure parsing line %d of input' % x))
cryo.puts(path, item_it)
else:
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
i = item[1] if opts.omit_offset else item
if opts.jsonl:
outp.printf(json.dumps(i, sort_keys=True))
elif opts.msgpack:
sys.stdout.write(s_msgpack.en(i))
else:
outp.printf(pprint.pformat(i))
return 0