def cell_populate(self, port, auth):
# Populate the cell with data
addr = ('127.0.0.1', port)
user = s_cryotank.CryoUser(auth, addr, timeout=2)
nodes = [(None, {'key': i}) for i in range(10)]
user.puts('test:hehe', nodes, 4)
self.len(10, list(user.slice('test:hehe', 0, 100)))
user.puts('test:haha', nodes, 4)
self.len(2, user.list())
def test_cryo_cell_daemon(self):
with self.getTestDir() as dirn:
celldir = os.path.join(dirn, 'cell')
port = random.randint(10000, 50000)
conf = {
'cells': [
(celldir, {
'ctor': 'synapse.cryotank.CryoCell',
'port': port,
'host': '127.0.0.1',
}),
],
}
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
with genfile(celldir, 'cell.lock') as fd:
self.true(checkLock(fd, 30))
authfp = os.path.join(celldir, 'cell.auth')
auth = s_msgpack.loadfile(authfp)
addr = ('127.0.0.1', port)
user = s_cryotank.CryoUser(auth, addr, timeout=2)
retn = user.list()
self.eq(retn, ())
user.puts('woot:woot', cryodata, timeout=2)
retn = user.list()
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list()
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
# ensure dmon cell processes are fini'd
for celldir, proc in dmon.cellprocs.items():
self.false(proc.is_alive())
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument('cryocell', help='The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list', default=False, action='store_true', help='List tanks in the remote cell and return')
pars.add_argument('--offset', default=0, type=int, help='Begin at offset index')
pars.add_argument('--size', default=10, type=int, help='How many items to display')
pars.add_argument('--timeout', default=10, type=int, help='The network timeout setting')
pars.add_argument('--authfile', help='Path to your auth file for the remote cell')
# TODO: make input mode using stdin...
# TODO: make --jsonl output form for writing to file
# TODO: make --no-index option that prints just the item
opts = pars.parse_args(argv)
if not opts.authfile:
outp.printf('Currently requires --authfile until neuron protocol is supported')
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
outp.printf('connecting to: %r' % (addr,))
cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
outp.printf(pprint.pformat(item))
return 0
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument(
'cryocell',
help=
'The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list',
default=False,
action='store_true',
help='List tanks in the remote cell and return')
pars.add_argument('--offset',
default=0,
type=int,
help='Begin at offset index')
pars.add_argument('--size',
default=10,
type=int,
help='How many items to display')
pars.add_argument('--timeout',
default=10,
type=int,
help='The network timeout setting')
pars.add_argument('--authfile',
help='Path to your auth file for the remote cell')
group = pars.add_mutually_exclusive_group()
group.add_argument('--jsonl',
action='store_true',
help='Input/Output items in jsonl format')
group.add_argument('--msgpack',
action='store_true',
help='Input/Output items in msgpack format')
pars.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='Verbose output')
pars.add_argument(
'--ingest',
'-i',
default=False,
action='store_true',
help=
'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format'
)
pars.add_argument(
'--omit-offset',
default=False,
action='store_true',
help=
"Don't output offsets of objects. This is recommended to be used when jsonl/msgpack"
" output is used.")
opts = pars.parse_args(argv)
if opts.verbose:
logger.setLevel(logging.INFO)
if not opts.authfile:
logger.error(
'Currently requires --authfile until neuron protocol is supported')
return 1
if opts.ingest and not opts.jsonl and not opts.msgpack:
logger.error(
'Must specify exactly one of --jsonl or --msgpack if --ingest is specified'
)
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
logger.info('connecting to: %r', addr)
cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
if opts.ingest:
if opts.msgpack:
fd = sys.stdin.buffer
item_it = _except_wrap(s_msgpack.iterfd(fd),
lambda x: 'Error parsing item %d' % x)
else:
fd = sys.stdin
item_it = _except_wrap((json.loads(s) for s in fd), lambda x:
('Failure parsing line %d of input' % x))
cryo.puts(path, item_it)
else:
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
i = item[1] if opts.omit_offset else item
if opts.jsonl:
outp.printf(json.dumps(i, sort_keys=True))
elif opts.msgpack:
sys.stdout.write(s_msgpack.en(i))
else:
outp.printf(pprint.pformat(i))
return 0
def test_cryo_cell(self):
with self.getTestDir() as dirn:
conf = {'host': '127.0.0.1'}
with s_cryotank.CryoCell(dirn, conf) as cell:
port = cell.getCellPort()
auth = cell.genUserAuth('*****@*****.**')
addr = ('127.0.0.1', port)
user = s_cryotank.CryoUser(auth, addr, timeout=2)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list()
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
metr = user.metrics('woot:woot', 0, 100, timeout=2)
self.len(2, metr)
self.eq(metr[0][1]['count'], 1)
user.puts('woot:woot', cryodata, timeout=2)
retn = list(user.slice('woot:woot', 2, 2))
self.len(2, retn)
self.eq(2, retn[0][0])
retn = list(user.rows('woot:woot', 0, 2))
self.len(2, retn)
self.eq(retn[0], (0, s_msgpack.en(cryodata[0])))
self.eq(retn[1], (1, s_msgpack.en(cryodata[1])))
# Reset chunksize
user._chunksize = s_cryotank.CryoUser._chunksize
user.puts('woot:hehe', cryodata, timeout=5)
user.puts('woot:hehe', cryodata, timeout=5)
retn = list(user.slice('woot:hehe', 1, 2))
retn = [val for indx, val in retn[::-1]]
self.eq(tuple(retn), cryodata)
self.len(2, (user.metrics('woot:hehe', 0, 100)))
listd = dict(user.list())
self.isin('woot:hehe', listd)
self.eq(user.last('woot:hehe'), (3, cryodata[1]))
# delete woot.hehe and then call apis on it
self.true(user.delete('woot:hehe'))
self.false(user.delete('woot:hehe'))
self.none(cell.tanks.get('woot:hehe'))
self.none(cell.names.get('woot:hehe'))
self.eq(list(user.slice('woot:hehe', 1, 2)), [])
self.eq(list(user.rows('woot:hehe', 1, 2)), [])
listd = dict(user.list())
self.notin('woot:hehe', listd)
self.none(user.metrics('woot:hehe', 0, 100))
self.none(user.last('woot:hehe'))
# Adding data re-adds the tank
user.puts('woot:hehe', cryodata, timeout=5)
self.len(1, (user.metrics('woot:hehe', 0, 100)))
# We can initialize a new tank directly with a custom map size
self.true(user.init('weee:imthebest', {'mapsize': 5558675309}))
self.false(user.init('woot:hehe'))
with self.getLoggerStream('synapse.cryotank') as stream:
self.false(user.init('weee:danktank', {'newp': 'hehe'}))
stream.seek(0)
mesgs = stream.read()
self.isin('Error making CryoTank', mesgs)
# Turn it back on
with s_cryotank.CryoCell(dirn, conf) as cell:
# auth and port persist
user = s_cryotank.CryoUser(auth, addr, timeout=2)
listd = dict(user.list())
self.len(3, listd)
self.isin('weee:imthebest', listd)
self.isin('woot:woot', listd)
self.isin('woot:hehe', listd)
self.istufo(user.last('woot:woot')[1])
self.istufo(user.last('woot:hehe')[1])
self.none(user.last('weee:imthebest'))
# Test empty puts
user.puts('woot:hehe', tuple())
listd = dict(user.list())
self.len(2, (user.metrics('woot:hehe', 0, 100)))
self.nn(user.metrics('woot:hehe', 0, 100))
self.nn(user.last('woot:hehe'))