async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.load',
description='Load data into a remote hive from a previous hivesave.')
pars.add_argument('--trim',
default=False,
action='store_true',
help='Trim all other hive nodes (DANGER!)')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to load.')
opts = pars.parse_args(argv)
if opts.yaml:
tree = s_common.yamlload(opts.filepath)
else:
tree = s_msgpack.loadfile(opts.filepath)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
await hive.loadHiveTree(tree, path=path, trim=opts.trim)
def genauth(opts, outp=s_output.stdout):
authpath = s_common.genpath(opts.authfile)
savepath = s_common.genpath(opts.savepath)
if not os.path.isfile(authpath):
outp.printf('auth file not found: %s' % (authpath, ))
return
auth = s_msgpack.loadfile(authpath)
addr = auth[1].get('neuron')
if addr is None:
outp.printf('auth file has no neuron info: %s' % (authpath, ))
return
celluser = s_cell.CellUser(auth)
with celluser.open(addr, timeout=20) as sess:
nuro = s_neuron.NeuronClient(sess)
auth = nuro.genCellAuth(opts.cellname, timeout=20)
s_msgpack.dumpfile(auth, savepath)
outp.printf('saved %s: %s' % (opts.cellname, savepath))
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.load',
description='Load data into a remote hive from a previous hivesave.')
pars.add_argument('--trim',
default=False,
action='store_true',
help='Trim all other hive nodes (DANGER!)')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to load.')
opts = pars.parse_args(argv)
if opts.yaml:
tree = s_common.yamlload(opts.filepath)
else:
tree = s_msgpack.loadfile(opts.filepath)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
classes = hive.sharinfo.get('classes', ())
try:
s_version.reqVersion(hive._getSynVers(), reqver)
if 'synapse.lib.hive.HiveApi' in classes:
await hive.loadHiveTree(tree, path=path, trim=opts.trim)
else:
todo = s_common.todo('loadHiveTree',
tree,
path=path,
trim=opts.trim)
await hive.dyncall('cell', todo)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
outp.printf(
f'Hive version {valu} is outside of the hive.load supported range ({reqver}).'
)
outp.printf(
f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.'
)
return 1
async def test_tools_hivesave(self):
with self.getTestDir() as dirn:
hivepath0 = os.path.join(dirn, 'hivesave0.mpk')
yamlpath0 = os.path.join(dirn, 'hivesave0.yaml')
async with self.getTestHiveDmon() as dmon:
hive = dmon.shared.get('hive')
await hive.set(('baz', 'faz'), 'visi')
hurl = self.getTestUrl(dmon, 'hive')
await s_hivesave.main([hurl, hivepath0])
tree = s_msgpack.loadfile(hivepath0)
self.eq('visi', tree['kids']['baz']['kids']['faz']['value'])
await s_hivesave.main(
['--path', 'baz', '--yaml', hurl, yamlpath0])
tree = s_common.yamlload(yamlpath0)
self.eq('visi', tree['kids']['faz']['value'])
hivepath1 = os.path.join(dirn, 'hivesave1.mpk')
yamlpath1 = os.path.join(dirn, 'hivesave1.yaml')
path = os.path.join(dirn, 'cell')
async with await s_cell.Cell.anit(path) as cell:
await cell.hive.set(('hehe', 'haha'), 20)
curl = cell.getLocalUrl()
await s_hivesave.main([curl, hivepath1])
tree = s_msgpack.loadfile(hivepath1)
self.eq(20, tree['kids']['hehe']['kids']['haha']['value'])
await s_hivesave.main(
['--path', 'hehe', '--yaml', curl, yamlpath1])
tree = s_common.yamlload(yamlpath1)
self.eq(20, tree['kids']['haha']['value'])
def test_cryo_cell_daemon(self):
with self.getTestDir() as dirn:
celldir = os.path.join(dirn, 'cell')
port = random.randint(10000, 50000)
conf = {
'cells': [
(celldir, {
'ctor': 'synapse.cryotank.CryoCell',
'port': port,
'host': 'localhost',
'bind': '127.0.0.1',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}),
],
}
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
with genfile(celldir, 'cell.lock') as fd:
self.true(checkLock(fd, 30))
authfp = os.path.join(celldir, 'cell.auth')
auth = s_msgpack.loadfile(authfp)
addr = ('127.0.0.1', port)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
retn = user.list(timeout=3)
self.eq(retn, ())
user.puts('woot:woot', cryodata, timeout=2)
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
# ensure dmon cell processes are fini'd
for celldir, proc in dmon.cellprocs.items():
self.false(proc.is_alive())
def _genSelfAuth(self):
path = self._path('cell.auth')
if os.path.isfile(path):
return s_msgpack.loadfile(path)
name = self._genCellName('root')
root = self.vault.genUserAuth(name)
s_msgpack.dumpfile(root, path)
path = self._path('user.auth')
name = self._genCellName('user')
user = self.vault.genUserAuth(name)
s_msgpack.dumpfile(user, path)
return root
def checkLoadfile(self, enfunc):
t0 = ('5678', {'key': 1})
t1 = ('1234', {'key': 'haha'})
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'oneobj.mpk')
fd.write(enfunc(t0))
fd.close()
fd = s_common.genfile(fdir, 'twoobjs.mpk')
for obj in (t0, t1):
fd.write(enfunc(obj))
fd.close()
data = s_msgpack.loadfile(s_common.genpath(fdir, 'oneobj.mpk'))
self.eq(data, ('5678', {'key': 1}))
# Files containing multiple objects are not supported
self.raises(msgpack.exceptions.ExtraData, s_msgpack.loadfile, s_common.genpath(fdir, 'twoobjs.mpk'))
def test_msgpack_loadfile(self):
t0 = ('5678', {'key': 1})
t1 = ('1234', {'key': 'haha'})
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'oneobj.mpk')
fd.write(s_msgpack.en(t0))
fd.close()
fd = s_common.genfile(fdir, 'twoobjs.mpk')
for obj in (t0, t1):
fd.write(s_msgpack.en(obj))
fd.close()
data = s_msgpack.loadfile(s_common.genpath(fdir, 'oneobj.mpk'))
self.eq(data, ('5678', {'key': 1}))
# Files containing multiple objects are not supported
self.raises(msgpack.exceptions.ExtraData, s_msgpack.loadfile, s_common.genpath(fdir, 'twoobjs.mpk'))
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument('cryocell', help='The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list', default=False, action='store_true', help='List tanks in the remote cell and return')
pars.add_argument('--offset', default=0, type=int, help='Begin at offset index')
pars.add_argument('--size', default=10, type=int, help='How many items to display')
pars.add_argument('--timeout', default=10, type=int, help='The network timeout setting')
pars.add_argument('--authfile', help='Path to your auth file for the remote cell')
# TODO: make input mode using stdin...
# TODO: make --jsonl output form for writing to file
# TODO: make --no-index option that prints just the item
opts = pars.parse_args(argv)
if not opts.authfile:
outp.printf('Currently requires --authfile until neuron protocol is supported')
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
outp.printf('connecting to: %r' % (addr,))
cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
outp.printf(pprint.pformat(item))
return 0
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument(
'cryocell',
help=
'The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list',
default=False,
action='store_true',
help='List tanks in the remote cell and return')
pars.add_argument('--offset',
default=0,
type=int,
help='Begin at offset index')
pars.add_argument('--size',
default=10,
type=int,
help='How many items to display')
pars.add_argument('--timeout',
default=10,
type=int,
help='The network timeout setting')
pars.add_argument('--authfile',
help='Path to your auth file for the remote cell')
group = pars.add_mutually_exclusive_group()
group.add_argument('--jsonl',
action='store_true',
help='Input/Output items in jsonl format')
group.add_argument('--msgpack',
action='store_true',
help='Input/Output items in msgpack format')
pars.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='Verbose output')
pars.add_argument(
'--ingest',
'-i',
default=False,
action='store_true',
help=
'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format'
)
pars.add_argument(
'--omit-offset',
default=False,
action='store_true',
help=
"Don't output offsets of objects. This is recommended to be used when jsonl/msgpack"
" output is used.")
opts = pars.parse_args(argv)
if opts.verbose:
logger.setLevel(logging.INFO)
if not opts.authfile:
logger.error(
'Currently requires --authfile until neuron protocol is supported')
return 1
if opts.ingest and not opts.jsonl and not opts.msgpack:
logger.error(
'Must specify exactly one of --jsonl or --msgpack if --ingest is specified'
)
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
logger.info('connecting to: %r', addr)
cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
if opts.ingest:
if opts.msgpack:
fd = sys.stdin.buffer
item_it = _except_wrap(s_msgpack.iterfd(fd),
lambda x: 'Error parsing item %d' % x)
else:
fd = sys.stdin
item_it = _except_wrap((json.loads(s) for s in fd), lambda x:
('Failure parsing line %d of input' % x))
cryo.puts(path, item_it)
else:
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
i = item[1] if opts.omit_offset else item
if opts.jsonl:
outp.printf(json.dumps(i, sort_keys=True))
elif opts.msgpack:
sys.stdout.write(s_msgpack.en(i))
else:
outp.printf(pprint.pformat(i))
return 0
