async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.load', description='Load data into a remote hive from a previous hivesave.') pars.add_argument('--trim', default=False, action='store_true', help='Trim all other hive nodes (DANGER!)') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to load.') opts = pars.parse_args(argv) if opts.yaml: tree = s_common.yamlload(opts.filepath) else: tree = s_msgpack.loadfile(opts.filepath) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: await hive.loadHiveTree(tree, path=path, trim=opts.trim)
def genauth(opts, outp=s_output.stdout): authpath = s_common.genpath(opts.authfile) savepath = s_common.genpath(opts.savepath) if not os.path.isfile(authpath): outp.printf('auth file not found: %s' % (authpath, )) return auth = s_msgpack.loadfile(authpath) addr = auth[1].get('neuron') if addr is None: outp.printf('auth file has no neuron info: %s' % (authpath, )) return celluser = s_cell.CellUser(auth) with celluser.open(addr, timeout=20) as sess: nuro = s_neuron.NeuronClient(sess) auth = nuro.genCellAuth(opts.cellname, timeout=20) s_msgpack.dumpfile(auth, savepath) outp.printf('saved %s: %s' % (opts.cellname, savepath))
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.load', description='Load data into a remote hive from a previous hivesave.') pars.add_argument('--trim', default=False, action='store_true', help='Trim all other hive nodes (DANGER!)') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to load.') opts = pars.parse_args(argv) if opts.yaml: tree = s_common.yamlload(opts.filepath) else: tree = s_msgpack.loadfile(opts.filepath) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: classes = hive.sharinfo.get('classes', ()) try: s_version.reqVersion(hive._getSynVers(), reqver) if 'synapse.lib.hive.HiveApi' in classes: await hive.loadHiveTree(tree, path=path, trim=opts.trim) else: todo = s_common.todo('loadHiveTree', tree, path=path, trim=opts.trim) await hive.dyncall('cell', todo) except s_exc.BadVersion as e: valu = s_version.fmtVersion(*e.get('valu')) outp.printf( f'Hive version {valu} is outside of the hive.load supported range ({reqver}).' ) outp.printf( f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.' ) return 1
async def test_tools_hivesave(self): with self.getTestDir() as dirn: hivepath0 = os.path.join(dirn, 'hivesave0.mpk') yamlpath0 = os.path.join(dirn, 'hivesave0.yaml') async with self.getTestHiveDmon() as dmon: hive = dmon.shared.get('hive') await hive.set(('baz', 'faz'), 'visi') hurl = self.getTestUrl(dmon, 'hive') await s_hivesave.main([hurl, hivepath0]) tree = s_msgpack.loadfile(hivepath0) self.eq('visi', tree['kids']['baz']['kids']['faz']['value']) await s_hivesave.main( ['--path', 'baz', '--yaml', hurl, yamlpath0]) tree = s_common.yamlload(yamlpath0) self.eq('visi', tree['kids']['faz']['value']) hivepath1 = os.path.join(dirn, 'hivesave1.mpk') yamlpath1 = os.path.join(dirn, 'hivesave1.yaml') path = os.path.join(dirn, 'cell') async with await s_cell.Cell.anit(path) as cell: await cell.hive.set(('hehe', 'haha'), 20) curl = cell.getLocalUrl() await s_hivesave.main([curl, hivepath1]) tree = s_msgpack.loadfile(hivepath1) self.eq(20, tree['kids']['hehe']['kids']['haha']['value']) await s_hivesave.main( ['--path', 'hehe', '--yaml', curl, yamlpath1]) tree = s_common.yamlload(yamlpath1) self.eq(20, tree['kids']['haha']['value'])
def test_cryo_cell_daemon(self): with self.getTestDir() as dirn: celldir = os.path.join(dirn, 'cell') port = random.randint(10000, 50000) conf = { 'cells': [ (celldir, { 'ctor': 'synapse.cryotank.CryoCell', 'port': port, 'host': 'localhost', 'bind': '127.0.0.1', 'defvals': { 'mapsize': s_iq.TEST_MAP_SIZE } }), ], } with s_daemon.Daemon() as dmon: dmon.loadDmonConf(conf) with genfile(celldir, 'cell.lock') as fd: self.true(checkLock(fd, 30)) authfp = os.path.join(celldir, 'cell.auth') auth = s_msgpack.loadfile(authfp) addr = ('127.0.0.1', port) cuser = s_cell.CellUser(auth) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) retn = user.list(timeout=3) self.eq(retn, ()) user.puts('woot:woot', cryodata, timeout=2) retn = user.list(timeout=3) self.eq(retn[0][1]['indx'], 2) self.eq(retn[0][0], 'woot:woot') self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz') retn = user.list(timeout=3) self.eq(retn[0][1]['indx'], 2) self.eq(retn[0][0], 'woot:woot') # ensure dmon cell processes are fini'd for celldir, proc in dmon.cellprocs.items(): self.false(proc.is_alive())
def _genSelfAuth(self): path = self._path('cell.auth') if os.path.isfile(path): return s_msgpack.loadfile(path) name = self._genCellName('root') root = self.vault.genUserAuth(name) s_msgpack.dumpfile(root, path) path = self._path('user.auth') name = self._genCellName('user') user = self.vault.genUserAuth(name) s_msgpack.dumpfile(user, path) return root
def checkLoadfile(self, enfunc): t0 = ('5678', {'key': 1}) t1 = ('1234', {'key': 'haha'}) with self.getTestDir() as fdir: fd = s_common.genfile(fdir, 'oneobj.mpk') fd.write(enfunc(t0)) fd.close() fd = s_common.genfile(fdir, 'twoobjs.mpk') for obj in (t0, t1): fd.write(enfunc(obj)) fd.close() data = s_msgpack.loadfile(s_common.genpath(fdir, 'oneobj.mpk')) self.eq(data, ('5678', {'key': 1})) # Files containing multiple objects are not supported self.raises(msgpack.exceptions.ExtraData, s_msgpack.loadfile, s_common.genpath(fdir, 'twoobjs.mpk'))
def test_msgpack_loadfile(self): t0 = ('5678', {'key': 1}) t1 = ('1234', {'key': 'haha'}) with self.getTestDir() as fdir: fd = s_common.genfile(fdir, 'oneobj.mpk') fd.write(s_msgpack.en(t0)) fd.close() fd = s_common.genfile(fdir, 'twoobjs.mpk') for obj in (t0, t1): fd.write(s_msgpack.en(obj)) fd.close() data = s_msgpack.loadfile(s_common.genpath(fdir, 'oneobj.mpk')) self.eq(data, ('5678', {'key': 1})) # Files containing multiple objects are not supported self.raises(msgpack.exceptions.ExtraData, s_msgpack.loadfile, s_common.genpath(fdir, 'twoobjs.mpk'))
def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser(prog='cryo.cat', description='display data items from a cryo cell') pars.add_argument('cryocell', help='The cell descriptor and cryo tank path (cell://<host:port>/<name>).') pars.add_argument('--list', default=False, action='store_true', help='List tanks in the remote cell and return') pars.add_argument('--offset', default=0, type=int, help='Begin at offset index') pars.add_argument('--size', default=10, type=int, help='How many items to display') pars.add_argument('--timeout', default=10, type=int, help='The network timeout setting') pars.add_argument('--authfile', help='Path to your auth file for the remote cell') # TODO: make input mode using stdin... # TODO: make --jsonl output form for writing to file # TODO: make --no-index option that prints just the item opts = pars.parse_args(argv) if not opts.authfile: outp.printf('Currently requires --authfile until neuron protocol is supported') return 1 authpath = s_common.genpath(opts.authfile) auth = s_msgpack.loadfile(authpath) netw, path = opts.cryocell[7:].split('/', 1) host, portstr = netw.split(':') addr = (host, int(portstr)) outp.printf('connecting to: %r' % (addr,)) cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout) if opts.list: for name, info in cryo.list(timeout=opts.timeout): outp.printf('%s: %r' % (name, info)) return 0 for item in cryo.slice(path, opts.offset, opts.size, opts.timeout): outp.printf(pprint.pformat(item)) return 0
def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='cryo.cat', description='display data items from a cryo cell') pars.add_argument( 'cryocell', help= 'The cell descriptor and cryo tank path (cell://<host:port>/<name>).') pars.add_argument('--list', default=False, action='store_true', help='List tanks in the remote cell and return') pars.add_argument('--offset', default=0, type=int, help='Begin at offset index') pars.add_argument('--size', default=10, type=int, help='How many items to display') pars.add_argument('--timeout', default=10, type=int, help='The network timeout setting') pars.add_argument('--authfile', help='Path to your auth file for the remote cell') group = pars.add_mutually_exclusive_group() group.add_argument('--jsonl', action='store_true', help='Input/Output items in jsonl format') group.add_argument('--msgpack', action='store_true', help='Input/Output items in msgpack format') pars.add_argument('--verbose', '-v', default=False, action='store_true', help='Verbose output') pars.add_argument( '--ingest', '-i', default=False, action='store_true', help= 'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format' ) pars.add_argument( '--omit-offset', default=False, action='store_true', help= "Don't output offsets of objects. This is recommended to be used when jsonl/msgpack" " output is used.") opts = pars.parse_args(argv) if opts.verbose: logger.setLevel(logging.INFO) if not opts.authfile: logger.error( 'Currently requires --authfile until neuron protocol is supported') return 1 if opts.ingest and not opts.jsonl and not opts.msgpack: logger.error( 'Must specify exactly one of --jsonl or --msgpack if --ingest is specified' ) return 1 authpath = s_common.genpath(opts.authfile) auth = s_msgpack.loadfile(authpath) netw, path = opts.cryocell[7:].split('/', 1) host, portstr = netw.split(':') addr = (host, int(portstr)) logger.info('connecting to: %r', addr) cryo = s_cryotank.CryoUser(auth, addr, timeout=opts.timeout) if opts.list: for name, info in cryo.list(timeout=opts.timeout): outp.printf('%s: %r' % (name, info)) return 0 if opts.ingest: if opts.msgpack: fd = sys.stdin.buffer item_it = _except_wrap(s_msgpack.iterfd(fd), lambda x: 'Error parsing item %d' % x) else: fd = sys.stdin item_it = _except_wrap((json.loads(s) for s in fd), lambda x: ('Failure parsing line %d of input' % x)) cryo.puts(path, item_it) else: for item in cryo.slice(path, opts.offset, opts.size, opts.timeout): i = item[1] if opts.omit_offset else item if opts.jsonl: outp.printf(json.dumps(i, sort_keys=True)) elif opts.msgpack: sys.stdout.write(s_msgpack.en(i)) else: outp.printf(pprint.pformat(i)) return 0