def genauth(opts, outp=s_output.stdout): authpath = s_common.genpath(opts.authfile) savepath = s_common.genpath(opts.savepath) if not os.path.isfile(authpath): outp.printf('auth file not found: %s' % (authpath, )) return auth = s_msgpack.loadfile(authpath) addr = auth[1].get('neuron') if addr is None: outp.printf('auth file has no neuron info: %s' % (authpath, )) return celluser = s_cell.CellUser(auth) with celluser.open(addr, timeout=20) as sess: nuro = s_neuron.NeuronClient(sess) auth = nuro.genCellAuth(opts.cellname, timeout=20) s_msgpack.dumpfile(auth, savepath) outp.printf('saved %s: %s' % (opts.cellname, savepath))
def test_doc_data(self): with self.getTestDir() as dirn: s_common.gendir(dirn, 'docdata', 'stuff') docdata = s_common.genpath(dirn, 'docdata') root = s_common.genpath(dirn, 'synapse', 'userguides') d = {'key': 'value'} s_common.jssave(d, docdata, 'data.json') s_common.yamlsave(d, docdata, 'data.yaml') s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk')) with s_common.genfile(docdata, 'stuff', 'data.txt') as fd: fd.write('beep'.encode()) with s_common.genfile(docdata, 'data.jsonl') as fd: fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') data = s_jupyter.getDocData('data.json', root) self.eq(data, d) data = s_jupyter.getDocData('data.yaml', root) self.eq(data, d) data = s_jupyter.getDocData('data.mpk', root) self.eq(data, d) data = s_jupyter.getDocData('stuff/data.txt', root) self.eq(data, b'beep') data = s_jupyter.getDocData('data.jsonl', root) self.eq(data, [d, d, d]) self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root) self.raises(ValueError, s_jupyter.getDocData, '../../../../../../etc/passwd', root)
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.save', description='Save tree data from a remote hive to file.') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to save.') opts = pars.parse_args(argv) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: tree = await hive.saveHiveTree(path=path) if opts.yaml: s_common.yamlsave(tree, opts.filepath) else: s_msgpack.dumpfile(tree, opts.filepath)
def postCell(self): self.cells = self.getCellDict('cells') path = self._path('admin.auth') if not os.path.exists(path): auth = self.genCellAuth('admin') s_msgpack.dumpfile(auth, path)
def _genSelfAuth(self): path = self._path('cell.auth') if os.path.isfile(path): return s_msgpack.loadfile(path) name = self._genCellName('root') root = self.vault.genUserAuth(name) s_msgpack.dumpfile(root, path) path = self._path('user.auth') name = self._genCellName('user') user = self.vault.genUserAuth(name) s_msgpack.dumpfile(user, path) return root
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.save', description='Save tree data from a remote hive to file.') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to save.') opts = pars.parse_args(argv) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: try: s_version.reqVersion(hive._getSynVers(), reqver) except s_exc.BadVersion as e: valu = s_version.fmtVersion(*e.get('valu')) outp.printf( f'Hive version {valu} is outside of the hive.save supported range ({reqver}).' ) outp.printf( f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.' ) return 1 tree = await hive.saveHiveTree(path=path) if opts.yaml: s_common.yamlsave(tree, opts.filepath) else: s_msgpack.dumpfile(tree, opts.filepath) return 0
async def test_tools_hiveload_vercheck(self): with self.getTestDir() as dirn: hivepath0 = os.path.join(dirn, 'hivesave0.mpk') s_msgpack.dumpfile(htree0, hivepath0) async with self.getTestHiveDmon() as dmon: hurl = self.getTestUrl(dmon, 'hive') argv = [hurl, hivepath0] def _getOldSynVers(self): return (0, 0, 0) with mock.patch('synapse.telepath.Proxy._getSynVers', _getOldSynVers): outp = self.getTestOutp() retn = await s_hiveload.main(argv, outp=outp) outp.expect( 'Hive version 0.0.0 is outside of the hive.load supported range' ) self.eq(1, retn)
async def test_tools_hiveload(self): with self.getTestDir() as dirn: hivepath0 = os.path.join(dirn, 'hivepath0.mpk') hivepath1 = os.path.join(dirn, 'hivepath1.mpk') yamlpath0 = os.path.join(dirn, 'hivepath0.yaml') s_msgpack.dumpfile(htree0, hivepath0) s_msgpack.dumpfile(htree1, hivepath1) s_common.yamlsave(htree0, yamlpath0) async with self.getTestHiveDmon() as dmon: hive = dmon.shared.get('hive') hurl = self.getTestUrl(dmon, 'hive') retn = await s_hiveload.main([hurl, hivepath0]) self.eq(0, retn) self.eq(20, await hive.get(('hehe', ))) self.eq(30, await hive.get(('haha', ))) self.eq('bar', await hive.get(('haha', 'foo'))) self.eq('faz', await hive.get(('haha', 'baz'))) retn = await s_hiveload.main([hurl, hivepath1]) self.eq(0, retn) self.eq(20, await hive.get(('hehe', ))) self.eq(30, await hive.get(('haha', ))) self.eq('bar', await hive.get(('haha', 'foo'))) self.eq('faz', await hive.get(('haha', 'baz'))) retn = await s_hiveload.main(['--trim', hurl, hivepath1]) self.eq(0, retn) self.eq(20, await hive.get(('hehe', ))) self.eq(30, await hive.get(('haha', ))) self.eq('faz', await hive.get(('haha', 'baz'))) self.none(await hive.get(('haha', 'foo'))) async with self.getTestHiveDmon() as dmon: hive = dmon.shared.get('hive') hurl = self.getTestUrl(dmon, 'hive') await s_hiveload.main( ['--path', 'v/i/s/i', '--yaml', hurl, yamlpath0]) self.eq('bar', await hive.get( ('v', 'i', 's', 'i', 'haha', 'foo'))) path = os.path.join(dirn, 'cell') async with await s_cell.Cell.anit(path) as cell: curl = cell.getLocalUrl() await s_hiveload.main(['--path', 'gronk', curl, hivepath0]) self.eq('bar', await cell.hive.get(('gronk', 'haha', 'foo'))) await s_hiveload.main( ['--trim', '--path', 'gronk', curl, hivepath1]) self.none(await cell.hive.get(('gronk', 'haha', 'foo'))) await s_hiveload.main( ['--path', 'v/i/s/i', '--yaml', curl, yamlpath0]) self.eq( 'bar', await cell.hive.get( ('v', 'i', 's', 'i', 'haha', 'foo')))
def getAxonCore(self, cortex_conf=None): ''' Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex. Notes: The following items are available in the TstEnv: * dirn: Temporary test directory. * axon_client: A Axon client object. * core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex shared by the Daemon. * dmon_port: Port the Daemon is listening on. * dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex. * core: A Cortex. * axon_sess: The client session for the Axon. * axon: The AxonCell. * blob: The BlobCell backing the Axon. * neuron: The Neuron. Args: cortex_conf (dict): Optional cortex config Yields: TstEnv: A TstEnv instance. ''' with self.getTestDir() as dirn: neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} neurpath = s_common.gendir(dirn, 'neuron') neur = s_neuron.Neuron(neurpath, neurconf) neurhost, neurport = neur.getCellAddr() blobpath = s_common.gendir(dirn, 'blob') blobconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'blob:mapsize': TEST_MAP_SIZE } blobauth = neur.genCellAuth('blob') s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth')) blob = s_axon.BlobCell(blobpath, blobconf) self.true(blob.cellpool.neurwait(timeout=3)) axonpath = s_common.gendir(dirn, 'axon') axonauth = neur.genCellAuth('axon') s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'axon:blobs': ('blob@localhost', ), 'axon:mapsize': TEST_MAP_SIZE } axon = s_axon.AxonCell(axonpath, axonconf) self.true(axon.cellpool.neurwait(timeout=3)) axonhost, axonport = axon.getCellAddr() # wait for the axon to have blob ready = False for i in range(30): if axon.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon_user = s_cell.CellUser(axonauth) axon_sess = axon_user.open((axonhost, axonport)) axon_client = s_axon.AxonClient(axon_sess) core = s_cortex.openurl('ram:///', conf=cortex_conf) self.addTstForms(core) cellpoolconf = { 'host': neurhost, 'port': neurport, 'auth': s_common.enbase64(s_msgpack.en(axonauth)) } core.setConfOpt('cellpool:conf', cellpoolconf) core.setConfOpt('axon:name', 'axon@localhost') dmon = s_daemon.Daemon() dmonlink = dmon.listen('tcp://127.0.0.1:0/') dmonport = dmonlink[1].get('port') dmon.share('core', core) coreurl = 'tcp://127.0.0.1:%d/core' % dmonport env = TstEnv() env.add('dirn', dirn) env.add('axon_client', axon_client) env.add('core_url', coreurl) env.add('dmon_port', dmonport) # Order matter for clean fini env.add('dmon', dmon, True) env.add('core', core, True) env.add('axon_sess', axon_sess, True) env.add('axon', axon, True) env.add('blob', blob, True) env.add('neuron', neur, True) try: yield env finally: env.fini()