def genauth(opts, outp=s_output.stdout):
authpath = s_common.genpath(opts.authfile)
savepath = s_common.genpath(opts.savepath)
if not os.path.isfile(authpath):
outp.printf('auth file not found: %s' % (authpath, ))
return
auth = s_msgpack.loadfile(authpath)
addr = auth[1].get('neuron')
if addr is None:
outp.printf('auth file has no neuron info: %s' % (authpath, ))
return
celluser = s_cell.CellUser(auth)
with celluser.open(addr, timeout=20) as sess:
nuro = s_neuron.NeuronClient(sess)
auth = nuro.genCellAuth(opts.cellname, timeout=20)
s_msgpack.dumpfile(auth, savepath)
outp.printf('saved %s: %s' % (opts.cellname, savepath))
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.save',
description='Save tree data from a remote hive to file.')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to save.')
opts = pars.parse_args(argv)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
tree = await hive.saveHiveTree(path=path)
if opts.yaml:
s_common.yamlsave(tree, opts.filepath)
else:
s_msgpack.dumpfile(tree, opts.filepath)
def postCell(self):
self.cells = self.getCellDict('cells')
path = self._path('admin.auth')
if not os.path.exists(path):
auth = self.genCellAuth('admin')
s_msgpack.dumpfile(auth, path)
def _genSelfAuth(self):
path = self._path('cell.auth')
if os.path.isfile(path):
return s_msgpack.loadfile(path)
name = self._genCellName('root')
root = self.vault.genUserAuth(name)
s_msgpack.dumpfile(root, path)
path = self._path('user.auth')
name = self._genCellName('user')
user = self.vault.genUserAuth(name)
s_msgpack.dumpfile(user, path)
return root
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.save',
description='Save tree data from a remote hive to file.')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to save.')
opts = pars.parse_args(argv)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
try:
s_version.reqVersion(hive._getSynVers(), reqver)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
outp.printf(
f'Hive version {valu} is outside of the hive.save supported range ({reqver}).'
)
outp.printf(
f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.'
)
return 1
tree = await hive.saveHiveTree(path=path)
if opts.yaml:
s_common.yamlsave(tree, opts.filepath)
else:
s_msgpack.dumpfile(tree, opts.filepath)
return 0
async def test_tools_hiveload_vercheck(self):
with self.getTestDir() as dirn:
hivepath0 = os.path.join(dirn, 'hivesave0.mpk')
s_msgpack.dumpfile(htree0, hivepath0)
async with self.getTestHiveDmon() as dmon:
hurl = self.getTestUrl(dmon, 'hive')
argv = [hurl, hivepath0]
def _getOldSynVers(self):
return (0, 0, 0)
with mock.patch('synapse.telepath.Proxy._getSynVers',
_getOldSynVers):
outp = self.getTestOutp()
retn = await s_hiveload.main(argv, outp=outp)
outp.expect(
'Hive version 0.0.0 is outside of the hive.load supported range'
)
self.eq(1, retn)
async def test_tools_hiveload(self):
with self.getTestDir() as dirn:
hivepath0 = os.path.join(dirn, 'hivepath0.mpk')
hivepath1 = os.path.join(dirn, 'hivepath1.mpk')
yamlpath0 = os.path.join(dirn, 'hivepath0.yaml')
s_msgpack.dumpfile(htree0, hivepath0)
s_msgpack.dumpfile(htree1, hivepath1)
s_common.yamlsave(htree0, yamlpath0)
async with self.getTestHiveDmon() as dmon:
hive = dmon.shared.get('hive')
hurl = self.getTestUrl(dmon, 'hive')
retn = await s_hiveload.main([hurl, hivepath0])
self.eq(0, retn)
self.eq(20, await hive.get(('hehe', )))
self.eq(30, await hive.get(('haha', )))
self.eq('bar', await hive.get(('haha', 'foo')))
self.eq('faz', await hive.get(('haha', 'baz')))
retn = await s_hiveload.main([hurl, hivepath1])
self.eq(0, retn)
self.eq(20, await hive.get(('hehe', )))
self.eq(30, await hive.get(('haha', )))
self.eq('bar', await hive.get(('haha', 'foo')))
self.eq('faz', await hive.get(('haha', 'baz')))
retn = await s_hiveload.main(['--trim', hurl, hivepath1])
self.eq(0, retn)
self.eq(20, await hive.get(('hehe', )))
self.eq(30, await hive.get(('haha', )))
self.eq('faz', await hive.get(('haha', 'baz')))
self.none(await hive.get(('haha', 'foo')))
async with self.getTestHiveDmon() as dmon:
hive = dmon.shared.get('hive')
hurl = self.getTestUrl(dmon, 'hive')
await s_hiveload.main(
['--path', 'v/i/s/i', '--yaml', hurl, yamlpath0])
self.eq('bar', await hive.get(
('v', 'i', 's', 'i', 'haha', 'foo')))
path = os.path.join(dirn, 'cell')
async with await s_cell.Cell.anit(path) as cell:
curl = cell.getLocalUrl()
await s_hiveload.main(['--path', 'gronk', curl, hivepath0])
self.eq('bar', await cell.hive.get(('gronk', 'haha', 'foo')))
await s_hiveload.main(
['--trim', '--path', 'gronk', curl, hivepath1])
self.none(await cell.hive.get(('gronk', 'haha', 'foo')))
await s_hiveload.main(
['--path', 'v/i/s/i', '--yaml', curl, yamlpath0])
self.eq(
'bar', await cell.hive.get(
('v', 'i', 's', 'i', 'haha', 'foo')))
def getAxonCore(self, cortex_conf=None):
'''
Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex.
Notes:
The following items are available in the TstEnv:
* dirn: Temporary test directory.
* axon_client: A Axon client object.
* core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex
shared by the Daemon.
* dmon_port: Port the Daemon is listening on.
* dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex.
* core: A Cortex.
* axon_sess: The client session for the Axon.
* axon: The AxonCell.
* blob: The BlobCell backing the Axon.
* neuron: The Neuron.
Args:
cortex_conf (dict): Optional cortex config
Yields:
TstEnv: A TstEnv instance.
'''
with self.getTestDir() as dirn:
neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
neurpath = s_common.gendir(dirn, 'neuron')
neur = s_neuron.Neuron(neurpath, neurconf)
neurhost, neurport = neur.getCellAddr()
blobpath = s_common.gendir(dirn, 'blob')
blobconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'blob:mapsize': TEST_MAP_SIZE
}
blobauth = neur.genCellAuth('blob')
s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth'))
blob = s_axon.BlobCell(blobpath, blobconf)
self.true(blob.cellpool.neurwait(timeout=3))
axonpath = s_common.gendir(dirn, 'axon')
axonauth = neur.genCellAuth('axon')
s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'axon:blobs': ('blob@localhost', ),
'axon:mapsize': TEST_MAP_SIZE
}
axon = s_axon.AxonCell(axonpath, axonconf)
self.true(axon.cellpool.neurwait(timeout=3))
axonhost, axonport = axon.getCellAddr()
# wait for the axon to have blob
ready = False
for i in range(30):
if axon.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon_user = s_cell.CellUser(axonauth)
axon_sess = axon_user.open((axonhost, axonport))
axon_client = s_axon.AxonClient(axon_sess)
core = s_cortex.openurl('ram:///', conf=cortex_conf)
self.addTstForms(core)
cellpoolconf = {
'host': neurhost,
'port': neurport,
'auth': s_common.enbase64(s_msgpack.en(axonauth))
}
core.setConfOpt('cellpool:conf', cellpoolconf)
core.setConfOpt('axon:name', 'axon@localhost')
dmon = s_daemon.Daemon()
dmonlink = dmon.listen('tcp://127.0.0.1:0/')
dmonport = dmonlink[1].get('port')
dmon.share('core', core)
coreurl = 'tcp://127.0.0.1:%d/core' % dmonport
env = TstEnv()
env.add('dirn', dirn)
env.add('axon_client', axon_client)
env.add('core_url', coreurl)
env.add('dmon_port', dmonport)
# Order matter for clean fini
env.add('dmon', dmon, True)
env.add('core', core, True)
env.add('axon_sess', axon_sess, True)
env.add('axon', axon, True)
env.add('blob', blob, True)
env.add('neuron', neur, True)
try:
yield env
finally:
env.fini()
