def main(argv):
'''
A tool for inititializing neuron options.
'''
p = argparse.ArgumentParser(prog='neutool')
p.add_argument('cortex',
default='ram:///',
help='Cortex URL for the neuron local storage')
#p.add_argument('--init-root', default=False, action='store_true', help='Initialize neuron root user')
#p.add_argument('--init-share', default=None, help='Initialize a shared object in the neuron')
p.add_argument('--init-name',
default=None,
help='Initialize the name for the neuron')
p.add_argument('--init-listen',
default=None,
help='Initialize a listening server in the neuron')
p.add_argument('--init-connect',
default=None,
help='Initialize a connect link in the neuron')
opts = p.parse_args(argv)
core = s_cortex.openurl(opts.cortex)
neuron = s_neuron.Neuron(core=core)
if opts.init_name:
neuron.setNeuProp('name', opts.init_name)
if opts.init_listen:
print('init listen: %s' % (opts.init_listen, ))
neuron.addNeuListen(opts.init_listen)
if opts.init_connect:
print('init connect: %s' % (opts.init_connect, ))
neuron.addNeuConnect(opts.init_connect)
evt = threading.Event()
def onfini():
evt.set()
neuron.onfini(onfini)
try:
evt.wait()
finally:
neuron.fini()
def neuron(dirn, conf=None):
return s_neuron.Neuron(dirn, conf=conf)
def test_neuron_neuron(self):
with self.getTestDir() as dirn:
steps = self.getTestSteps(('cell:reg',))
conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
path = s_common.gendir(dirn, 'neuron')
with s_neuron.Neuron(path, conf) as neur:
cdef = neur.getConfDef('port')
self.eq(s_neuron.defport, cdef[1].get('defval'))
def onreg(mesg):
steps.done('cell:reg')
neur.on('cell:reg', onreg)
self.eq(neur._genCellName('root'), 'root@localhost')
path = neur._path('admin.auth')
auth = s_msgpack.loadfile(path)
user = s_cell.CellUser(auth)
pool = s_cell.CellPool(neur.genUserAuth('foo'), neur.getCellAddr())
pool.neurok.wait(timeout=8)
self.true(pool.neurok.is_set())
with user.open(neur.getCellAddr()) as sess:
ncli = s_neuron.NeuronClient(sess)
auth = ncli.genCellAuth('cell00')
path = s_common.gendir(dirn, 'cell')
authpath = s_common.genpath(path, 'cell.auth')
s_msgpack.dumpfile(auth, authpath)
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
with s_cell.Cell(path, conf) as cell:
steps.wait('cell:reg', timeout=3)
steps.clear('cell:reg')
# we should be able to get a session to him in the pool...
wait = pool.waiter(1, 'cell:add')
pool.add('cell00@localhost')
self.nn(wait.wait(timeout=3))
self.nn(pool.get('cell00@localhost'))
ok, cells = sess.call(('cell:list', {}))
self.true(ok)
self.eq(cells[0][0], 'cell00@localhost')
ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'}))
self.true(ok)
self.eq(info['ctor'], 'synapse.lib.cell.Cell')
self.eq(info.get('addr'), cell.getCellAddr())
# he'll come up on a new port...
with TstCell(path, conf) as cell:
wait = pool.waiter(1, 'cell:add')
steps.wait('cell:reg', timeout=3)
steps.clear('cell:reg')
self.nn(wait.wait(timeout=3))
self.nn(pool.get('cell00@localhost'))
ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'}))
self.true(ok)
self.eq(info['ctor'], 'synapse.tests.test_neuron.TstCell')
mesg = ('cell:ping', {'data': 'hehe'})
self.eq(pool.get('cell00@localhost').call(mesg), 'hehe')
# since we have an active neuron, lets test the CLI tools here as well...
authpath = os.path.join(dirn, 'neuron', 'admin.auth')
savepath = os.path.join(dirn, 'woot.auth')
argv = ['genauth', authpath, 'woot', savepath]
outp = self.getTestOutp()
s_tools_neuron.main(argv, outp=outp)
self.true(outp.expect('saved woot'))
self.true(outp.expect('woot.auth'))
auth = s_msgpack.loadfile(savepath)
self.eq(auth[0], 'woot@localhost')
self.nn(auth[1].get('neuron'))
# Use wootauth for provisioning a test cell
steps.clear('cell:reg')
path = gendir(dirn, 'wootcell')
authpath = s_common.genpath(path, 'cell.auth')
s_msgpack.dumpfile(auth, authpath)
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
with s_cell.Cell(path, conf) as cell:
wait = pool.waiter(1, 'cell:add')
pool.add('woot@localhost')
steps.wait('cell:reg', timeout=3)
self.nn(wait.wait(timeout=3))
self.nn(pool.get('woot@localhost'))
mesg = ('cell:ping', {'data': 'w00t!'})
self.eq(pool.get('woot@localhost').call(mesg), 'w00t!')
pool.fini()
def test_axon_cell(self):
# implement as many tests as possible in this one
# since it *has* to use a neuron to work correctly
# put all the things that need fini() into a BusRef...
with self.getTestDir() as dirn:
with s_eventbus.BusRef() as bref:
# neur00 ############################################
# Set port to zero to allow a port to be automatically assigned during testing
conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
addr = neur.getCellAddr()
nport = addr[1] # Save the port for later use
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
authblob00 = neur.genCellAuth('blob00')
s_msgpack.dumpfile(authblob00, os.path.join(path, 'cell.auth'))
logger.debug('Bringing blob00 online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
mesg = ('blob:stat', {})
ok, retn = blob00sess.call(mesg, timeout=3)
self.true(ok)
self.eq(retn, {}) # Nothing there yet
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
authblob01 = neur.genCellAuth('blob01')
s_msgpack.dumpfile(authblob01, os.path.join(path, 'cell.auth'))
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01sess = user.open(blob01.getCellAddr(), timeout=3)
bref.put('blob01sess', blob01sess)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
blob = s_axon.BlobClient(blob00sess)
blob01c = s_axon.BlobClient(blob01sess)
self.eq((), tuple(axon.metrics()))
self.eq((), tuple(blob.metrics()))
self.len(1, axon.wants([asdfhash]))
# Asking for bytes prior to the bytes being present raises
self.genraises(RetnErr, axon.bytes, asdfhash, timeout=3)
self.eq(1, axon.save([b'asdfasdf'], timeout=3))
self.eq((), tuple(axon.metrics(offs=999999999)))
self.eq((), tuple(blob.metrics(offs=99999999, timeout=3)))
metrics = list(blob.metrics(timeout=3))
self.len(1, metrics)
self.eq(8, metrics[0][1].get('size'))
self.eq(1, metrics[0][1].get('blocks'))
self.len(0, axon.wants([asdfhash], timeout=3))
self.eq(b'asdfasdf', b''.join(axon.bytes(asdfhash, timeout=3)))
stat = axon.stat(timeout=3)
self.eq(1, stat.get('files'))
self.eq(8, stat.get('bytes'))
# lets see if the bytes made it to the blob clone...
self.nn(blob01wait.wait(timeout=10))
newp = os.urandom(32)
def loop():
s_common.spin(axon.bytes(newp))
self.raises(s_exc.RetnErr, loop)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(qwerhash, axon.upload([b'qwer', b'qwer'], timeout=3))
self.len(0, axon.wants([qwerhash]))
self.eq(b'qwerqwer', b''.join(axon.bytes(qwerhash, timeout=3)))
self.nn(blob01wait.wait(3))
retn = list(axon.metrics(0, timeout=3))
self.eq(retn[0][1].get('size'), 8)
self.eq(retn[0][1].get('cell'), 'blob00@localhost')
# Try uploading a large file
logger.debug('Large file test')
# Monkeypatch axon to a smaller blocksize
s_axon.blocksize = s_const.kibibyte
self.raises(RetnErr, axon.locs, bbufhash, timeout=3)
genr = s_common.chunks(bbuf, s_axon.blocksize)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(bbufhash, axon.upload(genr, timeout=3))
self.eq((), axon.wants([bbufhash], timeout=3))
# Then retrieve it
size = 0
gots = []
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
size += len(byts)
gots.append(byts)
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
try:
self.eq(size, len(bbuf))
self.eq(bbufhash, testhash.digest())
except Exception as e:
for byts in gots:
print(repr(byts))
print('SIZE: %d/%d' % (size, len(bbuf)))
raise
self.nn(blob01wait.wait(3))
locs = axon.locs(bbufhash, timeout=3)
self.len(1, locs)
self.isin('blob00', locs[0][0])
# Use the buid to retrieve the large file from blob01
buid = locs[0][1]
testhash = hashlib.sha256()
for byts in blob01c.bytes(buid, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try storing a empty file
logger.debug('Nullfile test')
axon.save([b''])
self.eq((), tuple(axon.wants([nullhash])))
# Then retrieve it
parts = []
for part in axon.bytes(nullhash):
parts.append(part)
self.eq([b''], parts)
logger.debug('Shutdown / restart blob01 test')
bref.pop('blob01')
blob01.fini()
self.true(blob01.isfini)
axon.save([b'hehehaha'], timeout=3)
self.eq((), axon.wants([hehahash], timeout=3))
# Now bring blob01 back online
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# Cloning should start up shortly
self.nn(blob01wait.wait(10))
# Let everything get shut down by the busref fini
logger.debug('Bringing everything back up')
with s_eventbus.BusRef() as bref:
# neur00 ############################################
conf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': nport
}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron Back online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
logger.debug('Bringing blob00 back online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
# Try retrieving a large file
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try saving a new file and a existing file to the cluster and ensure it is replicated
self.eq((ohmyhash, ),
axon.wants((ohmyhash, hehahash, nullhash), 3))
self.eq(1, axon.save([b'ohmyohmyy', b'']))
self.nn(blob01wait.wait(10))
def getAxonCore(self, cortex_conf=None):
'''
Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex.
Notes:
The following items are available in the TstEnv:
* dirn: Temporary test directory.
* axon_client: A Axon client object.
* core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex
shared by the Daemon.
* dmon_port: Port the Daemon is listening on.
* dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex.
* core: A Cortex.
* axon_sess: The client session for the Axon.
* axon: The AxonCell.
* blob: The BlobCell backing the Axon.
* neuron: The Neuron.
Args:
cortex_conf (dict): Optional cortex config
Yields:
TstEnv: A TstEnv instance.
'''
with self.getTestDir() as dirn:
neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
neurpath = s_common.gendir(dirn, 'neuron')
neur = s_neuron.Neuron(neurpath, neurconf)
neurhost, neurport = neur.getCellAddr()
blobpath = s_common.gendir(dirn, 'blob')
blobconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'blob:mapsize': TEST_MAP_SIZE
}
blobauth = neur.genCellAuth('blob')
s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth'))
blob = s_axon.BlobCell(blobpath, blobconf)
self.true(blob.cellpool.neurwait(timeout=3))
axonpath = s_common.gendir(dirn, 'axon')
axonauth = neur.genCellAuth('axon')
s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'axon:blobs': ('blob@localhost', ),
'axon:mapsize': TEST_MAP_SIZE
}
axon = s_axon.AxonCell(axonpath, axonconf)
self.true(axon.cellpool.neurwait(timeout=3))
axonhost, axonport = axon.getCellAddr()
# wait for the axon to have blob
ready = False
for i in range(30):
if axon.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon_user = s_cell.CellUser(axonauth)
axon_sess = axon_user.open((axonhost, axonport))
axon_client = s_axon.AxonClient(axon_sess)
core = s_cortex.openurl('ram:///', conf=cortex_conf)
self.addTstForms(core)
cellpoolconf = {
'host': neurhost,
'port': neurport,
'auth': s_common.enbase64(s_msgpack.en(axonauth))
}
core.setConfOpt('cellpool:conf', cellpoolconf)
core.setConfOpt('axon:name', 'axon@localhost')
dmon = s_daemon.Daemon()
dmonlink = dmon.listen('tcp://127.0.0.1:0/')
dmonport = dmonlink[1].get('port')
dmon.share('core', core)
coreurl = 'tcp://127.0.0.1:%d/core' % dmonport
env = TstEnv()
env.add('dirn', dirn)
env.add('axon_client', axon_client)
env.add('core_url', coreurl)
env.add('dmon_port', dmonport)
# Order matter for clean fini
env.add('dmon', dmon, True)
env.add('core', core, True)
env.add('axon_sess', axon_sess, True)
env.add('axon', axon, True)
env.add('blob', blob, True)
env.add('neuron', neur, True)
try:
yield env
finally:
env.fini()