def main(argv): ''' A tool for inititializing neuron options. ''' p = argparse.ArgumentParser(prog='neutool') p.add_argument('cortex', default='ram:///', help='Cortex URL for the neuron local storage') #p.add_argument('--init-root', default=False, action='store_true', help='Initialize neuron root user') #p.add_argument('--init-share', default=None, help='Initialize a shared object in the neuron') p.add_argument('--init-name', default=None, help='Initialize the name for the neuron') p.add_argument('--init-listen', default=None, help='Initialize a listening server in the neuron') p.add_argument('--init-connect', default=None, help='Initialize a connect link in the neuron') opts = p.parse_args(argv) core = s_cortex.openurl(opts.cortex) neuron = s_neuron.Neuron(core=core) if opts.init_name: neuron.setNeuProp('name', opts.init_name) if opts.init_listen: print('init listen: %s' % (opts.init_listen, )) neuron.addNeuListen(opts.init_listen) if opts.init_connect: print('init connect: %s' % (opts.init_connect, )) neuron.addNeuConnect(opts.init_connect) evt = threading.Event() def onfini(): evt.set() neuron.onfini(onfini) try: evt.wait() finally: neuron.fini()
def neuron(dirn, conf=None): return s_neuron.Neuron(dirn, conf=conf)
def test_neuron_neuron(self): with self.getTestDir() as dirn: steps = self.getTestSteps(('cell:reg',)) conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} path = s_common.gendir(dirn, 'neuron') with s_neuron.Neuron(path, conf) as neur: cdef = neur.getConfDef('port') self.eq(s_neuron.defport, cdef[1].get('defval')) def onreg(mesg): steps.done('cell:reg') neur.on('cell:reg', onreg) self.eq(neur._genCellName('root'), 'root@localhost') path = neur._path('admin.auth') auth = s_msgpack.loadfile(path) user = s_cell.CellUser(auth) pool = s_cell.CellPool(neur.genUserAuth('foo'), neur.getCellAddr()) pool.neurok.wait(timeout=8) self.true(pool.neurok.is_set()) with user.open(neur.getCellAddr()) as sess: ncli = s_neuron.NeuronClient(sess) auth = ncli.genCellAuth('cell00') path = s_common.gendir(dirn, 'cell') authpath = s_common.genpath(path, 'cell.auth') s_msgpack.dumpfile(auth, authpath) conf = {'host': 'localhost', 'bind': '127.0.0.1'} with s_cell.Cell(path, conf) as cell: steps.wait('cell:reg', timeout=3) steps.clear('cell:reg') # we should be able to get a session to him in the pool... wait = pool.waiter(1, 'cell:add') pool.add('cell00@localhost') self.nn(wait.wait(timeout=3)) self.nn(pool.get('cell00@localhost')) ok, cells = sess.call(('cell:list', {})) self.true(ok) self.eq(cells[0][0], 'cell00@localhost') ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'})) self.true(ok) self.eq(info['ctor'], 'synapse.lib.cell.Cell') self.eq(info.get('addr'), cell.getCellAddr()) # he'll come up on a new port... with TstCell(path, conf) as cell: wait = pool.waiter(1, 'cell:add') steps.wait('cell:reg', timeout=3) steps.clear('cell:reg') self.nn(wait.wait(timeout=3)) self.nn(pool.get('cell00@localhost')) ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'})) self.true(ok) self.eq(info['ctor'], 'synapse.tests.test_neuron.TstCell') mesg = ('cell:ping', {'data': 'hehe'}) self.eq(pool.get('cell00@localhost').call(mesg), 'hehe') # since we have an active neuron, lets test the CLI tools here as well... authpath = os.path.join(dirn, 'neuron', 'admin.auth') savepath = os.path.join(dirn, 'woot.auth') argv = ['genauth', authpath, 'woot', savepath] outp = self.getTestOutp() s_tools_neuron.main(argv, outp=outp) self.true(outp.expect('saved woot')) self.true(outp.expect('woot.auth')) auth = s_msgpack.loadfile(savepath) self.eq(auth[0], 'woot@localhost') self.nn(auth[1].get('neuron')) # Use wootauth for provisioning a test cell steps.clear('cell:reg') path = gendir(dirn, 'wootcell') authpath = s_common.genpath(path, 'cell.auth') s_msgpack.dumpfile(auth, authpath) conf = {'host': 'localhost', 'bind': '127.0.0.1'} with s_cell.Cell(path, conf) as cell: wait = pool.waiter(1, 'cell:add') pool.add('woot@localhost') steps.wait('cell:reg', timeout=3) self.nn(wait.wait(timeout=3)) self.nn(pool.get('woot@localhost')) mesg = ('cell:ping', {'data': 'w00t!'}) self.eq(pool.get('woot@localhost').call(mesg), 'w00t!') pool.fini()
def test_axon_cell(self): # implement as many tests as possible in this one # since it *has* to use a neuron to work correctly # put all the things that need fini() into a BusRef... with self.getTestDir() as dirn: with s_eventbus.BusRef() as bref: # neur00 ############################################ # Set port to zero to allow a port to be automatically assigned during testing conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} path = s_common.gendir(dirn, 'neuron') logger.debug('Bringing Neuron online') neur = s_neuron.Neuron(path, conf) bref.put('neur00', neur) root = neur.getCellAuth() addr = neur.getCellAddr() nport = addr[1] # Save the port for later use # blob00 ############################################ path = s_common.gendir(dirn, 'blob00') authblob00 = neur.genCellAuth('blob00') s_msgpack.dumpfile(authblob00, os.path.join(path, 'cell.auth')) logger.debug('Bringing blob00 online') conf = {'host': 'localhost', 'bind': '127.0.0.1'} blob00 = s_axon.BlobCell(path, conf) bref.put('blob00', blob00) self.true(blob00.cellpool.neurwait(timeout=3)) user = s_cell.CellUser(root) blob00sess = user.open(blob00.getCellAddr(), timeout=3) bref.put('blob00sess', blob00sess) mesg = ('blob:stat', {}) ok, retn = blob00sess.call(mesg, timeout=3) self.true(ok) self.eq(retn, {}) # Nothing there yet # blob01 ############################################ path = s_common.gendir(dirn, 'blob01') authblob01 = neur.genCellAuth('blob01') s_msgpack.dumpfile(authblob01, os.path.join(path, 'cell.auth')) blob01conf = dict(conf) blob01conf['blob:cloneof'] = 'blob00@localhost' logger.debug('Bringing blob01 online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01sess = user.open(blob01.getCellAddr(), timeout=3) bref.put('blob01sess', blob01sess) blob01wait = blob01.waiter(1, 'blob:clone:rows') # axon00 ############################################ path = s_common.gendir(dirn, 'axon00') authaxon00 = neur.genCellAuth('axon00') s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'axon:blobs': ('blob00@localhost', ), } logger.debug('Bringing axon00 online') axon00 = s_axon.AxonCell(path, axonconf) bref.put('axon00', axon00) self.true(axon00.cellpool.neurwait(timeout=3)) ##################################################### sess = user.open(axon00.getCellAddr(), timeout=3) bref.put('sess', sess) # wait for the axon to have blob00 ready = False for i in range(30): if axon00.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon = s_axon.AxonClient(sess) blob = s_axon.BlobClient(blob00sess) blob01c = s_axon.BlobClient(blob01sess) self.eq((), tuple(axon.metrics())) self.eq((), tuple(blob.metrics())) self.len(1, axon.wants([asdfhash])) # Asking for bytes prior to the bytes being present raises self.genraises(RetnErr, axon.bytes, asdfhash, timeout=3) self.eq(1, axon.save([b'asdfasdf'], timeout=3)) self.eq((), tuple(axon.metrics(offs=999999999))) self.eq((), tuple(blob.metrics(offs=99999999, timeout=3))) metrics = list(blob.metrics(timeout=3)) self.len(1, metrics) self.eq(8, metrics[0][1].get('size')) self.eq(1, metrics[0][1].get('blocks')) self.len(0, axon.wants([asdfhash], timeout=3)) self.eq(b'asdfasdf', b''.join(axon.bytes(asdfhash, timeout=3))) stat = axon.stat(timeout=3) self.eq(1, stat.get('files')) self.eq(8, stat.get('bytes')) # lets see if the bytes made it to the blob clone... self.nn(blob01wait.wait(timeout=10)) newp = os.urandom(32) def loop(): s_common.spin(axon.bytes(newp)) self.raises(s_exc.RetnErr, loop) blob01wait = blob01.waiter(1, 'blob:clone:rows') self.eq(qwerhash, axon.upload([b'qwer', b'qwer'], timeout=3)) self.len(0, axon.wants([qwerhash])) self.eq(b'qwerqwer', b''.join(axon.bytes(qwerhash, timeout=3))) self.nn(blob01wait.wait(3)) retn = list(axon.metrics(0, timeout=3)) self.eq(retn[0][1].get('size'), 8) self.eq(retn[0][1].get('cell'), 'blob00@localhost') # Try uploading a large file logger.debug('Large file test') # Monkeypatch axon to a smaller blocksize s_axon.blocksize = s_const.kibibyte self.raises(RetnErr, axon.locs, bbufhash, timeout=3) genr = s_common.chunks(bbuf, s_axon.blocksize) blob01wait = blob01.waiter(1, 'blob:clone:rows') self.eq(bbufhash, axon.upload(genr, timeout=3)) self.eq((), axon.wants([bbufhash], timeout=3)) # Then retrieve it size = 0 gots = [] testhash = hashlib.sha256() for byts in axon.bytes(bbufhash, timeout=3): size += len(byts) gots.append(byts) testhash.update(byts) self.eq(bbufhash, testhash.digest()) try: self.eq(size, len(bbuf)) self.eq(bbufhash, testhash.digest()) except Exception as e: for byts in gots: print(repr(byts)) print('SIZE: %d/%d' % (size, len(bbuf))) raise self.nn(blob01wait.wait(3)) locs = axon.locs(bbufhash, timeout=3) self.len(1, locs) self.isin('blob00', locs[0][0]) # Use the buid to retrieve the large file from blob01 buid = locs[0][1] testhash = hashlib.sha256() for byts in blob01c.bytes(buid, timeout=3): testhash.update(byts) self.eq(bbufhash, testhash.digest()) # Try storing a empty file logger.debug('Nullfile test') axon.save([b'']) self.eq((), tuple(axon.wants([nullhash]))) # Then retrieve it parts = [] for part in axon.bytes(nullhash): parts.append(part) self.eq([b''], parts) logger.debug('Shutdown / restart blob01 test') bref.pop('blob01') blob01.fini() self.true(blob01.isfini) axon.save([b'hehehaha'], timeout=3) self.eq((), axon.wants([hehahash], timeout=3)) # Now bring blob01 back online logger.debug('Bringing blob01 back online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01wait = blob01.waiter(1, 'blob:clone:rows') # Cloning should start up shortly self.nn(blob01wait.wait(10)) # Let everything get shut down by the busref fini logger.debug('Bringing everything back up') with s_eventbus.BusRef() as bref: # neur00 ############################################ conf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': nport } path = s_common.gendir(dirn, 'neuron') logger.debug('Bringing Neuron Back online') neur = s_neuron.Neuron(path, conf) bref.put('neur00', neur) root = neur.getCellAuth() # blob00 ############################################ path = s_common.gendir(dirn, 'blob00') logger.debug('Bringing blob00 back online') conf = {'host': 'localhost', 'bind': '127.0.0.1'} blob00 = s_axon.BlobCell(path, conf) bref.put('blob00', blob00) self.true(blob00.cellpool.neurwait(timeout=3)) user = s_cell.CellUser(root) blob00sess = user.open(blob00.getCellAddr(), timeout=3) bref.put('blob00sess', blob00sess) # blob01 ############################################ path = s_common.gendir(dirn, 'blob01') blob01conf = dict(conf) blob01conf['blob:cloneof'] = 'blob00@localhost' logger.debug('Bringing blob01 back online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01wait = blob01.waiter(1, 'blob:clone:rows') # axon00 ############################################ path = s_common.gendir(dirn, 'axon00') authaxon00 = neur.genCellAuth('axon00') s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'axon:blobs': ('blob00@localhost', ), } logger.debug('Bringing axon00 online') axon00 = s_axon.AxonCell(path, axonconf) bref.put('axon00', axon00) self.true(axon00.cellpool.neurwait(timeout=3)) ##################################################### sess = user.open(axon00.getCellAddr(), timeout=3) bref.put('sess', sess) # wait for the axon to have blob00 ready = False for i in range(30): if axon00.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon = s_axon.AxonClient(sess) # Try retrieving a large file testhash = hashlib.sha256() for byts in axon.bytes(bbufhash, timeout=3): testhash.update(byts) self.eq(bbufhash, testhash.digest()) # Try saving a new file and a existing file to the cluster and ensure it is replicated self.eq((ohmyhash, ), axon.wants((ohmyhash, hehahash, nullhash), 3)) self.eq(1, axon.save([b'ohmyohmyy', b''])) self.nn(blob01wait.wait(10))
def getAxonCore(self, cortex_conf=None): ''' Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex. Notes: The following items are available in the TstEnv: * dirn: Temporary test directory. * axon_client: A Axon client object. * core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex shared by the Daemon. * dmon_port: Port the Daemon is listening on. * dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex. * core: A Cortex. * axon_sess: The client session for the Axon. * axon: The AxonCell. * blob: The BlobCell backing the Axon. * neuron: The Neuron. Args: cortex_conf (dict): Optional cortex config Yields: TstEnv: A TstEnv instance. ''' with self.getTestDir() as dirn: neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} neurpath = s_common.gendir(dirn, 'neuron') neur = s_neuron.Neuron(neurpath, neurconf) neurhost, neurport = neur.getCellAddr() blobpath = s_common.gendir(dirn, 'blob') blobconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'blob:mapsize': TEST_MAP_SIZE } blobauth = neur.genCellAuth('blob') s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth')) blob = s_axon.BlobCell(blobpath, blobconf) self.true(blob.cellpool.neurwait(timeout=3)) axonpath = s_common.gendir(dirn, 'axon') axonauth = neur.genCellAuth('axon') s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'axon:blobs': ('blob@localhost', ), 'axon:mapsize': TEST_MAP_SIZE } axon = s_axon.AxonCell(axonpath, axonconf) self.true(axon.cellpool.neurwait(timeout=3)) axonhost, axonport = axon.getCellAddr() # wait for the axon to have blob ready = False for i in range(30): if axon.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon_user = s_cell.CellUser(axonauth) axon_sess = axon_user.open((axonhost, axonport)) axon_client = s_axon.AxonClient(axon_sess) core = s_cortex.openurl('ram:///', conf=cortex_conf) self.addTstForms(core) cellpoolconf = { 'host': neurhost, 'port': neurport, 'auth': s_common.enbase64(s_msgpack.en(axonauth)) } core.setConfOpt('cellpool:conf', cellpoolconf) core.setConfOpt('axon:name', 'axon@localhost') dmon = s_daemon.Daemon() dmonlink = dmon.listen('tcp://127.0.0.1:0/') dmonport = dmonlink[1].get('port') dmon.share('core', core) coreurl = 'tcp://127.0.0.1:%d/core' % dmonport env = TstEnv() env.add('dirn', dirn) env.add('axon_client', axon_client) env.add('core_url', coreurl) env.add('dmon_port', dmonport) # Order matter for clean fini env.add('dmon', dmon, True) env.add('core', core, True) env.add('axon_sess', axon_sess, True) env.add('axon', axon, True) env.add('blob', blob, True) env.add('neuron', neur, True) try: yield env finally: env.fini()