def genauth(opts, outp=s_output.stdout): authpath = s_common.genpath(opts.authfile) savepath = s_common.genpath(opts.savepath) if not os.path.isfile(authpath): outp.printf('auth file not found: %s' % (authpath, )) return auth = s_msgpack.loadfile(authpath) addr = auth[1].get('neuron') if addr is None: outp.printf('auth file has no neuron info: %s' % (authpath, )) return celluser = s_cell.CellUser(auth) with celluser.open(addr, timeout=20) as sess: nuro = s_neuron.NeuronClient(sess) auth = nuro.genCellAuth(opts.cellname, timeout=20) s_msgpack.dumpfile(auth, savepath) outp.printf('saved %s: %s' % (opts.cellname, savepath))
def test_cryo_cell_indexing(self): conf = { 'bind': '127.0.0.1', 'host': 'localhost', 'defvals': { 'mapsize': s_iq.TEST_MAP_SIZE } } with self.getTestDir() as dirn, s_cryotank.CryoCell(dirn, conf) as cell: addr = cell.getCellAddr() cuser = s_cell.CellUser(cell.genUserAuth('foo')) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) # Setting the _chunksize to 1 forces iteration on the client # side of puts, as well as the server-side. user._chunksize = 1 user.puts('woot:woot', cryodata, timeout=2) # Test index operations self.raises(s_exc.RetnErr, user.getIndices, 'notpresent') self.eq((), user.getIndices('woot:woot')) self.raises(s_exc.BadOperArg, user.addIndex, 'woot:woot', 'prop1', 'str', []) user.addIndex('woot:woot', 'prop1', 'str', ['0']) user.delIndex('woot:woot', 'prop1') self.raises(s_exc.RetnErr, user.delIndex, 'woot:woot', 'noexist') user.addIndex('woot:woot', 'prop1', 'str', ['0']) user.pauseIndex('woot:woot', 'prop1') user.pauseIndex('woot:woot') user.resumeIndex('woot:woot') self.eq([(1, 'baz'), (0, 'foo')], list(user.queryNormValu('woot:woot', 'prop1'))) self.eq([(1, 'baz')], list(user.queryNormValu('woot:woot', 'prop1', valu='b'))) self.eq([], list( user.queryNormValu('woot:woot', 'prop1', valu='bz', timeout=10))) self.eq([(1, { 'prop1': 'baz' })], (list( user.queryNormRecords('woot:woot', 'prop1', valu='b')))) self.eq([(1, s_msgpack.en(('baz', { 'faz': 20 })))], list(user.queryRows('woot:woot', 'prop1', valu='b'))) user.init('woot:boring', {'noindex': True}) self.raises(s_exc.RetnErr, user.getIndices, 'woot:boring')
def test_neuron_divide(self): with self.getTestDir() as dirn: celldirn = os.path.join(dirn, 'cell') # FIXME: this could randomly fail if the port is in use! port = random.randint(20000, 50000) conf = {'bind': '127.0.0.1', 'port': port, 'host': 'localhost', 'ctor': 'synapse.tests.test_neuron.TstCell'} # Preload the cell vault vdir = os.path.join(celldirn, 'vault.lmdb') with s_vault.shared(vdir) as vault: auth = vault.genUserAuth('*****@*****.**') proc = s_cell.divide(celldirn, conf) with genfile(celldirn, 'cell.lock') as fd: self.true(checkLock(fd, 30)) try: # Try connecting to the cell user = s_cell.CellUser(auth) addr = ('127.0.0.1', port) with user.open(addr, timeout=10) as sess: with sess.task(('cell:ping', {'data': 'haha'})) as chan: retn = chan.next(timeout=2) self.eq(retn, 'haha') retn = sess.call(('cell:ping', {'data': 'rofl'}), timeout=2) self.eq(retn, 'rofl') retn = sess.call(('cell:pong', {'data': 'rofl'}), timeout=2) self.eq(retn, {'mesg': 'pong', 'counter': 1}) retn = sess.call(('cell:pong', {'data': 'rofl'}), timeout=2) self.eq(retn, {'mesg': 'pong', 'counter': 2}) except Exception as e: logger.exception('Error during test!') # Clean up the proc proc.terminate() proc.join(1) raise else: time.sleep(0.01) proc.terminate() proc.join(30) self.false(proc.is_alive()) self.eq(proc.exitcode, 0)
def cell_populate(self, port, auth): # Populate the cell with data addr = ('127.0.0.1', port) cuser = s_cell.CellUser(auth) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) nodes = [(None, {'key': i}) for i in range(10)] user.puts('test:hehe', nodes, 4) self.len(10, list(user.slice('test:hehe', 0, 100))) user.puts('test:haha', nodes, 4) self.len(2, user.list())
def test_neuron_cell_notok(self): with self.getTestDir() as dirn: conf = {'bind': '127.0.0.1', 'host': 'localhost'} with s_cell.Cell(dirn, conf) as cell: user = s_cell.CellUser(cell.genUserAuth('foo')) addr = ('localhost', 1) self.raises(CellUserErr, user.open, addr, timeout=1) self.raises(CellUserErr, user.open, addr, timeout=-1)
def test_cryo_cell_daemon(self): with self.getTestDir() as dirn: celldir = os.path.join(dirn, 'cell') port = random.randint(10000, 50000) conf = { 'cells': [ (celldir, { 'ctor': 'synapse.cryotank.CryoCell', 'port': port, 'host': 'localhost', 'bind': '127.0.0.1', 'defvals': { 'mapsize': s_iq.TEST_MAP_SIZE } }), ], } with s_daemon.Daemon() as dmon: dmon.loadDmonConf(conf) with genfile(celldir, 'cell.lock') as fd: self.true(checkLock(fd, 30)) authfp = os.path.join(celldir, 'cell.auth') auth = s_msgpack.loadfile(authfp) addr = ('127.0.0.1', port) cuser = s_cell.CellUser(auth) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) retn = user.list(timeout=3) self.eq(retn, ()) user.puts('woot:woot', cryodata, timeout=2) retn = user.list(timeout=3) self.eq(retn[0][1]['indx'], 2) self.eq(retn[0][0], 'woot:woot') self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz') retn = user.list(timeout=3) self.eq(retn[0][1]['indx'], 2) self.eq(retn[0][0], 'woot:woot') # ensure dmon cell processes are fini'd for celldir, proc in dmon.cellprocs.items(): self.false(proc.is_alive())
def test_neuron_cell_openlist(self): with self.getTestDir() as dirn: conf = {'bind': '127.0.0.1', 'host': 'localhost'} cell = s_cell.Cell(dirn) with s_cell.Cell(dirn, conf) as cell: user = s_cell.CellUser(cell.genUserAuth('foo')) addr = list(cell.getCellAddr()) with user.open(addr, timeout=2) as sess: with sess.task(('cell:ping', {'data': 'haha'})) as chan: retn = chan.next(timeout=2) self.eq(retn, 'haha') retn = sess.call(('cell:ping', {'data': 'rofl'}), timeout=2) self.eq(retn, 'rofl')
def test_neuron_cell_authfail(self): ''' Make a separate cell dir and make sure it can't connect to the first one ''' with self.getTestDir() as dirn, self.getTestDir() as dirn2: conf = {'bind': '127.0.0.1', 'host': 'localhost'} with s_cell.Cell(dirn, conf) as cell, s_cell.Cell(dirn2) as newp: user = s_cell.CellUser(newp.getCellAuth()) addr = cell.getCellAddr() with self.getLoggerStream('synapse.lib.cell') as stream: self.raises(CellUserErr, user.open, addr, timeout=1) stream.seek(0) mesgs = stream.read() self.isin('got bad cert', mesgs)
def test_neuron_neuron(self): with self.getTestDir() as dirn: steps = self.getTestSteps(('cell:reg',)) conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} path = s_common.gendir(dirn, 'neuron') with s_neuron.Neuron(path, conf) as neur: cdef = neur.getConfDef('port') self.eq(s_neuron.defport, cdef[1].get('defval')) def onreg(mesg): steps.done('cell:reg') neur.on('cell:reg', onreg) self.eq(neur._genCellName('root'), 'root@localhost') path = neur._path('admin.auth') auth = s_msgpack.loadfile(path) user = s_cell.CellUser(auth) pool = s_cell.CellPool(neur.genUserAuth('foo'), neur.getCellAddr()) pool.neurok.wait(timeout=8) self.true(pool.neurok.is_set()) with user.open(neur.getCellAddr()) as sess: ncli = s_neuron.NeuronClient(sess) auth = ncli.genCellAuth('cell00') path = s_common.gendir(dirn, 'cell') authpath = s_common.genpath(path, 'cell.auth') s_msgpack.dumpfile(auth, authpath) conf = {'host': 'localhost', 'bind': '127.0.0.1'} with s_cell.Cell(path, conf) as cell: steps.wait('cell:reg', timeout=3) steps.clear('cell:reg') # we should be able to get a session to him in the pool... wait = pool.waiter(1, 'cell:add') pool.add('cell00@localhost') self.nn(wait.wait(timeout=3)) self.nn(pool.get('cell00@localhost')) ok, cells = sess.call(('cell:list', {})) self.true(ok) self.eq(cells[0][0], 'cell00@localhost') ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'})) self.true(ok) self.eq(info['ctor'], 'synapse.lib.cell.Cell') self.eq(info.get('addr'), cell.getCellAddr()) # he'll come up on a new port... with TstCell(path, conf) as cell: wait = pool.waiter(1, 'cell:add') steps.wait('cell:reg', timeout=3) steps.clear('cell:reg') self.nn(wait.wait(timeout=3)) self.nn(pool.get('cell00@localhost')) ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'})) self.true(ok) self.eq(info['ctor'], 'synapse.tests.test_neuron.TstCell') mesg = ('cell:ping', {'data': 'hehe'}) self.eq(pool.get('cell00@localhost').call(mesg), 'hehe') # since we have an active neuron, lets test the CLI tools here as well... authpath = os.path.join(dirn, 'neuron', 'admin.auth') savepath = os.path.join(dirn, 'woot.auth') argv = ['genauth', authpath, 'woot', savepath] outp = self.getTestOutp() s_tools_neuron.main(argv, outp=outp) self.true(outp.expect('saved woot')) self.true(outp.expect('woot.auth')) auth = s_msgpack.loadfile(savepath) self.eq(auth[0], 'woot@localhost') self.nn(auth[1].get('neuron')) # Use wootauth for provisioning a test cell steps.clear('cell:reg') path = gendir(dirn, 'wootcell') authpath = s_common.genpath(path, 'cell.auth') s_msgpack.dumpfile(auth, authpath) conf = {'host': 'localhost', 'bind': '127.0.0.1'} with s_cell.Cell(path, conf) as cell: wait = pool.waiter(1, 'cell:add') pool.add('woot@localhost') steps.wait('cell:reg', timeout=3) self.nn(wait.wait(timeout=3)) self.nn(pool.get('woot@localhost')) mesg = ('cell:ping', {'data': 'w00t!'}) self.eq(pool.get('woot@localhost').call(mesg), 'w00t!') pool.fini()
def test_cryo_cell(self): with self.getTestDir() as dirn: conf = {'bind': '127.0.0.1', 'host': 'localhost'} with s_cryotank.CryoCell(dirn, conf) as cell: addr = cell.getCellAddr() cuser = s_cell.CellUser(cell.genUserAuth('foo')) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) # Setting the _chunksize to 1 forces iteration on the client # side of puts, as well as the server-side. user._chunksize = 1 user.puts('woot:woot', cryodata, timeout=2) self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz') retn = user.list(timeout=3) self.eq(retn[0][1]['indx'], 2) self.eq(retn[0][0], 'woot:woot') metr = list(user.metrics('woot:woot', 0, 100, timeout=2)) self.len(2, metr) self.eq(metr[0][1]['count'], 1) user.puts('woot:woot', cryodata, timeout=2) retn = list(user.slice('woot:woot', 2, 2, timeout=3)) self.len(2, retn) self.eq(2, retn[0][0]) retn = list(user.rows('woot:woot', 0, 2, timeout=3)) self.len(2, retn) self.eq(retn[0], (0, s_msgpack.en(cryodata[0]))) self.eq(retn[1], (1, s_msgpack.en(cryodata[1]))) # Reset chunksize user._chunksize = s_cryotank.CryoClient._chunksize user.puts('woot:hehe', cryodata, timeout=5) user.puts('woot:hehe', cryodata, timeout=5) retn = list(user.slice('woot:hehe', 1, 2)) retn = [val for indx, val in retn[::-1]] self.eq(tuple(retn), cryodata) metr = list(user.metrics('woot:hehe', 0)) self.len(2, metr) listd = dict(user.list(timeout=3)) self.isin('woot:hehe', listd) self.eq(user.last('woot:hehe', timeout=3), (3, cryodata[1])) # delete woot.hehe and then call apis on it self.true(user.delete('woot:hehe')) self.false(user.delete('woot:hehe')) self.none(cell.tanks.get('woot:hehe')) self.none(cell.names.get('woot:hehe')) self.genraises(s_exc.RetnErr, user.slice, 'woot:hehe', 1, 2, timeout=3) self.genraises(s_exc.RetnErr, user.rows, 'woot:hehe', 1, 2, timeout=3) listd = dict(user.list(timeout=3)) self.notin('woot:hehe', listd) self.none(user.last('woot:hehe', timeout=3)) self.genraises(s_exc.RetnErr, user.metrics, 'woot:hehe', 0, 100, timeout=3) # Adding data re-adds the tank user._chunksize = 1000 user.puts('woot:hehe', cryodata, timeout=5) metr = list(user.metrics('woot:hehe', 0)) self.len(1, metr) # We can initialize a new tank directly with a custom map size self.true( user.init('weee:imthebest', {'mapsize': 5558675309})) self.false(user.init('woot:hehe')) # error when we specify an invalid config option self.raises(s_exc.RetnErr, user.init, 'weee:danktank', {'newp': 'hehe'}) # Turn it back on with s_cryotank.CryoCell(dirn, conf) as cell: addr = cell.getCellAddr() cuser = s_cell.CellUser(cell.genUserAuth('foo')) with cuser.open(addr, timeout=2) as sess: user = s_cryotank.CryoClient(sess) listd = dict(user.list(timeout=3)) self.len(3, listd) self.isin('weee:imthebest', listd) self.isin('woot:woot', listd) self.isin('woot:hehe', listd) self.istufo(user.last('woot:woot')[1]) self.istufo(user.last('woot:hehe')[1]) self.none(user.last('weee:imthebest')) # Test empty puts user.puts('woot:hehe', tuple()) listd = dict(user.list(timeout=3)) metr = list(user.metrics('woot:hehe', 0)) self.len(2, metr) self.nn(user.last('woot:hehe'))
def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='cryo.cat', description='display data items from a cryo cell') pars.add_argument( 'cryocell', help= 'The cell descriptor and cryo tank path (cell://<host:port>/<name>).') pars.add_argument('--list', default=False, action='store_true', help='List tanks in the remote cell and return') pars.add_argument('--offset', default=0, type=int, help='Begin at offset index') pars.add_argument('--size', default=10, type=int, help='How many items to display') pars.add_argument('--timeout', default=10, type=int, help='The network timeout setting') pars.add_argument('--authfile', help='Path to your auth file for the remote cell') group = pars.add_mutually_exclusive_group() group.add_argument('--jsonl', action='store_true', help='Input/Output items in jsonl format') group.add_argument('--msgpack', action='store_true', help='Input/Output items in msgpack format') pars.add_argument('--verbose', '-v', default=False, action='store_true', help='Verbose output') pars.add_argument( '--ingest', '-i', default=False, action='store_true', help= 'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format' ) pars.add_argument( '--omit-offset', default=False, action='store_true', help= "Don't output offsets of objects. This is recommended to be used when jsonl/msgpack" " output is used.") opts = pars.parse_args(argv) if opts.verbose: logger.setLevel(logging.INFO) if not opts.authfile: logger.error( 'Currently requires --authfile until neuron protocol is supported') return 1 if opts.ingest and not opts.jsonl and not opts.msgpack: logger.error( 'Must specify exactly one of --jsonl or --msgpack if --ingest is specified' ) return 1 authpath = s_common.genpath(opts.authfile) auth = s_msgpack.loadfile(authpath) netw, path = opts.cryocell[7:].split('/', 1) host, portstr = netw.split(':') addr = (host, int(portstr)) logger.info('connecting to: %r', addr) cuser = s_cell.CellUser(auth) with cuser.open(addr, timeout=opts.timeout) as sess: cryo = s_cryotank.CryoClient(sess) if opts.list: for name, info in cryo.list(timeout=opts.timeout): outp.printf('%s: %r' % (name, info)) return 0 if opts.ingest: if opts.msgpack: fd = sys.stdin.buffer item_it = _except_wrap(s_msgpack.iterfd(fd), lambda x: 'Error parsing item %d' % x) else: fd = sys.stdin item_it = _except_wrap( (json.loads(s) for s in fd), lambda x: ('Failure parsing line %d of input' % x)) cryo.puts(path, item_it) else: for item in cryo.slice(path, opts.offset, opts.size, opts.timeout): i = item[1] if opts.omit_offset else item if opts.jsonl: outp.printf(json.dumps(i, sort_keys=True)) elif opts.msgpack: sys.stdout.write(s_msgpack.en(i)) else: outp.printf(pprint.pformat(i)) return 0
def test_axon_cell(self): # implement as many tests as possible in this one # since it *has* to use a neuron to work correctly # put all the things that need fini() into a BusRef... with self.getTestDir() as dirn: with s_eventbus.BusRef() as bref: # neur00 ############################################ # Set port to zero to allow a port to be automatically assigned during testing conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} path = s_common.gendir(dirn, 'neuron') logger.debug('Bringing Neuron online') neur = s_neuron.Neuron(path, conf) bref.put('neur00', neur) root = neur.getCellAuth() addr = neur.getCellAddr() nport = addr[1] # Save the port for later use # blob00 ############################################ path = s_common.gendir(dirn, 'blob00') authblob00 = neur.genCellAuth('blob00') s_msgpack.dumpfile(authblob00, os.path.join(path, 'cell.auth')) logger.debug('Bringing blob00 online') conf = {'host': 'localhost', 'bind': '127.0.0.1'} blob00 = s_axon.BlobCell(path, conf) bref.put('blob00', blob00) self.true(blob00.cellpool.neurwait(timeout=3)) user = s_cell.CellUser(root) blob00sess = user.open(blob00.getCellAddr(), timeout=3) bref.put('blob00sess', blob00sess) mesg = ('blob:stat', {}) ok, retn = blob00sess.call(mesg, timeout=3) self.true(ok) self.eq(retn, {}) # Nothing there yet # blob01 ############################################ path = s_common.gendir(dirn, 'blob01') authblob01 = neur.genCellAuth('blob01') s_msgpack.dumpfile(authblob01, os.path.join(path, 'cell.auth')) blob01conf = dict(conf) blob01conf['blob:cloneof'] = 'blob00@localhost' logger.debug('Bringing blob01 online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01sess = user.open(blob01.getCellAddr(), timeout=3) bref.put('blob01sess', blob01sess) blob01wait = blob01.waiter(1, 'blob:clone:rows') # axon00 ############################################ path = s_common.gendir(dirn, 'axon00') authaxon00 = neur.genCellAuth('axon00') s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'axon:blobs': ('blob00@localhost', ), } logger.debug('Bringing axon00 online') axon00 = s_axon.AxonCell(path, axonconf) bref.put('axon00', axon00) self.true(axon00.cellpool.neurwait(timeout=3)) ##################################################### sess = user.open(axon00.getCellAddr(), timeout=3) bref.put('sess', sess) # wait for the axon to have blob00 ready = False for i in range(30): if axon00.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon = s_axon.AxonClient(sess) blob = s_axon.BlobClient(blob00sess) blob01c = s_axon.BlobClient(blob01sess) self.eq((), tuple(axon.metrics())) self.eq((), tuple(blob.metrics())) self.len(1, axon.wants([asdfhash])) # Asking for bytes prior to the bytes being present raises self.genraises(RetnErr, axon.bytes, asdfhash, timeout=3) self.eq(1, axon.save([b'asdfasdf'], timeout=3)) self.eq((), tuple(axon.metrics(offs=999999999))) self.eq((), tuple(blob.metrics(offs=99999999, timeout=3))) metrics = list(blob.metrics(timeout=3)) self.len(1, metrics) self.eq(8, metrics[0][1].get('size')) self.eq(1, metrics[0][1].get('blocks')) self.len(0, axon.wants([asdfhash], timeout=3)) self.eq(b'asdfasdf', b''.join(axon.bytes(asdfhash, timeout=3))) stat = axon.stat(timeout=3) self.eq(1, stat.get('files')) self.eq(8, stat.get('bytes')) # lets see if the bytes made it to the blob clone... self.nn(blob01wait.wait(timeout=10)) newp = os.urandom(32) def loop(): s_common.spin(axon.bytes(newp)) self.raises(s_exc.RetnErr, loop) blob01wait = blob01.waiter(1, 'blob:clone:rows') self.eq(qwerhash, axon.upload([b'qwer', b'qwer'], timeout=3)) self.len(0, axon.wants([qwerhash])) self.eq(b'qwerqwer', b''.join(axon.bytes(qwerhash, timeout=3))) self.nn(blob01wait.wait(3)) retn = list(axon.metrics(0, timeout=3)) self.eq(retn[0][1].get('size'), 8) self.eq(retn[0][1].get('cell'), 'blob00@localhost') # Try uploading a large file logger.debug('Large file test') # Monkeypatch axon to a smaller blocksize s_axon.blocksize = s_const.kibibyte self.raises(RetnErr, axon.locs, bbufhash, timeout=3) genr = s_common.chunks(bbuf, s_axon.blocksize) blob01wait = blob01.waiter(1, 'blob:clone:rows') self.eq(bbufhash, axon.upload(genr, timeout=3)) self.eq((), axon.wants([bbufhash], timeout=3)) # Then retrieve it size = 0 gots = [] testhash = hashlib.sha256() for byts in axon.bytes(bbufhash, timeout=3): size += len(byts) gots.append(byts) testhash.update(byts) self.eq(bbufhash, testhash.digest()) try: self.eq(size, len(bbuf)) self.eq(bbufhash, testhash.digest()) except Exception as e: for byts in gots: print(repr(byts)) print('SIZE: %d/%d' % (size, len(bbuf))) raise self.nn(blob01wait.wait(3)) locs = axon.locs(bbufhash, timeout=3) self.len(1, locs) self.isin('blob00', locs[0][0]) # Use the buid to retrieve the large file from blob01 buid = locs[0][1] testhash = hashlib.sha256() for byts in blob01c.bytes(buid, timeout=3): testhash.update(byts) self.eq(bbufhash, testhash.digest()) # Try storing a empty file logger.debug('Nullfile test') axon.save([b'']) self.eq((), tuple(axon.wants([nullhash]))) # Then retrieve it parts = [] for part in axon.bytes(nullhash): parts.append(part) self.eq([b''], parts) logger.debug('Shutdown / restart blob01 test') bref.pop('blob01') blob01.fini() self.true(blob01.isfini) axon.save([b'hehehaha'], timeout=3) self.eq((), axon.wants([hehahash], timeout=3)) # Now bring blob01 back online logger.debug('Bringing blob01 back online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01wait = blob01.waiter(1, 'blob:clone:rows') # Cloning should start up shortly self.nn(blob01wait.wait(10)) # Let everything get shut down by the busref fini logger.debug('Bringing everything back up') with s_eventbus.BusRef() as bref: # neur00 ############################################ conf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': nport } path = s_common.gendir(dirn, 'neuron') logger.debug('Bringing Neuron Back online') neur = s_neuron.Neuron(path, conf) bref.put('neur00', neur) root = neur.getCellAuth() # blob00 ############################################ path = s_common.gendir(dirn, 'blob00') logger.debug('Bringing blob00 back online') conf = {'host': 'localhost', 'bind': '127.0.0.1'} blob00 = s_axon.BlobCell(path, conf) bref.put('blob00', blob00) self.true(blob00.cellpool.neurwait(timeout=3)) user = s_cell.CellUser(root) blob00sess = user.open(blob00.getCellAddr(), timeout=3) bref.put('blob00sess', blob00sess) # blob01 ############################################ path = s_common.gendir(dirn, 'blob01') blob01conf = dict(conf) blob01conf['blob:cloneof'] = 'blob00@localhost' logger.debug('Bringing blob01 back online') blob01 = s_axon.BlobCell(path, blob01conf) bref.put('blob01', blob01) self.true(blob01.cellpool.neurwait(timeout=3)) blob01wait = blob01.waiter(1, 'blob:clone:rows') # axon00 ############################################ path = s_common.gendir(dirn, 'axon00') authaxon00 = neur.genCellAuth('axon00') s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'axon:blobs': ('blob00@localhost', ), } logger.debug('Bringing axon00 online') axon00 = s_axon.AxonCell(path, axonconf) bref.put('axon00', axon00) self.true(axon00.cellpool.neurwait(timeout=3)) ##################################################### sess = user.open(axon00.getCellAddr(), timeout=3) bref.put('sess', sess) # wait for the axon to have blob00 ready = False for i in range(30): if axon00.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon = s_axon.AxonClient(sess) # Try retrieving a large file testhash = hashlib.sha256() for byts in axon.bytes(bbufhash, timeout=3): testhash.update(byts) self.eq(bbufhash, testhash.digest()) # Try saving a new file and a existing file to the cluster and ensure it is replicated self.eq((ohmyhash, ), axon.wants((ohmyhash, hehahash, nullhash), 3)) self.eq(1, axon.save([b'ohmyohmyy', b''])) self.nn(blob01wait.wait(10))
def getAxonCore(self, cortex_conf=None): ''' Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex. Notes: The following items are available in the TstEnv: * dirn: Temporary test directory. * axon_client: A Axon client object. * core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex shared by the Daemon. * dmon_port: Port the Daemon is listening on. * dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex. * core: A Cortex. * axon_sess: The client session for the Axon. * axon: The AxonCell. * blob: The BlobCell backing the Axon. * neuron: The Neuron. Args: cortex_conf (dict): Optional cortex config Yields: TstEnv: A TstEnv instance. ''' with self.getTestDir() as dirn: neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0} neurpath = s_common.gendir(dirn, 'neuron') neur = s_neuron.Neuron(neurpath, neurconf) neurhost, neurport = neur.getCellAddr() blobpath = s_common.gendir(dirn, 'blob') blobconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'blob:mapsize': TEST_MAP_SIZE } blobauth = neur.genCellAuth('blob') s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth')) blob = s_axon.BlobCell(blobpath, blobconf) self.true(blob.cellpool.neurwait(timeout=3)) axonpath = s_common.gendir(dirn, 'axon') axonauth = neur.genCellAuth('axon') s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth')) axonconf = { 'host': 'localhost', 'bind': '127.0.0.1', 'port': 0, 'axon:blobs': ('blob@localhost', ), 'axon:mapsize': TEST_MAP_SIZE } axon = s_axon.AxonCell(axonpath, axonconf) self.true(axon.cellpool.neurwait(timeout=3)) axonhost, axonport = axon.getCellAddr() # wait for the axon to have blob ready = False for i in range(30): if axon.blobs.items(): ready = True break time.sleep(0.1) self.true(ready) axon_user = s_cell.CellUser(axonauth) axon_sess = axon_user.open((axonhost, axonport)) axon_client = s_axon.AxonClient(axon_sess) core = s_cortex.openurl('ram:///', conf=cortex_conf) self.addTstForms(core) cellpoolconf = { 'host': neurhost, 'port': neurport, 'auth': s_common.enbase64(s_msgpack.en(axonauth)) } core.setConfOpt('cellpool:conf', cellpoolconf) core.setConfOpt('axon:name', 'axon@localhost') dmon = s_daemon.Daemon() dmonlink = dmon.listen('tcp://127.0.0.1:0/') dmonport = dmonlink[1].get('port') dmon.share('core', core) coreurl = 'tcp://127.0.0.1:%d/core' % dmonport env = TstEnv() env.add('dirn', dirn) env.add('axon_client', axon_client) env.add('core_url', coreurl) env.add('dmon_port', dmonport) # Order matter for clean fini env.add('dmon', dmon, True) env.add('core', core, True) env.add('axon_sess', axon_sess, True) env.add('axon', axon, True) env.add('blob', blob, True) env.add('neuron', neur, True) try: yield env finally: env.fini()