def genauth(opts, outp=s_output.stdout):
authpath = s_common.genpath(opts.authfile)
savepath = s_common.genpath(opts.savepath)
if not os.path.isfile(authpath):
outp.printf('auth file not found: %s' % (authpath, ))
return
auth = s_msgpack.loadfile(authpath)
addr = auth[1].get('neuron')
if addr is None:
outp.printf('auth file has no neuron info: %s' % (authpath, ))
return
celluser = s_cell.CellUser(auth)
with celluser.open(addr, timeout=20) as sess:
nuro = s_neuron.NeuronClient(sess)
auth = nuro.genCellAuth(opts.cellname, timeout=20)
s_msgpack.dumpfile(auth, savepath)
outp.printf('saved %s: %s' % (opts.cellname, savepath))
def test_cryo_cell_indexing(self):
conf = {
'bind': '127.0.0.1',
'host': 'localhost',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}
with self.getTestDir() as dirn, s_cryotank.CryoCell(dirn,
conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
# Test index operations
self.raises(s_exc.RetnErr, user.getIndices, 'notpresent')
self.eq((), user.getIndices('woot:woot'))
self.raises(s_exc.BadOperArg, user.addIndex, 'woot:woot',
'prop1', 'str', [])
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.delIndex('woot:woot', 'prop1')
self.raises(s_exc.RetnErr, user.delIndex, 'woot:woot',
'noexist')
user.addIndex('woot:woot', 'prop1', 'str', ['0'])
user.pauseIndex('woot:woot', 'prop1')
user.pauseIndex('woot:woot')
user.resumeIndex('woot:woot')
self.eq([(1, 'baz'), (0, 'foo')],
list(user.queryNormValu('woot:woot', 'prop1')))
self.eq([(1, 'baz')],
list(user.queryNormValu('woot:woot', 'prop1',
valu='b')))
self.eq([],
list(
user.queryNormValu('woot:woot',
'prop1',
valu='bz',
timeout=10)))
self.eq([(1, {
'prop1': 'baz'
})], (list(
user.queryNormRecords('woot:woot', 'prop1', valu='b'))))
self.eq([(1, s_msgpack.en(('baz', {
'faz': 20
})))], list(user.queryRows('woot:woot', 'prop1', valu='b')))
user.init('woot:boring', {'noindex': True})
self.raises(s_exc.RetnErr, user.getIndices, 'woot:boring')
def test_neuron_divide(self):
with self.getTestDir() as dirn:
celldirn = os.path.join(dirn, 'cell')
# FIXME: this could randomly fail if the port is in use!
port = random.randint(20000, 50000)
conf = {'bind': '127.0.0.1',
'port': port,
'host': 'localhost',
'ctor': 'synapse.tests.test_neuron.TstCell'}
# Preload the cell vault
vdir = os.path.join(celldirn, 'vault.lmdb')
with s_vault.shared(vdir) as vault:
auth = vault.genUserAuth('*****@*****.**')
proc = s_cell.divide(celldirn, conf)
with genfile(celldirn, 'cell.lock') as fd:
self.true(checkLock(fd, 30))
try:
# Try connecting to the cell
user = s_cell.CellUser(auth)
addr = ('127.0.0.1', port)
with user.open(addr, timeout=10) as sess:
with sess.task(('cell:ping', {'data': 'haha'})) as chan:
retn = chan.next(timeout=2)
self.eq(retn, 'haha')
retn = sess.call(('cell:ping', {'data': 'rofl'}), timeout=2)
self.eq(retn, 'rofl')
retn = sess.call(('cell:pong', {'data': 'rofl'}), timeout=2)
self.eq(retn, {'mesg': 'pong', 'counter': 1})
retn = sess.call(('cell:pong', {'data': 'rofl'}), timeout=2)
self.eq(retn, {'mesg': 'pong', 'counter': 2})
except Exception as e:
logger.exception('Error during test!')
# Clean up the proc
proc.terminate()
proc.join(1)
raise
else:
time.sleep(0.01)
proc.terminate()
proc.join(30)
self.false(proc.is_alive())
self.eq(proc.exitcode, 0)
def cell_populate(self, port, auth):
# Populate the cell with data
addr = ('127.0.0.1', port)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
nodes = [(None, {'key': i}) for i in range(10)]
user.puts('test:hehe', nodes, 4)
self.len(10, list(user.slice('test:hehe', 0, 100)))
user.puts('test:haha', nodes, 4)
self.len(2, user.list())
def test_neuron_cell_notok(self):
with self.getTestDir() as dirn:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
with s_cell.Cell(dirn, conf) as cell:
user = s_cell.CellUser(cell.genUserAuth('foo'))
addr = ('localhost', 1)
self.raises(CellUserErr, user.open, addr, timeout=1)
self.raises(CellUserErr, user.open, addr, timeout=-1)
def test_cryo_cell_daemon(self):
with self.getTestDir() as dirn:
celldir = os.path.join(dirn, 'cell')
port = random.randint(10000, 50000)
conf = {
'cells': [
(celldir, {
'ctor': 'synapse.cryotank.CryoCell',
'port': port,
'host': 'localhost',
'bind': '127.0.0.1',
'defvals': {
'mapsize': s_iq.TEST_MAP_SIZE
}
}),
],
}
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
with genfile(celldir, 'cell.lock') as fd:
self.true(checkLock(fd, 30))
authfp = os.path.join(celldir, 'cell.auth')
auth = s_msgpack.loadfile(authfp)
addr = ('127.0.0.1', port)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
retn = user.list(timeout=3)
self.eq(retn, ())
user.puts('woot:woot', cryodata, timeout=2)
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
# ensure dmon cell processes are fini'd
for celldir, proc in dmon.cellprocs.items():
self.false(proc.is_alive())
def test_neuron_cell_openlist(self):
with self.getTestDir() as dirn:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
cell = s_cell.Cell(dirn)
with s_cell.Cell(dirn, conf) as cell:
user = s_cell.CellUser(cell.genUserAuth('foo'))
addr = list(cell.getCellAddr())
with user.open(addr, timeout=2) as sess:
with sess.task(('cell:ping', {'data': 'haha'})) as chan:
retn = chan.next(timeout=2)
self.eq(retn, 'haha')
retn = sess.call(('cell:ping', {'data': 'rofl'}), timeout=2)
self.eq(retn, 'rofl')
def test_neuron_cell_authfail(self):
'''
Make a separate cell dir and make sure it can't connect to the first one
'''
with self.getTestDir() as dirn, self.getTestDir() as dirn2:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
with s_cell.Cell(dirn, conf) as cell, s_cell.Cell(dirn2) as newp:
user = s_cell.CellUser(newp.getCellAuth())
addr = cell.getCellAddr()
with self.getLoggerStream('synapse.lib.cell') as stream:
self.raises(CellUserErr, user.open, addr, timeout=1)
stream.seek(0)
mesgs = stream.read()
self.isin('got bad cert', mesgs)
def test_neuron_neuron(self):
with self.getTestDir() as dirn:
steps = self.getTestSteps(('cell:reg',))
conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
path = s_common.gendir(dirn, 'neuron')
with s_neuron.Neuron(path, conf) as neur:
cdef = neur.getConfDef('port')
self.eq(s_neuron.defport, cdef[1].get('defval'))
def onreg(mesg):
steps.done('cell:reg')
neur.on('cell:reg', onreg)
self.eq(neur._genCellName('root'), 'root@localhost')
path = neur._path('admin.auth')
auth = s_msgpack.loadfile(path)
user = s_cell.CellUser(auth)
pool = s_cell.CellPool(neur.genUserAuth('foo'), neur.getCellAddr())
pool.neurok.wait(timeout=8)
self.true(pool.neurok.is_set())
with user.open(neur.getCellAddr()) as sess:
ncli = s_neuron.NeuronClient(sess)
auth = ncli.genCellAuth('cell00')
path = s_common.gendir(dirn, 'cell')
authpath = s_common.genpath(path, 'cell.auth')
s_msgpack.dumpfile(auth, authpath)
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
with s_cell.Cell(path, conf) as cell:
steps.wait('cell:reg', timeout=3)
steps.clear('cell:reg')
# we should be able to get a session to him in the pool...
wait = pool.waiter(1, 'cell:add')
pool.add('cell00@localhost')
self.nn(wait.wait(timeout=3))
self.nn(pool.get('cell00@localhost'))
ok, cells = sess.call(('cell:list', {}))
self.true(ok)
self.eq(cells[0][0], 'cell00@localhost')
ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'}))
self.true(ok)
self.eq(info['ctor'], 'synapse.lib.cell.Cell')
self.eq(info.get('addr'), cell.getCellAddr())
# he'll come up on a new port...
with TstCell(path, conf) as cell:
wait = pool.waiter(1, 'cell:add')
steps.wait('cell:reg', timeout=3)
steps.clear('cell:reg')
self.nn(wait.wait(timeout=3))
self.nn(pool.get('cell00@localhost'))
ok, info = sess.call(('cell:get', {'name': 'cell00@localhost'}))
self.true(ok)
self.eq(info['ctor'], 'synapse.tests.test_neuron.TstCell')
mesg = ('cell:ping', {'data': 'hehe'})
self.eq(pool.get('cell00@localhost').call(mesg), 'hehe')
# since we have an active neuron, lets test the CLI tools here as well...
authpath = os.path.join(dirn, 'neuron', 'admin.auth')
savepath = os.path.join(dirn, 'woot.auth')
argv = ['genauth', authpath, 'woot', savepath]
outp = self.getTestOutp()
s_tools_neuron.main(argv, outp=outp)
self.true(outp.expect('saved woot'))
self.true(outp.expect('woot.auth'))
auth = s_msgpack.loadfile(savepath)
self.eq(auth[0], 'woot@localhost')
self.nn(auth[1].get('neuron'))
# Use wootauth for provisioning a test cell
steps.clear('cell:reg')
path = gendir(dirn, 'wootcell')
authpath = s_common.genpath(path, 'cell.auth')
s_msgpack.dumpfile(auth, authpath)
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
with s_cell.Cell(path, conf) as cell:
wait = pool.waiter(1, 'cell:add')
pool.add('woot@localhost')
steps.wait('cell:reg', timeout=3)
self.nn(wait.wait(timeout=3))
self.nn(pool.get('woot@localhost'))
mesg = ('cell:ping', {'data': 'w00t!'})
self.eq(pool.get('woot@localhost').call(mesg), 'w00t!')
pool.fini()
def test_cryo_cell(self):
with self.getTestDir() as dirn:
conf = {'bind': '127.0.0.1', 'host': 'localhost'}
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
# Setting the _chunksize to 1 forces iteration on the client
# side of puts, as well as the server-side.
user._chunksize = 1
user.puts('woot:woot', cryodata, timeout=2)
self.eq(user.last('woot:woot', timeout=2)[1][0], 'baz')
retn = user.list(timeout=3)
self.eq(retn[0][1]['indx'], 2)
self.eq(retn[0][0], 'woot:woot')
metr = list(user.metrics('woot:woot', 0, 100, timeout=2))
self.len(2, metr)
self.eq(metr[0][1]['count'], 1)
user.puts('woot:woot', cryodata, timeout=2)
retn = list(user.slice('woot:woot', 2, 2, timeout=3))
self.len(2, retn)
self.eq(2, retn[0][0])
retn = list(user.rows('woot:woot', 0, 2, timeout=3))
self.len(2, retn)
self.eq(retn[0], (0, s_msgpack.en(cryodata[0])))
self.eq(retn[1], (1, s_msgpack.en(cryodata[1])))
# Reset chunksize
user._chunksize = s_cryotank.CryoClient._chunksize
user.puts('woot:hehe', cryodata, timeout=5)
user.puts('woot:hehe', cryodata, timeout=5)
retn = list(user.slice('woot:hehe', 1, 2))
retn = [val for indx, val in retn[::-1]]
self.eq(tuple(retn), cryodata)
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
listd = dict(user.list(timeout=3))
self.isin('woot:hehe', listd)
self.eq(user.last('woot:hehe', timeout=3),
(3, cryodata[1]))
# delete woot.hehe and then call apis on it
self.true(user.delete('woot:hehe'))
self.false(user.delete('woot:hehe'))
self.none(cell.tanks.get('woot:hehe'))
self.none(cell.names.get('woot:hehe'))
self.genraises(s_exc.RetnErr,
user.slice,
'woot:hehe',
1,
2,
timeout=3)
self.genraises(s_exc.RetnErr,
user.rows,
'woot:hehe',
1,
2,
timeout=3)
listd = dict(user.list(timeout=3))
self.notin('woot:hehe', listd)
self.none(user.last('woot:hehe', timeout=3))
self.genraises(s_exc.RetnErr,
user.metrics,
'woot:hehe',
0,
100,
timeout=3)
# Adding data re-adds the tank
user._chunksize = 1000
user.puts('woot:hehe', cryodata, timeout=5)
metr = list(user.metrics('woot:hehe', 0))
self.len(1, metr)
# We can initialize a new tank directly with a custom map size
self.true(
user.init('weee:imthebest', {'mapsize': 5558675309}))
self.false(user.init('woot:hehe'))
# error when we specify an invalid config option
self.raises(s_exc.RetnErr, user.init, 'weee:danktank',
{'newp': 'hehe'})
# Turn it back on
with s_cryotank.CryoCell(dirn, conf) as cell:
addr = cell.getCellAddr()
cuser = s_cell.CellUser(cell.genUserAuth('foo'))
with cuser.open(addr, timeout=2) as sess:
user = s_cryotank.CryoClient(sess)
listd = dict(user.list(timeout=3))
self.len(3, listd)
self.isin('weee:imthebest', listd)
self.isin('woot:woot', listd)
self.isin('woot:hehe', listd)
self.istufo(user.last('woot:woot')[1])
self.istufo(user.last('woot:hehe')[1])
self.none(user.last('weee:imthebest'))
# Test empty puts
user.puts('woot:hehe', tuple())
listd = dict(user.list(timeout=3))
metr = list(user.metrics('woot:hehe', 0))
self.len(2, metr)
self.nn(user.last('woot:hehe'))
def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='cryo.cat', description='display data items from a cryo cell')
pars.add_argument(
'cryocell',
help=
'The cell descriptor and cryo tank path (cell://<host:port>/<name>).')
pars.add_argument('--list',
default=False,
action='store_true',
help='List tanks in the remote cell and return')
pars.add_argument('--offset',
default=0,
type=int,
help='Begin at offset index')
pars.add_argument('--size',
default=10,
type=int,
help='How many items to display')
pars.add_argument('--timeout',
default=10,
type=int,
help='The network timeout setting')
pars.add_argument('--authfile',
help='Path to your auth file for the remote cell')
group = pars.add_mutually_exclusive_group()
group.add_argument('--jsonl',
action='store_true',
help='Input/Output items in jsonl format')
group.add_argument('--msgpack',
action='store_true',
help='Input/Output items in msgpack format')
pars.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='Verbose output')
pars.add_argument(
'--ingest',
'-i',
default=False,
action='store_true',
help=
'Reverses direction: feeds cryotank from stdin in msgpack or jsonl format'
)
pars.add_argument(
'--omit-offset',
default=False,
action='store_true',
help=
"Don't output offsets of objects. This is recommended to be used when jsonl/msgpack"
" output is used.")
opts = pars.parse_args(argv)
if opts.verbose:
logger.setLevel(logging.INFO)
if not opts.authfile:
logger.error(
'Currently requires --authfile until neuron protocol is supported')
return 1
if opts.ingest and not opts.jsonl and not opts.msgpack:
logger.error(
'Must specify exactly one of --jsonl or --msgpack if --ingest is specified'
)
return 1
authpath = s_common.genpath(opts.authfile)
auth = s_msgpack.loadfile(authpath)
netw, path = opts.cryocell[7:].split('/', 1)
host, portstr = netw.split(':')
addr = (host, int(portstr))
logger.info('connecting to: %r', addr)
cuser = s_cell.CellUser(auth)
with cuser.open(addr, timeout=opts.timeout) as sess:
cryo = s_cryotank.CryoClient(sess)
if opts.list:
for name, info in cryo.list(timeout=opts.timeout):
outp.printf('%s: %r' % (name, info))
return 0
if opts.ingest:
if opts.msgpack:
fd = sys.stdin.buffer
item_it = _except_wrap(s_msgpack.iterfd(fd),
lambda x: 'Error parsing item %d' % x)
else:
fd = sys.stdin
item_it = _except_wrap(
(json.loads(s) for s in fd), lambda x:
('Failure parsing line %d of input' % x))
cryo.puts(path, item_it)
else:
for item in cryo.slice(path, opts.offset, opts.size, opts.timeout):
i = item[1] if opts.omit_offset else item
if opts.jsonl:
outp.printf(json.dumps(i, sort_keys=True))
elif opts.msgpack:
sys.stdout.write(s_msgpack.en(i))
else:
outp.printf(pprint.pformat(i))
return 0
def test_axon_cell(self):
# implement as many tests as possible in this one
# since it *has* to use a neuron to work correctly
# put all the things that need fini() into a BusRef...
with self.getTestDir() as dirn:
with s_eventbus.BusRef() as bref:
# neur00 ############################################
# Set port to zero to allow a port to be automatically assigned during testing
conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
addr = neur.getCellAddr()
nport = addr[1] # Save the port for later use
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
authblob00 = neur.genCellAuth('blob00')
s_msgpack.dumpfile(authblob00, os.path.join(path, 'cell.auth'))
logger.debug('Bringing blob00 online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
mesg = ('blob:stat', {})
ok, retn = blob00sess.call(mesg, timeout=3)
self.true(ok)
self.eq(retn, {}) # Nothing there yet
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
authblob01 = neur.genCellAuth('blob01')
s_msgpack.dumpfile(authblob01, os.path.join(path, 'cell.auth'))
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01sess = user.open(blob01.getCellAddr(), timeout=3)
bref.put('blob01sess', blob01sess)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
blob = s_axon.BlobClient(blob00sess)
blob01c = s_axon.BlobClient(blob01sess)
self.eq((), tuple(axon.metrics()))
self.eq((), tuple(blob.metrics()))
self.len(1, axon.wants([asdfhash]))
# Asking for bytes prior to the bytes being present raises
self.genraises(RetnErr, axon.bytes, asdfhash, timeout=3)
self.eq(1, axon.save([b'asdfasdf'], timeout=3))
self.eq((), tuple(axon.metrics(offs=999999999)))
self.eq((), tuple(blob.metrics(offs=99999999, timeout=3)))
metrics = list(blob.metrics(timeout=3))
self.len(1, metrics)
self.eq(8, metrics[0][1].get('size'))
self.eq(1, metrics[0][1].get('blocks'))
self.len(0, axon.wants([asdfhash], timeout=3))
self.eq(b'asdfasdf', b''.join(axon.bytes(asdfhash, timeout=3)))
stat = axon.stat(timeout=3)
self.eq(1, stat.get('files'))
self.eq(8, stat.get('bytes'))
# lets see if the bytes made it to the blob clone...
self.nn(blob01wait.wait(timeout=10))
newp = os.urandom(32)
def loop():
s_common.spin(axon.bytes(newp))
self.raises(s_exc.RetnErr, loop)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(qwerhash, axon.upload([b'qwer', b'qwer'], timeout=3))
self.len(0, axon.wants([qwerhash]))
self.eq(b'qwerqwer', b''.join(axon.bytes(qwerhash, timeout=3)))
self.nn(blob01wait.wait(3))
retn = list(axon.metrics(0, timeout=3))
self.eq(retn[0][1].get('size'), 8)
self.eq(retn[0][1].get('cell'), 'blob00@localhost')
# Try uploading a large file
logger.debug('Large file test')
# Monkeypatch axon to a smaller blocksize
s_axon.blocksize = s_const.kibibyte
self.raises(RetnErr, axon.locs, bbufhash, timeout=3)
genr = s_common.chunks(bbuf, s_axon.blocksize)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(bbufhash, axon.upload(genr, timeout=3))
self.eq((), axon.wants([bbufhash], timeout=3))
# Then retrieve it
size = 0
gots = []
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
size += len(byts)
gots.append(byts)
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
try:
self.eq(size, len(bbuf))
self.eq(bbufhash, testhash.digest())
except Exception as e:
for byts in gots:
print(repr(byts))
print('SIZE: %d/%d' % (size, len(bbuf)))
raise
self.nn(blob01wait.wait(3))
locs = axon.locs(bbufhash, timeout=3)
self.len(1, locs)
self.isin('blob00', locs[0][0])
# Use the buid to retrieve the large file from blob01
buid = locs[0][1]
testhash = hashlib.sha256()
for byts in blob01c.bytes(buid, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try storing a empty file
logger.debug('Nullfile test')
axon.save([b''])
self.eq((), tuple(axon.wants([nullhash])))
# Then retrieve it
parts = []
for part in axon.bytes(nullhash):
parts.append(part)
self.eq([b''], parts)
logger.debug('Shutdown / restart blob01 test')
bref.pop('blob01')
blob01.fini()
self.true(blob01.isfini)
axon.save([b'hehehaha'], timeout=3)
self.eq((), axon.wants([hehahash], timeout=3))
# Now bring blob01 back online
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# Cloning should start up shortly
self.nn(blob01wait.wait(10))
# Let everything get shut down by the busref fini
logger.debug('Bringing everything back up')
with s_eventbus.BusRef() as bref:
# neur00 ############################################
conf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': nport
}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron Back online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
logger.debug('Bringing blob00 back online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
# Try retrieving a large file
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try saving a new file and a existing file to the cluster and ensure it is replicated
self.eq((ohmyhash, ),
axon.wants((ohmyhash, hehahash, nullhash), 3))
self.eq(1, axon.save([b'ohmyohmyy', b'']))
self.nn(blob01wait.wait(10))
def getAxonCore(self, cortex_conf=None):
'''
Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex.
Notes:
The following items are available in the TstEnv:
* dirn: Temporary test directory.
* axon_client: A Axon client object.
* core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex
shared by the Daemon.
* dmon_port: Port the Daemon is listening on.
* dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex.
* core: A Cortex.
* axon_sess: The client session for the Axon.
* axon: The AxonCell.
* blob: The BlobCell backing the Axon.
* neuron: The Neuron.
Args:
cortex_conf (dict): Optional cortex config
Yields:
TstEnv: A TstEnv instance.
'''
with self.getTestDir() as dirn:
neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
neurpath = s_common.gendir(dirn, 'neuron')
neur = s_neuron.Neuron(neurpath, neurconf)
neurhost, neurport = neur.getCellAddr()
blobpath = s_common.gendir(dirn, 'blob')
blobconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'blob:mapsize': TEST_MAP_SIZE
}
blobauth = neur.genCellAuth('blob')
s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth'))
blob = s_axon.BlobCell(blobpath, blobconf)
self.true(blob.cellpool.neurwait(timeout=3))
axonpath = s_common.gendir(dirn, 'axon')
axonauth = neur.genCellAuth('axon')
s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'axon:blobs': ('blob@localhost', ),
'axon:mapsize': TEST_MAP_SIZE
}
axon = s_axon.AxonCell(axonpath, axonconf)
self.true(axon.cellpool.neurwait(timeout=3))
axonhost, axonport = axon.getCellAddr()
# wait for the axon to have blob
ready = False
for i in range(30):
if axon.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon_user = s_cell.CellUser(axonauth)
axon_sess = axon_user.open((axonhost, axonport))
axon_client = s_axon.AxonClient(axon_sess)
core = s_cortex.openurl('ram:///', conf=cortex_conf)
self.addTstForms(core)
cellpoolconf = {
'host': neurhost,
'port': neurport,
'auth': s_common.enbase64(s_msgpack.en(axonauth))
}
core.setConfOpt('cellpool:conf', cellpoolconf)
core.setConfOpt('axon:name', 'axon@localhost')
dmon = s_daemon.Daemon()
dmonlink = dmon.listen('tcp://127.0.0.1:0/')
dmonport = dmonlink[1].get('port')
dmon.share('core', core)
coreurl = 'tcp://127.0.0.1:%d/core' % dmonport
env = TstEnv()
env.add('dirn', dirn)
env.add('axon_client', axon_client)
env.add('core_url', coreurl)
env.add('dmon_port', dmonport)
# Order matter for clean fini
env.add('dmon', dmon, True)
env.add('core', core, True)
env.add('axon_sess', axon_sess, True)
env.add('axon', axon, True)
env.add('blob', blob, True)
env.add('neuron', neur, True)
try:
yield env
finally:
env.fini()
