def test_axon_cell(self):
with self.getTestDir() as dirn:
conf = {'host': '127.0.0.1'}
with s_axon.AxonCell(dirn, conf=conf) as cell:
port = cell.getCellPort()
auth = cell.genUserAuth('*****@*****.**')
addr = ('127.0.0.1', port)
axon = s_axon.AxonUser(auth, addr, timeout=4)
self.true(axon.eat(b'asdfasdf'))
self.false(axon.eat(b'asdfasdf'))
self.true(axon.has('md5', asdfmd5))
byts = b''.join(axon.bytes('md5', asdfmd5))
self.eq(byts, b'asdfasdf')
def axon(dirn, conf=None):
return s_axon.AxonCell(dirn, conf=conf)
def test_axon_cell(self):
# implement as many tests as possible in this one
# since it *has* to use a neuron to work correctly
# put all the things that need fini() into a BusRef...
with self.getTestDir() as dirn:
with s_eventbus.BusRef() as bref:
# neur00 ############################################
# Set port to zero to allow a port to be automatically assigned during testing
conf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
addr = neur.getCellAddr()
nport = addr[1] # Save the port for later use
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
authblob00 = neur.genCellAuth('blob00')
s_msgpack.dumpfile(authblob00, os.path.join(path, 'cell.auth'))
logger.debug('Bringing blob00 online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
mesg = ('blob:stat', {})
ok, retn = blob00sess.call(mesg, timeout=3)
self.true(ok)
self.eq(retn, {}) # Nothing there yet
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
authblob01 = neur.genCellAuth('blob01')
s_msgpack.dumpfile(authblob01, os.path.join(path, 'cell.auth'))
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01sess = user.open(blob01.getCellAddr(), timeout=3)
bref.put('blob01sess', blob01sess)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
blob = s_axon.BlobClient(blob00sess)
blob01c = s_axon.BlobClient(blob01sess)
self.eq((), tuple(axon.metrics()))
self.eq((), tuple(blob.metrics()))
self.len(1, axon.wants([asdfhash]))
# Asking for bytes prior to the bytes being present raises
self.genraises(RetnErr, axon.bytes, asdfhash, timeout=3)
self.eq(1, axon.save([b'asdfasdf'], timeout=3))
self.eq((), tuple(axon.metrics(offs=999999999)))
self.eq((), tuple(blob.metrics(offs=99999999, timeout=3)))
metrics = list(blob.metrics(timeout=3))
self.len(1, metrics)
self.eq(8, metrics[0][1].get('size'))
self.eq(1, metrics[0][1].get('blocks'))
self.len(0, axon.wants([asdfhash], timeout=3))
self.eq(b'asdfasdf', b''.join(axon.bytes(asdfhash, timeout=3)))
stat = axon.stat(timeout=3)
self.eq(1, stat.get('files'))
self.eq(8, stat.get('bytes'))
# lets see if the bytes made it to the blob clone...
self.nn(blob01wait.wait(timeout=10))
newp = os.urandom(32)
def loop():
s_common.spin(axon.bytes(newp))
self.raises(s_exc.RetnErr, loop)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(qwerhash, axon.upload([b'qwer', b'qwer'], timeout=3))
self.len(0, axon.wants([qwerhash]))
self.eq(b'qwerqwer', b''.join(axon.bytes(qwerhash, timeout=3)))
self.nn(blob01wait.wait(3))
retn = list(axon.metrics(0, timeout=3))
self.eq(retn[0][1].get('size'), 8)
self.eq(retn[0][1].get('cell'), 'blob00@localhost')
# Try uploading a large file
logger.debug('Large file test')
# Monkeypatch axon to a smaller blocksize
s_axon.blocksize = s_const.kibibyte
self.raises(RetnErr, axon.locs, bbufhash, timeout=3)
genr = s_common.chunks(bbuf, s_axon.blocksize)
blob01wait = blob01.waiter(1, 'blob:clone:rows')
self.eq(bbufhash, axon.upload(genr, timeout=3))
self.eq((), axon.wants([bbufhash], timeout=3))
# Then retrieve it
size = 0
gots = []
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
size += len(byts)
gots.append(byts)
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
try:
self.eq(size, len(bbuf))
self.eq(bbufhash, testhash.digest())
except Exception as e:
for byts in gots:
print(repr(byts))
print('SIZE: %d/%d' % (size, len(bbuf)))
raise
self.nn(blob01wait.wait(3))
locs = axon.locs(bbufhash, timeout=3)
self.len(1, locs)
self.isin('blob00', locs[0][0])
# Use the buid to retrieve the large file from blob01
buid = locs[0][1]
testhash = hashlib.sha256()
for byts in blob01c.bytes(buid, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try storing a empty file
logger.debug('Nullfile test')
axon.save([b''])
self.eq((), tuple(axon.wants([nullhash])))
# Then retrieve it
parts = []
for part in axon.bytes(nullhash):
parts.append(part)
self.eq([b''], parts)
logger.debug('Shutdown / restart blob01 test')
bref.pop('blob01')
blob01.fini()
self.true(blob01.isfini)
axon.save([b'hehehaha'], timeout=3)
self.eq((), axon.wants([hehahash], timeout=3))
# Now bring blob01 back online
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# Cloning should start up shortly
self.nn(blob01wait.wait(10))
# Let everything get shut down by the busref fini
logger.debug('Bringing everything back up')
with s_eventbus.BusRef() as bref:
# neur00 ############################################
conf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': nport
}
path = s_common.gendir(dirn, 'neuron')
logger.debug('Bringing Neuron Back online')
neur = s_neuron.Neuron(path, conf)
bref.put('neur00', neur)
root = neur.getCellAuth()
# blob00 ############################################
path = s_common.gendir(dirn, 'blob00')
logger.debug('Bringing blob00 back online')
conf = {'host': 'localhost', 'bind': '127.0.0.1'}
blob00 = s_axon.BlobCell(path, conf)
bref.put('blob00', blob00)
self.true(blob00.cellpool.neurwait(timeout=3))
user = s_cell.CellUser(root)
blob00sess = user.open(blob00.getCellAddr(), timeout=3)
bref.put('blob00sess', blob00sess)
# blob01 ############################################
path = s_common.gendir(dirn, 'blob01')
blob01conf = dict(conf)
blob01conf['blob:cloneof'] = 'blob00@localhost'
logger.debug('Bringing blob01 back online')
blob01 = s_axon.BlobCell(path, blob01conf)
bref.put('blob01', blob01)
self.true(blob01.cellpool.neurwait(timeout=3))
blob01wait = blob01.waiter(1, 'blob:clone:rows')
# axon00 ############################################
path = s_common.gendir(dirn, 'axon00')
authaxon00 = neur.genCellAuth('axon00')
s_msgpack.dumpfile(authaxon00, os.path.join(path, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'axon:blobs': ('blob00@localhost', ),
}
logger.debug('Bringing axon00 online')
axon00 = s_axon.AxonCell(path, axonconf)
bref.put('axon00', axon00)
self.true(axon00.cellpool.neurwait(timeout=3))
#####################################################
sess = user.open(axon00.getCellAddr(), timeout=3)
bref.put('sess', sess)
# wait for the axon to have blob00
ready = False
for i in range(30):
if axon00.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon = s_axon.AxonClient(sess)
# Try retrieving a large file
testhash = hashlib.sha256()
for byts in axon.bytes(bbufhash, timeout=3):
testhash.update(byts)
self.eq(bbufhash, testhash.digest())
# Try saving a new file and a existing file to the cluster and ensure it is replicated
self.eq((ohmyhash, ),
axon.wants((ohmyhash, hehahash, nullhash), 3))
self.eq(1, axon.save([b'ohmyohmyy', b'']))
self.nn(blob01wait.wait(10))
def getAxonCore(self, cortex_conf=None):
'''
Get a TstEnv instance which is preconfigured with a Neuron, Blob, Axon, Daemon and Cortex.
Notes:
The following items are available in the TstEnv:
* dirn: Temporary test directory.
* axon_client: A Axon client object.
* core_url: The Telepath URL to the Cortex so a connection can be made to the Cortex
shared by the Daemon.
* dmon_port: Port the Daemon is listening on.
* dmon: A Daemon which is listening on 127.0.0.1:0. It is preconfigured to share the Cortex.
* core: A Cortex.
* axon_sess: The client session for the Axon.
* axon: The AxonCell.
* blob: The BlobCell backing the Axon.
* neuron: The Neuron.
Args:
cortex_conf (dict): Optional cortex config
Yields:
TstEnv: A TstEnv instance.
'''
with self.getTestDir() as dirn:
neurconf = {'host': 'localhost', 'bind': '127.0.0.1', 'port': 0}
neurpath = s_common.gendir(dirn, 'neuron')
neur = s_neuron.Neuron(neurpath, neurconf)
neurhost, neurport = neur.getCellAddr()
blobpath = s_common.gendir(dirn, 'blob')
blobconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'blob:mapsize': TEST_MAP_SIZE
}
blobauth = neur.genCellAuth('blob')
s_msgpack.dumpfile(blobauth, os.path.join(blobpath, 'cell.auth'))
blob = s_axon.BlobCell(blobpath, blobconf)
self.true(blob.cellpool.neurwait(timeout=3))
axonpath = s_common.gendir(dirn, 'axon')
axonauth = neur.genCellAuth('axon')
s_msgpack.dumpfile(axonauth, os.path.join(axonpath, 'cell.auth'))
axonconf = {
'host': 'localhost',
'bind': '127.0.0.1',
'port': 0,
'axon:blobs': ('blob@localhost', ),
'axon:mapsize': TEST_MAP_SIZE
}
axon = s_axon.AxonCell(axonpath, axonconf)
self.true(axon.cellpool.neurwait(timeout=3))
axonhost, axonport = axon.getCellAddr()
# wait for the axon to have blob
ready = False
for i in range(30):
if axon.blobs.items():
ready = True
break
time.sleep(0.1)
self.true(ready)
axon_user = s_cell.CellUser(axonauth)
axon_sess = axon_user.open((axonhost, axonport))
axon_client = s_axon.AxonClient(axon_sess)
core = s_cortex.openurl('ram:///', conf=cortex_conf)
self.addTstForms(core)
cellpoolconf = {
'host': neurhost,
'port': neurport,
'auth': s_common.enbase64(s_msgpack.en(axonauth))
}
core.setConfOpt('cellpool:conf', cellpoolconf)
core.setConfOpt('axon:name', 'axon@localhost')
dmon = s_daemon.Daemon()
dmonlink = dmon.listen('tcp://127.0.0.1:0/')
dmonport = dmonlink[1].get('port')
dmon.share('core', core)
coreurl = 'tcp://127.0.0.1:%d/core' % dmonport
env = TstEnv()
env.add('dirn', dirn)
env.add('axon_client', axon_client)
env.add('core_url', coreurl)
env.add('dmon_port', dmonport)
# Order matter for clean fini
env.add('dmon', dmon, True)
env.add('core', core, True)
env.add('axon_sess', axon_sess, True)
env.add('axon', axon, True)
env.add('blob', blob, True)
env.add('neuron', neur, True)
try:
yield env
finally:
env.fini()