def test_service_byname(self):
sbus = s_service.SvcBus()
woot0 = Woot()
with s_daemon.Daemon() as dmon:
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
with s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port) as prox:
iden = prox.runSynSvc('foo0', woot0)
self.eq(prox.callByName('foo0', gentask('foo', 20)), 30)
def __init__(self, datadir, **opts):
s_eventbus.EventBus.__init__(self)
self.datadir = gendir(datadir)
self.opts = opts
self.lock = threading.Lock()
self.axons = {}
self.axonbus = None
self.axonforks = {}
self.onfini(self._onAxonHostFini)
self.opts.setdefault('autorun', 0) # how many axons to auto-start
self.opts.setdefault('axonbus', '') # url to axonbus
self.opts.setdefault('bytemax',
terabyte) # by default make each Axon 1 Terabyte
self.opts.setdefault('syncmax', gigabyte * 10) #
self.opts.setdefault(
'hostname',
s_thishost.get('hostname')) # allow override for testing
url = self.opts.get('axonbus')
if url:
self.axonbus = s_service.openurl(url)
self.axonbus.runSynSvc(guid(), self)
for name in os.listdir(self.datadir):
if not name.endswith('.axon'):
continue
iden, _ = name.split('.', 1)
self._fireAxonIden(iden)
# fire auto-run axons
auto = self.opts.get('autorun')
while len(self.axons) < auto:
self.add()
def test_service_byname(self):
sbus = s_service.SvcBus()
woot0 = Woot()
dmon = s_daemon.Daemon()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)
iden = prox.runSynSvc('foo0', woot0)
self.assertEqual( prox.callByName('foo0', 'foo',20), 30 )
dmon.fini()
def test_service_getTagProxy(self):
woot0 = Woot()
with s_daemon.Daemon() as dmon:
sbus = s_service.SvcBus()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
with s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port) as prox:
prox.runSynSvc('foo0', woot0, tags=['bar0'])
tagprox = prox.getTagProxy('bar0')
self.eq(next(tagprox.foo(20), None), 30)
def test_service_getNameProxy(self):
woot0 = Woot()
with s_daemon.Daemon() as dmon:
sbus = s_service.SvcBus()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
with s_service.openurl('tcp://127.0.0.1/syn.svcbus',
port=port) as prox:
prox.runSynSvc('foo0', woot0)
nameprox = prox.getNameProxy('foo0')
self.assertEqual(nameprox.foo(20), 30)
def test_service_getTagProxy(self):
sbus = s_service.SvcBus()
woot0 = Woot()
dmon = s_daemon.Daemon()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)
prox.runSynSvc('foo0', woot0, tags=['bar0'])
tagprox = prox.getTagProxy('bar0')
self.assertEqual(next(tagprox.foo(20), None), 30)
dmon.fini()
def __init__(self, datadir, **opts):
s_config.Config.__init__(self)
self.datadir = s_common.gendir(datadir)
self.lock = threading.Lock()
self.axons = {} # iden -> Axon mapping.
self.axonbus = None
self.axonforks = {}
self.cloneaxons = [] # List of idens which are clones.
self.onfini(self._onAxonHostFini)
self.setConfOpts(opts)
self._axonconfs = [_name for _name, _ in Axon._axon_confdefs()]
# track the total number of bytes which may be used by axons for startup operations
self.usedspace = 0
for name in os.listdir(self.datadir):
if not name.endswith('.axon'):
continue
iden, _ = name.split('.', 1)
self._fireAxonIden(iden)
# fire auto-run axons
auto = self.getConfOpt('axonhost:autorun')
while (len(self.axons) - len(self.cloneaxons)) < auto:
self.add()
url = self.getConfOpt('axon:axonbus')
if url:
self.axonbus = s_service.openurl(url)
self.onfini(self.axonbus.fini)
self.axonbus.runSynSvc(s_common.guid(), self)
def __init__(self, datadir, **opts):
s_eventbus.EventBus.__init__(self)
self.datadir = gendir(datadir)
self.opts = opts
self.lock = threading.Lock()
self.axons = {}
self.axonbus = None
self.axonforks = {}
self.onfini( self._onAxonHostFini )
self.opts.setdefault('autorun',0) # how many axons to auto-start
self.opts.setdefault('axonbus','') # url to axonbus
self.opts.setdefault('bytemax',terabyte) # by default make each Axon 1 Terabyte
self.opts.setdefault('syncmax',gigabyte * 10) #
self.opts.setdefault('hostname', s_thishost.get('hostname') ) # allow override for testing
url = self.opts.get('axonbus')
if url:
self.axonbus = s_service.openurl(url)
self.axonbus.runSynSvc(guid(),self)
for name in os.listdir(self.datadir):
if not name.endswith('.axon'):
continue
iden,_ = name.split('.',1)
self._fireAxonIden(iden)
# fire auto-run axons
auto = self.opts.get('autorun')
while len(self.axons) < auto:
self.add()
def test_service_proxysugar(self):
sbus = s_service.SvcBus()
woot0 = Woot()
dmon = s_daemon.Daemon()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)
iden = prox.runSynSvc('foo0', woot0, tags=('hehe.haha',))
res = list( prox['foo0'].foo(90) )
self.assertEqual( len(res), 1 )
self.assertEqual( res[0], 100 )
dmon.fini()
def __init__(self, axondir, **opts):
s_eventbus.EventBus.__init__(self)
self.inprog = {}
self.axondir = gendir(axondir)
self.clonedir = gendir(axondir, 'clones')
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.opts = opts
self.axonbus = None
self.iden = self.opts.get('iden')
self.tags = self.opts.get('tags', ())
self.opts.setdefault('ro', False)
self.opts.setdefault('clone', '') # are we a clone?
self.opts.setdefault('clones', 2) # how many clones do we want?
self.opts.setdefault('axonbus', '') # do we have an axon svcbus?
self.opts.setdefault('hostname', s_thishost.get('hostname'))
self.opts.setdefault(
'listen',
'tcp://0.0.0.0:0/axon') # our default "ephemeral" listener
# if we're a clone, we're read-only and have no clones
if self.opts.get('clone'):
self.opts['ro'] = True
self.opts['clones'] = 0
self.opts.setdefault('synckeep', threedays)
self.opts.setdefault('syncsize', gigabyte * 10)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
fd = genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.opts.get('listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.opts)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:sync', self.heap.sync)
self.syncact.act('core:sync', self.core.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('core:sync', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
# model details for the actual byte blobs
self.core.addTufoForm('axon:blob', ptype='guid')
self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
self.core.addTufoProp('axon:blob',
'sha256',
ptype='hash:sha256',
req=True)
self.core.addTufoProp('axon:blob',
'sha512',
ptype='hash:sha512',
req=True)
self.core.addTufoForm('axon:clone', ptype='guid')
dirname = gendir(axondir, 'sync')
syncopts = self.opts.get('syncopts', {})
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.opts.get('clone'):
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.opts.get('axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
def __init__(self, axondir, **opts):
s_config.Config.__init__(self)
self.inprog = {}
self.axondir = s_common.gendir(axondir)
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.axonbus = None
self.setConfOpts(opts)
self.iden = self.getConfOpt('axon:iden')
self.tags = self.getConfOpt('axon:tags')
# if we're a clone, we're read-only and have no clones
if self.getConfOpt('axon:clone'):
self.setConfOpt('axon:ro', 1)
self.setConfOpt('axon:clones', 0)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
self.core.setConfOpt('modules',
(('synapse.models.axon.AxonMod', {}), ))
self.core.setConfOpt('caching', 1)
self._fs_mkdir_root() # create the fs root
self.flock = threading.Lock()
fd = s_common.genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.getConfOpt('axon:listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.getConfOpts())
self.on('syn:conf:set', self._onSetConfigableValu)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('splice', self.core.splice)
self.syncact.act('heap:sync', self.heap.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('splice', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.getConfOpt('axon:clone'):
dirname = s_common.gendir(axondir, 'sync')
syncopts = self.getConfOpt('axon:syncopts')
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.getConfOpt('axon:axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
self.onfini(self.axonbus.fini)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
def test_axon_cluster_cortex(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
localguid = guid()
busurl = 'local://%s/axons' % localguid
hahaurl = 'local://%s/haha' % localguid
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
dmon.share('haha', {})
svcprox = s_service.openurl(busurl)
axcluster = s_axon.AxonCluster(svcprox)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:clones': 1,
'axon:bytemax': s_axon.megabyte,
}
axfo0 = host0.add(**props)
axcluster._waitWrAxons(1, 4)
# Ensure our axfo0 was cloned to someone in the cluster
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
axon0._waitClonesReady(timeout=16)
foundclone = False
if host1.cloneaxons:
foundclone = True
if host2.cloneaxons:
foundclone = True
self.true(foundclone)
axon0.fini() # fini the proxy object
core = s_cortex.openurl('ram://')
core.setConfOpt('axon:url', busurl)
self.false(axcluster.has('md5', craphash))
self.false(axcluster.has('md5', asdfhash))
node = core.formNodeByBytes(b'asdfasdf', name='asdf')
self.eq(node[1].get('file:bytes'), asdfhash_iden)
self.eq(node[1].get('file:bytes:md5'), asdfhash)
self.true(axcluster.has('md5', asdfhash))
self.true(axcluster.has('guid', asdfhash_iden))
fd = io.BytesIO(b'visi')
node = core.formNodeByFd(fd, name='visi.bin')
self.eq(node[1].get('file:bytes:size'), 4)
self.eq(node[1].get('file:bytes:name'), 'visi.bin')
self.eq(node[1].get('file:bytes'), '442f602ecf8230b2a59a44b4f845be27')
self.eq(node[1].get('file:bytes:md5'), '1b2e93225959e3722efed95e1731b764')
self.true(axcluster.has('md5', '1b2e93225959e3722efed95e1731b764'))
self.true(axcluster.has('guid', '442f602ecf8230b2a59a44b4f845be27'))
host0.fini()
host1.fini()
host2.fini()
svcprox.fini()
dmon.fini()
def test_axon_cluster(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
svcprox = s_service.openurl(busurl)
axcluster = s_axon.AxonCluster(svcprox)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:clones': 1,
'axon:bytemax': s_axon.megabyte,
}
axfo0 = host0.add(**props)
axcluster._waitWrAxons(1, 4)
# Ensure our axfo0 was cloned to someone in the cluster
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
axon0._waitClonesReady(timeout=16)
foundclone = False
if host1.cloneaxons:
foundclone = True
if host2.cloneaxons:
foundclone = True
self.true(foundclone)
axon0.fini() # fini the proxy object
self.false(axcluster.has('md5', craphash))
self.false(axcluster.has('md5', asdfhash))
buf = b'asdfasdf'
iden = axcluster.alloc(len(buf))
self.nn(axcluster.chunk(iden, buf))
self.false(axcluster.has('md5', craphash))
self.true(axcluster.has('md5', asdfhash))
self.true(axcluster.has('guid', asdfhash_iden))
blobs = axcluster.find('md5', craphash)
self.eq(len(blobs), 0)
time.sleep(0.2) # Yield to axon threads
blobs = axcluster.find('md5', asdfhash)
# We have two blobs for the same hash since the clone of axfo0 is up on host1/host2
self.eq(len(blobs), 2)
blob = blobs[0]
byts = b''.join(axcluster.iterblob(blob))
self.eq(byts, buf)
blob[1].pop('.axon')
byts = b''.join(axcluster.iterblob(blob))
self.eq(byts, buf)
self.nn(axcluster.wants('md5', craphash, len(buf)))
self.none(axcluster.wants('md5', asdfhash, len(buf)))
# Eat bytes via AxonMixin APIs
byts = 'pennywise'.encode()
blob = axcluster.eatbytes(byts)
self.nn(blob)
self.isin('.new', blob[1])
blob = axcluster.eatbytes(byts)
self.notin('.new', blob[1])
buf = io.BytesIO('dancing clown'.encode())
blob = axcluster.eatfd(buf)
self.nn(blob)
self.isin('.new', blob[1])
blob = axcluster.eatfd(buf)
self.notin('.new', blob[1])
host0.fini()
host1.fini()
host2.fini()
svcprox.fini()
dmon.fini()
def test_axon_host_spinbackup(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
hstcfg = {
"vars": {
"hcfg0": {
"axon:hostname": "host0",
"axon:bytemax": 1024000000,
"axonhost:maxsize": 10240000000,
"axonhost:autorun": 1,
"axon:clones": 1,
},
"hcfg1": {
"axon:hostname": "host1",
"axon:bytemax": 1024000000,
"axonhost:maxsize": 10240000000,
"axonhost:autorun": 1,
"axon:clones": 1,
}
},
"ctors": [
[
"host0",
"ctor://synapse.axon.AxonHost(dir0, **hcfg0)"
],
[
"host1",
"ctor://synapse.axon.AxonHost(dir1, **hcfg1)"
]
],
"share": [
[
"host0",
{
"onfini": True
}
],
[
"host1",
{
"onfini": True
}
]
]
}
svccfg = {
"comment": "dmon file for axon stresstest",
"ctors": [
[
"axonbus",
"ctor://synapse.lib.service.SvcBus()",
{}
]
],
"share": [
[
"axonbus",
{
"onfini": True
}
]
]
}
with self.getTestDir() as dirname:
hostdir0 = gendir(dirname, 'host0')
hostdir1 = gendir(dirname, 'host1')
hstcfg['vars']['dir0'] = hostdir0
hstcfg['vars']['dir1'] = hostdir1
with s_daemon.Daemon() as svcdmon:
svcdmon.loadDmonConf(svccfg)
link = svcdmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
busurl = 'tcp://127.0.0.1:{}/axonbus'.format(port)
hstcfg['vars']['hcfg0']['axon:axonbus'] = busurl
hstcfg['vars']['hcfg1']['axon:axonbus'] = busurl
svcbus = s_service.openurl('tcp://127.0.0.1:0/axonbus', port=port) # type: s_service.SvcProxy
with s_daemon.Daemon() as axondmon:
w = svcbus.waiter(6, 'syn:svc:init')
axondmon.loadDmonConf(hstcfg)
w.wait(15)
first_axons = svcbus.getSynSvcsByTag(s_axon.axontag)
self.eq(len(first_axons), 4)
# Close the proxy
svcbus.fini()
# Spin the AxonHost back up
# This does exercise a behavior in the AxonHost to always give
# preference for its :axonbus configuration option over that of
# its Axon's. While this scenario is present in the unit test,
# in a real migration which could involve changing the :axonbus,
# this ensures that the children Axons of the AxonHost are updated
# to point to the new bus.
with s_daemon.Daemon() as svcdmon:
svcdmon.loadDmonConf(svccfg)
link = svcdmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
busurl = 'tcp://127.0.0.1:{}/axonbus'.format(port)
hstcfg['vars']['hcfg0']['axon:axonbus'] = busurl
hstcfg['vars']['hcfg1']['axon:axonbus'] = busurl
svcbus = s_service.openurl('tcp://127.0.0.1:0/axonbus', port=port) # type: s_service.SvcProxy
with s_daemon.Daemon() as axondmon:
w = svcbus.waiter(6, 'syn:svc:init')
axondmon.loadDmonConf(hstcfg)
w.wait(15)
axons = svcbus.getSynSvcsByTag(s_axon.axontag)
self.eq(len(axons), 4)
# Ensure these are the same axons we had first created
self.eq({axn[1].get('name') for axn in axons}, {axn[1].get('name') for axn in first_axons})
# Close the proxy
svcbus.fini()
def __init__(self, axondir, **opts):
s_eventbus.EventBus.__init__(self)
self.inprog = {}
self.axondir = gendir(axondir)
self.clonedir = gendir(axondir,'clones')
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event() # set once all clones are up and running
self.opts = opts
self.axonbus = None
self.iden = self.opts.get('iden')
self.tags = self.opts.get('tags',())
self.opts.setdefault('ro',False)
self.opts.setdefault('clone','') # are we a clone?
self.opts.setdefault('clones',2) # how many clones do we want?
self.opts.setdefault('axonbus','') # do we have an axon svcbus?
self.opts.setdefault('hostname', s_thishost.get('hostname') )
self.opts.setdefault('listen','tcp://0.0.0.0:0/axon') # our default "ephemeral" listener
# if we're a clone, we're read-only and have no clones
if self.opts.get('clone'):
self.opts['ro'] = True
self.opts['clones'] = 0
self.opts.setdefault('synckeep',threedays)
self.opts.setdefault('syncsize',gigabyte*10)
self.core = opts.get('core')
if self.core == None:
savefile = os.path.join(self.axondir,'axon.save')
self.core = s_cortex.openurl('ram:///',savefile=savefile)
self.onfini( self.core.fini )
fd = genfile(axondir,'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.opts.get('listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden,{})
self.axthrs = set()
self.setAxonInfo('link',self.link)
self.setAxonInfo('opts',self.opts)
self.dmon.share('axon',self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:sync', self.heap.sync )
self.syncact.act('core:sync', self.core.sync )
# wrap core/heap sync events as axon:sync events
self.core.on('core:sync', self._fireAxonSync )
self.heap.on('heap:sync', self._fireAxonSync )
# model details for the actual byte blobs
self.core.addTufoForm('axon:blob',ptype='guid')
self.core.addTufoProp('axon:blob','ver',ptype='int',defval=1)
self.core.addTufoProp('axon:blob','off', ptype='int',req=True)
self.core.addTufoProp('axon:blob','size', ptype='int',req=True)
self.core.addTufoProp('axon:blob','md5', ptype='hash:md5',req=True)
self.core.addTufoProp('axon:blob','sha1', ptype='hash:sha1',req=True)
self.core.addTufoProp('axon:blob','sha256', ptype='hash:sha256',req=True)
self.core.addTufoProp('axon:blob','sha512', ptype='hash:sha512',req=True)
self.core.addTufoForm('axon:clone',ptype='guid')
dirname = gendir(axondir,'sync')
syncopts = self.opts.get('syncopts',{})
self.syncdir = None
# if we're not a clone, create a sync dir
if not self.opts.get('clone'):
self.syncdir = s_persist.Dir(dirname,**syncopts)
self.onfini( self.syncdir.fini )
self.on('axon:sync', self.syncdir.add )
self.onfini( self._onAxonFini )
self.onfini( self.core.fini )
self.onfini( self.heap.fini )
self.onfini( self.dmon.fini )
# share last to avoid startup races
busurl = self.opts.get('axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
props = {'link':self.link,'tags':self.tags}
self.axonbus.runSynSvc(self.iden,self,**props)
self._fireAxonClones()
def test_axon_cluster(self):
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
svcprox = s_service.openurl(busurl)
axcluster = s_axon.AxonCluster(svcprox)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
opts = {
'axonbus': busurl,
}
host0 = s_axon.AxonHost(dir0, hostname='host0', **opts)
host1 = s_axon.AxonHost(dir1, hostname='host1', **opts)
host2 = s_axon.AxonHost(dir2, hostname='host2', **opts)
props = {
'clones': 1,
'syncmax': s_axon.megabyte,
'bytemax': s_axon.megabyte,
}
axfo0 = host0.add(**props)
self.assertFalse(axcluster.has('md5', craphash))
self.assertFalse(axcluster.has('md5', asdfhash))
buf = b'asdfasdf'
iden = axcluster.alloc(len(buf))
self.assertIsNotNone(axcluster.chunk(iden, buf))
self.assertFalse(axcluster.has('md5', craphash))
self.assertTrue(axcluster.has('md5', asdfhash))
blobs = axcluster.find('md5', craphash)
self.assertEqual(len(blobs), 0)
blobs = axcluster.find('md5', asdfhash)
self.assertEqual(len(blobs), 1)
blob = blobs[0]
byts = b''.join(axcluster.iterblob(blob))
self.assertEqual(byts, buf)
blob[1].pop('.axon')
byts = b''.join(axcluster.iterblob(blob))
self.assertEqual(byts, buf)
self.assertIsNone(axcluster.wants('md5', asdfhash, len(buf)))
self.assertIsNotNone(axcluster.wants('md5', craphash, len(buf)))
host0.fini()
host1.fini()
host2.fini()
dmon.fini()