def test_persist_pump(self):
events = []
iden = guid()
wait = threading.Event()
def pumpfunc(x):
events.append(x)
if len(events) >= 4:
wait.set()
with self.getTestDir() as dirname:
pdir = s_persist.Dir(dirname)
pdir.add(('foo', {}))
pdir.add(('bar', {}))
pdir.pump(iden, pumpfunc)
pdir.add(('hehe', {}))
pdir.add(('haha', {}))
wait.wait(timeout=2)
self.true(wait.is_set())
pdir.fini()
def test_persist_dir(self):
with self.getTestDir() as dirname:
opts = {
'filemax': 1024,
}
pdir = s_persist.Dir(dirname, **opts)
# trigger storage file alloc
pdir.add(b'V' * 2000)
pdir.add(b'I' * 2000)
pdir.add(b'S' * 2000)
# get all caught up and gen one real-time
items = []
ev0 = threading.Event()
ev1 = threading.Event()
def pumploop():
for noff, item in pdir.items(0):
items.append((noff, item))
if len(items) == 3:
ev0.set()
if len(items) == 4:
ev1.set()
thr = worker(pumploop)
ev0.wait(timeout=3)
self.true(ev0.is_set())
pdir.add(b'VISI')
ev1.wait(timeout=3)
self.true(ev1.is_set())
pdir.fini()
self.eq(items[3][1], b'VISI')
self.eq(items[0][1], b'V' * 2000)
def __init__(self, axondir, **opts):
s_eventbus.EventBus.__init__(self)
self.inprog = {}
self.axondir = gendir(axondir)
self.clonedir = gendir(axondir, 'clones')
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.opts = opts
self.axonbus = None
self.iden = self.opts.get('iden')
self.tags = self.opts.get('tags', ())
self.opts.setdefault('ro', False)
self.opts.setdefault('clone', '') # are we a clone?
self.opts.setdefault('clones', 2) # how many clones do we want?
self.opts.setdefault('axonbus', '') # do we have an axon svcbus?
self.opts.setdefault('hostname', s_thishost.get('hostname'))
self.opts.setdefault(
'listen',
'tcp://0.0.0.0:0/axon') # our default "ephemeral" listener
# if we're a clone, we're read-only and have no clones
if self.opts.get('clone'):
self.opts['ro'] = True
self.opts['clones'] = 0
self.opts.setdefault('synckeep', threedays)
self.opts.setdefault('syncsize', gigabyte * 10)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
fd = genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.opts.get('listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.opts)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:sync', self.heap.sync)
self.syncact.act('core:sync', self.core.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('core:sync', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
# model details for the actual byte blobs
self.core.addTufoForm('axon:blob', ptype='guid')
self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
self.core.addTufoProp('axon:blob',
'sha256',
ptype='hash:sha256',
req=True)
self.core.addTufoProp('axon:blob',
'sha512',
ptype='hash:sha512',
req=True)
self.core.addTufoForm('axon:clone', ptype='guid')
dirname = gendir(axondir, 'sync')
syncopts = self.opts.get('syncopts', {})
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.opts.get('clone'):
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.opts.get('axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
def __init__(self, axondir, **opts):
s_config.Config.__init__(self)
self.inprog = {}
self.axondir = s_common.gendir(axondir)
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.axonbus = None
self.setConfOpts(opts)
self.iden = self.getConfOpt('axon:iden')
self.tags = self.getConfOpt('axon:tags')
# if we're a clone, we're read-only and have no clones
if self.getConfOpt('axon:clone'):
self.setConfOpt('axon:ro', 1)
self.setConfOpt('axon:clones', 0)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
self.core.setConfOpt('modules',
(('synapse.models.axon.AxonMod', {}), ))
self.core.setConfOpt('caching', 1)
self._fs_mkdir_root() # create the fs root
self.flock = threading.Lock()
fd = s_common.genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.getConfOpt('axon:listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.getConfOpts())
self.on('syn:conf:set', self._onSetConfigableValu)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('splice', self.core.splice)
self.syncact.act('heap:sync', self.heap.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('splice', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.getConfOpt('axon:clone'):
dirname = s_common.gendir(axondir, 'sync')
syncopts = self.getConfOpt('axon:syncopts')
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.getConfOpt('axon:axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
self.onfini(self.axonbus.fini)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
