def test_persist_pump(self): events = [] iden = guid() wait = threading.Event() def pumpfunc(x): events.append(x) if len(events) >= 4: wait.set() with self.getTestDir() as dirname: pdir = s_persist.Dir(dirname) pdir.add(('foo', {})) pdir.add(('bar', {})) pdir.pump(iden, pumpfunc) pdir.add(('hehe', {})) pdir.add(('haha', {})) wait.wait(timeout=2) self.true(wait.is_set()) pdir.fini()
def test_persist_dir(self): with self.getTestDir() as dirname: opts = { 'filemax': 1024, } pdir = s_persist.Dir(dirname, **opts) # trigger storage file alloc pdir.add(b'V' * 2000) pdir.add(b'I' * 2000) pdir.add(b'S' * 2000) # get all caught up and gen one real-time items = [] ev0 = threading.Event() ev1 = threading.Event() def pumploop(): for noff, item in pdir.items(0): items.append((noff, item)) if len(items) == 3: ev0.set() if len(items) == 4: ev1.set() thr = worker(pumploop) ev0.wait(timeout=3) self.true(ev0.is_set()) pdir.add(b'VISI') ev1.wait(timeout=3) self.true(ev1.is_set()) pdir.fini() self.eq(items[3][1], b'VISI') self.eq(items[0][1], b'V' * 2000)
def __init__(self, axondir, **opts): s_eventbus.EventBus.__init__(self) self.inprog = {} self.axondir = gendir(axondir) self.clonedir = gendir(axondir, 'clones') self.clones = {} self.cloneinfo = {} self.clonehosts = set() self.clonelock = threading.Lock() self.readyclones = set() # iden of each clone added as it comes up self.clonesready = threading.Event( ) # set once all clones are up and running self.opts = opts self.axonbus = None self.iden = self.opts.get('iden') self.tags = self.opts.get('tags', ()) self.opts.setdefault('ro', False) self.opts.setdefault('clone', '') # are we a clone? self.opts.setdefault('clones', 2) # how many clones do we want? self.opts.setdefault('axonbus', '') # do we have an axon svcbus? self.opts.setdefault('hostname', s_thishost.get('hostname')) self.opts.setdefault( 'listen', 'tcp://0.0.0.0:0/axon') # our default "ephemeral" listener # if we're a clone, we're read-only and have no clones if self.opts.get('clone'): self.opts['ro'] = True self.opts['clones'] = 0 self.opts.setdefault('synckeep', threedays) self.opts.setdefault('syncsize', gigabyte * 10) corepath = os.path.join(self.axondir, 'axon.db') self.core = s_cortex.openurl('sqlite:///%s' % corepath) fd = genfile(axondir, 'axon.heap') self.link = None self.heap = s_heap.Heap(fd) self.dmon = s_daemon.Daemon() lisn = self.opts.get('listen') if lisn: self.link = self.dmon.listen(lisn) self.axfo = (self.iden, {}) self.axthrs = set() self.setAxonInfo('link', self.link) self.setAxonInfo('opts', self.opts) self.dmon.share('axon', self) # create a reactor to unwrap core/heap sync events self.syncact = s_reactor.Reactor() self.syncact.act('heap:sync', self.heap.sync) self.syncact.act('core:sync', self.core.sync) # wrap core/heap sync events as axon:sync events self.core.on('core:sync', self._fireAxonSync) self.heap.on('heap:sync', self._fireAxonSync) # model details for the actual byte blobs self.core.addTufoForm('axon:blob', ptype='guid') self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True) self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True) self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True) self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True) self.core.addTufoProp('axon:blob', 'sha256', ptype='hash:sha256', req=True) self.core.addTufoProp('axon:blob', 'sha512', ptype='hash:sha512', req=True) self.core.addTufoForm('axon:clone', ptype='guid') dirname = gendir(axondir, 'sync') syncopts = self.opts.get('syncopts', {}) self.syncdir = None self.onfini(self._onAxonFini) self.onfini(self.core.fini) self.onfini(self.heap.fini) self.onfini(self.dmon.fini) # if we're not a clone, create a sync dir if not self.opts.get('clone'): self.syncdir = s_persist.Dir(dirname, **syncopts) self.onfini(self.syncdir.fini) self.on('axon:sync', self.syncdir.add) self.axcthr = None # share last to avoid startup races busurl = self.opts.get('axonbus') if busurl: self.axonbus = s_service.openurl(busurl) props = {'link': self.link, 'tags': self.tags} self.axonbus.runSynSvc(self.iden, self, **props) self.axcthr = self._fireAxonClones()
def __init__(self, axondir, **opts): s_config.Config.__init__(self) self.inprog = {} self.axondir = s_common.gendir(axondir) self.clones = {} self.cloneinfo = {} self.clonehosts = set() self.clonelock = threading.Lock() self.readyclones = set() # iden of each clone added as it comes up self.clonesready = threading.Event( ) # set once all clones are up and running self.axonbus = None self.setConfOpts(opts) self.iden = self.getConfOpt('axon:iden') self.tags = self.getConfOpt('axon:tags') # if we're a clone, we're read-only and have no clones if self.getConfOpt('axon:clone'): self.setConfOpt('axon:ro', 1) self.setConfOpt('axon:clones', 0) corepath = os.path.join(self.axondir, 'axon.db') self.core = s_cortex.openurl('sqlite:///%s' % corepath) self.core.setConfOpt('modules', (('synapse.models.axon.AxonMod', {}), )) self.core.setConfOpt('caching', 1) self._fs_mkdir_root() # create the fs root self.flock = threading.Lock() fd = s_common.genfile(axondir, 'axon.heap') self.link = None self.heap = s_heap.Heap(fd) self.dmon = s_daemon.Daemon() lisn = self.getConfOpt('axon:listen') if lisn: self.link = self.dmon.listen(lisn) self.axfo = (self.iden, {}) self.axthrs = set() self.setAxonInfo('link', self.link) self.setAxonInfo('opts', self.getConfOpts()) self.on('syn:conf:set', self._onSetConfigableValu) self.dmon.share('axon', self) # create a reactor to unwrap core/heap sync events self.syncact = s_reactor.Reactor() self.syncact.act('splice', self.core.splice) self.syncact.act('heap:sync', self.heap.sync) # wrap core/heap sync events as axon:sync events self.core.on('splice', self._fireAxonSync) self.heap.on('heap:sync', self._fireAxonSync) self.syncdir = None self.onfini(self._onAxonFini) self.onfini(self.core.fini) self.onfini(self.heap.fini) self.onfini(self.dmon.fini) # if we're not a clone, create a sync dir if not self.getConfOpt('axon:clone'): dirname = s_common.gendir(axondir, 'sync') syncopts = self.getConfOpt('axon:syncopts') self.syncdir = s_persist.Dir(dirname, **syncopts) self.onfini(self.syncdir.fini) self.on('axon:sync', self.syncdir.add) self.axcthr = None # share last to avoid startup races busurl = self.getConfOpt('axon:axonbus') if busurl: self.axonbus = s_service.openurl(busurl) self.onfini(self.axonbus.fini) props = {'link': self.link, 'tags': self.tags} self.axonbus.runSynSvc(self.iden, self, **props) self.axcthr = self._fireAxonClones()