def test_heap_save(self):
#self.thisHostMust(platform='linux')
msgs = []
fd0 = tempfile.TemporaryFile()
heap0 = s_heap.Heap(fd0)
heap0.on('heap:sync', msgs.append)
off0 = heap0.alloc(8)
off1 = heap0.alloc(8)
# do interlaced writes
heap0.writeoff(off0, b'asdf')
heap0.writeoff(off1, b'hehe')
heap0.writeoff(off0 + 4, b'qwer')
heap0.writeoff(off1 + 4, b'haha')
fd1 = tempfile.TemporaryFile()
heap1 = s_heap.Heap(fd1)
heap1.syncs(msgs)
self.assertEqual(heap0.readoff(off0, 8), heap1.readoff(off0, 8))
self.assertEqual(heap0.readoff(off1, 8), heap1.readoff(off1, 8))
heap0.fini()
heap1.fini()
def test_heap_readiter(self):
#self.thisHostMust(platform='linux')
fd = tempfile.TemporaryFile()
with s_heap.Heap(fd) as heap:
rand = os.urandom(2048)
off = heap.alloc(2048)
heap.writeoff(off, rand)
blocks = [b for b in heap.readiter(off, 2048, itersize=9)]
byts = b''.join(blocks)
self.assertEqual(rand, byts)
def test_heap_resize(self):
fd = tempfile.TemporaryFile()
with s_heap.Heap(fd) as heap:
pagesize = heap.pagesize
self.assertEqual(heap.size(), heap.pagesize)
blocks = []
while heap.size() == heap.pagesize:
# NOTE test assumes pages are at least 1k
blocks.append(heap.alloc(1024))
self.assertEqual(heap.size(), heap.pagesize * 2)
def test_heap_base(self):
# FIXME test these on windows...
#self.thisHostMust(platform='linux')
fd = tempfile.TemporaryFile()
heap = s_heap.Heap(fd)
self.eq(heap.size(), heap.pagesize)
off0 = heap.alloc(8)
off1 = heap.alloc(8)
# do interlaced writes
heap.writeoff(off0, b'asdf')
heap.writeoff(off1, b'hehe')
heap.writeoff(off0 + 4, b'qwer')
heap.writeoff(off1 + 4, b'haha')
self.assertEqual(heap.readoff(off0, 8), b'asdfqwer')
self.assertEqual(heap.readoff(off1, 8), b'hehehaha')
heap.fini()
def __init__(self, axondir, **opts):
s_eventbus.EventBus.__init__(self)
self.inprog = {}
self.axondir = gendir(axondir)
self.clonedir = gendir(axondir, 'clones')
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.opts = opts
self.axonbus = None
self.iden = self.opts.get('iden')
self.tags = self.opts.get('tags', ())
self.opts.setdefault('ro', False)
self.opts.setdefault('clone', '') # are we a clone?
self.opts.setdefault('clones', 2) # how many clones do we want?
self.opts.setdefault('axonbus', '') # do we have an axon svcbus?
self.opts.setdefault('hostname', s_thishost.get('hostname'))
self.opts.setdefault(
'listen',
'tcp://0.0.0.0:0/axon') # our default "ephemeral" listener
# if we're a clone, we're read-only and have no clones
if self.opts.get('clone'):
self.opts['ro'] = True
self.opts['clones'] = 0
self.opts.setdefault('synckeep', threedays)
self.opts.setdefault('syncsize', gigabyte * 10)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
fd = genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.opts.get('listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.opts)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:sync', self.heap.sync)
self.syncact.act('core:sync', self.core.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('core:sync', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
# model details for the actual byte blobs
self.core.addTufoForm('axon:blob', ptype='guid')
self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
self.core.addTufoProp('axon:blob',
'sha256',
ptype='hash:sha256',
req=True)
self.core.addTufoProp('axon:blob',
'sha512',
ptype='hash:sha512',
req=True)
self.core.addTufoForm('axon:clone', ptype='guid')
dirname = gendir(axondir, 'sync')
syncopts = self.opts.get('syncopts', {})
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.opts.get('clone'):
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.opts.get('axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
def __init__(self, axondir, **opts):
s_config.Config.__init__(self)
self.inprog = {}
self.axondir = s_common.gendir(axondir)
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.axonbus = None
self.setConfOpts(opts)
self.iden = self.getConfOpt('axon:iden')
self.tags = self.getConfOpt('axon:tags')
# if we're a clone, we're read-only and have no clones
if self.getConfOpt('axon:clone'):
self.setConfOpt('axon:ro', 1)
self.setConfOpt('axon:clones', 0)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
self.core.setConfOpt('modules',
(('synapse.models.axon.AxonMod', {}), ))
self.core.setConfOpt('caching', 1)
self._fs_mkdir_root() # create the fs root
self.flock = threading.Lock()
fd = s_common.genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.getConfOpt('axon:listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.getConfOpts())
self.on('syn:conf:set', self._onSetConfigableValu)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('splice', self.core.splice)
self.syncact.act('heap:sync', self.heap.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('splice', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.getConfOpt('axon:clone'):
dirname = s_common.gendir(axondir, 'sync')
syncopts = self.getConfOpt('axon:syncopts')
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.getConfOpt('axon:axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
self.onfini(self.axonbus.fini)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()