Пример #1
0
    def test_heap_save(self):

        #self.thisHostMust(platform='linux')

        msgs = []

        fd0 = tempfile.TemporaryFile()
        heap0 = s_heap.Heap(fd0)

        heap0.on('heap:sync', msgs.append)

        off0 = heap0.alloc(8)
        off1 = heap0.alloc(8)

        # do interlaced writes
        heap0.writeoff(off0, b'asdf')
        heap0.writeoff(off1, b'hehe')

        heap0.writeoff(off0 + 4, b'qwer')
        heap0.writeoff(off1 + 4, b'haha')

        fd1 = tempfile.TemporaryFile()
        heap1 = s_heap.Heap(fd1)

        heap1.syncs(msgs)

        self.assertEqual(heap0.readoff(off0, 8), heap1.readoff(off0, 8))
        self.assertEqual(heap0.readoff(off1, 8), heap1.readoff(off1, 8))

        heap0.fini()
        heap1.fini()
Пример #2
0
    def test_heap_readiter(self):
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()

        with s_heap.Heap(fd) as heap:

            rand = os.urandom(2048)
            off = heap.alloc(2048)
            heap.writeoff(off, rand)

            blocks = [b for b in heap.readiter(off, 2048, itersize=9)]
            byts = b''.join(blocks)

            self.assertEqual(rand, byts)
Пример #3
0
    def test_heap_resize(self):

        fd = tempfile.TemporaryFile()

        with s_heap.Heap(fd) as heap:

            pagesize = heap.pagesize
            self.assertEqual(heap.size(), heap.pagesize)

            blocks = []
            while heap.size() == heap.pagesize:
                # NOTE test assumes pages are at least 1k
                blocks.append(heap.alloc(1024))

            self.assertEqual(heap.size(), heap.pagesize * 2)
Пример #4
0
    def test_heap_base(self):

        # FIXME test these on windows...
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()
        heap = s_heap.Heap(fd)

        self.eq(heap.size(), heap.pagesize)

        off0 = heap.alloc(8)
        off1 = heap.alloc(8)

        # do interlaced writes
        heap.writeoff(off0, b'asdf')
        heap.writeoff(off1, b'hehe')

        heap.writeoff(off0 + 4, b'qwer')
        heap.writeoff(off1 + 4, b'haha')

        self.assertEqual(heap.readoff(off0, 8), b'asdfqwer')
        self.assertEqual(heap.readoff(off1, 8), b'hehehaha')

        heap.fini()
Пример #5
0
    def __init__(self, axondir, **opts):
        s_eventbus.EventBus.__init__(self)

        self.inprog = {}
        self.axondir = gendir(axondir)
        self.clonedir = gendir(axondir, 'clones')

        self.clones = {}
        self.cloneinfo = {}
        self.clonehosts = set()
        self.clonelock = threading.Lock()

        self.readyclones = set()  # iden of each clone added as it comes up
        self.clonesready = threading.Event(
        )  # set once all clones are up and running

        self.opts = opts
        self.axonbus = None

        self.iden = self.opts.get('iden')
        self.tags = self.opts.get('tags', ())

        self.opts.setdefault('ro', False)
        self.opts.setdefault('clone', '')  # are we a clone?
        self.opts.setdefault('clones', 2)  # how many clones do we want?
        self.opts.setdefault('axonbus', '')  # do we have an axon svcbus?

        self.opts.setdefault('hostname', s_thishost.get('hostname'))

        self.opts.setdefault(
            'listen',
            'tcp://0.0.0.0:0/axon')  # our default "ephemeral" listener

        # if we're a clone, we're read-only and have no clones
        if self.opts.get('clone'):
            self.opts['ro'] = True
            self.opts['clones'] = 0

        self.opts.setdefault('synckeep', threedays)
        self.opts.setdefault('syncsize', gigabyte * 10)

        corepath = os.path.join(self.axondir, 'axon.db')
        self.core = s_cortex.openurl('sqlite:///%s' % corepath)

        fd = genfile(axondir, 'axon.heap')

        self.link = None
        self.heap = s_heap.Heap(fd)
        self.dmon = s_daemon.Daemon()

        lisn = self.opts.get('listen')
        if lisn:
            self.link = self.dmon.listen(lisn)

        self.axfo = (self.iden, {})

        self.axthrs = set()

        self.setAxonInfo('link', self.link)
        self.setAxonInfo('opts', self.opts)

        self.dmon.share('axon', self)

        # create a reactor to unwrap core/heap sync events
        self.syncact = s_reactor.Reactor()
        self.syncact.act('heap:sync', self.heap.sync)
        self.syncact.act('core:sync', self.core.sync)

        # wrap core/heap sync events as axon:sync events
        self.core.on('core:sync', self._fireAxonSync)
        self.heap.on('heap:sync', self._fireAxonSync)

        # model details for the actual byte blobs
        self.core.addTufoForm('axon:blob', ptype='guid')
        self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
        self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)

        self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
        self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
        self.core.addTufoProp('axon:blob',
                              'sha256',
                              ptype='hash:sha256',
                              req=True)
        self.core.addTufoProp('axon:blob',
                              'sha512',
                              ptype='hash:sha512',
                              req=True)

        self.core.addTufoForm('axon:clone', ptype='guid')

        dirname = gendir(axondir, 'sync')
        syncopts = self.opts.get('syncopts', {})

        self.syncdir = None

        self.onfini(self._onAxonFini)

        self.onfini(self.core.fini)
        self.onfini(self.heap.fini)
        self.onfini(self.dmon.fini)

        # if we're not a clone, create a sync dir
        if not self.opts.get('clone'):
            self.syncdir = s_persist.Dir(dirname, **syncopts)
            self.onfini(self.syncdir.fini)

            self.on('axon:sync', self.syncdir.add)

        self.axcthr = None

        # share last to avoid startup races
        busurl = self.opts.get('axonbus')
        if busurl:
            self.axonbus = s_service.openurl(busurl)

            props = {'link': self.link, 'tags': self.tags}
            self.axonbus.runSynSvc(self.iden, self, **props)

            self.axcthr = self._fireAxonClones()
Пример #6
0
    def __init__(self, axondir, **opts):
        s_config.Config.__init__(self)

        self.inprog = {}
        self.axondir = s_common.gendir(axondir)

        self.clones = {}
        self.cloneinfo = {}
        self.clonehosts = set()
        self.clonelock = threading.Lock()

        self.readyclones = set()  # iden of each clone added as it comes up
        self.clonesready = threading.Event(
        )  # set once all clones are up and running

        self.axonbus = None

        self.setConfOpts(opts)

        self.iden = self.getConfOpt('axon:iden')
        self.tags = self.getConfOpt('axon:tags')

        # if we're a clone, we're read-only and have no clones
        if self.getConfOpt('axon:clone'):
            self.setConfOpt('axon:ro', 1)
            self.setConfOpt('axon:clones', 0)

        corepath = os.path.join(self.axondir, 'axon.db')
        self.core = s_cortex.openurl('sqlite:///%s' % corepath)
        self.core.setConfOpt('modules',
                             (('synapse.models.axon.AxonMod', {}), ))
        self.core.setConfOpt('caching', 1)

        self._fs_mkdir_root()  # create the fs root
        self.flock = threading.Lock()

        fd = s_common.genfile(axondir, 'axon.heap')

        self.link = None
        self.heap = s_heap.Heap(fd)
        self.dmon = s_daemon.Daemon()

        lisn = self.getConfOpt('axon:listen')
        if lisn:
            self.link = self.dmon.listen(lisn)

        self.axfo = (self.iden, {})

        self.axthrs = set()

        self.setAxonInfo('link', self.link)
        self.setAxonInfo('opts', self.getConfOpts())
        self.on('syn:conf:set', self._onSetConfigableValu)

        self.dmon.share('axon', self)

        # create a reactor to unwrap core/heap sync events
        self.syncact = s_reactor.Reactor()
        self.syncact.act('splice', self.core.splice)
        self.syncact.act('heap:sync', self.heap.sync)

        # wrap core/heap sync events as axon:sync events
        self.core.on('splice', self._fireAxonSync)
        self.heap.on('heap:sync', self._fireAxonSync)

        self.syncdir = None

        self.onfini(self._onAxonFini)

        self.onfini(self.core.fini)
        self.onfini(self.heap.fini)
        self.onfini(self.dmon.fini)

        # if we're not a clone, create a sync dir
        if not self.getConfOpt('axon:clone'):
            dirname = s_common.gendir(axondir, 'sync')
            syncopts = self.getConfOpt('axon:syncopts')
            self.syncdir = s_persist.Dir(dirname, **syncopts)
            self.onfini(self.syncdir.fini)

            self.on('axon:sync', self.syncdir.add)

        self.axcthr = None

        # share last to avoid startup races
        busurl = self.getConfOpt('axon:axonbus')
        if busurl:
            self.axonbus = s_service.openurl(busurl)
            self.onfini(self.axonbus.fini)

            props = {'link': self.link, 'tags': self.tags}
            self.axonbus.runSynSvc(self.iden, self, **props)
            self.axcthr = self._fireAxonClones()