Exemplo n.º 1
0
    def test_service_byname(self):
        sbus = s_service.SvcBus()

        woot0 = Woot()

        with s_daemon.Daemon() as dmon:

            dmon.share('syn.svcbus', sbus, fini=True)

            link = dmon.listen('tcp://127.0.0.1:0/')

            port = link[1].get('port')

            with s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port) as prox:

                iden = prox.runSynSvc('foo0', woot0)

                self.eq(prox.callByName('foo0', gentask('foo', 20)), 30)
Exemplo n.º 2
0
    def __init__(self, datadir, **opts):
        s_eventbus.EventBus.__init__(self)

        self.datadir = gendir(datadir)

        self.opts = opts
        self.lock = threading.Lock()

        self.axons = {}
        self.axonbus = None
        self.axonforks = {}

        self.onfini(self._onAxonHostFini)

        self.opts.setdefault('autorun', 0)  # how many axons to auto-start
        self.opts.setdefault('axonbus', '')  # url to axonbus

        self.opts.setdefault('bytemax',
                             terabyte)  # by default make each Axon 1 Terabyte
        self.opts.setdefault('syncmax', gigabyte * 10)  #

        self.opts.setdefault(
            'hostname',
            s_thishost.get('hostname'))  # allow override for testing

        url = self.opts.get('axonbus')
        if url:
            self.axonbus = s_service.openurl(url)
            self.axonbus.runSynSvc(guid(), self)

        for name in os.listdir(self.datadir):

            if not name.endswith('.axon'):
                continue

            iden, _ = name.split('.', 1)

            self._fireAxonIden(iden)

        # fire auto-run axons
        auto = self.opts.get('autorun')
        while len(self.axons) < auto:
            self.add()
Exemplo n.º 3
0
    def test_service_byname(self):
        sbus = s_service.SvcBus()

        woot0 = Woot()

        dmon = s_daemon.Daemon()
        dmon.share('syn.svcbus', sbus, fini=True)

        link = dmon.listen('tcp://127.0.0.1:0/')

        port = link[1].get('port')

        prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)

        iden = prox.runSynSvc('foo0', woot0)

        self.assertEqual( prox.callByName('foo0', 'foo',20), 30 )

        dmon.fini()
Exemplo n.º 4
0
    def test_service_getTagProxy(self):

        woot0 = Woot()

        with s_daemon.Daemon() as dmon:

            sbus = s_service.SvcBus()
            dmon.share('syn.svcbus', sbus, fini=True)

            link = dmon.listen('tcp://127.0.0.1:0/')

            port = link[1].get('port')

            with s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port) as prox:

                prox.runSynSvc('foo0', woot0, tags=['bar0'])

                tagprox = prox.getTagProxy('bar0')
                self.eq(next(tagprox.foo(20), None), 30)
Exemplo n.º 5
0
    def test_service_getNameProxy(self):

        woot0 = Woot()

        with s_daemon.Daemon() as dmon:

            sbus = s_service.SvcBus()
            dmon.share('syn.svcbus', sbus, fini=True)

            link = dmon.listen('tcp://127.0.0.1:0/')

            port = link[1].get('port')

            with s_service.openurl('tcp://127.0.0.1/syn.svcbus',
                                   port=port) as prox:

                prox.runSynSvc('foo0', woot0)

                nameprox = prox.getNameProxy('foo0')
                self.assertEqual(nameprox.foo(20), 30)
Exemplo n.º 6
0
    def test_service_getTagProxy(self):
        sbus = s_service.SvcBus()

        woot0 = Woot()

        dmon = s_daemon.Daemon()
        dmon.share('syn.svcbus', sbus, fini=True)

        link = dmon.listen('tcp://127.0.0.1:0/')

        port = link[1].get('port')

        prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)

        prox.runSynSvc('foo0', woot0, tags=['bar0'])

        tagprox = prox.getTagProxy('bar0')
        self.assertEqual(next(tagprox.foo(20), None), 30)

        dmon.fini()
Exemplo n.º 7
0
    def __init__(self, datadir, **opts):
        s_config.Config.__init__(self)

        self.datadir = s_common.gendir(datadir)

        self.lock = threading.Lock()

        self.axons = {}  # iden -> Axon mapping.
        self.axonbus = None
        self.axonforks = {}
        self.cloneaxons = []  # List of idens which are clones.

        self.onfini(self._onAxonHostFini)

        self.setConfOpts(opts)

        self._axonconfs = [_name for _name, _ in Axon._axon_confdefs()]

        # track the total number of bytes which may be used by axons for startup operations
        self.usedspace = 0

        for name in os.listdir(self.datadir):

            if not name.endswith('.axon'):
                continue

            iden, _ = name.split('.', 1)

            self._fireAxonIden(iden)

        # fire auto-run axons
        auto = self.getConfOpt('axonhost:autorun')
        while (len(self.axons) - len(self.cloneaxons)) < auto:
            self.add()

        url = self.getConfOpt('axon:axonbus')
        if url:
            self.axonbus = s_service.openurl(url)
            self.onfini(self.axonbus.fini)
            self.axonbus.runSynSvc(s_common.guid(), self)
Exemplo n.º 8
0
    def __init__(self, datadir, **opts):
        s_eventbus.EventBus.__init__(self)

        self.datadir = gendir(datadir)

        self.opts = opts
        self.lock = threading.Lock()

        self.axons = {}
        self.axonbus = None
        self.axonforks = {}

        self.onfini( self._onAxonHostFini )

        self.opts.setdefault('autorun',0)               # how many axons to auto-start
        self.opts.setdefault('axonbus','')              # url to axonbus

        self.opts.setdefault('bytemax',terabyte)        # by default make each Axon 1 Terabyte
        self.opts.setdefault('syncmax',gigabyte * 10)   # 

        self.opts.setdefault('hostname', s_thishost.get('hostname') ) # allow override for testing

        url = self.opts.get('axonbus')
        if url:
            self.axonbus = s_service.openurl(url)
            self.axonbus.runSynSvc(guid(),self)

        for name in os.listdir(self.datadir):

            if not name.endswith('.axon'):
                continue

            iden,_ = name.split('.',1)

            self._fireAxonIden(iden)

        # fire auto-run axons
        auto = self.opts.get('autorun')
        while len(self.axons) < auto:
            self.add()
Exemplo n.º 9
0
    def test_service_proxysugar(self):
        sbus = s_service.SvcBus()

        woot0 = Woot()

        dmon = s_daemon.Daemon()
        dmon.share('syn.svcbus', sbus, fini=True)

        link = dmon.listen('tcp://127.0.0.1:0/')

        port = link[1].get('port')

        prox = s_service.openurl('tcp://127.0.0.1/syn.svcbus', port=port)

        iden = prox.runSynSvc('foo0', woot0, tags=('hehe.haha',))

        res = list( prox['foo0'].foo(90) )

        self.assertEqual( len(res), 1 )
        self.assertEqual( res[0], 100 )

        dmon.fini()
Exemplo n.º 10
0
    def __init__(self, axondir, **opts):
        s_eventbus.EventBus.__init__(self)

        self.inprog = {}
        self.axondir = gendir(axondir)
        self.clonedir = gendir(axondir, 'clones')

        self.clones = {}
        self.cloneinfo = {}
        self.clonehosts = set()
        self.clonelock = threading.Lock()

        self.readyclones = set()  # iden of each clone added as it comes up
        self.clonesready = threading.Event(
        )  # set once all clones are up and running

        self.opts = opts
        self.axonbus = None

        self.iden = self.opts.get('iden')
        self.tags = self.opts.get('tags', ())

        self.opts.setdefault('ro', False)
        self.opts.setdefault('clone', '')  # are we a clone?
        self.opts.setdefault('clones', 2)  # how many clones do we want?
        self.opts.setdefault('axonbus', '')  # do we have an axon svcbus?

        self.opts.setdefault('hostname', s_thishost.get('hostname'))

        self.opts.setdefault(
            'listen',
            'tcp://0.0.0.0:0/axon')  # our default "ephemeral" listener

        # if we're a clone, we're read-only and have no clones
        if self.opts.get('clone'):
            self.opts['ro'] = True
            self.opts['clones'] = 0

        self.opts.setdefault('synckeep', threedays)
        self.opts.setdefault('syncsize', gigabyte * 10)

        corepath = os.path.join(self.axondir, 'axon.db')
        self.core = s_cortex.openurl('sqlite:///%s' % corepath)

        fd = genfile(axondir, 'axon.heap')

        self.link = None
        self.heap = s_heap.Heap(fd)
        self.dmon = s_daemon.Daemon()

        lisn = self.opts.get('listen')
        if lisn:
            self.link = self.dmon.listen(lisn)

        self.axfo = (self.iden, {})

        self.axthrs = set()

        self.setAxonInfo('link', self.link)
        self.setAxonInfo('opts', self.opts)

        self.dmon.share('axon', self)

        # create a reactor to unwrap core/heap sync events
        self.syncact = s_reactor.Reactor()
        self.syncact.act('heap:sync', self.heap.sync)
        self.syncact.act('core:sync', self.core.sync)

        # wrap core/heap sync events as axon:sync events
        self.core.on('core:sync', self._fireAxonSync)
        self.heap.on('heap:sync', self._fireAxonSync)

        # model details for the actual byte blobs
        self.core.addTufoForm('axon:blob', ptype='guid')
        self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
        self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)

        self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
        self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
        self.core.addTufoProp('axon:blob',
                              'sha256',
                              ptype='hash:sha256',
                              req=True)
        self.core.addTufoProp('axon:blob',
                              'sha512',
                              ptype='hash:sha512',
                              req=True)

        self.core.addTufoForm('axon:clone', ptype='guid')

        dirname = gendir(axondir, 'sync')
        syncopts = self.opts.get('syncopts', {})

        self.syncdir = None

        self.onfini(self._onAxonFini)

        self.onfini(self.core.fini)
        self.onfini(self.heap.fini)
        self.onfini(self.dmon.fini)

        # if we're not a clone, create a sync dir
        if not self.opts.get('clone'):
            self.syncdir = s_persist.Dir(dirname, **syncopts)
            self.onfini(self.syncdir.fini)

            self.on('axon:sync', self.syncdir.add)

        self.axcthr = None

        # share last to avoid startup races
        busurl = self.opts.get('axonbus')
        if busurl:
            self.axonbus = s_service.openurl(busurl)

            props = {'link': self.link, 'tags': self.tags}
            self.axonbus.runSynSvc(self.iden, self, **props)

            self.axcthr = self._fireAxonClones()
Exemplo n.º 11
0
    def __init__(self, axondir, **opts):
        s_config.Config.__init__(self)

        self.inprog = {}
        self.axondir = s_common.gendir(axondir)

        self.clones = {}
        self.cloneinfo = {}
        self.clonehosts = set()
        self.clonelock = threading.Lock()

        self.readyclones = set()  # iden of each clone added as it comes up
        self.clonesready = threading.Event(
        )  # set once all clones are up and running

        self.axonbus = None

        self.setConfOpts(opts)

        self.iden = self.getConfOpt('axon:iden')
        self.tags = self.getConfOpt('axon:tags')

        # if we're a clone, we're read-only and have no clones
        if self.getConfOpt('axon:clone'):
            self.setConfOpt('axon:ro', 1)
            self.setConfOpt('axon:clones', 0)

        corepath = os.path.join(self.axondir, 'axon.db')
        self.core = s_cortex.openurl('sqlite:///%s' % corepath)
        self.core.setConfOpt('modules',
                             (('synapse.models.axon.AxonMod', {}), ))
        self.core.setConfOpt('caching', 1)

        self._fs_mkdir_root()  # create the fs root
        self.flock = threading.Lock()

        fd = s_common.genfile(axondir, 'axon.heap')

        self.link = None
        self.heap = s_heap.Heap(fd)
        self.dmon = s_daemon.Daemon()

        lisn = self.getConfOpt('axon:listen')
        if lisn:
            self.link = self.dmon.listen(lisn)

        self.axfo = (self.iden, {})

        self.axthrs = set()

        self.setAxonInfo('link', self.link)
        self.setAxonInfo('opts', self.getConfOpts())
        self.on('syn:conf:set', self._onSetConfigableValu)

        self.dmon.share('axon', self)

        # create a reactor to unwrap core/heap sync events
        self.syncact = s_reactor.Reactor()
        self.syncact.act('splice', self.core.splice)
        self.syncact.act('heap:sync', self.heap.sync)

        # wrap core/heap sync events as axon:sync events
        self.core.on('splice', self._fireAxonSync)
        self.heap.on('heap:sync', self._fireAxonSync)

        self.syncdir = None

        self.onfini(self._onAxonFini)

        self.onfini(self.core.fini)
        self.onfini(self.heap.fini)
        self.onfini(self.dmon.fini)

        # if we're not a clone, create a sync dir
        if not self.getConfOpt('axon:clone'):
            dirname = s_common.gendir(axondir, 'sync')
            syncopts = self.getConfOpt('axon:syncopts')
            self.syncdir = s_persist.Dir(dirname, **syncopts)
            self.onfini(self.syncdir.fini)

            self.on('axon:sync', self.syncdir.add)

        self.axcthr = None

        # share last to avoid startup races
        busurl = self.getConfOpt('axon:axonbus')
        if busurl:
            self.axonbus = s_service.openurl(busurl)
            self.onfini(self.axonbus.fini)

            props = {'link': self.link, 'tags': self.tags}
            self.axonbus.runSynSvc(self.iden, self, **props)
            self.axcthr = self._fireAxonClones()
Exemplo n.º 12
0
    def test_axon_cluster_cortex(self):
        self.skipLongTest()
        self.thisHostMustNot(platform='windows')

        localguid = guid()
        busurl = 'local://%s/axons' % localguid
        hahaurl = 'local://%s/haha' % localguid

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)
        dmon.share('haha', {})

        svcprox = s_service.openurl(busurl)

        axcluster = s_axon.AxonCluster(svcprox)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })
            host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })
            host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })

            props = {
                'axon:clones': 1,
                'axon:bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            axcluster._waitWrAxons(1, 4)

            # Ensure our axfo0 was cloned to someone in the cluster
            axon0 = s_telepath.openlink(axfo0[1].get('link'))  # type: s_axon.Axon
            axon0._waitClonesReady(timeout=16)
            foundclone = False
            if host1.cloneaxons:
                foundclone = True
            if host2.cloneaxons:
                foundclone = True
            self.true(foundclone)
            axon0.fini()  # fini the proxy object

            core = s_cortex.openurl('ram://')
            core.setConfOpt('axon:url', busurl)

            self.false(axcluster.has('md5', craphash))
            self.false(axcluster.has('md5', asdfhash))

            node = core.formNodeByBytes(b'asdfasdf', name='asdf')
            self.eq(node[1].get('file:bytes'), asdfhash_iden)
            self.eq(node[1].get('file:bytes:md5'), asdfhash)

            self.true(axcluster.has('md5', asdfhash))
            self.true(axcluster.has('guid', asdfhash_iden))

            fd = io.BytesIO(b'visi')
            node = core.formNodeByFd(fd, name='visi.bin')
            self.eq(node[1].get('file:bytes:size'), 4)
            self.eq(node[1].get('file:bytes:name'), 'visi.bin')
            self.eq(node[1].get('file:bytes'), '442f602ecf8230b2a59a44b4f845be27')
            self.eq(node[1].get('file:bytes:md5'), '1b2e93225959e3722efed95e1731b764')

            self.true(axcluster.has('md5', '1b2e93225959e3722efed95e1731b764'))
            self.true(axcluster.has('guid', '442f602ecf8230b2a59a44b4f845be27'))

            host0.fini()
            host1.fini()
            host2.fini()

        svcprox.fini()
        dmon.fini()
Exemplo n.º 13
0
    def test_axon_cluster(self):
        self.skipLongTest()
        self.thisHostMustNot(platform='windows')

        busurl = 'local://%s/axons' % guid()

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)
        svcprox = s_service.openurl(busurl)

        axcluster = s_axon.AxonCluster(svcprox)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })
            host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })
            host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
                                             'axon:axonbus': busurl,
                                             'axon:bytemax': s_axon.megabyte * 100,
                                             })

            props = {
                'axon:clones': 1,
                'axon:bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            axcluster._waitWrAxons(1, 4)

            # Ensure our axfo0 was cloned to someone in the cluster
            axon0 = s_telepath.openlink(axfo0[1].get('link'))  # type: s_axon.Axon
            axon0._waitClonesReady(timeout=16)
            foundclone = False
            if host1.cloneaxons:
                foundclone = True
            if host2.cloneaxons:
                foundclone = True
            self.true(foundclone)
            axon0.fini()  # fini the proxy object

            self.false(axcluster.has('md5', craphash))
            self.false(axcluster.has('md5', asdfhash))

            buf = b'asdfasdf'
            iden = axcluster.alloc(len(buf))
            self.nn(axcluster.chunk(iden, buf))

            self.false(axcluster.has('md5', craphash))
            self.true(axcluster.has('md5', asdfhash))
            self.true(axcluster.has('guid', asdfhash_iden))

            blobs = axcluster.find('md5', craphash)
            self.eq(len(blobs), 0)

            time.sleep(0.2)  # Yield to axon threads
            blobs = axcluster.find('md5', asdfhash)
            # We have two blobs for the same hash since the clone of axfo0 is up on host1/host2
            self.eq(len(blobs), 2)

            blob = blobs[0]
            byts = b''.join(axcluster.iterblob(blob))
            self.eq(byts, buf)

            blob[1].pop('.axon')
            byts = b''.join(axcluster.iterblob(blob))
            self.eq(byts, buf)

            self.nn(axcluster.wants('md5', craphash, len(buf)))
            self.none(axcluster.wants('md5', asdfhash, len(buf)))

            # Eat bytes via AxonMixin APIs
            byts = 'pennywise'.encode()

            blob = axcluster.eatbytes(byts)
            self.nn(blob)
            self.isin('.new', blob[1])
            blob = axcluster.eatbytes(byts)
            self.notin('.new', blob[1])

            buf = io.BytesIO('dancing clown'.encode())
            blob = axcluster.eatfd(buf)
            self.nn(blob)
            self.isin('.new', blob[1])
            blob = axcluster.eatfd(buf)
            self.notin('.new', blob[1])

            host0.fini()
            host1.fini()
            host2.fini()

        svcprox.fini()
        dmon.fini()
Exemplo n.º 14
0
    def test_axon_host_spinbackup(self):
        self.skipLongTest()
        self.thisHostMustNot(platform='windows')

        hstcfg = {
            "vars": {
                "hcfg0": {
                    "axon:hostname": "host0",
                    "axon:bytemax": 1024000000,
                    "axonhost:maxsize": 10240000000,
                    "axonhost:autorun": 1,
                    "axon:clones": 1,
                },
                "hcfg1": {
                    "axon:hostname": "host1",
                    "axon:bytemax": 1024000000,
                    "axonhost:maxsize": 10240000000,
                    "axonhost:autorun": 1,
                    "axon:clones": 1,
                }
            },
            "ctors": [
                [
                    "host0",
                    "ctor://synapse.axon.AxonHost(dir0, **hcfg0)"
                ],
                [
                    "host1",
                    "ctor://synapse.axon.AxonHost(dir1, **hcfg1)"
                ]
            ],
            "share": [
                [
                    "host0",
                    {
                        "onfini": True
                    }
                ],
                [
                    "host1",
                    {
                        "onfini": True
                    }
                ]
            ]
        }

        svccfg = {
            "comment": "dmon file for axon stresstest",
            "ctors": [
                [
                    "axonbus",
                    "ctor://synapse.lib.service.SvcBus()",
                    {}
                ]
            ],
            "share": [
                [
                    "axonbus",
                    {
                        "onfini": True
                    }
                ]
            ]
        }

        with self.getTestDir() as dirname:
            hostdir0 = gendir(dirname, 'host0')
            hostdir1 = gendir(dirname, 'host1')
            hstcfg['vars']['dir0'] = hostdir0
            hstcfg['vars']['dir1'] = hostdir1

            with s_daemon.Daemon() as svcdmon:
                svcdmon.loadDmonConf(svccfg)
                link = svcdmon.listen('tcp://127.0.0.1:0/')
                port = link[1].get('port')
                busurl = 'tcp://127.0.0.1:{}/axonbus'.format(port)
                hstcfg['vars']['hcfg0']['axon:axonbus'] = busurl
                hstcfg['vars']['hcfg1']['axon:axonbus'] = busurl
                svcbus = s_service.openurl('tcp://127.0.0.1:0/axonbus', port=port)  # type: s_service.SvcProxy

                with s_daemon.Daemon() as axondmon:
                    w = svcbus.waiter(6, 'syn:svc:init')
                    axondmon.loadDmonConf(hstcfg)
                    w.wait(15)
                    first_axons = svcbus.getSynSvcsByTag(s_axon.axontag)
                    self.eq(len(first_axons), 4)

                # Close the proxy
                svcbus.fini()

            # Spin the AxonHost back up
            # This does exercise a behavior in the AxonHost to always give
            # preference for its :axonbus configuration option over that of
            # its Axon's. While this scenario is present in the unit test,
            # in a real migration which could involve changing the :axonbus,
            # this ensures that the children Axons of the AxonHost are updated
            # to point to the new bus.

            with s_daemon.Daemon() as svcdmon:
                svcdmon.loadDmonConf(svccfg)
                link = svcdmon.listen('tcp://127.0.0.1:0/')
                port = link[1].get('port')
                busurl = 'tcp://127.0.0.1:{}/axonbus'.format(port)
                hstcfg['vars']['hcfg0']['axon:axonbus'] = busurl
                hstcfg['vars']['hcfg1']['axon:axonbus'] = busurl
                svcbus = s_service.openurl('tcp://127.0.0.1:0/axonbus', port=port)  # type: s_service.SvcProxy

                with s_daemon.Daemon() as axondmon:
                    w = svcbus.waiter(6, 'syn:svc:init')
                    axondmon.loadDmonConf(hstcfg)
                    w.wait(15)
                    axons = svcbus.getSynSvcsByTag(s_axon.axontag)
                    self.eq(len(axons), 4)
                    # Ensure these are the same axons we had first created
                    self.eq({axn[1].get('name') for axn in axons}, {axn[1].get('name') for axn in first_axons})

                # Close the proxy
                svcbus.fini()
Exemplo n.º 15
0
    def __init__(self, axondir, **opts):
        s_eventbus.EventBus.__init__(self)

        self.inprog = {}
        self.axondir = gendir(axondir)
        self.clonedir = gendir(axondir,'clones')

        self.clones = {}
        self.cloneinfo = {}
        self.clonehosts = set()
        self.clonelock = threading.Lock()

        self.readyclones = set()                # iden of each clone added as it comes up
        self.clonesready = threading.Event()    # set once all clones are up and running

        self.opts = opts
        self.axonbus = None

        self.iden = self.opts.get('iden')
        self.tags = self.opts.get('tags',())

        self.opts.setdefault('ro',False)
        self.opts.setdefault('clone','')   # are we a clone?
        self.opts.setdefault('clones',2)   # how many clones do we want?
        self.opts.setdefault('axonbus','')  # do we have an axon svcbus?

        self.opts.setdefault('hostname', s_thishost.get('hostname') )

        self.opts.setdefault('listen','tcp://0.0.0.0:0/axon') # our default "ephemeral" listener

        # if we're a clone, we're read-only and have no clones
        if self.opts.get('clone'):
            self.opts['ro'] = True
            self.opts['clones'] = 0

        self.opts.setdefault('synckeep',threedays)
        self.opts.setdefault('syncsize',gigabyte*10)

        self.core = opts.get('core')

        if self.core == None:
            savefile = os.path.join(self.axondir,'axon.save')
            self.core = s_cortex.openurl('ram:///',savefile=savefile)
            self.onfini( self.core.fini )

        fd = genfile(axondir,'axon.heap')

        self.link = None
        self.heap = s_heap.Heap(fd)
        self.dmon = s_daemon.Daemon()

        lisn = self.opts.get('listen')
        if lisn:
            self.link = self.dmon.listen(lisn)

        self.axfo = (self.iden,{})

        self.axthrs = set()

        self.setAxonInfo('link',self.link)
        self.setAxonInfo('opts',self.opts)

        self.dmon.share('axon',self)

        # create a reactor to unwrap core/heap sync events
        self.syncact = s_reactor.Reactor()
        self.syncact.act('heap:sync', self.heap.sync )
        self.syncact.act('core:sync', self.core.sync )

        # wrap core/heap sync events as axon:sync events
        self.core.on('core:sync', self._fireAxonSync )
        self.heap.on('heap:sync', self._fireAxonSync )

        # model details for the actual byte blobs
        self.core.addTufoForm('axon:blob',ptype='guid')
        self.core.addTufoProp('axon:blob','ver',ptype='int',defval=1)
        self.core.addTufoProp('axon:blob','off', ptype='int',req=True)
        self.core.addTufoProp('axon:blob','size', ptype='int',req=True)

        self.core.addTufoProp('axon:blob','md5', ptype='hash:md5',req=True)
        self.core.addTufoProp('axon:blob','sha1', ptype='hash:sha1',req=True)
        self.core.addTufoProp('axon:blob','sha256', ptype='hash:sha256',req=True)
        self.core.addTufoProp('axon:blob','sha512', ptype='hash:sha512',req=True)

        self.core.addTufoForm('axon:clone',ptype='guid')

        dirname = gendir(axondir,'sync')
        syncopts = self.opts.get('syncopts',{})

        self.syncdir = None

        # if we're not a clone, create a sync dir
        if not self.opts.get('clone'):
            self.syncdir = s_persist.Dir(dirname,**syncopts)
            self.onfini( self.syncdir.fini )

            self.on('axon:sync', self.syncdir.add )

        self.onfini( self._onAxonFini )
        self.onfini( self.core.fini )
        self.onfini( self.heap.fini )
        self.onfini( self.dmon.fini )

        # share last to avoid startup races
        busurl = self.opts.get('axonbus')
        if busurl:
            self.axonbus = s_service.openurl(busurl)

            props = {'link':self.link,'tags':self.tags}
            self.axonbus.runSynSvc(self.iden,self,**props)

            self._fireAxonClones()
Exemplo n.º 16
0
    def test_axon_cluster(self):

        self.thisHostMustNot(platform='windows')

        busurl = 'local://%s/axons' % guid()

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)
        svcprox = s_service.openurl(busurl)

        axcluster = s_axon.AxonCluster(svcprox)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            opts = {
                'axonbus': busurl,
            }

            host0 = s_axon.AxonHost(dir0, hostname='host0', **opts)
            host1 = s_axon.AxonHost(dir1, hostname='host1', **opts)
            host2 = s_axon.AxonHost(dir2, hostname='host2', **opts)

            props = {
                'clones': 1,
                'syncmax': s_axon.megabyte,
                'bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            self.assertFalse(axcluster.has('md5', craphash))
            self.assertFalse(axcluster.has('md5', asdfhash))

            buf = b'asdfasdf'
            iden = axcluster.alloc(len(buf))
            self.assertIsNotNone(axcluster.chunk(iden, buf))

            self.assertFalse(axcluster.has('md5', craphash))
            self.assertTrue(axcluster.has('md5', asdfhash))

            blobs = axcluster.find('md5', craphash)
            self.assertEqual(len(blobs), 0)

            blobs = axcluster.find('md5', asdfhash)
            self.assertEqual(len(blobs), 1)

            blob = blobs[0]
            byts = b''.join(axcluster.iterblob(blob))
            self.assertEqual(byts, buf)

            blob[1].pop('.axon')
            byts = b''.join(axcluster.iterblob(blob))
            self.assertEqual(byts, buf)

            self.assertIsNone(axcluster.wants('md5', asdfhash, len(buf)))
            self.assertIsNotNone(axcluster.wants('md5', craphash, len(buf)))

            host0.fini()
            host1.fini()
            host2.fini()

        dmon.fini()