def test_cortex_meta_query_perm(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) def newp(event): event[1]['allow'] = False meta.on('meta:query:rows', newp) meta.on('meta:query:join', newp) id1 = guidstr() id2 = guidstr() rows = [ (id1, 'foo', 10, 10), (id1, 'haha', 10, 10), (id2, 'foo', 20, 20), (id2, 'haha', 20, 20), ] meta.addMetaRows('foo.bar', rows) rows = meta.getRowsByQuery('foo:foo') self.assertEqual(len(rows), 0) rows = meta.getJoinByQuery('foo:foo') self.assertEqual(len(rows), 0)
def test_cortex_meta_del(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.delCortex('foo.bar') self.assertIsNone(meta.getCortex('foo.bar')) meta.fini()
def test_cortex_meta_query_parser(self): meta = s_cortex.MetaCortex() qinfo = meta._parseQuery('foo:bar') self.assertEqual(qinfo.get('tag'), 'foo') self.assertEqual(qinfo.get('prop'), 'bar') qinfo = meta._parseQuery('foo:bar=30') self.assertEqual(qinfo.get('tag'), 'foo') self.assertEqual(qinfo.get('prop'), 'bar') self.assertEqual(qinfo.get('valu'), 30) qinfo = meta._parseQuery('foo:bar@2015,2016=30') self.assertEqual(qinfo.get('tag'), 'foo') self.assertEqual(qinfo.get('prop'), 'bar') self.assertEqual(qinfo.get('valu'), 30) #self.assertEqual( qinfo.get('mintime'), 30 ) #self.assertEqual( qinfo.get('maxtime'), 30 ) qinfo = meta._parseQuery('foo:bar#100="hehe"') self.assertEqual(qinfo.get('tag'), 'foo') self.assertEqual(qinfo.get('prop'), 'bar') self.assertEqual(qinfo.get('valu'), 'hehe') self.assertEqual(qinfo.get('limit'), 100) qinfo = meta._parseQuery('foo:bar#100*range=(10,30)') self.assertEqual(qinfo.get('by'), 'range') self.assertEqual(qinfo.get('tag'), 'foo') self.assertEqual(qinfo.get('prop'), 'bar') self.assertEqual(qinfo.get('valu'), (10, 30)) self.assertEqual(qinfo.get('limit'), 100) meta.fini()
def test_cortex_meta_query_event(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) def metaqueryrows(event): query = event[1].get('query') query['limit'] = 1 def metaqueryjoin(event): query = event[1].get('query') query['valu'] = 99 meta.on('meta:query:rows', metaqueryrows) meta.on('meta:query:join', metaqueryjoin) id1 = guidstr() id2 = guidstr() rows = [ (id1, 'foo', 10, 10), (id1, 'haha', 10, 10), (id2, 'foo', 20, 20), (id2, 'haha', 20, 20), ] meta.addMetaRows('foo.bar', rows) rows = meta.getRowsByQuery('foo:foo') self.assertEqual(len(rows), 1) rows = meta.getJoinByQuery('foo:foo') self.assertEqual(len(rows), 0)
def test_cortex_meta_getnames(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) names = meta.getCortexNames() names.sort() self.assertEqual(('foo.bar', 'foo.baz'), tuple(names)) meta.fini()
def test_cortex_meta(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) self.assertIsNotNone(meta.getCortex('foo.bar')) self.assertIsNotNone(meta.getCortex('foo.baz')) self.assertEqual(len(meta.getCortexes('woot')), 2) self.assertEqual(len(meta.getCortexes('woot.hoho')), 1) self.assertEqual(len(meta.getCortexes('woot.hehe')), 1) meta.fini()
def test_cortex_meta_addmeta(self): id1 = guidstr() meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) rows = ( (id1, 'woot', 20, 80), (id1, 'foo', 'bar', 80), ) meta.addMetaRows('foo.bar', rows) self.assertEqual(meta.getSizeByQuery('foo:woot'), 1) meta.fini()
def newp_cortex_meta_corapi(self): id1 = guid() meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) rows = ( (id1, 'woot', 20, 80), (id1, 'foo', 'bar', 80), ) meta.addMetaRows('foo.bar', rows) self.assertEqual(meta.callCorApi('foo.bar', 'getSizeByProp', 'woot'), 1) meta.fini()
def test_cortex_notok(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hehe', )) id1 = guidstr() meta.addMetaRows('foo.bar', ( (id1, 'foo', 10, 10), (id1, 'haha', 10, 10), )) id2 = guidstr() meta.addMetaRows('foo.baz', ( (id2, 'foo', 20, 20), (id2, 'haha', 20, 20), )) core = meta.getCortex('foo.bar') oldmeth = core.getRowsByProp def dorked(*args, **kwargs): raise Exception('hi') core.isok = False core.getRowsByProp = dorked rows = meta.getRowsByQuery('foo:foo') self.assertEqual(len(rows), 1) self.assertFalse(meta.coreok.get('foo.bar')) core.isok = True core.getRowsByProp = oldmeth meta._tryCoreOk('foo.bar') self.assertTrue(meta.coreok.get('foo.bar')) rows = meta.getRowsByQuery('foo:foo') self.assertEqual(len(rows), 2)
def test_cortex_meta_badname(self): meta = s_cortex.MetaCortex() self.assertRaises(s_cortex.InvalidParam, meta.addCortex, 30, 'ram:///')
def test_cortex_meta_noname(self): meta = s_cortex.MetaCortex() self.assertRaises(s_cortex.NoSuchName, meta.addMetaRows, 'hehe', []) meta.fini()
def test_cortex_meta_query(self): meta = s_cortex.MetaCortex() meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', )) meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', )) core0 = meta.getCortex('foo.bar') core1 = meta.getCortex('foo.baz') id0 = guidstr() id1 = guidstr() id2 = guidstr() id3 = guidstr() rows0 = ( (id0, 'a', 10, 99), (id1, 'x', 10, 80), (id1, 'ha', 'ho', 80), ) rows1 = ( (id2, 'c', 10, 99), (id3, 'x', 10, 80), (id3, 'ha', 'ho', 80), ) j0 = core0.addRows(rows0, async=True) j1 = core1.addRows(rows1, async=True) self.assertTrue(core0.waitForJob(j0, timeout=3)) self.assertTrue(core1.waitForJob(j1, timeout=3)) tufos = meta.getTufosByQuery('foo:x=10') self.assertEqual(len(tufos), 2) tfdict = dict(tufos) self.assertEqual(tfdict.get(id1).get('ha'), 'ho') self.assertEqual(tfdict.get(id3).get('ha'), 'ho') res = meta.getRowsByQuery('foo.bar:ha="ho"') self.assertEqual(len(res), 1) res = meta.getRowsByQuery('foo:ha') size = meta.getSizeByQuery('foo:ha') self.assertEqual(size, 2) self.assertEqual(len(res), 2) res = meta.getRowsByQuery('foo:x=10') size = meta.getSizeByQuery('foo:x=10') self.assertEqual(size, 2) self.assertEqual(len(res), 2) res = meta.getRowsByQuery('foo.baz:x=10') size = meta.getSizeByQuery('foo.baz:x=10') self.assertEqual(size, 1) self.assertEqual(len(res), 1) res = meta.getRowsByQuery('foo:x*range=(8,40)') size = meta.getSizeByQuery('foo:x*range=(8,40)') self.assertEqual(size, 2) self.assertEqual(len(res), 2) res = meta.getJoinByQuery('foo:x=10') self.assertEqual(len(res), 4) size = meta.getSizeByQuery('newp:newp=3') self.assertEqual(size, 0) rows = meta.getRowsByQuery('newp:newp=3') self.assertEqual(len(rows), 0) join = meta.getJoinByQuery('newp:newp=3') self.assertEqual(len(join), 0) meta.fini()
def __init__(self, statefd=None): self.sched = s_threads.Sched() self.asyncpool = s_async.AsyncBoss() self.peerbus = s_eventbus.EventBus() self.peersocks = {} # sockets directly connected to peers self.peergraph = collections.defaultdict(set) self.routecache = {} # persistant data in these... self.neuinfo = { 'poolsize': 10, # default to 10 neuron threads } self.neucores = {} self.neusocks = {} # <guid> : NeuSock() self.neushares = {} self.runinfo = collections.defaultdict(dict) self.peerinfo = collections.defaultdict(dict) self.metacore = s_cortex.MetaCortex() s_daemon.Daemon.__init__(self) self.onfini(self.sched.fini) self.onfini(self.peerbus.fini) self.onfini(self.asyncpool.fini) self.on('link:sock:init', self._onNeuSockInit) self.on('neu:peer:init', self._onNeuPeerInit) self.on('neu:peer:fini', self._onNeuPeerFini) # application layer peer messages... self.peerbus.on('ping', self._onPeerBusPing) self.peerbus.on('pong', self._onPeerBusPong) self.setMesgMethod('neu:data', self._onMesgNeuData) self.setMesgMethod('neu:link:syn', self._onMesgNeuLinkSyn) self.setMesgMethod('neu:link:synack', self._onMesgNeuLinkSynAck) self.setMesgMethod('neu:link:ack', self._onMesgNeuLinkAck) self.setMesgMethod('neu:peer:link:up', self._onMesgNeuPeerLinkUp) self.setMesgMethod('neu:peer:link:down', self._onMesgNeuPeerLinkDown) def setrsakey(event): valu = event[1].get('valu') self.rsakey = RSA.importKey(valu) self.pubkey = self.rsakey.publickey() self.pkcs15 = PKCS15.PKCS115_SigScheme(self.rsakey) def setauthmod(event): valu = event[1].get('valu') if valu == None: self.setAuthModule(None) return name, args, kwargs = valu auth = s_dyndeps.getDynLocal(name)(*args, **kwargs) self.setAuthModule(auth) def setpoolsize(event): valu = event[1].get('valu') self.asyncpool.setPoolSize(valu) self.on('neu:info:rsakey', setrsakey) self.on('neu:info:authmod', setauthmod) self.on('neu:info:poolsize', setpoolsize) if statefd: self.loadStateFd(statefd) #self._loadNeuAuthModule() self._loadNeuMetaCortex() #self._loadNeuSharedObjects() # check if we're brand new... if self.neuinfo.get('ident') == None: # making a shiney new neuron! self.setNeuInfo('ident', guid()) self.ident = self.getNeuInfo('ident') poolsize = self.getNeuInfo('poolsize') self.asyncpool.setPoolSize(poolsize)