def test_cortex_meta_query_perm(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
def newp(event):
event[1]['allow'] = False
meta.on('meta:query:rows', newp)
meta.on('meta:query:join', newp)
id1 = guidstr()
id2 = guidstr()
rows = [
(id1, 'foo', 10, 10),
(id1, 'haha', 10, 10),
(id2, 'foo', 20, 20),
(id2, 'haha', 20, 20),
]
meta.addMetaRows('foo.bar', rows)
rows = meta.getRowsByQuery('foo:foo')
self.assertEqual(len(rows), 0)
rows = meta.getJoinByQuery('foo:foo')
self.assertEqual(len(rows), 0)
def test_cortex_meta_del(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.delCortex('foo.bar')
self.assertIsNone(meta.getCortex('foo.bar'))
meta.fini()
def test_cortex_meta_query_parser(self):
meta = s_cortex.MetaCortex()
qinfo = meta._parseQuery('foo:bar')
self.assertEqual(qinfo.get('tag'), 'foo')
self.assertEqual(qinfo.get('prop'), 'bar')
qinfo = meta._parseQuery('foo:bar=30')
self.assertEqual(qinfo.get('tag'), 'foo')
self.assertEqual(qinfo.get('prop'), 'bar')
self.assertEqual(qinfo.get('valu'), 30)
qinfo = meta._parseQuery('foo:bar@2015,2016=30')
self.assertEqual(qinfo.get('tag'), 'foo')
self.assertEqual(qinfo.get('prop'), 'bar')
self.assertEqual(qinfo.get('valu'), 30)
#self.assertEqual( qinfo.get('mintime'), 30 )
#self.assertEqual( qinfo.get('maxtime'), 30 )
qinfo = meta._parseQuery('foo:bar#100="hehe"')
self.assertEqual(qinfo.get('tag'), 'foo')
self.assertEqual(qinfo.get('prop'), 'bar')
self.assertEqual(qinfo.get('valu'), 'hehe')
self.assertEqual(qinfo.get('limit'), 100)
qinfo = meta._parseQuery('foo:bar#100*range=(10,30)')
self.assertEqual(qinfo.get('by'), 'range')
self.assertEqual(qinfo.get('tag'), 'foo')
self.assertEqual(qinfo.get('prop'), 'bar')
self.assertEqual(qinfo.get('valu'), (10, 30))
self.assertEqual(qinfo.get('limit'), 100)
meta.fini()
def test_cortex_meta_query_event(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
def metaqueryrows(event):
query = event[1].get('query')
query['limit'] = 1
def metaqueryjoin(event):
query = event[1].get('query')
query['valu'] = 99
meta.on('meta:query:rows', metaqueryrows)
meta.on('meta:query:join', metaqueryjoin)
id1 = guidstr()
id2 = guidstr()
rows = [
(id1, 'foo', 10, 10),
(id1, 'haha', 10, 10),
(id2, 'foo', 20, 20),
(id2, 'haha', 20, 20),
]
meta.addMetaRows('foo.bar', rows)
rows = meta.getRowsByQuery('foo:foo')
self.assertEqual(len(rows), 1)
rows = meta.getJoinByQuery('foo:foo')
self.assertEqual(len(rows), 0)
def test_cortex_meta_getnames(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
names = meta.getCortexNames()
names.sort()
self.assertEqual(('foo.bar', 'foo.baz'), tuple(names))
meta.fini()
def test_cortex_meta(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
self.assertIsNotNone(meta.getCortex('foo.bar'))
self.assertIsNotNone(meta.getCortex('foo.baz'))
self.assertEqual(len(meta.getCortexes('woot')), 2)
self.assertEqual(len(meta.getCortexes('woot.hoho')), 1)
self.assertEqual(len(meta.getCortexes('woot.hehe')), 1)
meta.fini()
def test_cortex_meta_addmeta(self):
id1 = guidstr()
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
rows = (
(id1, 'woot', 20, 80),
(id1, 'foo', 'bar', 80),
)
meta.addMetaRows('foo.bar', rows)
self.assertEqual(meta.getSizeByQuery('foo:woot'), 1)
meta.fini()
def newp_cortex_meta_corapi(self):
id1 = guid()
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
rows = (
(id1, 'woot', 20, 80),
(id1, 'foo', 'bar', 80),
)
meta.addMetaRows('foo.bar', rows)
self.assertEqual(meta.callCorApi('foo.bar', 'getSizeByProp', 'woot'),
1)
meta.fini()
def test_cortex_notok(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hehe', ))
id1 = guidstr()
meta.addMetaRows('foo.bar', (
(id1, 'foo', 10, 10),
(id1, 'haha', 10, 10),
))
id2 = guidstr()
meta.addMetaRows('foo.baz', (
(id2, 'foo', 20, 20),
(id2, 'haha', 20, 20),
))
core = meta.getCortex('foo.bar')
oldmeth = core.getRowsByProp
def dorked(*args, **kwargs):
raise Exception('hi')
core.isok = False
core.getRowsByProp = dorked
rows = meta.getRowsByQuery('foo:foo')
self.assertEqual(len(rows), 1)
self.assertFalse(meta.coreok.get('foo.bar'))
core.isok = True
core.getRowsByProp = oldmeth
meta._tryCoreOk('foo.bar')
self.assertTrue(meta.coreok.get('foo.bar'))
rows = meta.getRowsByQuery('foo:foo')
self.assertEqual(len(rows), 2)
def test_cortex_meta_badname(self):
meta = s_cortex.MetaCortex()
self.assertRaises(s_cortex.InvalidParam, meta.addCortex, 30, 'ram:///')
def test_cortex_meta_noname(self):
meta = s_cortex.MetaCortex()
self.assertRaises(s_cortex.NoSuchName, meta.addMetaRows, 'hehe', [])
meta.fini()
def test_cortex_meta_query(self):
meta = s_cortex.MetaCortex()
meta.addCortex('foo.bar', 'ram:///', tags=('woot.hehe', ))
meta.addCortex('foo.baz', 'ram:///', tags=('woot.hoho', ))
core0 = meta.getCortex('foo.bar')
core1 = meta.getCortex('foo.baz')
id0 = guidstr()
id1 = guidstr()
id2 = guidstr()
id3 = guidstr()
rows0 = (
(id0, 'a', 10, 99),
(id1, 'x', 10, 80),
(id1, 'ha', 'ho', 80),
)
rows1 = (
(id2, 'c', 10, 99),
(id3, 'x', 10, 80),
(id3, 'ha', 'ho', 80),
)
j0 = core0.addRows(rows0, async=True)
j1 = core1.addRows(rows1, async=True)
self.assertTrue(core0.waitForJob(j0, timeout=3))
self.assertTrue(core1.waitForJob(j1, timeout=3))
tufos = meta.getTufosByQuery('foo:x=10')
self.assertEqual(len(tufos), 2)
tfdict = dict(tufos)
self.assertEqual(tfdict.get(id1).get('ha'), 'ho')
self.assertEqual(tfdict.get(id3).get('ha'), 'ho')
res = meta.getRowsByQuery('foo.bar:ha="ho"')
self.assertEqual(len(res), 1)
res = meta.getRowsByQuery('foo:ha')
size = meta.getSizeByQuery('foo:ha')
self.assertEqual(size, 2)
self.assertEqual(len(res), 2)
res = meta.getRowsByQuery('foo:x=10')
size = meta.getSizeByQuery('foo:x=10')
self.assertEqual(size, 2)
self.assertEqual(len(res), 2)
res = meta.getRowsByQuery('foo.baz:x=10')
size = meta.getSizeByQuery('foo.baz:x=10')
self.assertEqual(size, 1)
self.assertEqual(len(res), 1)
res = meta.getRowsByQuery('foo:x*range=(8,40)')
size = meta.getSizeByQuery('foo:x*range=(8,40)')
self.assertEqual(size, 2)
self.assertEqual(len(res), 2)
res = meta.getJoinByQuery('foo:x=10')
self.assertEqual(len(res), 4)
size = meta.getSizeByQuery('newp:newp=3')
self.assertEqual(size, 0)
rows = meta.getRowsByQuery('newp:newp=3')
self.assertEqual(len(rows), 0)
join = meta.getJoinByQuery('newp:newp=3')
self.assertEqual(len(join), 0)
meta.fini()
def __init__(self, statefd=None):
self.sched = s_threads.Sched()
self.asyncpool = s_async.AsyncBoss()
self.peerbus = s_eventbus.EventBus()
self.peersocks = {} # sockets directly connected to peers
self.peergraph = collections.defaultdict(set)
self.routecache = {}
# persistant data in these...
self.neuinfo = {
'poolsize': 10, # default to 10 neuron threads
}
self.neucores = {}
self.neusocks = {} # <guid> : NeuSock()
self.neushares = {}
self.runinfo = collections.defaultdict(dict)
self.peerinfo = collections.defaultdict(dict)
self.metacore = s_cortex.MetaCortex()
s_daemon.Daemon.__init__(self)
self.onfini(self.sched.fini)
self.onfini(self.peerbus.fini)
self.onfini(self.asyncpool.fini)
self.on('link:sock:init', self._onNeuSockInit)
self.on('neu:peer:init', self._onNeuPeerInit)
self.on('neu:peer:fini', self._onNeuPeerFini)
# application layer peer messages...
self.peerbus.on('ping', self._onPeerBusPing)
self.peerbus.on('pong', self._onPeerBusPong)
self.setMesgMethod('neu:data', self._onMesgNeuData)
self.setMesgMethod('neu:link:syn', self._onMesgNeuLinkSyn)
self.setMesgMethod('neu:link:synack', self._onMesgNeuLinkSynAck)
self.setMesgMethod('neu:link:ack', self._onMesgNeuLinkAck)
self.setMesgMethod('neu:peer:link:up', self._onMesgNeuPeerLinkUp)
self.setMesgMethod('neu:peer:link:down', self._onMesgNeuPeerLinkDown)
def setrsakey(event):
valu = event[1].get('valu')
self.rsakey = RSA.importKey(valu)
self.pubkey = self.rsakey.publickey()
self.pkcs15 = PKCS15.PKCS115_SigScheme(self.rsakey)
def setauthmod(event):
valu = event[1].get('valu')
if valu == None:
self.setAuthModule(None)
return
name, args, kwargs = valu
auth = s_dyndeps.getDynLocal(name)(*args, **kwargs)
self.setAuthModule(auth)
def setpoolsize(event):
valu = event[1].get('valu')
self.asyncpool.setPoolSize(valu)
self.on('neu:info:rsakey', setrsakey)
self.on('neu:info:authmod', setauthmod)
self.on('neu:info:poolsize', setpoolsize)
if statefd:
self.loadStateFd(statefd)
#self._loadNeuAuthModule()
self._loadNeuMetaCortex()
#self._loadNeuSharedObjects()
# check if we're brand new...
if self.neuinfo.get('ident') == None:
# making a shiney new neuron!
self.setNeuInfo('ident', guid())
self.ident = self.getNeuInfo('ident')
poolsize = self.getNeuInfo('poolsize')
self.asyncpool.setPoolSize(poolsize)
