def test_cortex_sync(self): core0 = s_cortex.openurl('ram://') core1 = s_cortex.openurl('ram://') core0.on('core:sync', core1.sync ) tufo0 = core0.formTufoByProp('foo','bar',baz='faz') tufo1 = core1.getTufoByProp('foo','bar') self.assertEqual( tufo1[1].get('foo'), 'bar' ) self.assertEqual( tufo1[1].get('foo:baz'), 'faz' ) tufo0 = core0.addTufoTag(tufo0,'hehe') tufo1 = core1.getTufoByProp('foo','bar') self.assertTrue( s_tags.tufoHasTag(tufo1,'hehe') ) core0.delTufoTag(tufo0,'hehe') tufo1 = core1.getTufoByProp('foo','bar') self.assertFalse( s_tags.tufoHasTag(tufo1,'hehe') ) core0.delTufo(tufo0) tufo1 = core1.getTufoByProp('foo','bar') self.assertIsNone( tufo1 )
def test_pki_base(self): core0 = s_cortex.openurl('ram:///') pki0 = s_pki.PkiStor(core0) uidn = guid() root = pki0.genRootToken(bits=512, save=True) dork = pki0.genUserToken('*****@*****.**', bits=512, save=True) visi = pki0.genUserToken('*****@*****.**', can=('sign:cert',), bits=512, save=True) iden = root[0] self.assertIsNotNone( iden ) self.assertEqual(visi[1].get('user'), '*****@*****.**') vcrt = pki0.genTokenCert(visi, signas=iden) pki0.setTokenCert(visi[0], vcrt, save=True) dcrt = s_pki.initTokenCert(dork) dcrt = pki0.signTokenCert(visi[0], dcrt) core1 = s_cortex.openurl('ram:///') pki1 = s_pki.PkiStor(core1) self.assertIsNone( pki1.loadCertToken( dcrt ) ) pki1.setTokenTufo(root) self.assertIsNotNone( pki1.loadCertToken( dcrt ) ) foob = b'foob' iden = root[0] sign = pki0.genByteSign(iden, b'blob') toks = tuple( pki0.iterTokenTufos() ) self.assertEqual( len(toks), 3) self.assertTrue( pki1.isValidSign(iden, sign, b'blob') ) self.assertFalse( pki1.isValidSign(iden, sign, foob) ) self.assertFalse( pki1.isValidSign(foob, sign, b'blob') ) self.assertFalse( pki1.isValidSign(iden, foob, b'blob') ) pki0.fini() core0.fini() pki1.fini() core1.fini()
def test_cortex_ramhost(self): core0 = s_cortex.openurl('ram:///foobar') core1 = s_cortex.openurl('ram:///foobar') self.assertEqual( id(core0), id(core1) ) core0.fini() core0 = s_cortex.openurl('ram:///foobar') core1 = s_cortex.openurl('ram:///bazfaz') self.assertNotEqual( id(core0), id(core1) ) core0.fini() core1.fini() core0 = s_cortex.openurl('ram:///') core1 = s_cortex.openurl('ram:///') self.assertNotEqual( id(core0), id(core1) ) core0.fini() core1.fini() core0 = s_cortex.openurl('ram://') core1 = s_cortex.openurl('ram://') self.assertNotEqual( id(core0), id(core1) ) core0.fini() core1.fini()
def benchmark_cortex(test_data, url, cleanup_func, num_threads=1): core = s_cortex.openurl(url) _prepopulate_core(core, test_data.prepop_rows) g = {'_addRows': _addRows, '_getTufosByIdens': _getTufosByIdens, 'core': core, 'test_data': test_data, '_getTufoByPropVal': _getTufoByPropVal, '_run_x': _run_x} do_it('_addRows', 'test_data.rows', num_threads, g, 1, 1, len(test_data.rows)) if cleanup_func: del core core = s_cortex.openurl(url) g['core'] = core do_it('_getTufosByIdens', 'test_data.idens', num_threads, g, 2, 5, NUM_TUFOS) do_it('_getTufoByPropVal', 'test_data.props', num_threads, g, 2, 5, NUM_TUFOS) if cleanup_func: cleanup_func()
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument('--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument('--configable-opts', action='store_true', default=False, help='Generate RST docs of the Configable classes in Synapse.') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) fd = None if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) if opts.doc_model: with s_cortex.openurl(opts.cortex) as core: return docModel(outp, fd, core) if opts.configable_opts: return docConfigables(outp, fd)
def test_storm_setprop(self): with s_cortex.openurl('ram:///') as core: core.setConfOpt('enforce', 1) # relative key/val syntax, explicitly relative vals node = core.formTufoByProp('inet:netuser', 'vertex.link/pennywise') node = core.formTufoByProp('inet:netuser', 'vertex.link/visi') node = core.eval('inet:netuser=vertex.link/pennywise setprop(:realname="Robert Gray")')[0] self.eq(node[1].get('inet:netuser'), 'vertex.link/pennywise') self.eq(node[1].get('inet:netuser:realname'), 'robert gray') # Can set multiple props at once cmd = 'inet:netuser=vertex.link/pennywise setprop(:seen:min="2000", :seen:max="2017")' node = core.eval(cmd)[0] self.nn(node[1].get('inet:netuser:seen:min')) self.nn(node[1].get('inet:netuser:seen:max')) # old / bad syntax fails # kwlist key/val syntax is no longer valid in setprop() node = core.formTufoByProp('inet:fqdn', 'vertex.link') bad_cmd = 'inet:fqdn=vertex.link setprop(created="2016-05-05",updated="2017/05/05")' self.raises(BadSyntaxError, core.eval, bad_cmd) # a rel prop which isn't valid for the node is bad bad_cmd = 'inet:fqdn=vertex.link setprop(:typocreated="2016-05-05")' self.raises(BadSyntaxError, core.eval, bad_cmd) # full prop syntax is not acceptable bad_cmd = 'inet:netuser=vertex.link/pennywise setprop(inet:netuser:signup="1970-01-01")' self.raises(BadSyntaxError, core.eval, bad_cmd)
def test_storm_lifts_by(self): # Test various lifts by handlers with s_cortex.openurl('ram:///') as core: # type: s_common.Cortex node1 = core.formTufoByProp('inet:dns:a', 'woot.com/1.2.3.4') node2 = core.formTufoByProp('inet:fqdn', 'vertex.vis') node3 = core.formTufoByProp('inet:url', 'https://vertex.link') node4 = core.formTufoByProp('inet:netuser', 'clowntown.link/pennywise') core.addTufoDark(node1, 'hehe', 'haha') core.addTufoDark(node2, 'hehe', 'haha') core.addTufoDark(node3, 'hehe', 'haha') core.addTufoTags(node1, ['aka.foo.bar.baz', 'aka.duck.quack.loud', 'src.clowntown']) core.addTufoTags(node2, ['aka.foo.duck.baz', 'aka.duck.quack.loud', 'src.clowntown']) core.addTufoTags(node3, ['aka.foo.bar.knight', 'aka.duck.sound.loud', 'src.clowntown']) # Lift by tags nodes = core.eval('inet:dns:a*tag=src.clowntown') self.eq(len(nodes), 1) # Lift by type nodes = core.eval('inet:user*type=pennywise') self.eq(len(nodes), 2) # Lift by inet:cidr nodes = core.eval('inet:dns:a:ipv4*inet:cidr=1.2.0.0/16') self.eq(len(nodes), 1) # Lift by dark nodes = core.eval('hehe*dark=haha') self.eq(len(nodes), 3)
def test_storm_delnode(self): with s_cortex.openurl('ram:///') as core: node = core.formTufoByProp('inet:ipv4', 0x01020304) core.eval('inet:ipv4=1.2.3.4 delnode()') self.nn(core.getTufoByProp('inet:ipv4', 0x01020304)) core.eval('inet:ipv4=1.2.3.4 delnode(force=1)') self.none(core.getTufoByProp('inet:ipv4', 0x01020304))
def test_sess_current(self): core = s_cortex.openurl('ram:///') cura = s_session.Curator(core=core) sess = cura.new() iden = sess.iden sess.put('woot',10) with sess: woot = s_session.current() self.eq(sess.iden,woot.iden) self.eq( woot.get('woot'), 10 ) sess.put('haha',30,save=False) self.eq( woot.get('haha'), 30 ) cura.fini() cura = s_session.Curator(core=core) sess = cura.get(iden) self.eq(sess.get('woot'), 10) self.eq(sess.get('haha'), None) core.fini()
def test_cortex_fire_set(self): core = s_cortex.openurl('ram://') props = {'foo:bar':'lol'} tufo = core.formTufoByProp('foo', 'hehe', bar='lol') events = ['tufo:set','tufo:props:foo','tufo:set:foo:bar'] wait = self.getTestWait(core,len(events),*events) core.setTufoProps(tufo,bar='hah') evts = wait.wait() self.assertEqual( evts[0][0], 'tufo:set') self.assertEqual( evts[0][1]['tufo'][0], tufo[0]) self.assertEqual( evts[0][1]['props']['foo:bar'], 'hah' ) self.assertEqual( evts[1][0], 'tufo:props:foo') self.assertEqual( evts[1][1]['tufo'][0], tufo[0]) self.assertEqual( evts[1][1]['props']['foo:bar'], 'hah' ) self.assertEqual( evts[2][0], 'tufo:set:foo:bar') self.assertEqual( evts[2][1]['tufo'][0], tufo[0]) self.assertEqual( evts[2][1]['valu'], 'hah' ) core.fini()
def test_cortex_caching_under_limit(self): with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar', qwer=10) tufo1 = core.formTufoByProp('foo','baz', qwer=10) core.setConfOpt('caching',1) tufs0 = core.getTufosByProp('foo:qwer', limit=9) tufs1 = core.getTufosByProp('foo:qwer', valu=10, limit=9) self.assertEqual( len(tufs0), 2 ) self.assertEqual( len(tufs1), 2 ) # when an entry is deleted from a cache result that was under it's limit # it should be removed but *not* invalidated core.delTufo(tufo0) self.assertIsNotNone( core.cache_bykey.get( ('foo:qwer',None,9) ) ) self.assertIsNotNone( core.cache_bykey.get( ('foo:qwer',10,9) ) ) tufs0 = core.getTufosByProp('foo:qwer', limit=9) tufs1 = core.getTufosByProp('foo:qwer', valu=10, limit=9) self.assertEqual( len(tufs0), 1 ) self.assertEqual( len(tufs1), 1 )
def test_cortex_xact_deadlock(self): N = 100 prop = 'testform' fd = tempfile.NamedTemporaryFile() dmon = s_daemon.Daemon() pool = s_threads.Pool(size=4, maxsize=8) wait = s_eventbus.Waiter(pool, 1, 'pool:work:fini') with s_cortex.openurl('sqlite:///%s' % fd.name) as core: def populate(): for i in range(N): #print('wrote %d tufos' % i) core.formTufoByProp(prop, str(i), **{}) dmon.share('core', core) link = dmon.listen('tcp://127.0.0.1:0/core') prox = s_telepath.openurl('tcp://127.0.0.1:%d/core' % link[1]['port']) pool.wrap(populate)() for i in range(N): tufos = prox.getTufosByProp(prop) #print('got %d tufos' % len(tufos)) wait.wait() pool.fini()
def test_ingest_embed_props(self): with s_cortex.openurl('ram://') as core: info = { "embed": [ { "props": {"sfx": 1}, "nodes": [ ["inet:fqdn", [ "com", "net", "org" ]], ], } ] } gest = s_ingest.Ingest(info) gest.ingest(core) self.nn(core.getTufoByProp('inet:fqdn', 'com')) self.nn(core.getTufoByProp('inet:fqdn', 'net')) self.nn(core.getTufoByProp('inet:fqdn', 'org')) self.eq(3, len(core.eval('inet:fqdn:sfx=1')))
def test_ingest_embed_pernode_tagsprops(self): with s_cortex.openurl('ram://') as core: info = { "embed": [ { "nodes": [ ["inet:fqdn", [ ["link", {"props": {"tld": 1}}], ]], ["inet:netuser", [ ["rootkit.com/metr0", {"props": {"email": "*****@*****.**"}}], ["twitter.com/invisig0th", {"props": {"email": "*****@*****.**"}}] ]], ["inet:email", [ ["*****@*****.**", {"tags": ["foo.bar", "baz.faz"]}] ]] ] } ] } gest = s_ingest.Ingest(info) gest.ingest(core) self.nn(core.getTufoByProp('inet:netuser', 'rootkit.com/metr0')) self.nn(core.getTufoByProp('inet:netuser', 'twitter.com/invisig0th')) self.eq(1, len(core.eval('inet:netuser:email="*****@*****.**"'))) self.eq(1, len(core.eval('inet:netuser:email="*****@*****.**"'))) node = core.eval('inet:email*tag=foo.bar')[0] self.eq(node[1].get('inet:email'), '*****@*****.**')
def test_ingest_embed_nodes(self): with s_cortex.openurl('ram://') as core: info = { "embed": [ { "nodes": [ ["inet:fqdn", [ "woot.com", "vertex.link" ]], ["inet:ipv4", [ "1.2.3.4", 0x05060708, ]], ] } ] } gest = s_ingest.Ingest(info) gest.ingest(core) self.nn(core.getTufoByProp('inet:ipv4', 0x01020304)) self.nn(core.getTufoByProp('inet:ipv4', 0x05060708)) self.nn(core.getTufoByProp('inet:fqdn', 'woot.com')) self.nn(core.getTufoByProp('inet:fqdn', 'vertex.link'))
def test_ingest_embed_tags(self): with s_cortex.openurl('ram://') as core: info = { "embed": [ { "tags": [ "hehe.haha.hoho" ], "nodes":[ ["inet:fqdn", [ "rofl.com", "laughitup.edu" ]], ["inet:email", [ "*****@*****.**" ]] ] } ] } gest = s_ingest.Ingest(info) gest.ingest(core) self.nn(core.getTufoByProp('inet:fqdn', 'rofl.com')) self.nn(core.getTufoByProp('inet:fqdn', 'laughitup.edu')) self.nn(core.getTufoByProp('inet:email', '*****@*****.**')) self.eq(2, len(core.eval('inet:fqdn*tag=hehe.haha.hoho'))) self.eq(1, len(core.eval('inet:email*tag=hehe.haha.hoho')))
def test_ingest_addFormat(self): def _fmt_woot_old(fd, info): yield 'old.bad' def _fmt_woot(fd, info): yield 'woot' opts = {'mode': 'r', 'encoding': 'utf8'} s_ingest.addFormat('woot', _fmt_woot_old, opts) self.nn(s_ingest.fmtyielders.get('woot')) s_ingest.addFormat('woot', _fmt_woot, opts) # last write wins with s_cortex.openurl('ram://') as core: with self.getTestDir() as path: wootpath = os.path.join(path, 'woot.woot') with genfile(wootpath) as fd: fd.write(b'this is irrelevant, we always yield woot :-)') info = { 'sources': ( (wootpath, {'open': {'format': 'woot'}, 'ingest': { 'tags': ['hehe.haha'], 'forms': [ ('inet:fqdn', {}), ] }}), ) } gest = s_ingest.Ingest(info) gest.ingest(core) self.nn(core.getTufoByProp('inet:fqdn', 'woot'))
def test_ingest_tag_template_whif(self): data = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]} info = {'ingest': { 'iters': [ ["foo/*", { 'vars': [['zoom', {'path': 'fqdn'}]], 'tags': [ {'iter': 'haha/*', 'vars': [ ['tag', {'regex': '^foo'}], ], 'template': 'zoom.{{tag}}'} ], 'forms': [('inet:fqdn', {'path': 'fqdn'})], }], ], }} with s_cortex.openurl('ram://') as core: gest = s_ingest.Ingest(info) gest.ingest(core, data=data) node = core.getTufoByProp('inet:fqdn', 'vertex.link') self.true(s_tufo.tagged(node, 'zoom.foofoo')) self.false(s_tufo.tagged(node, 'zoom.barbar'))
def test_ingest_condtag(self): data = {'foo': [{'fqdn': 'vertex.link', 'hehe': 3}]} info = {'ingest': { 'iters': [ ["foo/*", { 'vars': [['hehe', {'path': 'hehe'}]], 'tags': [{'value': 'hehe.haha', 'cond': 'hehe != 3'}], 'forms': [('inet:fqdn', {'path': 'fqdn'})], }], ], }} with s_cortex.openurl('ram://') as core: gest = s_ingest.Ingest(info) gest.ingest(core, data=data) node = core.getTufoByProp('inet:fqdn', 'vertex.link') self.false(s_tufo.tagged(node, 'hehe.haha')) data['foo'][0]['hehe'] = 9 gest.ingest(core, data=data) node = core.getTufoByProp('inet:fqdn', 'vertex.link') self.true(s_tufo.tagged(node, 'hehe.haha'))
def test_datamodel_cortex(self): core = s_cortex.openurl('ram:///') core.addTufoForm('foo') core.addTufoProp('foo', 'bar', ptype='int', defval=10) core.formTufoByProp('foo', 'hehe') core.formTufoByProp('foo', 'haha') core.formTufoByProp('foo', 'blah', bar=99) tufo0 = core.formTufoByProp('foo', 'hehe') self.assertEqual(tufo0[1].get('foo:bar'), 10) core.setTufoProp(tufo0, 'bar', 30) self.assertEqual(tufo0[1].get('foo:bar'), 30) tufo1 = core.formTufoByProp('foo', 'hehe') self.assertEqual(tufo0[0], tufo1[0]) tufos = core.getTufosByProp('foo') self.assertEqual(len(tufos), 3) tufos = core.getTufosByProp('foo:bar', valu=30, limit=20) self.assertEqual(len(tufos), 1) tufos = core.getTufosByProp('foo:bar', valu=99, limit=20) self.assertEqual(len(tufos), 1)
def test_ingest_iter_object(self): data = { 'foo': { 'boosh': { 'fqdn': 'vertex.link' }, 'woot': { 'fqdn': 'foo.bario' } } } info = {'ingest': { 'iters': [ ["foo/*", { 'vars': [ ['bar', {'path': '0'}], ['fqdn', {'path': '1/fqdn'}] ], 'forms': [('inet:fqdn', {'template': '{{bar}}.{{fqdn}}'})] }], ], }} with s_cortex.openurl('ram://') as core: gest = s_ingest.Ingest(info) gest.ingest(core, data=data) node = core.getTufoByProp('inet:fqdn', 'boosh.vertex.link') self.nn(node) node = core.getTufoByProp('inet:fqdn', 'woot.foo.bario') self.nn(node)
def test_cortex_caching_del_tufo(self): with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar', qwer=10) tufo1 = core.formTufoByProp('foo','baz', qwer=10) core.setConfOpt('caching',1) tufs0 = core.getTufosByProp('foo:qwer') tufs1 = core.getTufosByProp('foo:qwer', valu=10) tufs2 = core.getTufosByProp('foo:qwer', valu=11) self.assertEqual( len(tufs0), 2 ) self.assertEqual( len(tufs1), 2 ) self.assertEqual( len(tufs2), 0 ) core.delTufo( tufo0 ) #tufo2 = core.formTufoByProp('foo','lol', qwer=10) tufs0 = core.getTufosByProp('foo:qwer') tufs1 = core.getTufosByProp('foo:qwer', valu=10) tufs2 = core.getTufosByProp('foo:qwer', valu=11) self.assertEqual( len(tufs0), 1 ) self.assertEqual( len(tufs1), 1 ) self.assertEqual( len(tufs2), 0 )
def test_ingest_cortex_registration(self): data1 = {'foo': [{'fqdn': 'vertex.link', 'haha': ['barbar', 'foofoo']}]} data2 = {'foo': [{'fqdn': 'weallfloat.com', 'haha': ['fooboat', 'sewer']}]} data3 = {'foo': [{'fqdn': 'woot.com', 'haha': ['fooboat', 'sewer']}]} ingest_def = {'ingest': { 'iters': [ ["foo/*", { 'vars': [['zoom', {'path': 'fqdn'}]], 'tags': [ {'iter': 'haha/*', 'vars': [ ['tag', {'regex': '^foo'}], ], 'template': 'zoom.{{tag}}'} ], 'forms': [('inet:fqdn', {'path': 'fqdn'})], }], ], }} ingest_def2 = {'ingest': { 'iters': [ ["foo/*", { 'vars': [['zoom', {'path': 'fqdn'}]], 'forms': [('inet:fqdn', {'path': 'fqdn'})], }], ], }} gest = s_ingest.Ingest(ingest_def) gest2 = s_ingest.Ingest(ingest_def2) with s_cortex.openurl('ram:///') as core: ret1 = s_ingest.register_ingest(core=core, gest=gest, evtname='ingest:test') ret2 = s_ingest.register_ingest(core=core, gest=gest2, evtname='ingest:test2', ret_func=True) self.none(ret1) self.true(callable(ret2)) # Dump data into the core an event at a time. core.fire('ingest:test', data=data1) node = core.getTufoByProp('inet:fqdn', 'vertex.link') self.true(isinstance(node, tuple)) self.true(s_tufo.tagged(node, 'zoom.foofoo')) self.false(s_tufo.tagged(node, 'zoom.barbar')) core.fire('ingest:test', data=data2) node = core.getTufoByProp('inet:fqdn', 'weallfloat.com') self.true(isinstance(node, tuple)) self.true(s_tufo.tagged(node, 'zoom.fooboat')) self.false(s_tufo.tagged(node, 'zoom.sewer')) # Try another ingest attached to the core. This won't have any tags applied. core.fire('ingest:test2', data=data3) node = core.getTufoByProp('inet:fqdn', 'woot.com') self.true(isinstance(node, tuple)) self.false(s_tufo.tagged(node, 'zoom.fooboat')) self.false(s_tufo.tagged(node, 'zoom.sewer'))
def test_cortex_tlib_persistence(self): with self.getTestDir() as path: savefile = genpath(path,'savefile.mpk') with s_cortex.openurl('ram://',savefile=savefile) as core: core.formTufoByProp('syn:type','foo',subof='bar') core.formTufoByProp('syn:type','bar',ctor='synapse.tests.test_cortex.FakeType') self.assertEqual( core.getTypeParse('foo','30')[0], 30 ) self.assertEqual( core.getTypeParse('bar','30')[0], 30 ) with s_cortex.openurl('ram://',savefile=savefile) as core: self.assertEqual( core.getTypeParse('foo','30')[0], 30 ) self.assertEqual( core.getTypeParse('bar','30')[0], 30 )
def test_model_crypto_201708231712(self): byts = self.getRev0DbByts() with self.getTestDir() as temp: finl = os.path.join(temp, 'test.db') with open(finl, 'wb') as fd: fd.write(byts) url = 'sqlite:///%s' % finl with s_cortex.openstore(url) as store: prop = '.:modl:vers:crypto' valu = 0 rows = store.getRowsByProp(prop) if rows: iden = rows[0][0] else: iden = guid() store.setRowsByIdProp(iden, prop, valu) # Open the cortex, applying the data model updates # Validate our nodes now have the correct data with s_cortex.openurl(url) as core: modlrev = core.getModlVers('crypto') self.ge(modlrev, 201708231712) pdef = core.getTufoByProp('syn:prop', 'rsa:key:mod') self.eq(pdef[1].get('syn:prop:ptype'), 'str:hex') self.notin('syn:prop:pytpe', pdef[1])
def test_coretex_logging(self): with s_cortex.openurl('ram:///') as core: core.setConfOpt('log:save',1) try: raise NoSuchPath(path='foo/bar') except NoSuchPath as exc: core.logCoreExc(exc,subsys='hehe') tufo = core.getTufoByProp('syn:log:subsys',valu='hehe') self.eq( tufo[1].get('syn:log:subsys'), 'hehe' ) self.eq( tufo[1].get('syn:log:exc'), 'synapse.exc.NoSuchPath' ) self.eq( tufo[1].get('syn:log:info:path'), 'foo/bar' ) self.assertIsNotNone( tufo[1].get('syn:log:time') ) core.setConfOpt('log:level', logging.ERROR) try: raise NoSuchPath(path='foo/bar') except NoSuchPath as exc: core.logCoreExc(exc,subsys='haha', level=logging.WARNING) self.assertIsNone( core.getTufoByProp('syn:log:subsys', valu='haha') )
def test_storm_tag_query(self): # Ensure that non-glob tag filters operate as expected. with s_cortex.openurl('ram:///') as core: # type: s_common.Cortex node1 = core.formTufoByProp('inet:dns:a', 'woot.com/1.2.3.4') node2 = core.formTufoByProp('inet:dns:a', 'vertex.vis/5.6.7.8') node3 = core.formTufoByProp('inet:dns:a', 'vertex.link/5.6.7.8') core.addTufoTags(node1, ['aka.foo.bar.baz', 'aka.duck.quack.loud', 'src.clowntown']) core.addTufoTags(node2, ['aka.foo.duck.baz', 'aka.duck.quack.loud', 'src.clowntown']) core.addTufoTags(node3, ['aka.foo.bar.knight', 'aka.duck.sound.loud', 'src.clowntown']) nodes = core.eval('inet:dns:a +#src.clowntown') self.eq(len(nodes), 3) nodes = core.eval('inet:dns:a +#src') self.eq(len(nodes), 3) nodes = core.eval('inet:dns:a +#aka.duck.quack') self.eq(len(nodes), 2) nodes = core.eval('inet:dns:a +#aka.foo.bar.knight') self.eq(len(nodes), 1) nodes = core.eval('inet:dns:a +#src.internet') self.eq(len(nodes), 0) nodes = core.eval('inet:dns:a -#aka.foo.bar') self.eq(len(nodes), 1)
def test_cortex_by_type(self): with s_cortex.openurl('ram://') as core: core.addTufoForm('foo') core.addTufoProp('foo','min', ptype='time:epoch:min') core.addTufoProp('foo','max', ptype='time:epoch:max') core.addTufoForm('bar') core.addTufoProp('bar','min', ptype='time:epoch:min') core.addTufoProp('bar','max', ptype='time:epoch:max') core.addTufoForm('baz') core.addTufoProp('baz','min', ptype='time:epoch') core.addTufoProp('baz','max', ptype='time:epoch') props = {'min':20,'max':20} tufo0 = core.formTufoByProp('foo', 'hurr', **props) tufo1 = core.formTufoByProp('bar', 'durr', **props) tufo2 = core.formTufoByProp('baz', 'durr', **props) want = tuple(sorted([tufo0[0],tufo1[0]])) res0 = core.getTufosByPropType('time:epoch:min', valu=20) self.eq( tuple(sorted([r[0] for r in res0])), want )
def test_cortex_stats(self): rows = [ (guid(), 'foo:bar', 1, 99), (guid(), 'foo:bar', 2, 99), (guid(), 'foo:bar', 3, 99), (guid(), 'foo:bar', 5, 99), (guid(), 'foo:bar', 8, 99), (guid(), 'foo:bar', 13, 99), (guid(), 'foo:bar', 21, 99), ] core = s_cortex.openurl('ram://') core.addRows(rows) self.eq( core.getStatByProp('sum','foo:bar'), 53 ) self.eq( core.getStatByProp('count','foo:bar'), 7 ) self.eq( core.getStatByProp('min','foo:bar'), 1 ) self.eq( core.getStatByProp('max','foo:bar'), 21 ) self.eq( core.getStatByProp('mean','foo:bar'), 7.571428571428571 ) self.eq( core.getStatByProp('any','foo:bar'), 1) self.eq( core.getStatByProp('all','foo:bar'), 1) histo = core.getStatByProp('histo','foo:bar') self.eq( histo.get(13), 1 ) self.assertRaises( NoSuchStat, core.getStatByProp, 'derp', 'inet:ipv4' )
def test_cortex_stats(self): rows = [ (guid(), 'foo:bar', 1, 99), (guid(), 'foo:bar', 2, 99), (guid(), 'foo:bar', 3, 99), (guid(), 'foo:bar', 5, 99), (guid(), 'foo:bar', 8, 99), (guid(), 'foo:bar', 13, 99), (guid(), 'foo:bar', 21, 99), ] core = s_cortex.openurl('ram://') core.addRows(rows) self.assertEqual( core.getStatByProp('sum','foo:bar'), 53 ) self.assertEqual( core.getStatByProp('count','foo:bar'), 7 ) self.assertEqual( core.getStatByProp('min','foo:bar'), 1 ) self.assertEqual( core.getStatByProp('max','foo:bar'), 21 ) self.assertEqual( core.getStatByProp('average','foo:bar'), 7.571428571428571 ) self.assertEqual( core.getStatByProp('any','foo:bar'), True) self.assertEqual( core.getStatByProp('all','foo:bar'), True) histo = core.getStatByProp('histo','foo:bar') self.assertEqual( histo.get(13), 1 )
def opencore(url, **opts): ''' Construct a UserAuth object around the given cortex URL. ''' import synapse.cortex as s_cortex core = s_cortex.openurl(url, **opts) return UserAuth(core)
def test_cortex_stats(self): rows = [ (guid(), "foo:bar", 1, 99), (guid(), "foo:bar", 2, 99), (guid(), "foo:bar", 3, 99), (guid(), "foo:bar", 5, 99), (guid(), "foo:bar", 8, 99), (guid(), "foo:bar", 13, 99), (guid(), "foo:bar", 21, 99), ] core = s_cortex.openurl("ram://") core.addRows(rows) self.assertEqual(core.getStatByProp("sum", "foo:bar"), 53) self.assertEqual(core.getStatByProp("count", "foo:bar"), 7) self.assertEqual(core.getStatByProp("min", "foo:bar"), 1) self.assertEqual(core.getStatByProp("max", "foo:bar"), 21) self.assertEqual(core.getStatByProp("average", "foo:bar"), 7.571428571428571) self.assertEqual(core.getStatByProp("any", "foo:bar"), True) self.assertEqual(core.getStatByProp("all", "foo:bar"), True) histo = core.getStatByProp("histo", "foo:bar") self.assertEqual(histo.get(13), 1)
def test_cortex_tufo_pop(self): with s_cortex.openurl('ram://') as core: foo0 = core.formTufoByProp('foo','bar',woot='faz') foo1 = core.formTufoByProp('foo','baz',woot='faz') self.assertEqual( 2, len(core.popTufosByProp('foo:woot', valu='faz'))) self.assertEqual( 0, len(core.getTufosByProp('foo')))
def test_datamodel_cortex(self): core = s_cortex.openurl('ram:///') core.addTufoForm('foo') core.addTufoProp('foo', 'bar', ptype='int', defval=10) core.formTufoByProp('foo','hehe') core.formTufoByProp('foo','haha') core.formTufoByProp('foo','blah', bar=99) tufo0 = core.formTufoByProp('foo','hehe') self.assertEqual( tufo0[1].get('foo:bar'), 10 ) core.setTufoProp(tufo0,'bar',30) self.assertEqual( tufo0[1].get('foo:bar'), 30 ) tufo1 = core.formTufoByProp('foo','hehe') self.assertEqual( tufo0[0], tufo1[0] ) tufos = core.getTufosByProp('foo') self.assertEqual( len(tufos) , 3 ) tufos = core.getTufosByProp('foo:bar', valu=30, limit=20) self.assertEqual( len(tufos) , 1 ) tufos = core.getTufosByProp('foo:bar', valu=99, limit=20) self.assertEqual( len(tufos) , 1 )
def getDmonCore(self): ''' Context manager to make a ram:/// cortex which has test models loaded into it and shared via daemon. Yields: s_cores_common.Cortex: A proxy object to the Ram backed cortex with test models. ''' dmon = s_daemon.Daemon() core = s_cortex.openurl('ram:///') self.addTstForms(core) link = dmon.listen('tcp://127.0.0.1:0/') dmon.share('core00', core) port = link[1].get('port') prox = s_telepath.openurl('tcp://127.0.0.1/core00', port=port) s_scope.set('syn:test:link', link) s_scope.set('syn:cmd:core', prox) try: yield prox except: # pragma: no cover raise finally: prox.fini() core.fini() dmon.fini()
def test_cortex_tags(self): core = s_cortex.openurl('ram://') core.addTufoForm('foo') hehe = core.formTufoByProp('foo','hehe') core.addTufoTag(hehe,'lulz.rofl') lulz = core.getTufoByProp('syn:tag','lulz') self.assertIsNone( lulz[1].get('syn:tag:up') ) self.assertEqual( lulz[1].get('syn:tag:doc'), '') self.assertEqual( lulz[1].get('syn:tag:title'), '') self.assertEqual( lulz[1].get('syn:tag:depth'), 0 ) rofl = core.getTufoByProp('syn:tag','lulz.rofl') self.assertEqual( rofl[1].get('syn:tag:doc'), '') self.assertEqual( rofl[1].get('syn:tag:title'), '') self.assertEqual( rofl[1].get('syn:tag:up'), 'lulz' ) self.assertEqual( rofl[1].get('syn:tag:depth'), 1 ) core.delTufo(lulz) # tag and subs should be wiped self.assertIsNone( core.getTufoByProp('syn:tag','lulz') ) self.assertIsNone( core.getTufoByProp('syn:tag','lulz.rofl') ) self.assertEqual( len(core.getTufosByTag('foo','lulz')), 0 ) self.assertEqual( len(core.getTufosByTag('foo','lulz.rofl')), 0 ) core.fini()
def test_cortex_caching(self): with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar', asdf=2) tufo1 = core.formTufoByProp('foo','baz', asdf=2) answ0 = core.getTufosByProp('foo') answ1 = core.getTufosByProp('foo', valu='bar') self.eq( len(answ0), 2 ) self.eq( len(answ1), 1 ) self.eq( len(core.cache_fifo), 0 ) self.eq( len(core.cache_bykey), 0 ) self.eq( len(core.cache_byiden), 0 ) self.eq( len(core.cache_byprop), 0 ) core.setConfOpt('caching',1) self.eq( core.caching, 1 ) answ0 = core.getTufosByProp('foo') self.eq( len(answ0), 2 ) self.eq( len(core.cache_fifo), 1 ) self.eq( len(core.cache_bykey), 1 ) self.eq( len(core.cache_byiden), 2 ) self.eq( len(core.cache_byprop), 1 )
def getSwarmEnv(self): tenv = TestEnv() core0 = s_cortex.openurl('ram://') core1 = s_cortex.openurl('ram://') tenv.add('core0',core0,fini=True) tenv.add('core1',core1,fini=True) tufo0 = core0.formTufoByProp('foo:bar','baz',vvv='visi') tufo1 = core0.formTufoByProp('foo:bar','faz',vvv='visi') tufo2 = core1.formTufoByProp('foo:bar','lol',vvv='visi') tufo3 = core1.formTufoByProp('foo:bar','hai',vvv='visi') tufo4 = core0.formTufoByProp('zzz:woot',10,vvv='visi') tufo5 = core1.formTufoByProp('zzz:woot',12,vvv='romp') tenv.add('tufo0',tufo0) tenv.add('tufo1',tufo1) tenv.add('tufo2',tufo2) tenv.add('tufo3',tufo3) dmon = s_daemon.Daemon() link = dmon.listen('tcp://127.0.0.1:0') tenv.add('link',link) tenv.add('dmon',dmon,fini=True) port = link[1].get('port') svcbus = s_service.SvcBus() tenv.add('svcbus',svcbus,fini=True) dmon.share('syn.svcbus',svcbus) svcrmi = s_telepath.openurl('tcp://127.0.0.1/syn.svcbus', port=port) tenv.add('svcrmi',svcrmi,fini=True) s_service.runSynSvc('cortex',core0,svcrmi,tags=('hehe.haha',)) s_service.runSynSvc('cortex',core1,svcrmi,tags=('hehe.hoho',)) runt = s_runtime.Runtime(svcrmi) tenv.add('runt',runt,fini=True) return tenv
def test_cortex_tufo_setprop(self): core = s_cortex.openurl("ram://") foob = core.formTufoByProp("foo", "bar", baz="faz") self.assertEqual(foob[1].get("foo:baz"), "faz") core.setTufoProp(foob, "baz", "zap") self.assertEqual(len(core.getTufosByProp("foo:baz", valu="zap")), 1)
def getSwarmEnv(self): tenv = TestEnv() core0 = s_cortex.openurl("ram://") core1 = s_cortex.openurl("ram://") tenv.add("core0", core0, fini=True) tenv.add("core1", core1, fini=True) tufo0 = core0.formTufoByProp("foo:bar", "baz", vvv="visi") tufo1 = core0.formTufoByProp("foo:bar", "faz", vvv="visi") tufo2 = core1.formTufoByProp("foo:bar", "lol", vvv="visi") tufo3 = core1.formTufoByProp("foo:bar", "hai", vvv="visi") tufo4 = core0.formTufoByProp("zzz:woot", 10, vvv="visi") tufo5 = core1.formTufoByProp("zzz:woot", 12, vvv="romp") tenv.add("tufo0", tufo0) tenv.add("tufo1", tufo1) tenv.add("tufo2", tufo2) tenv.add("tufo3", tufo3) dmon = s_daemon.Daemon() link = dmon.listen("tcp://127.0.0.1:0") tenv.add("link", link) tenv.add("dmon", dmon, fini=True) port = link[1].get("port") svcbus = s_service.SvcBus() tenv.add("svcbus", svcbus, fini=True) dmon.share("syn.svcbus", svcbus) svcrmi = s_telepath.openurl("tcp://127.0.0.1/syn.svcbus", port=port) tenv.add("svcrmi", svcrmi, fini=True) s_service.runSynSvc("cortex", core0, svcrmi, tags=("hehe.haha",)) s_service.runSynSvc("cortex", core1, svcrmi, tags=("hehe.hoho",)) runt = s_runtime.Runtime(svcrmi) tenv.add("runt", runt, fini=True) return tenv
def test_cortex_tufo_setprop(self): core = s_cortex.openurl('ram://') foob = core.formTufoByProp('foo','bar',baz='faz') self.assertEqual( foob[1].get('foo:baz'), 'faz' ) core.setTufoProp(foob,'baz','zap') self.assertEqual( len(core.getTufosByProp('foo:baz',valu='zap')), 1 )
def test_cortex_ramtyperange(self): with s_cortex.openurl('ram://') as core: core.formTufoByProp('foo:bar',10) core.formTufoByProp('foo:bar','baz') tufs = core.getTufosBy('range','foo:bar', (5,15)) self.eq( len(tufs), 1 )
def test_cortex_dict(self): core = s_cortex.openurl('ram://') core.addTufoForm('foo:bar', ptype='int') core.addTufoForm('baz:faz', ptype='int', defval=22) modl = core.getModelDict() self.assertEqual( modl['forms'][-2:], ['foo:bar','baz:faz'] ) self.assertEqual( modl['props']['foo:bar'][1]['ptype'], 'int') self.assertEqual( modl['props']['baz:faz'][1]['defval'], 22)
def test_cortex_caching_atlimit(self): with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar', qwer=10) tufo1 = core.formTufoByProp('foo','baz', qwer=10) core.setConfOpt('caching',1) tufs0 = core.getTufosByProp('foo:qwer', limit=2) tufs1 = core.getTufosByProp('foo:qwer', valu=10, limit=2) self.assertEqual( len(tufs0), 2 ) self.assertEqual( len(tufs1), 2 ) # when an entry is deleted from a cache result that was at it's limit # it should be fully invalidated core.delTufo(tufo0) self.assertIsNone( core.cache_bykey.get( ('foo:qwer',None,2) ) ) self.assertIsNone( core.cache_bykey.get( ('foo:qwer',10,2) ) ) with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar', qwer=10) tufo1 = core.formTufoByProp('foo','baz', qwer=10) core.setConfOpt('caching',1) tufs0 = core.getTufosByProp('foo:qwer', limit=2) tufs1 = core.getTufosByProp('foo:qwer', valu=10, limit=2) self.assertEqual( len(tufs0), 2 ) self.assertEqual( len(tufs1), 2 ) tufo2 = core.formTufoByProp('foo','baz', qwer=10) # when an entry is added from a cache result that was at it's limit # it should *not* be invalidated self.assertIsNotNone( core.cache_bykey.get( ('foo:qwer',None,2) ) ) self.assertIsNotNone( core.cache_bykey.get( ('foo:qwer',10,2) ) )
def test_cache_tufo_prop(self): core = s_cortex.openurl('ram:///') cache = s_cache.TufoPropCache(core,'woot') tufo = core.formTufoByProp('woot','haha', lolol=10) newfo = cache.get('haha') self.assertIsNotNone(newfo) self.assertEqual(newfo[1].get('woot:lolol'), 10)
def test_pki_idenbyuser(self): cor = s_cortex.openurl('ram:///') pki = s_pki.PkiStor(cor) tokn = pki.genUserToken('visi', bits=512) self.assertIsNone( pki.getIdenByUser('newp') ) self.assertEqual( tokn[0], pki.getIdenByUser('visi') ) pki.fini() cor.fini()
def test_pki_idenbyhost(self): cor = s_cortex.openurl('ram:///') pki = s_pki.PkiStor(cor) tokn = pki.genHostToken('visi.kenshoto.com', bits=512) self.assertIsNone( pki.getIdenByHost('newp.newp.com') ) self.assertEqual( tokn[0], pki.getIdenByHost('visi.kenshoto.com') ) pki.fini() cor.fini()
def test_cortex_caching_new(self): with s_cortex.openurl('ram://') as core: core.setConfOpt('caching',1) tufo0 = core.formTufoByProp('foo','bar') tufo1 = core.formTufoByProp('foo','bar') self.assertTrue(tufo0[1].get('.new')) self.assertFalse(tufo1[1].get('.new'))
def test_cortex_caching_oneref(self): with s_cortex.openurl('ram://') as core: tufo0 = core.formTufoByProp('foo','bar') core.setConfOpt('caching',1) ref0 = core.getTufosByProp('foo',valu='bar')[0] ref1 = core.getTufosByProp('foo',valu='bar')[0] self.eq( id(ref0), id(ref1) )
def test_cortex_savefd(self): fd = s_compat.BytesIO() core0 = s_cortex.openurl('ram://', savefd=fd) t0 = core0.formTufoByProp('foo','one', baz='faz') t1 = core0.formTufoByProp('foo','two', baz='faz') core0.setTufoProps(t0,baz='gronk') core0.delTufoByProp('foo','two') core0.fini() fd.seek(0) core1 = s_cortex.openurl('ram://', savefd=fd) self.assertIsNone( core1.getTufoByProp('foo','two') ) t0 = core1.getTufoByProp('foo','one') self.assertIsNotNone( t0 ) self.assertEqual( t0[1].get('foo:baz'), 'gronk' )
def test_cortex_savefd(self): fd = s_compat.BytesIO() core0 = s_cortex.openurl("ram://", savefd=fd) t0 = core0.formTufoByProp("foo", "one", baz="faz") t1 = core0.formTufoByProp("foo", "two", baz="faz") core0.setTufoProps(t0, baz="gronk") core0.delTufoByProp("foo", "two") core0.fini() fd.seek(0) core1 = s_cortex.openurl("ram://", savefd=fd) self.assertIsNone(core1.getTufoByProp("foo", "two")) t0 = core1.getTufoByProp("foo", "one") self.assertIsNotNone(t0) self.assertEqual(t0[1].get("foo:baz"), "gronk")
def getUserPki(): # TODO env var # TODO agent? ''' Return the current/default PkiStor for the current user. ''' with s_glob.lock: if s_glob.pki == None: core = s_cortex.openurl(pkicore) s_glob.pki = PkiStor(core) return s_glob.pki