def test_telepath_yielder(self): class YieldTest: def woot(self): yield 10 yield 20 yield 30 dmon = s_daemon.Daemon() link = dmon.listen('tcp://127.0.0.1:0/hehe') dmon.share('hehe', YieldTest() ) prox = s_telepath.openlink(link) items = [] for item in prox.woot(): items.append(item) self.assertEqual( tuple(items), (10,20,30) ) self.assertEqual( len(dmon._dmon_yields), 0 ) prox.fini() dmon.fini()
def _getSvcAxon(self, iden): svcfo = self.svcprox.getSynSvc(iden) if svcfo == None: return None axon = self.axons.get(iden) if axon == None: link = svcfo[1].get('link') if link == None: return None def onfini(): self.axons.pop(iden,None) try: # copy before we frob it #link = (link[0],dict(link[1])) #link[1]['once'] = True axon = s_telepath.openlink(link) self.axons[iden] = axon axon.onfini( onfini ) except Exception as e: return None return axon
def _getSvcAxon(self, iden): svcfo = self.svcprox.getSynSvc(iden) if svcfo == None: return None axon = self.axons.get(iden) if axon == None: link = svcfo[1].get('link') if link == None: return None def onfini(): self.axons.pop(iden, None) try: # copy before we frob it #link = (link[0],dict(link[1])) #link[1]['once'] = True axon = s_telepath.openlink(link) self.axons[iden] = axon axon.onfini(onfini) except Exception as e: return None return axon
def test_telepath_basics(self): env = self.getFooEnv() foo = s_telepath.openlink(env.link) self.true(s_telepath.isProxy(foo)) self.false(s_telepath.isProxy(self)) # Test magic methods self.true(bool(foo) is True) self.true(foo == foo) self.false(foo == 1) self.true(foo != 1) s = time.time() for i in range(1000): foo.speed() e = time.time() self.eq(foo.bar(10, 20), 30) self.raises(NoSuchMeth, foo.faz, 10, 20) self.raises(SynErr, foo.baz, 10, 20) foo.fini() # We have fini'd the Proxy resources self.true(foo._tele_boss.isfini) self.true(foo._tele_sock.isfini) self.true(foo._tele_plex.isfini) env.fini()
def test_telepath_yielder(self): class YieldTest: def woot(self): yield 10 yield 20 yield 30 dmon = s_daemon.Daemon() link = dmon.listen('tcp://127.0.0.1:0/hehe') dmon.share('hehe', YieldTest()) prox = s_telepath.openlink(link) items = [] for item in prox.woot(): items.append(item) self.eq(tuple(items), (10, 20, 30)) self.eq(len(dmon._dmon_yields), 0) prox.fini() dmon.fini()
def test_axon_cluster(self): self.thisHostMustNot(platform='windows') busurl = 'local://%s/axons' % guid() dmon = s_daemon.Daemon() dmon.listen(busurl) dmon.share('axons', s_service.SvcBus(), fini=True) with self.getTestDir() as datadir: dir0 = gendir(datadir,'host0') dir1 = gendir(datadir,'host1') dir2 = gendir(datadir,'host2') opts = { 'axonbus':busurl, } host0 = s_axon.AxonHost(dir0,hostname='host0',**opts) host1 = s_axon.AxonHost(dir1,hostname='host1',**opts) host2 = s_axon.AxonHost(dir2,hostname='host2',**opts) props = { 'clones':2, 'syncmax':s_axon.megabyte, 'bytemax':s_axon.megabyte, } axfo0 = host0.add(**props) axon0 = s_telepath.openlink( axfo0[1].get('link') ) # wait for clones to come online self.assertTrue( axon0._waitClonesReady(timeout=2) ) #self.assertIsNotNone( usage.get('total') ) #axon = host.axons.get(iden) iden = axon0.alloc(100) blob = axon0.chunk(iden,b'V'*100) self.assertIsNotNone(blob) self.assertTrue( axon0.has( 'md5', blob[1].get('hash:md5') ) ) self.assertTrue( axon0.has( 'sha1', blob[1].get('hash:sha1') ) ) self.assertTrue( axon0.has( 'sha256', blob[1].get('hash:sha256') ) ) axon0.fini() host0.fini() host1.fini() host2.fini() dmon.fini()
def openlink(link): ''' Open a cortex via a link tuple. ''' ctor = corctors.get(link[0]) if ctor is None: return s_telepath.openlink(link) return ctor(link)
def test_telepath_call(self): dmon, link = self.getFooServ() foo = s_telepath.openlink(link) job = foo.call('bar', 10, 20) self.assertIsNotNone(job) self.assertEqual(foo.syncjob(job), 30) foo.fini() dmon.fini()
def test_telepath_call(self): dmon,link = self.getFooServ() foo = s_telepath.openlink(link) job = foo.call('bar', 10, 20) self.assertIsNotNone( job ) self.assertEqual( foo.sync(job), 30 ) foo.fini() dmon.fini()
def test_telepath_call(self): dmon, link = self.getFooServ() foo = s_telepath.openlink(link) job = foo.call('bar', 10, 20) self.nn(job) self.eq(foo.syncjob(job), 30) foo.fini() dmon.fini()
def test_crypto_zerosig(self): dmon = s_daemon.Daemon() dmon.share('foo',Foo()) link = dmon.listen('tcp://127.0.0.1:0/foo?zerosig=1') prox = s_telepath.openlink(link) self.assertEqual( prox.bar(), 'baz' ) prox.fini() dmon.fini()
def test_crypto_rc4(self): dmon = s_daemon.Daemon() dmon.share('foo', Foo()) link = dmon.listen('tcp://127.0.0.1:0/foo?rc4key=asdfasdf') prox = s_telepath.openlink(link) self.eq(prox.bar(), 'baz') prox.fini() dmon.fini()
def test_crypto_zerosig(self): dmon = s_daemon.Daemon() dmon.share('foo', Foo()) link = dmon.listen('tcp://127.0.0.1:0/foo?zerosig=1') prox = s_telepath.openlink(link) self.assertEqual(prox.bar(), 'baz') prox.fini() dmon.fini()
def _fireAxonClone(self, iden, poff): # axon iden is persistent ( and used as svc name ) clonefo = self.cloneinfo.get(iden) with poff: while not self.isfini: try: svcfo = self.axonbus.getSynSvcByName(iden) if not svcfo: time.sleep(0.3) continue link = svcfo[1].get('link') if link is None: # pragma: no cover raise s_common.LinkErr('No Axon clone Link For: %s' % (iden, )) with s_telepath.openlink(link) as axon: self._addCloneReady(iden) self.clones[iden] = axon #if oldp == None: #self.cloned.release() self.fire('syn:axon:clone:ready', iden=iden) off = poff.get() for noff, item in self.syncdir.items(off): axon.sync(item) poff.set(noff) clonefo['off'] = noff except Exception as e: # pragma: no cover logger.exception('Axon %s, clone iden %s (_fireAxonClone)', self.iden, iden) if self.isfini: break time.sleep(1)
def test_telepath_surrogate(self): dmon, link = self.getFooServ() foo = s_telepath.openlink(link) bads = '\u01cb\ufffd\ud842\ufffd\u0012' t0 = ('1234', {'key': bads}) # Shovel a malformed UTF8 string with an unpaired surrogate over telepath ret = foo.echo(t0) self.eq(ret, t0) foo.fini() dmon.fini()
def test_telepath_gzip(self): with s_daemon.Daemon() as dmon: link = dmon.listen('tcp://127.0.0.1:0/foo') dmon.share('foo', Foo()) with s_telepath.openlink(link) as prox: wait0 = self.getTestWait(prox, 1, 'sock:gzip') #wait1 = self.getTestWait(dmon., 1, 'sock:gzip') ping = 'V' * 50000 self.eq(prox.echo(ping), ping) wait0.wait(timeout=2)
def test_telepath_basics(self): env = self.getFooEnv() foo = s_telepath.openlink(env.link) s = time.time() for i in range(1000): foo.speed() e = time.time() self.assertEqual(foo.bar(10, 20), 30) self.assertRaises(JobErr, foo.faz, 10, 20) self.assertRaises(JobErr, foo.baz, 10, 20) foo.fini() env.fini()
def test_telepath_basics(self): env = self.getFooEnv() foo = s_telepath.openlink(env.link) s = time.time() for i in range(1000): foo.speed() e = time.time() self.assertEqual( foo.bar(10,20), 30 ) self.assertRaises( JobErr, foo.faz, 10, 20 ) self.assertRaises( JobErr, foo.baz, 10, 20 ) foo.fini() env.fini()
def _fireAxonClone(self, iden, poff): # axon iden is persistent ( and used as svc name ) clonefo = self.cloneinfo.get(iden) with poff: while not self.isfini: try: svcfo = self.axonbus.getSynSvcByName(iden) link = svcfo[1].get('link') if link == None: raise Exception('NoLinkFor: %s' % (iden,)) with s_telepath.openlink(link) as axon: self._addCloneReady(iden) self.clones[iden] = axon #if oldp == None: #self.cloned.release() off = poff.get() for noff,item in self.syncdir.items(off): axon.sync(item) poff.set(noff) clonefo['off'] = noff except Exception as e: logger.exception('_fireAxonClone') if self.isfini: break time.sleep(1)
def _fireAxonClone(self, iden, poff): # axon iden is persistent ( and used as svc name ) clonefo = self.cloneinfo.get(iden) with poff: while not self.isfini: try: svcfo = self.axonbus.getSynSvcByName(iden) link = svcfo[1].get('link') if link == None: raise Exception('NoLinkFor: %s' % (iden, )) with s_telepath.openlink(link) as axon: self._addCloneReady(iden) self.clones[iden] = axon #if oldp == None: #self.cloned.release() off = poff.get() for noff, item in self.syncdir.items(off): axon.sync(item) poff.set(noff) clonefo['off'] = noff except Exception as e: logger.exception('_fireAxonClone') if self.isfini: break time.sleep(1)
def test_telepath_basics(self): env = self.getFooEnv() foo = s_telepath.openlink(env.link) self.true(s_telepath.isProxy(foo)) self.false(s_telepath.isProxy(self)) s = time.time() for i in range(1000): foo.speed() e = time.time() self.eq(foo.bar(10, 20), 30) self.raises(NoSuchMeth, foo.faz, 10, 20) self.raises(SynErr, foo.baz, 10, 20) foo.fini() env.fini()
def test_axon_host_clone(self): self.skipLongTest() self.thisHostMustNot(platform='windows') busurl = 'local://%s/axons' % guid() dmon = s_daemon.Daemon() dmon.listen(busurl) dmon.share('axons', s_service.SvcBus(), fini=True) with self.getTestDir() as datadir: dir0 = gendir(datadir, 'host0') dir1 = gendir(datadir, 'host1') dir2 = gendir(datadir, 'host2') host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) props = { 'axon:bytemax': s_axon.megabyte * 50, } axfo0 = host0.add(**props) axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon self.true(axon0._waitClonesReady(timeout=16)) self.notin(axfo0[0], host0.cloneaxons) # get refs to axon0's clones ciden1 = host1.cloneaxons[0] axonc1 = host1.axons.get(ciden1) # type: s_axon.Axon self.eq(axonc1.getConfOpt('axon:clone:iden'), axfo0[0]) ciden2 = host2.cloneaxons[0] axonc2 = host2.axons.get(ciden2) # type: s_axon.Axon self.eq(axonc2.getConfOpt('axon:clone:iden'), axfo0[0]) # Ensure axon:clone events have fired on the cores iden = axon0.alloc(100) cv = b'V' * 100 blob = axon0.chunk(iden, cv) # We need to check the clones of axon0 to ensure that the clones have the data too time.sleep(1) self.nn(blob) self.true(axon0.has('md5', blob[1].get('axon:blob:md5'))) self.true(axon0.has('sha1', blob[1].get('axon:blob:sha1'))) self.true(axon0.has('sha256', blob[1].get('axon:blob:sha256'))) axonbyts = b''.join(_byts for _byts in axon0.iterblob(blob)) self.eq(axonbyts, cv) self.true(axonc1.has('md5', blob[1].get('axon:blob:md5'))) self.true(axonc1.has('sha1', blob[1].get('axon:blob:sha1'))) self.true(axonc1.has('sha256', blob[1].get('axon:blob:sha256'))) axonbyts = b''.join(_byts for _byts in axonc1.iterblob(blob)) self.eq(axonbyts, cv) self.true(axonc2.has('md5', blob[1].get('axon:blob:md5'))) self.true(axonc2.has('sha1', blob[1].get('axon:blob:sha1'))) self.true(axonc2.has('sha256', blob[1].get('axon:blob:sha256'))) axonbyts = b''.join(_byts for _byts in axonc2.iterblob(blob)) self.eq(axonbyts, cv) # Now write a large amount of data to axon0 and ensure that data is replicated blobs = [] n = 5 tststr = 'deadb33f' * 100000 for i in range(n): byts = tststr + 'lol' * i byts = byts.encode() blob = axon0.eatbytes(byts) blobs.append(blob) self.eq(len(blobs), n) time.sleep(2) for i, blob in enumerate(blobs): form, pprop = s_tufo.ndef(blob) ret = axonc1.byiden(pprop) self.nn(ret) ret = axonc2.byiden(pprop) self.nn(ret) # Add axons to the other two hosts and see that they clone over to the other hosts axfo1 = host1.add(**props) axon1 = s_telepath.openlink(axfo1[1].get('link')) # type: s_axon.Axon self.true(axon1._waitClonesReady(timeout=16)) axfo2 = host2.add(**props) axon2 = s_telepath.openlink(axfo2[1].get('link')) # type: s_axon.Axon self.true(axon2._waitClonesReady(timeout=16)) self.eq(len(host0.axons), 3) self.eq(len(host1.axons), 3) self.eq(len(host2.axons), 3) self.eq(len(host0.cloneaxons), 2) self.eq(len(host1.cloneaxons), 2) self.eq(len(host2.cloneaxons), 2) # Fini the proxy objects axonc1.fini() axonc2.fini() axon0.fini() axon1.fini() axon2.fini() # Fini the hosts host0.fini() host1.fini() host2.fini() # Ensure the axonhost fini'd its objects self.true(host0.axonbus.isfini) for axon in host0.axons.values(): self.true(axon.isfini) dmon.fini()
def test_axon_cluster(self): self.skipLongTest() self.thisHostMustNot(platform='windows') busurl = 'local://%s/axons' % guid() dmon = s_daemon.Daemon() dmon.listen(busurl) dmon.share('axons', s_service.SvcBus(), fini=True) svcprox = s_service.openurl(busurl) axcluster = s_axon.AxonCluster(svcprox) with self.getTestDir() as datadir: dir0 = gendir(datadir, 'host0') dir1 = gendir(datadir, 'host1') dir2 = gendir(datadir, 'host2') host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) props = { 'axon:clones': 1, 'axon:bytemax': s_axon.megabyte, } axfo0 = host0.add(**props) axcluster._waitWrAxons(1, 4) # Ensure our axfo0 was cloned to someone in the cluster axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon axon0._waitClonesReady(timeout=16) foundclone = False if host1.cloneaxons: foundclone = True if host2.cloneaxons: foundclone = True self.true(foundclone) axon0.fini() # fini the proxy object self.false(axcluster.has('md5', craphash)) self.false(axcluster.has('md5', asdfhash)) buf = b'asdfasdf' iden = axcluster.alloc(len(buf)) self.nn(axcluster.chunk(iden, buf)) self.false(axcluster.has('md5', craphash)) self.true(axcluster.has('md5', asdfhash)) self.true(axcluster.has('guid', asdfhash_iden)) blobs = axcluster.find('md5', craphash) self.eq(len(blobs), 0) time.sleep(0.2) # Yield to axon threads blobs = axcluster.find('md5', asdfhash) # We have two blobs for the same hash since the clone of axfo0 is up on host1/host2 self.eq(len(blobs), 2) blob = blobs[0] byts = b''.join(axcluster.iterblob(blob)) self.eq(byts, buf) blob[1].pop('.axon') byts = b''.join(axcluster.iterblob(blob)) self.eq(byts, buf) self.nn(axcluster.wants('md5', craphash, len(buf))) self.none(axcluster.wants('md5', asdfhash, len(buf))) # Eat bytes via AxonMixin APIs byts = 'pennywise'.encode() blob = axcluster.eatbytes(byts) self.nn(blob) self.isin('.new', blob[1]) blob = axcluster.eatbytes(byts) self.notin('.new', blob[1]) buf = io.BytesIO('dancing clown'.encode()) blob = axcluster.eatfd(buf) self.nn(blob) self.isin('.new', blob[1]) blob = axcluster.eatfd(buf) self.notin('.new', blob[1]) host0.fini() host1.fini() host2.fini() svcprox.fini() dmon.fini()
def test_axon_cluster_cortex(self): self.skipLongTest() self.thisHostMustNot(platform='windows') localguid = guid() busurl = 'local://%s/axons' % localguid hahaurl = 'local://%s/haha' % localguid dmon = s_daemon.Daemon() dmon.listen(busurl) dmon.share('axons', s_service.SvcBus(), fini=True) dmon.share('haha', {}) svcprox = s_service.openurl(busurl) axcluster = s_axon.AxonCluster(svcprox) with self.getTestDir() as datadir: dir0 = gendir(datadir, 'host0') dir1 = gendir(datadir, 'host1') dir2 = gendir(datadir, 'host2') host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2', 'axon:axonbus': busurl, 'axon:bytemax': s_axon.megabyte * 100, }) props = { 'axon:clones': 1, 'axon:bytemax': s_axon.megabyte, } axfo0 = host0.add(**props) axcluster._waitWrAxons(1, 4) # Ensure our axfo0 was cloned to someone in the cluster axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon axon0._waitClonesReady(timeout=16) foundclone = False if host1.cloneaxons: foundclone = True if host2.cloneaxons: foundclone = True self.true(foundclone) axon0.fini() # fini the proxy object core = s_cortex.openurl('ram://') core.setConfOpt('axon:url', busurl) self.false(axcluster.has('md5', craphash)) self.false(axcluster.has('md5', asdfhash)) node = core.formNodeByBytes(b'asdfasdf', name='asdf') self.eq(node[1].get('file:bytes'), asdfhash_iden) self.eq(node[1].get('file:bytes:md5'), asdfhash) self.true(axcluster.has('md5', asdfhash)) self.true(axcluster.has('guid', asdfhash_iden)) fd = io.BytesIO(b'visi') node = core.formNodeByFd(fd, name='visi.bin') self.eq(node[1].get('file:bytes:size'), 4) self.eq(node[1].get('file:bytes:name'), 'visi.bin') self.eq(node[1].get('file:bytes'), '442f602ecf8230b2a59a44b4f845be27') self.eq(node[1].get('file:bytes:md5'), '1b2e93225959e3722efed95e1731b764') self.true(axcluster.has('md5', '1b2e93225959e3722efed95e1731b764')) self.true(axcluster.has('guid', '442f602ecf8230b2a59a44b4f845be27')) host0.fini() host1.fini() host2.fini() svcprox.fini() dmon.fini()