def test_telepath_yielder(self):
class YieldTest:
def woot(self):
yield 10
yield 20
yield 30
dmon = s_daemon.Daemon()
link = dmon.listen('tcp://127.0.0.1:0/hehe')
dmon.share('hehe', YieldTest() )
prox = s_telepath.openlink(link)
items = []
for item in prox.woot():
items.append(item)
self.assertEqual( tuple(items), (10,20,30) )
self.assertEqual( len(dmon._dmon_yields), 0 )
prox.fini()
dmon.fini()
def _getSvcAxon(self, iden):
svcfo = self.svcprox.getSynSvc(iden)
if svcfo == None:
return None
axon = self.axons.get(iden)
if axon == None:
link = svcfo[1].get('link')
if link == None:
return None
def onfini():
self.axons.pop(iden,None)
try:
# copy before we frob it
#link = (link[0],dict(link[1]))
#link[1]['once'] = True
axon = s_telepath.openlink(link)
self.axons[iden] = axon
axon.onfini( onfini )
except Exception as e:
return None
return axon
def _getSvcAxon(self, iden):
svcfo = self.svcprox.getSynSvc(iden)
if svcfo == None:
return None
axon = self.axons.get(iden)
if axon == None:
link = svcfo[1].get('link')
if link == None:
return None
def onfini():
self.axons.pop(iden, None)
try:
# copy before we frob it
#link = (link[0],dict(link[1]))
#link[1]['once'] = True
axon = s_telepath.openlink(link)
self.axons[iden] = axon
axon.onfini(onfini)
except Exception as e:
return None
return axon
def test_telepath_basics(self):
env = self.getFooEnv()
foo = s_telepath.openlink(env.link)
self.true(s_telepath.isProxy(foo))
self.false(s_telepath.isProxy(self))
# Test magic methods
self.true(bool(foo) is True)
self.true(foo == foo)
self.false(foo == 1)
self.true(foo != 1)
s = time.time()
for i in range(1000):
foo.speed()
e = time.time()
self.eq(foo.bar(10, 20), 30)
self.raises(NoSuchMeth, foo.faz, 10, 20)
self.raises(SynErr, foo.baz, 10, 20)
foo.fini()
# We have fini'd the Proxy resources
self.true(foo._tele_boss.isfini)
self.true(foo._tele_sock.isfini)
self.true(foo._tele_plex.isfini)
env.fini()
def test_telepath_yielder(self):
class YieldTest:
def woot(self):
yield 10
yield 20
yield 30
dmon = s_daemon.Daemon()
link = dmon.listen('tcp://127.0.0.1:0/hehe')
dmon.share('hehe', YieldTest())
prox = s_telepath.openlink(link)
items = []
for item in prox.woot():
items.append(item)
self.eq(tuple(items), (10, 20, 30))
self.eq(len(dmon._dmon_yields), 0)
prox.fini()
dmon.fini()
def test_axon_cluster(self):
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir,'host0')
dir1 = gendir(datadir,'host1')
dir2 = gendir(datadir,'host2')
opts = {
'axonbus':busurl,
}
host0 = s_axon.AxonHost(dir0,hostname='host0',**opts)
host1 = s_axon.AxonHost(dir1,hostname='host1',**opts)
host2 = s_axon.AxonHost(dir2,hostname='host2',**opts)
props = {
'clones':2,
'syncmax':s_axon.megabyte,
'bytemax':s_axon.megabyte,
}
axfo0 = host0.add(**props)
axon0 = s_telepath.openlink( axfo0[1].get('link') )
# wait for clones to come online
self.assertTrue( axon0._waitClonesReady(timeout=2) )
#self.assertIsNotNone( usage.get('total') )
#axon = host.axons.get(iden)
iden = axon0.alloc(100)
blob = axon0.chunk(iden,b'V'*100)
self.assertIsNotNone(blob)
self.assertTrue( axon0.has( 'md5', blob[1].get('hash:md5') ) )
self.assertTrue( axon0.has( 'sha1', blob[1].get('hash:sha1') ) )
self.assertTrue( axon0.has( 'sha256', blob[1].get('hash:sha256') ) )
axon0.fini()
host0.fini()
host1.fini()
host2.fini()
dmon.fini()
def test_axon_cluster(self):
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir,'host0')
dir1 = gendir(datadir,'host1')
dir2 = gendir(datadir,'host2')
opts = {
'axonbus':busurl,
}
host0 = s_axon.AxonHost(dir0,hostname='host0',**opts)
host1 = s_axon.AxonHost(dir1,hostname='host1',**opts)
host2 = s_axon.AxonHost(dir2,hostname='host2',**opts)
props = {
'clones':2,
'syncmax':s_axon.megabyte,
'bytemax':s_axon.megabyte,
}
axfo0 = host0.add(**props)
axon0 = s_telepath.openlink( axfo0[1].get('link') )
# wait for clones to come online
self.assertTrue( axon0._waitClonesReady(timeout=2) )
#self.assertIsNotNone( usage.get('total') )
#axon = host.axons.get(iden)
iden = axon0.alloc(100)
blob = axon0.chunk(iden,b'V'*100)
self.assertIsNotNone(blob)
self.assertTrue( axon0.has( 'md5', blob[1].get('hash:md5') ) )
self.assertTrue( axon0.has( 'sha1', blob[1].get('hash:sha1') ) )
self.assertTrue( axon0.has( 'sha256', blob[1].get('hash:sha256') ) )
axon0.fini()
host0.fini()
host1.fini()
host2.fini()
dmon.fini()
def openlink(link):
'''
Open a cortex via a link tuple.
'''
ctor = corctors.get(link[0])
if ctor is None:
return s_telepath.openlink(link)
return ctor(link)
def test_telepath_call(self):
dmon, link = self.getFooServ()
foo = s_telepath.openlink(link)
job = foo.call('bar', 10, 20)
self.assertIsNotNone(job)
self.assertEqual(foo.syncjob(job), 30)
foo.fini()
dmon.fini()
def test_telepath_call(self):
dmon,link = self.getFooServ()
foo = s_telepath.openlink(link)
job = foo.call('bar', 10, 20)
self.assertIsNotNone( job )
self.assertEqual( foo.sync(job), 30 )
foo.fini()
dmon.fini()
def test_telepath_call(self):
dmon, link = self.getFooServ()
foo = s_telepath.openlink(link)
job = foo.call('bar', 10, 20)
self.nn(job)
self.eq(foo.syncjob(job), 30)
foo.fini()
dmon.fini()
def test_crypto_zerosig(self):
dmon = s_daemon.Daemon()
dmon.share('foo',Foo())
link = dmon.listen('tcp://127.0.0.1:0/foo?zerosig=1')
prox = s_telepath.openlink(link)
self.assertEqual( prox.bar(), 'baz' )
prox.fini()
dmon.fini()
def test_crypto_rc4(self):
dmon = s_daemon.Daemon()
dmon.share('foo', Foo())
link = dmon.listen('tcp://127.0.0.1:0/foo?rc4key=asdfasdf')
prox = s_telepath.openlink(link)
self.eq(prox.bar(), 'baz')
prox.fini()
dmon.fini()
def test_crypto_zerosig(self):
dmon = s_daemon.Daemon()
dmon.share('foo', Foo())
link = dmon.listen('tcp://127.0.0.1:0/foo?zerosig=1')
prox = s_telepath.openlink(link)
self.assertEqual(prox.bar(), 'baz')
prox.fini()
dmon.fini()
def _fireAxonClone(self, iden, poff):
# axon iden is persistent ( and used as svc name )
clonefo = self.cloneinfo.get(iden)
with poff:
while not self.isfini:
try:
svcfo = self.axonbus.getSynSvcByName(iden)
if not svcfo:
time.sleep(0.3)
continue
link = svcfo[1].get('link')
if link is None: # pragma: no cover
raise s_common.LinkErr('No Axon clone Link For: %s' %
(iden, ))
with s_telepath.openlink(link) as axon:
self._addCloneReady(iden)
self.clones[iden] = axon
#if oldp == None:
#self.cloned.release()
self.fire('syn:axon:clone:ready', iden=iden)
off = poff.get()
for noff, item in self.syncdir.items(off):
axon.sync(item)
poff.set(noff)
clonefo['off'] = noff
except Exception as e: # pragma: no cover
logger.exception('Axon %s, clone iden %s (_fireAxonClone)',
self.iden, iden)
if self.isfini:
break
time.sleep(1)
def test_telepath_surrogate(self):
dmon, link = self.getFooServ()
foo = s_telepath.openlink(link)
bads = '\u01cb\ufffd\ud842\ufffd\u0012'
t0 = ('1234', {'key': bads})
# Shovel a malformed UTF8 string with an unpaired surrogate over telepath
ret = foo.echo(t0)
self.eq(ret, t0)
foo.fini()
dmon.fini()
def test_telepath_gzip(self):
with s_daemon.Daemon() as dmon:
link = dmon.listen('tcp://127.0.0.1:0/foo')
dmon.share('foo', Foo())
with s_telepath.openlink(link) as prox:
wait0 = self.getTestWait(prox, 1, 'sock:gzip')
#wait1 = self.getTestWait(dmon., 1, 'sock:gzip')
ping = 'V' * 50000
self.eq(prox.echo(ping), ping)
wait0.wait(timeout=2)
def test_telepath_basics(self):
env = self.getFooEnv()
foo = s_telepath.openlink(env.link)
s = time.time()
for i in range(1000):
foo.speed()
e = time.time()
self.assertEqual(foo.bar(10, 20), 30)
self.assertRaises(JobErr, foo.faz, 10, 20)
self.assertRaises(JobErr, foo.baz, 10, 20)
foo.fini()
env.fini()
def test_telepath_basics(self):
env = self.getFooEnv()
foo = s_telepath.openlink(env.link)
s = time.time()
for i in range(1000):
foo.speed()
e = time.time()
self.assertEqual( foo.bar(10,20), 30 )
self.assertRaises( JobErr, foo.faz, 10, 20 )
self.assertRaises( JobErr, foo.baz, 10, 20 )
foo.fini()
env.fini()
def _fireAxonClone(self, iden, poff):
# axon iden is persistent ( and used as svc name )
clonefo = self.cloneinfo.get(iden)
with poff:
while not self.isfini:
try:
svcfo = self.axonbus.getSynSvcByName(iden)
link = svcfo[1].get('link')
if link == None:
raise Exception('NoLinkFor: %s' % (iden,))
with s_telepath.openlink(link) as axon:
self._addCloneReady(iden)
self.clones[iden] = axon
#if oldp == None:
#self.cloned.release()
off = poff.get()
for noff,item in self.syncdir.items(off):
axon.sync(item)
poff.set(noff)
clonefo['off'] = noff
except Exception as e:
logger.exception('_fireAxonClone')
if self.isfini:
break
time.sleep(1)
def _fireAxonClone(self, iden, poff):
# axon iden is persistent ( and used as svc name )
clonefo = self.cloneinfo.get(iden)
with poff:
while not self.isfini:
try:
svcfo = self.axonbus.getSynSvcByName(iden)
link = svcfo[1].get('link')
if link == None:
raise Exception('NoLinkFor: %s' % (iden, ))
with s_telepath.openlink(link) as axon:
self._addCloneReady(iden)
self.clones[iden] = axon
#if oldp == None:
#self.cloned.release()
off = poff.get()
for noff, item in self.syncdir.items(off):
axon.sync(item)
poff.set(noff)
clonefo['off'] = noff
except Exception as e:
logger.exception('_fireAxonClone')
if self.isfini:
break
time.sleep(1)
def test_telepath_basics(self):
env = self.getFooEnv()
foo = s_telepath.openlink(env.link)
self.true(s_telepath.isProxy(foo))
self.false(s_telepath.isProxy(self))
s = time.time()
for i in range(1000):
foo.speed()
e = time.time()
self.eq(foo.bar(10, 20), 30)
self.raises(NoSuchMeth, foo.faz, 10, 20)
self.raises(SynErr, foo.baz, 10, 20)
foo.fini()
env.fini()
def test_axon_host_clone(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:bytemax': s_axon.megabyte * 50,
}
axfo0 = host0.add(**props)
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
self.true(axon0._waitClonesReady(timeout=16))
self.notin(axfo0[0], host0.cloneaxons)
# get refs to axon0's clones
ciden1 = host1.cloneaxons[0]
axonc1 = host1.axons.get(ciden1) # type: s_axon.Axon
self.eq(axonc1.getConfOpt('axon:clone:iden'), axfo0[0])
ciden2 = host2.cloneaxons[0]
axonc2 = host2.axons.get(ciden2) # type: s_axon.Axon
self.eq(axonc2.getConfOpt('axon:clone:iden'), axfo0[0])
# Ensure axon:clone events have fired on the cores
iden = axon0.alloc(100)
cv = b'V' * 100
blob = axon0.chunk(iden, cv)
# We need to check the clones of axon0 to ensure that the clones have the data too
time.sleep(1)
self.nn(blob)
self.true(axon0.has('md5', blob[1].get('axon:blob:md5')))
self.true(axon0.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axon0.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axon0.iterblob(blob))
self.eq(axonbyts, cv)
self.true(axonc1.has('md5', blob[1].get('axon:blob:md5')))
self.true(axonc1.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axonc1.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axonc1.iterblob(blob))
self.eq(axonbyts, cv)
self.true(axonc2.has('md5', blob[1].get('axon:blob:md5')))
self.true(axonc2.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axonc2.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axonc2.iterblob(blob))
self.eq(axonbyts, cv)
# Now write a large amount of data to axon0 and ensure that data is replicated
blobs = []
n = 5
tststr = 'deadb33f' * 100000
for i in range(n):
byts = tststr + 'lol' * i
byts = byts.encode()
blob = axon0.eatbytes(byts)
blobs.append(blob)
self.eq(len(blobs), n)
time.sleep(2)
for i, blob in enumerate(blobs):
form, pprop = s_tufo.ndef(blob)
ret = axonc1.byiden(pprop)
self.nn(ret)
ret = axonc2.byiden(pprop)
self.nn(ret)
# Add axons to the other two hosts and see that they clone over to the other hosts
axfo1 = host1.add(**props)
axon1 = s_telepath.openlink(axfo1[1].get('link')) # type: s_axon.Axon
self.true(axon1._waitClonesReady(timeout=16))
axfo2 = host2.add(**props)
axon2 = s_telepath.openlink(axfo2[1].get('link')) # type: s_axon.Axon
self.true(axon2._waitClonesReady(timeout=16))
self.eq(len(host0.axons), 3)
self.eq(len(host1.axons), 3)
self.eq(len(host2.axons), 3)
self.eq(len(host0.cloneaxons), 2)
self.eq(len(host1.cloneaxons), 2)
self.eq(len(host2.cloneaxons), 2)
# Fini the proxy objects
axonc1.fini()
axonc2.fini()
axon0.fini()
axon1.fini()
axon2.fini()
# Fini the hosts
host0.fini()
host1.fini()
host2.fini()
# Ensure the axonhost fini'd its objects
self.true(host0.axonbus.isfini)
for axon in host0.axons.values():
self.true(axon.isfini)
dmon.fini()
def test_axon_cluster(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
svcprox = s_service.openurl(busurl)
axcluster = s_axon.AxonCluster(svcprox)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:clones': 1,
'axon:bytemax': s_axon.megabyte,
}
axfo0 = host0.add(**props)
axcluster._waitWrAxons(1, 4)
# Ensure our axfo0 was cloned to someone in the cluster
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
axon0._waitClonesReady(timeout=16)
foundclone = False
if host1.cloneaxons:
foundclone = True
if host2.cloneaxons:
foundclone = True
self.true(foundclone)
axon0.fini() # fini the proxy object
self.false(axcluster.has('md5', craphash))
self.false(axcluster.has('md5', asdfhash))
buf = b'asdfasdf'
iden = axcluster.alloc(len(buf))
self.nn(axcluster.chunk(iden, buf))
self.false(axcluster.has('md5', craphash))
self.true(axcluster.has('md5', asdfhash))
self.true(axcluster.has('guid', asdfhash_iden))
blobs = axcluster.find('md5', craphash)
self.eq(len(blobs), 0)
time.sleep(0.2) # Yield to axon threads
blobs = axcluster.find('md5', asdfhash)
# We have two blobs for the same hash since the clone of axfo0 is up on host1/host2
self.eq(len(blobs), 2)
blob = blobs[0]
byts = b''.join(axcluster.iterblob(blob))
self.eq(byts, buf)
blob[1].pop('.axon')
byts = b''.join(axcluster.iterblob(blob))
self.eq(byts, buf)
self.nn(axcluster.wants('md5', craphash, len(buf)))
self.none(axcluster.wants('md5', asdfhash, len(buf)))
# Eat bytes via AxonMixin APIs
byts = 'pennywise'.encode()
blob = axcluster.eatbytes(byts)
self.nn(blob)
self.isin('.new', blob[1])
blob = axcluster.eatbytes(byts)
self.notin('.new', blob[1])
buf = io.BytesIO('dancing clown'.encode())
blob = axcluster.eatfd(buf)
self.nn(blob)
self.isin('.new', blob[1])
blob = axcluster.eatfd(buf)
self.notin('.new', blob[1])
host0.fini()
host1.fini()
host2.fini()
svcprox.fini()
dmon.fini()
def test_axon_cluster_cortex(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
localguid = guid()
busurl = 'local://%s/axons' % localguid
hahaurl = 'local://%s/haha' % localguid
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
dmon.share('haha', {})
svcprox = s_service.openurl(busurl)
axcluster = s_axon.AxonCluster(svcprox)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:clones': 1,
'axon:bytemax': s_axon.megabyte,
}
axfo0 = host0.add(**props)
axcluster._waitWrAxons(1, 4)
# Ensure our axfo0 was cloned to someone in the cluster
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
axon0._waitClonesReady(timeout=16)
foundclone = False
if host1.cloneaxons:
foundclone = True
if host2.cloneaxons:
foundclone = True
self.true(foundclone)
axon0.fini() # fini the proxy object
core = s_cortex.openurl('ram://')
core.setConfOpt('axon:url', busurl)
self.false(axcluster.has('md5', craphash))
self.false(axcluster.has('md5', asdfhash))
node = core.formNodeByBytes(b'asdfasdf', name='asdf')
self.eq(node[1].get('file:bytes'), asdfhash_iden)
self.eq(node[1].get('file:bytes:md5'), asdfhash)
self.true(axcluster.has('md5', asdfhash))
self.true(axcluster.has('guid', asdfhash_iden))
fd = io.BytesIO(b'visi')
node = core.formNodeByFd(fd, name='visi.bin')
self.eq(node[1].get('file:bytes:size'), 4)
self.eq(node[1].get('file:bytes:name'), 'visi.bin')
self.eq(node[1].get('file:bytes'), '442f602ecf8230b2a59a44b4f845be27')
self.eq(node[1].get('file:bytes:md5'), '1b2e93225959e3722efed95e1731b764')
self.true(axcluster.has('md5', '1b2e93225959e3722efed95e1731b764'))
self.true(axcluster.has('guid', '442f602ecf8230b2a59a44b4f845be27'))
host0.fini()
host1.fini()
host2.fini()
svcprox.fini()
dmon.fini()
