def test_daemon_sessconf(self):
with self.getTestDir() as dirname:
dmon = s_daemon.Daemon()
conf = {
'sessions': {
'maxtime': 99999,
'savefile': os.path.join(dirname, 'sessions.sql3'),
},
}
dmon.loadDmonConf(conf)
sess0 = dmon.getNewSess()
iden = sess0.iden
sess0.put('woot', 10)
dmon.fini()
dmon = s_daemon.Daemon()
dmon.loadDmonConf(conf)
sess1 = dmon.getSessByIden(iden)
self.eq(sess1.get('woot'), 10)
dmon.fini()
def test_webapp_dmon(self):
class FooServer(s_webapp.WebApp):
def __init__(self, core):
s_webapp.WebApp.__init__(self, **{})
self.core = core
s_dyndeps.addDynAlias('FooServer', FooServer)
with s_daemon.Daemon() as core_dmon:
with self.getRamCore() as core:
core_dmon.share('core', core)
link = core_dmon.listen('tcp://127.0.0.1:0/')
linkurl = 'tcp://127.0.0.1:%d/core' % link[1]['port']
with s_daemon.Daemon() as dmon:
config = {
'vars': {
'linkurl': linkurl,
},
'ctors': [
('core', 'ctor://synapse.cortex.openurl(linkurl)'),
('webapp', 'ctor://FooServer(core)')
]
}
dmon.loadDmonConf(config)
def test_daemon_ctor_config(self):
conf = {
'ctors': (
('foo', 'ctor://synapse.cortex.openurl("ram://")', {'config': 'woot'}),
('bar', 'ctor://synapse.cortex.openurl("ram://")', {'configs': ('woot', 'blah')}),
),
'configs': {
'woot': {},
}
}
with s_daemon.Daemon() as dmon:
self.raises(NoSuchConf, dmon.loadDmonConf, conf)
conf['configs']['blah'] = {'newp': 1}
with s_daemon.Daemon() as dmon:
self.raises(NoSuchOpt, dmon.loadDmonConf, conf)
conf['configs']['blah'].pop('newp', None)
conf['configs']['blah']['caching'] = 'TRUE'
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
core = dmon.locs.get('bar')
self.eq(core.caching, 1)
def test_telepath_events(self):
with s_daemon.Daemon() as dmon:
with s_service.SvcBus() as sbus:
urlt = 'tcp://127.0.0.1:%d'
url = urlt % (0)
link = dmon.listen(url)
port = link[1].get('port')
url = urlt % (port)
dmon.share('sbus', sbus)
proxy0 = s_telepath.openurl(url + '/sbus')
proxy1 = s_telepath.openurl(url + '/sbus')
counters = [0, 0]
def count(offset):
def on(*args, **kwargs):
counters[offset] += 1
return on
proxy0.on('tufo:tag:add', count(0))
proxy1.on('tufo:tag:add', count(1))
wait = s_eventbus.Waiter(proxy1, 1, 'tufo:tag:add')
proxy0.fire('tufo:tag:add',
tag='tagu',
tufo=('iden', {
'prop': 'valu'
}))
wait.wait()
self.assertEqual(counters[0], 1)
self.assertEqual(counters[1], 1)
def getHiveEnv(self):
#FIXME remove when synapse link local:// supports windows
self.thisHostMustNot(platform='windows')
dmon = s_daemon.Daemon()
queen = s_hivemind.Queen()
dmon.share('syn.queen', queen)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
qprox = s_telepath.openurl('tcp://127.0.0.1/syn.queen', port=port)
hive = s_hivemind.Hive(qprox, size=32)
drone = s_hivemind.Drone(qprox, size=2)
env = TestEnv()
# order matters for fini calls...
env.add('hive', hive, fini=True)
env.add('drone', drone, fini=True)
env.add('qprox', qprox, fini=True)
env.add('queen', queen, fini=True)
env.add('dmon', dmon, fini=True)
return env
def test_model_file_bytes_axon(self):
self.skipLongTest()
with self.getAxonCore() as env:
with s_daemon.Daemon() as dmon:
dmonlink = dmon.listen('tcp://127.0.0.1:0/')
dmonport = dmonlink[1].get('port')
dmon.share('core', env.core)
coreurl = 'tcp://127.0.0.1:%d/core' % dmonport
core = s_telepath.openurl(coreurl)
node = core.formNodeByBytes(b'visi', name='visi.bin')
self.eq(node[1].get('file:bytes:size'), 4)
self.eq(node[1].get('file:bytes:name'), 'visi.bin')
self.eq(node[1].get('file:bytes'), '442f602ecf8230b2a59a44b4f845be27')
self.eq(node[1].get('file:bytes:md5'), '1b2e93225959e3722efed95e1731b764')
self.eq(node[1].get('file:bytes:sha1'), '93de0c7d579384feb3561aa504acd8f23f388040')
self.eq(node[1].get('file:bytes:sha256'), 'e45bbb7e03acacf4d1cca4c16af1ec0c51d777d10e53ed3155bd3d8deb398f3f')
self.eq(node[1].get('file:bytes:sha512'), '8238be12bcc3c10da7e07dbea528e9970dc809c07c5aef545a14e5e8d2038563b'
'29c2e818d167b06e6a33412e6beb8347fcc44520691347aea9ee21fcf804e39')
fd = io.BytesIO(b'foobar')
node = core.formNodeByFd(fd, name='foobar.exe')
self.eq(node[1].get('file:bytes:size'), 6)
self.eq(node[1].get('file:bytes:name'), 'foobar.exe')
def test_service_base(self):
with s_daemon.Daemon() as dmon:
sbus = s_service.SvcBus()
dmon.share('syn.svcbus', sbus, fini=True)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
with s_telepath.openurl('tcp://127.0.0.1/syn.svcbus',
port=port) as prox:
woot = Woot()
s_service.runSynSvc('syn.woot', woot, prox)
svcs = prox.getSynSvcs()
self.eq(len(svcs), 1)
self.eq(svcs[0][1].get('name'), 'syn.woot')
dyntask = s_async.newtask('foo', 10, y=30)
job = prox.callx(svcs[0][0], dyntask)
self.eq(prox.syncjob(job), 40)
def test_link_ssl_auth(self):
# FIXME some kind of cert validation diffs in *py* vers killed us
cafile = getTestPath('ca.crt')
keyfile = getTestPath('server.key')
certfile = getTestPath('server.crt')
userkey = getTestPath('user.key')
usercert = getTestPath('user.crt')
with s_daemon.Daemon() as dmon:
dmon.share('foobar', FooBar())
link = dmon.listen('ssl://localhost:0/',
cafile=cafile,
keyfile=keyfile,
certfile=certfile)
port = link[1].get('port')
url = 'ssl://localhost/foobar'
with s_telepath.openurl(url,
port=port,
cafile=cafile,
keyfile=userkey,
certfile=usercert) as foo:
self.assertEqual(foo.foo(), 'bar')
def test_telepath_reconnect(self):
tenv = self.getFooEnv()
port = tenv.link[1].get('port')
prox = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)
url = 'tcp://127.0.0.1:%d/foo' % (port, )
self.eq(prox.bar(10, 20), 30)
waiter = self.getTestWait(prox, 1, 'tele:sock:init')
# shut down the daemon
tenv.dmon.fini()
dmon = s_daemon.Daemon()
dmon.share('foo', Foo())
dmon.listen(url)
waiter.wait()
self.eq(prox.bar(10, 20), 30)
prox.fini()
dmon.fini()
tenv.fini()
def test_service_base(self):
sbus = s_service.SvcBus()
dmon = s_daemon.Daemon()
dmon.share('syn.svcbus', sbus)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
prox = s_telepath.openurl('tcp://127.0.0.1/syn.svcbus', port=port)
woot = Woot()
s_service.runSynSvc('syn.woot', woot, prox)
svcs = prox.getSynSvcs()
self.assertEqual( len(svcs), 1 )
self.assertEqual( svcs[0][1].get('name'), 'syn.woot' )
dyntask = s_async.newtask('foo',10,y=30)
job = prox.callx(svcs[0][0], dyntask)
self.assertEqual( prox.sync(job), 40 )
prox.fini()
dmon.fini()
sbus.fini()
def test_cortex_xact_deadlock(self):
N = 100
prop = 'testform'
fd = tempfile.NamedTemporaryFile()
dmon = s_daemon.Daemon()
pool = s_threads.Pool(size=4, maxsize=8)
wait = s_eventbus.Waiter(pool, 1, 'pool:work:fini')
with s_cortex.openurl('sqlite:///%s' % fd.name) as core:
def populate():
for i in range(N):
#print('wrote %d tufos' % i)
core.formTufoByProp(prop, str(i), **{})
dmon.share('core', core)
link = dmon.listen('tcp://127.0.0.1:0/core')
prox = s_telepath.openurl('tcp://127.0.0.1:%d/core' % link[1]['port'])
pool.wrap(populate)()
for i in range(N):
tufos = prox.getTufosByProp(prop)
#print('got %d tufos' % len(tufos))
wait.wait()
pool.fini()
def __init__(self, queen, **config):
# NOTE: queen must *always* be a telepath proxy
EventBus.__init__(self)
self.iden = s_common.guid()
self.slots = {}
self.slocs = {}
self.queen = queen
self.config = config
# FIXME maybe put our hostname etc in config?
self.queen.on('tele:sock:init', self._onTeleSockInit)
self.queen.on('hive:tell:%s' % self.iden, self._onHiveTell)
self.localurl = 'local://%s/syn.queen' % self.iden
# each worker has a local:// daemon
self.dmon = s_daemon.Daemon()
self.dmon.listen(self.localurl)
self.dmon.share('syn.queen', queen)
self.on('hive:slot:run', self._onHiveSlotRun)
self.on('hive:slot:fini', self._onHiveSlotFini)
self._initQueenProxy()
def test_daemon_conf_fork(self):
self.thisHostMustNot(platform='windows')
iden = guid()
conf = {
'forks': (('fork0', {
'ctors':
(('haha', 'ctor://synapse.tests.test_daemon.Woot()'), ),
'share': (('haha', {}), ),
'listen': ('local://%s' % (iden, ), ),
}), ),
}
dmon = s_daemon.Daemon()
dmon.loadDmonConf(conf)
prox = s_telepath.openurl('local://%s/haha?retry=20' % (iden, ))
pid0 = prox.pid()
self.ne(pid0, os.getpid())
prox.fini()
#dmon.killDmonFork('fork0')
#prox = s_telepath.openurl('local://%s/haha?retry=6' % (iden,))
#pid1 = prox.pid()
#self.assertNotEqual( pid0, pid1 )
#self.assertNotEqual( pid1, os.getpid() )
#prox.fini()
dmon.fini()
def test_daemon_ctor_dmonurl(self):
conf = {
'ctors': [
('thing0', 'synapse.tests.test_daemon.Blah', {
'lulz': 'hehe'
}),
('thing1', 'test:check:blah', {}),
],
}
self.raises(NoSuchName, s_telepath.openurl, 'dmon://thing0')
with s_daemon.Daemon() as dmon:
def checkblah(conf):
self.eq(dmon.locs.get('thing0'),
s_telepath.openurl('dmon://thing0'))
self.raises(NoSuchName, s_telepath.openurl, 'dmon://newp0/')
return 1
s_dyndeps.addDynAlias('test:check:blah', checkblah)
dmon.loadDmonConf(conf)
s_dyndeps.delDynAlias('test:check:blah')
def test_telepath_yielder(self):
class YieldTest:
def woot(self):
yield 10
yield 20
yield 30
dmon = s_daemon.Daemon()
link = dmon.listen('tcp://127.0.0.1:0/hehe')
dmon.share('hehe', YieldTest())
prox = s_telepath.openlink(link)
items = []
for item in prox.woot():
items.append(item)
self.eq(tuple(items), (10, 20, 30))
self.eq(len(dmon._dmon_yields), 0)
prox.fini()
dmon.fini()
def getFooServ(self):
dmon = s_daemon.Daemon()
link = dmon.listen('tcp://127.0.0.1:0/foo')
dmon.share('foo', Foo())
return dmon, link
def test_crypto_rc4(self):
link1 = s_link.chopLinkUrl('tcp://127.0.0.1:0?rc4key=asdfasdf')
data = {}
def wootmesg(sock, mesg):
data['foo'] = mesg[1].get('foo')
return ('woot2', {})
daemon = s_daemon.Daemon()
daemon.setMesgMethod('woot1', wootmesg)
daemon.runLinkServer(link1)
relay = s_link.initLinkRelay(link1)
client = relay.initLinkClient()
repl = client.sendAndRecv('woot1', foo=2)
self.assertEqual(repl[0], 'woot2')
self.assertEqual(data.get('foo'), 2)
client.fini()
daemon.fini()
def getDmonCore(self):
'''
Context manager to make a ram:/// cortex which has test models loaded into it and shared via daemon.
Yields:
s_cores_common.Cortex: A proxy object to the Ram backed cortex with test models.
'''
dmon = s_daemon.Daemon()
core = s_cortex.openurl('ram:///')
self.addTstForms(core)
link = dmon.listen('tcp://127.0.0.1:0/')
dmon.share('core00', core)
port = link[1].get('port')
prox = s_telepath.openurl('tcp://127.0.0.1/core00', port=port)
s_scope.set('syn:test:link', link)
s_scope.set('syn:cmd:core', prox)
try:
yield prox
except: # pragma: no cover
raise
finally:
prox.fini()
core.fini()
dmon.fini()
def test_daemon_on(self):
class Foo:
def bar(self):
return 'baz'
dmon = s_daemon.Daemon()
link = dmon.listen('tcp://127.0.0.1:0/')
bus = s_eventbus.EventBus()
foo = Foo()
dmon.share('bus', bus)
dmon.share('foo', foo)
port = link[1].get('port')
bprox = s_telepath.openurl('tcp://127.0.0.1/bus', port=port)
fprox = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)
evt = threading.Event()
def woot(mesg):
evt.set()
bprox.on('woot', woot)
fprox.on('woot', woot)
bus.fire('woot')
evt.wait(timeout=2)
fprox.off('woot', woot)
self.true(evt.is_set())
def getFooEnv(self, url='tcp://127.0.0.1:0/foo'):
env = TstEnv()
env.add('dmon', s_daemon.Daemon(), fini=True)
env.add('link', env.dmon.listen(url))
env.dmon.share('foo', Foo())
return env
def getFooServ(self):
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/foo')
daemon = s_daemon.Daemon()
daemon.runLinkServer(link)
daemon.addSharedObject('foo',Foo())
return daemon,link
def test_axon_cluster(self):
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir,'host0')
dir1 = gendir(datadir,'host1')
dir2 = gendir(datadir,'host2')
opts = {
'axonbus':busurl,
}
host0 = s_axon.AxonHost(dir0,hostname='host0',**opts)
host1 = s_axon.AxonHost(dir1,hostname='host1',**opts)
host2 = s_axon.AxonHost(dir2,hostname='host2',**opts)
props = {
'clones':2,
'syncmax':s_axon.megabyte,
'bytemax':s_axon.megabyte,
}
axfo0 = host0.add(**props)
axon0 = s_telepath.openlink( axfo0[1].get('link') )
# wait for clones to come online
self.assertTrue( axon0._waitClonesReady(timeout=2) )
#self.assertIsNotNone( usage.get('total') )
#axon = host.axons.get(iden)
iden = axon0.alloc(100)
blob = axon0.chunk(iden,b'V'*100)
self.assertIsNotNone(blob)
self.assertTrue( axon0.has( 'md5', blob[1].get('hash:md5') ) )
self.assertTrue( axon0.has( 'sha1', blob[1].get('hash:sha1') ) )
self.assertTrue( axon0.has( 'sha256', blob[1].get('hash:sha256') ) )
axon0.fini()
host0.fini()
host1.fini()
host2.fini()
dmon.fini()
def test_axon_clone_large(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
dir3 = gendir(datadir, 'host3')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:clones': 1,
'axonhost:autorun': 2,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:clones': 1,
'axonhost:autorun': 2,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:clones': 1,
'axonhost:autorun': 2,
'axon:bytemax': s_axon.megabyte * 100,
})
host3 = s_axon.AxonHost(dir3, **{'axon:hostname': 'host3',
'axon:axonbus': busurl,
'axon:clones': 1,
'axonhost:autorun': 2,
'axon:bytemax': s_axon.megabyte * 100,
})
time.sleep(3)
ta = [len(host.axons) for host in [host0, host1, host2, host3]]
tc = [len(host.cloneaxons) for host in [host0, host1, host2, host3]]
total_axons = sum(ta)
total_clones = sum(tc)
self.eq(total_axons, 16)
self.eq(total_clones, 8)
host0.fini()
host1.fini()
host2.fini()
host3.fini()
dmon.fini()
def test_service_dmon_conf(self):
conf0 = {
'ctors': [
['svcbus', 'ctor://synapse.lib.service.SvcBus()', {}],
],
'share': [
['svcbus', {}],
],
}
with s_daemon.Daemon() as dmon0:
dmon0.loadDmonConf(conf0)
link = dmon0.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
conf1 = {
'ctors': [
['svcbus',
'tcp://127.0.0.1:%d/svcbus' % port, {}],
['ebus', 'ctor://synapse.eventbus.EventBus()', {}],
],
'services': [
['svcbus', [
['ebus', {
'woot': 'heh'
}],
]],
],
}
with s_daemon.Daemon() as dmon1:
dmon1.loadDmonConf(conf1)
proxy = s_telepath.openurl('tcp://127.0.0.1/svcbus', port=port)
dmon1.onfini(proxy.fini)
svcfo = proxy.getSynSvcsByTag('ebus')[0]
self.eq(svcfo[1].get('woot'), 'heh')
def test_telepath_clientside(self):
with s_daemon.Daemon() as dmon:
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
dmon.share('foo', Foo())
with s_telepath.openurl('tcp://127.0.0.1/foo', port=port) as prox:
self.eq(prox.localthing(20), 20)
self.eq(prox.localthing(30), 30)
def test_telepath_events(self):
with s_daemon.Daemon() as dmon:
with s_service.SvcBus() as sbus:
urlt = 'tcp://127.0.0.1:%d'
url = urlt % (0)
link = dmon.listen(url)
port = link[1].get('port')
url = urlt % (port)
dmon.share('sbus', sbus)
proxy0 = s_telepath.openurl(url + '/sbus')
proxy1 = s_telepath.openurl(url + '/sbus')
counters = collections.defaultdict(int)
def count(name):
def onmesg(mesg):
counters[name] += 1
return onmesg
proxy0.on('foo:bar', count('p0'))
proxy1.on('foo:bar', count('p1'))
func = count('f0')
proxy1.on('foo:bar', func, tag='hehe')
wait = proxy1.waiter(1, 'foo:bar')
proxy0.fire('foo:bar', tag='tagu', tufo=('iden', {'prop': 'valu'}))
self.nn(wait.wait(timeout=2))
self.eq(counters['p0'], 1)
self.eq(counters['p1'], 1)
self.eq(counters['f0'], 0)
wait = proxy1.waiter(1, 'foo:bar')
proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
self.nn(wait.wait(timeout=2))
self.eq(counters['p0'], 2)
self.eq(counters['p1'], 2)
self.eq(counters['f0'], 1)
proxy1.off('foo:bar', func)
wait = proxy1.waiter(1, 'foo:bar')
proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
self.nn(wait.wait(timeout=2))
self.eq(counters['p0'], 3)
self.eq(counters['p1'], 3)
self.eq(counters['f0'], 1)
proxy0.fini()
proxy1.fini()
def test_crypto_rc4(self):
dmon = s_daemon.Daemon()
dmon.share('foo', Foo())
link = dmon.listen('tcp://127.0.0.1:0/foo?rc4key=asdfasdf')
prox = s_telepath.openlink(link)
self.eq(prox.bar(), 'baz')
prox.fini()
dmon.fini()
def test_daemon_timeout(self):
daemon = s_daemon.Daemon()
link = daemon.listen('tcp://127.0.0.1:0/?timeout=0.1')
relay = s_link.getLinkRelay(link)
sock = relay.connect()
self.eq(sock.recvobj(), None)
sock.fini()
daemon.fini()
def test_crypto_zerosig(self):
dmon = s_daemon.Daemon()
dmon.share('foo', Foo())
link = dmon.listen('tcp://127.0.0.1:0/foo?zerosig=1')
prox = s_telepath.openlink(link)
self.assertEqual(prox.bar(), 'baz')
prox.fini()
dmon.fini()
def test_model_file_bytes_axon(self):
fd = io.BytesIO(b'foobar')
# create an cortex with access to an axon
with self.getTestDir() as dirname:
conf = {
'ctors': (
('axon00', 'syn:axon', {
'datadir': dirname
}),
('core00', 'syn:cortex', {
'url': 'ram:///',
'axon:url': 'dmon://axon00'
}),
),
'share': (('core00', {}), ),
}
with s_daemon.Daemon() as dmon:
dmon.loadDmonConf(conf)
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
core = s_telepath.openurl('tcp://127.0.0.1/core00', port=port)
node = core.formNodeByBytes(b'visi', name='visi.bin')
self.eq(node[1].get('file:bytes:size'), 4)
self.eq(node[1].get('file:bytes:name'), 'visi.bin')
self.eq(node[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(node[1].get('file:bytes:md5'),
'1b2e93225959e3722efed95e1731b764')
self.eq(node[1].get('file:bytes:sha1'),
'93de0c7d579384feb3561aa504acd8f23f388040')
self.eq(
node[1].get('file:bytes:sha256'),
'e45bbb7e03acacf4d1cca4c16af1ec0c51d777d10e53ed3155bd3d8deb398f3f'
)
self.eq(
node[1].get('file:bytes:sha512'),
'8238be12bcc3c10da7e07dbea528e9970dc809c07c5aef545a14e5e8d2038563b29c2e818d167b06e6a33412e6beb8347fcc44520691347aea9ee21fcf804e39'
)
node = core.formNodeByFd(fd, name='foobar.exe')
self.eq(node[1].get('file:bytes:size'), 6)
self.eq(node[1].get('file:bytes:name'), 'foobar.exe')