def test_link_tcp_client(self):
data = {'count':0}
def sockinit(event):
data['count'] += 1
mesgs = []
def wootmesg(sock,mesg):
mesgs.append(mesg)
return tufo('woot',foo=mesg[1].get('id'))
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/?zerosig=1')
#link = tufo('tcp',host='127.0.0.1',port=0)
daemon = s_daemon.Daemon()
daemon.on('link:sock:init',sockinit)
daemon.setMesgMethod('woot',wootmesg)
daemon.runLinkServer(link)
relay = s_link.initLinkRelay(link)
link[1]['trans'] = True
client = relay.initLinkClient()
reply = client.sendAndRecv('woot',id=1)
self.assertEqual( reply[1].get('foo'), 1 )
reply = client.sendAndRecv('woot',id=2)
self.assertEqual( reply[1].get('foo'), 2 )
self.assertEqual( data.get('count'), 3)
client.fini()
daemon.fini()
def test_crypto_rc4(self):
link1 = s_link.chopLinkUrl('tcp://127.0.0.1:0?rc4key=asdfasdf')
data = {}
def wootmesg(sock, mesg):
data['foo'] = mesg[1].get('foo')
return ('woot2', {})
daemon = s_daemon.Daemon()
daemon.setMesgMethod('woot1', wootmesg)
daemon.runLinkServer(link1)
relay = s_link.initLinkRelay(link1)
client = relay.initLinkClient()
repl = client.sendAndRecv('woot1', foo=2)
self.assertEqual(repl[0], 'woot2')
self.assertEqual(data.get('foo'), 2)
client.fini()
daemon.fini()
def openurl(url, **opts):
'''
Construct a Telepath Proxy from a URL.
Args:
url (str): A URL to parser into a link tufo.
**opts: Additional options which are added to the link tufo.
Examples:
Get a proxy object from a URL, call a function then close the object::
foo = openurl('tcp://1.2.3.4:90/foo')
foo.dostuff(30) # call remote method
foo.fini()
Get a proxy object as a context manager, which will result in the object being automatically closed::
with openurl('tcp://1.2.3.4:90/foo') as foo:
foo.dostuff(30)
Returns:
Proxy: A Proxy object for calling remote tasks.
'''
#plex = opts.pop('plex',None)
#return openclass(Proxy,url,**opts)
link = s_link.chopLinkUrl(url)
link[1].update(opts)
return openlink(link)
def listen(self, linkurl, **opts):
'''
Create and run a link server by url.
Example:
link = dmon.listen('tcp://127.0.0.1:8888')
Notes:
* Returns the parsed link tufo
'''
link = s_link.chopLinkUrl(linkurl)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
sock = relay.listen()
sock.on('link:sock:init', self.dist )
self.plex.addPlexSock(sock)
self._dmon_links.append(link)
return link
def openclass(cls, url, **opts):
plex = opts.pop('plex',None)
link = s_link.chopLinkUrl(url)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
return cls(relay, plex=plex)
def openclass(cls, url, **opts):
plex = opts.pop('plex',None)
link = s_link.chopLinkUrl(url)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
return cls(relay, plex=plex)
def getFooServ(self):
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/foo')
daemon = s_daemon.Daemon()
daemon.runLinkServer(link)
daemon.addSharedObject('foo',Foo())
return daemon,link
def getFooServ(self):
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/foo')
daemon = s_daemon.Daemon()
daemon.runLinkServer(link)
daemon.addSharedObject('foo',Foo())
return daemon,link
def getPgConn(self):
'''
Get a psycopg2 connection object.
The PG database connected to is derived from the SYN_TEST_PG_DB
environmental variable.
Returns:
psycopg2.connection: Raw psycopg2 connection object.
'''
db = os.getenv('SYN_TEST_PG_DB')
if not db:
raise unittest.SkipTest('no SYN_TEST_PG_DB envar')
try:
import psycopg2
except ImportError:
raise unittest.SkipTest('psycopg2 not installed.')
url = 'postgres://%s' % db
link = s_link.chopLinkUrl(url)
def _initDbInfo(link):
dbinfo = {}
path = link[1].get('path')
if path:
parts = [p for p in path.split('/') if p]
if parts:
dbinfo['database'] = parts[0]
host = link[1].get('host')
if host is not None:
dbinfo['host'] = host
port = link[1].get('port')
if port is not None:
dbinfo['port'] = port
user = link[1].get('user')
if user is not None:
dbinfo['user'] = user
passwd = link[1].get('passwd')
if passwd is not None:
dbinfo['password'] = passwd
return dbinfo
dbinfo = _initDbInfo(link)
conn = psycopg2.connect(**dbinfo)
return conn
def openurl(url, conf=None, storconf=None, **opts):
'''
Construct or reference a cortex by url.
This will open a cortex if there is a registered handler for the URL
otherwise it will attempt to connect to the URL via Telepath.
If telepath is used, any configable options passed via openurl will
not be automatically set.
Args:
url (str): URL which is parsed in order to connect to.
conf (dict): Configable options passed to the Cortex.
storconf (dict): Configable options passed to the storage layer.
**opts (dict): Additional options added to the link tufo from the
parsed URL.
Examples:
Open up a ram backed cortex::
core = openurl('ram:///')
Open up a remote cortex over telepath::
core = openurl('tcp://1.2.3.4:10000/core)
Notes:
The following handlers are registered by default:
* ram://
* sqlite:///<db>
* lmdb:///<db>
* postgres://[[<passwd>:]<user>@][<host>]/[<db>][/<table>]
For SQL databases, the default table name is "syncortex"
Returns:
s_cores_common.Cortex: Cortex object or a telepath proxy for a Cortex.
'''
# Todo
# auditfd=<fd>
# auditfile=<filename>
if not conf:
conf = {}
if not storconf:
storconf = {}
link = s_link.chopLinkUrl(url)
link[1].update(opts)
return openlink(link, conf, storconf)
def test_link_fromurl(self):
url = 'tcp://*****:*****@127.0.0.1:9999/foo?rc4key=wootwoot&retry=20'
link = s_link.chopLinkUrl(url)
self.assertEqual(link[0], 'tcp')
self.assertEqual(link[1].get('port'), 9999)
self.assertEqual(link[1].get('path'), '/foo')
self.assertEqual(link[1].get('retry'), 20)
self.assertEqual(link[1].get('host'), '127.0.0.1')
self.assertEqual(link[1].get('rc4key'), b'wootwoot')
self.assertEqual(link[1].get('user'), 'visi')
self.assertEqual(link[1].get('passwd'), 'secret')
def getProxy(url):
'''
Construct a telepath proxy from a url.
Example:
foo = getProxy('tcp://1.2.3.4:90/foo')
foo.dostuff(30) # call remote method
'''
link = s_link.chopLinkUrl(url)
return Proxy(link)
def test_link_fromurl(self):
url = 'tcp://*****:*****@127.0.0.1:9999/foo?rc4key=wootwoot&timeout=30'
link = s_link.chopLinkUrl(url)
self.assertEqual(link[0],'tcp')
self.assertEqual(link[1].get('port'),9999)
self.assertEqual(link[1].get('path'),'/foo')
self.assertEqual(link[1].get('timeout'),30)
self.assertEqual(link[1].get('host'),'127.0.0.1')
self.assertEqual(link[1].get('rc4key'),b'wootwoot')
self.assertEqual(link[1]['authinfo'].get('user'),'visi')
self.assertEqual(link[1]['authinfo'].get('passwd'),'secret')
def test_link_client_retry(self):
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/?retry=2')
daemon = s_daemon.Daemon()
daemon.runLinkServer(link)
relay = s_link.initLinkRelay(link)
client = relay.initLinkClient()
daemon.fini()
self.assertRaises(s_link.RetryExceeded, client.sendAndRecv, 'woot')
client.fini()
def test_link_client_retry(self):
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/?retry=2')
daemon = s_daemon.Daemon()
daemon.runLinkServer(link)
relay = s_link.initLinkRelay(link)
client = relay.initLinkClient()
daemon.fini()
self.assertRaises( s_link.RetryExceeded, client.sendAndRecv, 'woot' )
client.fini()
def test_cortex_postgres(self):
db = os.getenv('SYN_COR_PG_DB')
if db == None:
raise unittest.SkipTest('no SYN_COR_PG_DB')
table = 'syn_test_%s' % guidstr()
link = s_link.chopLinkUrl('postgres:///%s/%s' % (db,table))
core = s_cortex.openlink(link)
try:
self.runcore( core )
self.runrange( core )
finally:
with core.cursor() as c:
c.execute('DROP TABLE %s' % (table,))
def test_cortex_postgres(self):
db = os.getenv('SYN_COR_PG_DB')
if db == None:
raise unittest.SkipTest('no SYN_COR_PG_DB')
table = 'syn_test_%s' % guidstr()
link = s_link.chopLinkUrl('postgres:///%s/%s' % (db, table))
core = s_cortex.openlink(link)
try:
self.runcore(core)
self.runrange(core)
finally:
with core.cursor() as c:
c.execute('DROP TABLE %s' % (table, ))
def openurl(url, **opts):
'''
Construct a telepath proxy from a url.
Example:
foo = openurl('tcp://1.2.3.4:90/foo')
foo.dostuff(30) # call remote method
'''
#plex = opts.pop('plex',None)
#return openclass(Proxy,url,**opts)
link = s_link.chopLinkUrl(url)
link[1].update(opts)
return openlink(link)
def test_cortex_postgres(self):
db = os.getenv("SYN_COR_PG_DB")
if db == None:
raise unittest.SkipTest("no SYN_COR_PG_DB")
table = "syn_test_%s" % guid()
link = s_link.chopLinkUrl("postgres:///%s/%s" % (db, table))
core = s_cortex.openlink(link)
try:
self.runcore(core)
self.runjson(core)
self.runrange(core)
finally:
with core.cursor() as c:
c.execute("DROP TABLE %s" % (table,))
def openurl(url,**opts):
'''
Construct a telepath proxy from a url.
Example:
foo = openurl('tcp://1.2.3.4:90/foo')
foo.dostuff(30) # call remote method
'''
plex = opts.pop('plex',None)
link = s_link.chopLinkUrl(url)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
return Proxy(relay, plex=plex)
def test_cortex_tufo_by_postgres(self):
db = os.getenv('SYN_COR_PG_DB')
if db == None:
raise unittest.SkipTest('no SYN_COR_PG_DB')
table = 'syn_test_%s' % guid()
link = s_link.chopLinkUrl('postgres:///%s/%s' % (db,table))
core = s_cortex.openlink(link)
fooa = core.formTufoByProp('foo','bar',p0=4)
foob = core.formTufoByProp('foo','baz',p0=5)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [4])), 1)
fooc = core.formTufoByProp('foo','faz',p0=5)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [5])), 2)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [4,5])), 3)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [5], limit=1)), 1)
def test_cortex_tufo_by_postgres(self):
db = os.getenv('SYN_COR_PG_DB')
if db == None:
raise unittest.SkipTest('no SYN_COR_PG_DB')
table = 'syn_test_%s' % guid()
link = s_link.chopLinkUrl('postgres:///%s/%s' % (db,table))
core = s_cortex.openlink(link)
fooa = core.formTufoByProp('foo','bar',p0=4)
foob = core.formTufoByProp('foo','baz',p0=5)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [4])), 1)
fooc = core.formTufoByProp('foo','faz',p0=5)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [5])), 2)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [4,5])), 3)
self.assertEqual( len(core.getTufosBy('in', 'foo:p0', [5], limit=1)), 1)
def openurl(url):
'''
Construct or reference a cortex by url.
Example:
core = openurl('ram://')
Notes:
* ram://
* sqlite3:///<db>
* postgres://[[<passwd>:]<user>@][<host>]/[<db>][/<table>]
* default table name: syncortex
Todo:
auditfd=<fd>
auditfile=<filename>
'''
link = s_link.chopLinkUrl(url)
return openlink(link)
def openstore(url, storconf=None, **opts):
'''
Opens or creates a Cortex Storage object by URL.
This does not attempt to open Storage objects over telepath.
Args:
url (str): URL which is parsed in order to connect to.
storconf (dict): Configable options passed to the storage layer.
**opts (dict): Additional options added to the link tufo from the
parsed URL.
Example:
Opening a object and adding a row::
tick = s_common.now()
url = 'sqlite:///./derry.db'
store = openstore(url)
store.addRows([('1234', 'syn:test', 'what', tick)])
Returns:
synapse.cores.storage.Storage: A storage object implementing a specific backend.
Raises:
NoSuchImpl: If the requested protocol has no storage implementation.
'''
if not storconf:
storconf = {}
link = s_link.chopLinkUrl(url)
link[1].update(opts)
ctor = storectors.get(link[0])
if ctor is None:
raise s_common.NoSuchImpl(
name=link[0],
mesg='No storage ctor registered for {}'.format(link[0]))
return ctor(link, **storconf)
def openurl(url):
'''
Construct or reference a cortex by url.
Example:
core = openurl('ram://')
Notes:
* ram://
* sqlite3:///<db>
* postgres://[[<passwd>:]<user>@][<host>]/[<db>][/<table>]
* default table name: syncortex
Todo:
auditfd=<fd>
auditfile=<filename>
'''
link = s_link.chopLinkUrl(url)
return openlink(link)
def connect(self, url, **opts):
'''
Connect to a peer neuron
This will attempt to bring up a permanent connection
and reconnect if it is torn down.
'''
if self.isfini:
return
link = s_link.chopLinkUrl(url)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
sock = relay.connect()
if sock == None:
self.sched.insec(1, self.connect, url, **opts)
return None
self.runPlexSock(sock)
sock.tx(tufo('peer:syn', iden=self.iden, mesh=self.mesh))
def connect(self, url, **opts):
'''
Connect to a peer neuron
This will attempt to bring up a permanent connection
and reconnect if it is torn down.
'''
if self.isfini:
return
link = s_link.chopLinkUrl(url)
link[1].update(opts)
relay = s_link.getLinkRelay(link)
sock = relay.connect()
if sock == None:
self.sched.insec( 1, self.connect, url, **opts )
return None
self.runPlexSock(sock)
sock.tx( tufo('peer:syn', iden=self.iden, mesh=self.mesh) )
def main(argv):
p = argparse.ArgumentParser(prog='server')
p.add_argument('--initmod',help='python module name for daemon init callback')
p.add_argument('--cortex', action='append', default=[], help='cortex name,url to share for RMI')
p.add_argument('linkurl',nargs='+',help='link urls to bind/listen')
opts = p.parse_args(argv)
daemon = s_daemon.Daemon()
# possibly load/share a cortex or two
for nameurl in opts.cortex:
name,url = nameurl.split(',',1)
core = s_cortex.openurl(url)
daemon.addSharedObject(name,core)
# fire up requested link servers
for url in opts.linkurl:
link = s_link.chopLinkUrl(url)
daemon.runLinkServer(link)
if opts.initmod:
mod = importlib.import_module(opts.initmod)
meth = getattr(mod,'initDaemon',None)
if meth == None:
print('error: initmod (%s) has no initDaemon() function!')
return
# call back the daemon init module
meth(daemon)
try:
daemon.wait()
except KeyboardInterrupt as e:
print('ctrl-c caught: shutting down')
daemon.fini()
def test_link_tcp_client(self):
data = {'count': 0}
def sockinit(event):
data['count'] += 1
mesgs = []
def wootmesg(sock, mesg):
mesgs.append(mesg)
return tufo('woot', foo=mesg[1].get('id'))
link = s_link.chopLinkUrl('tcp://127.0.0.1:0/?zerosig=1')
#link = tufo('tcp',host='127.0.0.1',port=0)
daemon = s_daemon.Daemon()
daemon.on('link:sock:init', sockinit)
daemon.setMesgMethod('woot', wootmesg)
daemon.runLinkServer(link)
relay = s_link.initLinkRelay(link)
link[1]['trans'] = True
client = relay.initLinkClient()
reply = client.sendAndRecv('woot', id=1)
self.assertEqual(reply[1].get('foo'), 1)
reply = client.sendAndRecv('woot', id=2)
self.assertEqual(reply[1].get('foo'), 2)
self.assertEqual(data.get('count'), 3)
client.fini()
daemon.fini()
def main(argv):
p = argparse.ArgumentParser(prog='server')
p.add_argument('--initmod',help='python module name for daemon init callback')
p.add_argument('linkurl',nargs='+',help='link urls to bind/listen')
opts = p.parse_args(argv)
daemon = s_telepath.Daemon()
for url in opts.linkurl:
link = s_link.chopLinkUrl(url)
daemon.runLinkServer(link)
if opts.initmod:
mod = importlib.import_module(opts.initmod)
meth = getattr(mod,'initDaemon',None)
if meth == None:
print('error: initmod (%s) has no initDaemon() function!')
return
# call back the daemon init module
meth(daemon)
daemon.wait()
def test_crypto_rc4(self):
link1 = s_link.chopLinkUrl('tcp://127.0.0.1:0?rc4key=asdfasdf')
data = {}
def wootmesg(sock,mesg):
data['foo'] = mesg[1].get('foo')
return ('woot2',{})
daemon = s_daemon.Daemon()
daemon.setMesgMethod('woot1',wootmesg)
daemon.runLinkServer(link1)
relay = s_link.initLinkRelay(link1)
client = relay.initLinkClient()
repl = client.sendAndRecv('woot1',foo=2)
self.assertEqual( repl[0], 'woot2' )
self.assertEqual( data.get('foo'), 2)
client.fini()
daemon.fini()
def test_link_pool(self):
link = s_link.chopLinkUrl('foo://*****:*****@[email protected]?poolsize=7&poolmax=-1')
self.assertEqual( link[1].get('poolsize'), 7 )
self.assertEqual( link[1].get('poolmax'), -1 )
def test_link_pool(self):
link = s_link.chopLinkUrl(
'foo://*****:*****@[email protected]?poolsize=7&poolmax=-1')
self.assertEqual(link[1].get('poolsize'), 7)
self.assertEqual(link[1].get('poolmax'), -1)
def test_neuron_peering(self):
neu1 = s_neuron.Daemon()
neu2 = s_neuron.Daemon()
neu3 = s_neuron.Daemon()
neu1.setNeuInfo('rsakey',rsa1)
neu2.setNeuInfo('rsakey',rsa2)
neu3.setNeuInfo('rsakey',rsa3)
cert1 = neu1.genPeerCert()
cert2 = neu2.genPeerCert()
cert3 = neu3.genPeerCert()
neu1.addPeerCert(cert2)
neu1.addPeerCert(cert3)
neu2.addPeerCert(cert1)
neu2.addPeerCert(cert3)
neu3.addPeerCert(cert1)
neu3.addPeerCert(cert2)
link1 = s_link.chopLinkUrl('tcp://127.0.0.1:0')
#link1 = s_common.tufo('tcp',listen=('0.0.0.0',0))
neu1.runLinkServer(link1)
evt1 = threading.Event()
def peer1init(event):
evt1.set()
evt2 = threading.Event()
def peer2init(event):
evt2.set()
evt3 = threading.Event()
def peer3init(event):
evt3.set()
neu1.on('neu:peer:init',peer1init)
neu2.on('neu:peer:init',peer2init)
neu3.on('neu:peer:init',peer3init)
neu2.runLinkPeer(link1)
self.assertTrue( evt1.wait(3) )
self.assertTrue( evt2.wait(3) )
neu3.runLinkPeer(link1)
self.assertTrue( evt3.wait(3) )
rtt1 = neu1.syncPingPeer(neu2.ident)
rtt2 = neu2.syncPingPeer(neu1.ident)
# check that 3 can reach 2 via 1
rtt3 = neu3.syncPingPeer(neu2.ident)
self.assertIsNotNone(rtt1)
self.assertIsNotNone(rtt2)
self.assertIsNotNone(rtt3)
neu1.fini()
neu2.fini()
neu3.fini()
