def test_link_chopurl(self):
info = s_urlhelp.chopurl('foo://woot.com:99/foo/bar')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), 99)
self.assertEqual(info.get('host'), 'woot.com')
self.assertEqual(info.get('path'), '/foo/bar')
info = s_urlhelp.chopurl('foo://*****:*****@woot.com')
self.assertEqual(info.get('user'), 'visi')
self.assertEqual(info.get('passwd'), 'secret')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), None)
self.assertEqual(info.get('host'), 'woot.com')
info = s_urlhelp.chopurl('foo://[2607:f8b0:4004:806::1014]:99/foo/bar?baz=faz&gronk=woot')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), 99)
self.assertEqual(info.get('host'), '2607:f8b0:4004:806::1014')
self.assertEqual(info.get('path'), '/foo/bar')
self.assertEqual(info.get('query').get('baz'), 'faz')
self.assertEqual(info.get('query').get('gronk'), 'woot')
info = s_urlhelp.chopurl('foo://*****:*****@[email protected]')
self.assertEqual( info.get('user'), 'visi')
self.assertEqual( info.get('passwd'), 'c@t')
self.assertEqual( info.get('host'), 'woot.com')
def test_link_chopurl(self):
info = s_urlhelp.chopurl('foo://woot.com:99/foo/bar')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), 99)
self.assertEqual(info.get('host'), 'woot.com')
self.assertEqual(info.get('path'), '/foo/bar')
info = s_urlhelp.chopurl('foo://*****:*****@woot.com')
self.assertEqual(info.get('user'), 'visi')
self.assertEqual(info.get('passwd'), 'secret')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), None)
self.assertEqual(info.get('host'), 'woot.com')
info = s_urlhelp.chopurl('foo://[2607:f8b0:4004:806::1014]:99/foo/bar?baz=faz&gronk=woot')
self.assertEqual(info.get('scheme'), 'foo')
self.assertEqual(info.get('port'), 99)
self.assertEqual(info.get('host'), '2607:f8b0:4004:806::1014')
self.assertEqual(info.get('path'), '/foo/bar')
self.assertEqual(info.get('query').get('baz'), 'faz')
self.assertEqual(info.get('query').get('gronk'), 'woot')
info = s_urlhelp.chopurl('foo://*****:*****@[email protected]')
self.assertEqual( info.get('user'), 'visi')
self.assertEqual( info.get('passwd'), 'c@t')
self.assertEqual( info.get('host'), 'woot.com')
def chopurl(url, **opts):
if isinstance(url, str):
if url.find('://') == -1:
newurl = alias(url)
if newurl is None:
raise s_exc.BadUrl(mesg=f':// not found in [{url}] and no alias found!',
url=url)
url = newurl
info = s_urlhelp.chopurl(url)
# flatten query params into info
query = info.pop('query', None)
if query is not None:
info.update(query)
elif isinstance(url, dict):
info = dict(url)
else:
mesg = 'telepath.chopurl() requires a str or dict.'
raise s_exc.BadArg(mesg)
info.update(opts)
return info
async def listen(self, url, **opts):
'''
Bind and listen on the given host/port with possible SSL.
Args:
host (str): A hostname or IP address.
port (int): The TCP port to bind.
'''
info = s_urlhelp.chopurl(url, **opts)
info.update(opts)
host = info.get('host')
port = info.get('port')
sslctx = None
if info.get('scheme') == 'ssl':
sslctx = self.certdir.getServerSSLContext(hostname=host)
server = await s_link.listen(host, port, self._onLinkInit, ssl=sslctx)
self.listenservers.append(server)
ret = server.sockets[0].getsockname()
if self.addr is None:
self.addr = ret
return ret
async def listen(self, url, **opts):
'''
Bind and listen on the given host/port with possible SSL.
Args:
host (str): A hostname or IP address.
port (int): The TCP port to bind.
'''
info = s_urlhelp.chopurl(url, **opts)
info.update(opts)
scheme = info.get('scheme')
if scheme == 'unix':
path = info.get('path')
try:
server = await s_link.unixlisten(path, self._onLinkInit)
except Exception as e:
if 'path too long' in str(e):
logger.error(
'unix:// exceeds OS supported UNIX socket path length: %s',
path)
raise
else:
host = info.get('host')
port = info.get('port')
sslctx = None
if scheme == 'ssl':
caname = None
hostname = None
query = info.get('query')
if query is not None:
hostname = query.get('hostname', host)
caname = query.get('ca')
sslctx = self.certdir.getServerSSLContext(hostname=hostname,
caname=caname)
server = await s_link.listen(host,
port,
self._onLinkInit,
ssl=sslctx)
self.listenservers.append(server)
ret = server.sockets[0].getsockname()
if self.addr is None:
self.addr = ret
return ret
def chopLinkUrl(url):
'''
Parse a link tuple from a url.
Example:
link = chopLinkUrl('tcp://1.2.3.4:80/')
Notes:
* url parameters become link properties
* user:passwd@host syntax is used for authdata
'''
urlinfo = s_urlhelp.chopurl(url)
scheme = urlinfo.get('scheme')
link = (scheme, {})
link[1]['url'] = url
link[1]['host'] = urlinfo.get('host')
link[1]['port'] = urlinfo.get('port')
link[1]['path'] = urlinfo.get('path')
link[1]['user'] = urlinfo.get('user')
link[1]['passwd'] = urlinfo.get('passwd')
query = urlinfo.get('query', {})
timeout = query.pop('timeout', None)
if timeout is not None:
link[1]['timeout'] = float(timeout)
poolmax = query.pop('poolmax', None)
if poolmax is not None:
link[1]['poolmax'] = int(poolmax)
poolsize = query.pop('poolsize', None)
if poolsize is not None:
link[1]['poolsize'] = int(poolsize)
rc4key = query.pop('rc4key', None)
if rc4key is not None:
link[1]['rc4key'] = rc4key.encode('utf8')
zerosig = query.pop('zerosig', None)
if zerosig is not None:
link[1]['zerosig'] = True
retry = query.pop('retry', None)
if retry is not None:
link[1]['retry'] = int(retry, 0)
link[1].update(query)
return link
def chopLinkUrl(url):
'''
Parse a link tuple from a url.
Example:
link = chopLinkUrl('tcp://1.2.3.4:80/')
Notes:
* url parameters become link properties
* user:passwd@host syntax is used for authdata
'''
urlinfo = s_urlhelp.chopurl(url)
scheme = urlinfo.get('scheme')
link = (scheme,{})
link[1]['url'] = url
link[1]['host'] = urlinfo.get('host')
link[1]['port'] = urlinfo.get('port')
link[1]['path'] = urlinfo.get('path')
link[1]['user'] = urlinfo.get('user')
link[1]['passwd'] = urlinfo.get('passwd')
query = urlinfo.get('query',{})
timeout = query.pop('timeout',None)
if timeout != None:
link[1]['timeout'] = float(timeout)
poolmax = query.pop('poolmax',None)
if poolmax != None:
link[1]['poolmax'] = int(poolmax)
poolsize = query.pop('poolsize',None)
if poolsize != None:
link[1]['poolsize'] = int(poolsize)
rc4key = query.pop('rc4key',None)
if rc4key != None:
link[1]['rc4key'] = rc4key.encode('utf8')
zerosig = query.pop('zerosig',None)
if zerosig != None:
link[1]['zerosig'] = True
retry = query.pop('retry',None)
if retry != None:
link[1]['retry'] = int(retry,0)
link[1].update(query)
return link
async def listen(self, url, **opts):
'''
Bind and listen on the given host/port with possible SSL.
Args:
host (str): A hostname or IP address.
port (int): The TCP port to bind.
'''
info = s_urlhelp.chopurl(url, **opts)
info.update(opts)
scheme = info.get('scheme')
if scheme == 'unix':
path = info.get('path')
try:
server = await s_link.unixlisten(path, self._onLinkInit)
except Exception as e:
if 'path too long' in str(e):
logger.error(f'unix:// exceeds OS supported UNIX socket path length: {path}')
raise
else:
host = info.get('host')
port = info.get('port')
sslctx = None
if scheme == 'ssl':
sslctx = self.certdir.getServerSSLContext(hostname=host)
server = await s_link.listen(host, port, self._onLinkInit, ssl=sslctx)
self.listenservers.append(server)
ret = server.sockets[0].getsockname()
if self.addr is None:
self.addr = ret
return ret
async def openurl(url, **opts):
'''
Open a URL to a remote telepath object.
Args:
url (str): A telepath URL.
**opts (dict): Telepath connect options.
Returns:
(synapse.telepath.Proxy): A telepath proxy object.
The telepath proxy may then be used for sync or async calls:
proxy = openurl(url)
value = proxy.getFooThing()
... or ...
proxy = await openurl(url)
valu = await proxy.getFooThing()
... or ...
async with await openurl(url) as proxy:
valu = await proxy.getFooThing()
'''
if url.find('://') == -1:
newurl = alias(url)
if newurl is None:
raise s_exc.BadUrl(f':// not found in [{url}] and no alias found!')
url = newurl
info = s_urlhelp.chopurl(url)
info.update(opts)
host = info.get('host')
port = info.get('port')
auth = None
user = info.get('user')
if user is not None:
passwd = info.get('passwd')
auth = (user, {'passwd': passwd})
scheme = info.get('scheme')
if scheme == 'cell':
# cell:///path/to/celldir:share
# cell://rel/path/to/celldir:share
path = info.get('path')
name = info.get('name', '*')
# support cell://<relpath>/<to>/<cell>
# by detecting host...
host = info.get('host')
if host:
path = path.strip('/')
path = os.path.join(host, path)
if ':' in path:
path, name = path.split(':')
full = os.path.join(path, 'sock')
link = await s_link.unixconnect(full)
elif scheme == 'unix':
# unix:///path/to/sock:share
path, name = info.get('path').split(':')
link = await s_link.unixconnect(path)
else:
path = info.get('path')
name = info.get('name', path[1:])
sslctx = None
if scheme == 'ssl':
certpath = info.get('certdir')
certdir = s_certdir.CertDir(certpath)
sslctx = certdir.getClientSSLContext()
link = await s_link.connect(host, port, ssl=sslctx)
prox = await Proxy.anit(link, name)
prox.onfini(link)
try:
await prox.handshake(auth=auth)
except Exception:
await prox.fini()
raise
return prox
async def test_discovery_consul(self):
# Example data from https://www.consul.io/api/catalog.html
consul_data = [{
"ID": "40e4a748-2192-161a-0510-9bf59fe950b5",
"Node": "foobar",
"Address": "192.168.10.10",
"Datacenter": "dc1",
"TaggedAddresses": {
"lan": "192.168.10.10",
"wan": "10.0.10.10"
},
"NodeMeta": {
"somekey": "somevalue"
},
"CreateIndex": 51,
"ModifyIndex": 51,
"ServiceAddress": "172.17.0.3",
"ServiceEnableTagOverride": False,
"ServiceID": "32a2a47f7992:nodea:5000",
"ServiceName": "foobar",
"ServicePort": 5000,
"ServiceMeta": {
"foobar_meta_value": "baz"
},
"ServiceTaggedAddresses": {
"lan": {
"address": "172.17.0.3",
"port": 5000
},
"wan": {
"address": "198.18.0.1",
"port": 512
}
},
"ServiceTags": ["tacos"],
"ServiceProxy": {
"DestinationServiceName": "",
"DestinationServiceID": "",
"LocalServiceAddress": "",
"LocalServicePort": 0,
"Config": None,
"Upstreams": None
},
"ServiceConnect": {
"Native": False,
"Proxy": None
},
"Namespace": "default"
}]
with self.getTestDir() as dirn:
async with await s_cell.Cell.anit(dirn) as cell:
root = await cell.auth.getUserByName('root')
await root.setPasswd('root')
cell.consul_data = consul_data
hhost, hport = await cell.addHttpsPort(0)
thost, tport = await cell.dmon.listen('tcp://127.0.0.1:0')
cell.addHttpApi('/v1/catalog/service/(.*)', ConsulV1Handler,
{'cell': cell})
consul = f'https://127.0.0.1:{hport}'
burl = f'tcp+consul://root:root@foobar/*?consul_nosslverify=1&consul={consul}'
info = s_urlhelp.chopurl(burl)
self.eq(info.get('host'), 'foobar')
self.none(info.get('port'))
# Discovery functions update info dict in place.
await s_telepath.disc_consul(info)
self.eq(info.get('host'), '192.168.10.10')
self.eq(info.get('port'), 5000)
self.eq(info.get('original_host'), 'foobar')
# Support tag_address and service_tag_address
url = burl + '&consul_tag_address=wan'
info = s_urlhelp.chopurl(url)
await s_telepath.disc_consul(info)
self.eq(info.get('host'), '10.0.10.10')
self.eq(info.get('port'), 5000)
self.eq(info.get('original_host'), 'foobar')
url = burl + '&consul_service_tag_address=wan'
info = s_urlhelp.chopurl(url)
await s_telepath.disc_consul(info)
self.eq(info.get('host'), '198.18.0.1')
self.eq(info.get('port'), 512)
self.eq(info.get('original_host'), 'foobar')
# Support servicetag based port finding
url = burl + '&consul_tag=tacos'
info = s_urlhelp.chopurl(url)
await s_telepath.disc_consul(info)
self.eq(info.get('host'), '192.168.10.10')
self.eq(info.get('port'), 5000)
self.eq(info.get('original_host'), 'foobar')
# Punch in our host/port info into consul_data so we can do a end to end test
consul_data[0]['Address'] = thost
consul_data[0]['ServicePort'] = tport
async with await s_telepath.openurl(burl) as proxy:
self.isin('synapse.lib.cell.CellApi', proxy._getClasses())
# Sad path - non-existing service name.
with self.raises(s_exc.BadUrl) as cm:
url = f'tcp+consul://root:root@newp/*?consul_nosslverify=1&consul={consul}'
await s_telepath.disc_consul(s_urlhelp.chopurl(url))
self.eq(cm.exception.get('name'), 'newp')
# Sad path - multiple tag resolution.
with self.raises(s_exc.BadUrl) as cm:
url = burl + '&consul_tag_address=wan'
url = url + '&consul_service_tag_address=lan'
await s_telepath.disc_consul(s_urlhelp.chopurl(url))
self.eq(cm.exception.get('consul_tag_address'), 'wan')
self.eq(cm.exception.get('consul_service_tag_address'), 'lan')
# Sad path - ssl verification failure
with self.raises(s_exc.BadUrl) as cm:
url = f'tcp+consul://root:root@foobar/*?consul={consul}'
await s_telepath.disc_consul(s_urlhelp.chopurl(url))
self.isin('certificate verify failed',
cm.exception.get('mesg'))
self.isinstance(cm.exception.__context__,
ssl.SSLCertVerificationError)
# Sad path - iterating through results and having a
# non-existent consul_tag value.
new_data = consul_data[0].copy()
new_data['ServiceTags'] = []
cell.consul_data.insert(0, new_data)
with self.raises(s_exc.BadUrl) as cm:
url = burl + '&consul_tag=burritos'
await s_telepath.disc_consul(s_urlhelp.chopurl(url))
self.eq('burritos', cm.exception.get('tag'))
def test_urlchop(self):
url = 'http://vertex.link:8080/hehe.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'port': 8080,
'host': 'vertex.link',
'path': '/hehe.html',
},
info
)
url = 'tcp://*****:*****@vertex.link/'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'user': '******',
'host': 'vertex.link',
'path': '/',
'passwd': 'candy',
},
info
)
url = 'tcp://[email protected]'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'user': '******',
'host': 'vertex.link',
'path': '',
},
info
)
url = 'tcp://1.2.3.4:8080/api/v1/wow?key=valu&foo=bar'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'host': '1.2.3.4',
'port': 8080,
'path': '/api/v1/wow',
'query': {'key': 'valu',
'foo': 'bar',
}
},
info
)
url = 'http://[1fff:0:a88:85a3::ac1f]:8001/index.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'host': '1fff:0:a88:85a3::ac1f',
'port': 8001,
'path': '/index.html',
},
info
)
url = 'http://::1/index.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'host': '::1',
'path': '/index.html',
},
info
)
self.raises(s_exc.BadUrl, s_urlhelp.chopurl,
'www.vertex.link')
async def openurl(url, **opts):
'''
Open a URL to a remote telepath object.
Args:
url (str): A telepath URL.
**opts (dict): Telepath connect options.
Returns:
(synapse.telepath.Proxy): A telepath proxy object.
The telepath proxy may then be used for sync or async calls:
proxy = openurl(url)
value = proxy.getFooThing()
... or ...
proxy = await openurl(url)
valu = await proxy.getFooThing()
... or ...
async with await openurl(url) as proxy:
valu = await proxy.getFooThing()
'''
if url.find('://') == -1:
newurl = alias(url)
if newurl is None:
raise s_exc.BadUrl(
mesg=f':// not found in [{url}] and no alias found!', url=url)
url = newurl
info = s_urlhelp.chopurl(url)
info.update(opts)
scheme = info.get('scheme')
if '+' in scheme:
scheme, disc = scheme.split('+', 1)
# Discovery protocols modify info dict inband?
if disc == 'consul':
await disc_consul(info)
else:
raise s_exc.BadUrl(mesg=f'Unknown discovery protocol [{disc}].',
disc=disc)
host = info.get('host')
port = info.get('port')
auth = None
user = info.get('user')
if user is not None:
passwd = info.get('passwd')
auth = (user, {'passwd': passwd})
if scheme == 'cell':
# cell:///path/to/celldir:share
# cell://rel/path/to/celldir:share
path = info.get('path')
name = info.get('name', '*')
# support cell://<relpath>/<to>/<cell>
# by detecting host...
host = info.get('host')
if host:
path = path.strip('/')
path = os.path.join(host, path)
if ':' in path:
path, name = path.split(':')
full = os.path.join(path, 'sock')
link = await s_link.unixconnect(full)
elif scheme == 'unix':
# unix:///path/to/sock:share
name = '*'
path = info.get('path')
if ':' in path:
path, name = path.split(':')
link = await s_link.unixconnect(path)
else:
path = info.get('path')
name = info.get('name', path[1:])
sslctx = None
if scheme == 'ssl':
certname = None
certpath = None
certdir = opts.get('certdir')
query = info.get('query')
if query is not None:
certpath = query.get('certdir')
certname = query.get('certname')
if certdir is None:
certdir = s_certdir.CertDir(certpath)
sslctx = certdir.getClientSSLContext(certname=certname)
link = await s_link.connect(host, port, ssl=sslctx)
prox = await Proxy.anit(link, name)
prox.onfini(link)
try:
await prox.handshake(auth=auth)
except Exception:
await prox.fini()
raise
return prox
async def openurl(url, **opts):
'''
Open a URL to a remote telepath object.
Args:
url (str): A telepath URL.
**opts (dict): Telepath connect options.
Returns:
(synapse.telepath.Proxy): A telepath proxy object.
The telepath proxy may then be used for sync or async calls:
proxy = openurl(url)
value = proxy.getFooThing()
... or ...
proxy = await openurl(url)
valu = await proxy.getFooThing()
... or ...
async with await openurl(url) as proxy:
valu = await proxy.getFooThing()
'''
if url.find('://') == -1:
newurl = alias(url)
if newurl is None:
raise s_exc.BadUrl(f':// not found in [{url}] and no alias found!')
url = newurl
info = s_urlhelp.chopurl(url)
info.update(opts)
host = info.get('host')
port = info.get('port')
name = info.get('path')[1:]
auth = None
user = info.get('user')
if user is not None:
passwd = info.get('passwd')
auth = (user, {'passwd': passwd})
sslctx = None
if info.get('scheme') == 'ssl':
certpath = info.get('certdir')
certdir = s_certdir.CertDir(certpath)
sslctx = certdir.getClientSSLContext()
link = await s_link.connect(host, port, ssl=sslctx)
prox = await Proxy.anit(link, name)
prox.onfini(link)
try:
await prox.handshake(auth=auth)
except Exception as e:
await prox.fini()
raise
return prox
def test_urlchop(self):
url = 'http://vertex.link:8080/hehe.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'port': 8080,
'host': 'vertex.link',
'path': '/hehe.html',
},
info
)
url = 'tcp://*****:*****@vertex.link/'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'user': '******',
'host': 'vertex.link',
'path': '/',
'passwd': 'candy',
},
info
)
url = 'tcp://[email protected]'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'user': '******',
'host': 'vertex.link',
'path': '',
},
info
)
url = 'tcp://1.2.3.4:8080/api/v1/wow?key=valu&foo=bar'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'tcp',
'host': '1.2.3.4',
'port': 8080,
'path': '/api/v1/wow',
'query': {'key': 'valu',
'foo': 'bar',
}
},
info
)
url = 'http://[1fff:0:a88:85a3::ac1f]:8001/index.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'host': '1fff:0:a88:85a3::ac1f',
'port': 8001,
'path': '/index.html',
},
info
)
url = 'http://::1/index.html'
info = s_urlhelp.chopurl(url)
self.eq({'scheme': 'http',
'host': '::1',
'path': '/index.html',
},
info
)
self.raises(s_exc.BadUrl, s_urlhelp.chopurl,
'www.vertex.link')
