def test_link_chopurl(self): info = s_urlhelp.chopurl('foo://woot.com:99/foo/bar') self.assertEqual(info.get('scheme'), 'foo') self.assertEqual(info.get('port'), 99) self.assertEqual(info.get('host'), 'woot.com') self.assertEqual(info.get('path'), '/foo/bar') info = s_urlhelp.chopurl('foo://*****:*****@woot.com') self.assertEqual(info.get('user'), 'visi') self.assertEqual(info.get('passwd'), 'secret') self.assertEqual(info.get('scheme'), 'foo') self.assertEqual(info.get('port'), None) self.assertEqual(info.get('host'), 'woot.com') info = s_urlhelp.chopurl('foo://[2607:f8b0:4004:806::1014]:99/foo/bar?baz=faz&gronk=woot') self.assertEqual(info.get('scheme'), 'foo') self.assertEqual(info.get('port'), 99) self.assertEqual(info.get('host'), '2607:f8b0:4004:806::1014') self.assertEqual(info.get('path'), '/foo/bar') self.assertEqual(info.get('query').get('baz'), 'faz') self.assertEqual(info.get('query').get('gronk'), 'woot') info = s_urlhelp.chopurl('foo://*****:*****@[email protected]') self.assertEqual( info.get('user'), 'visi') self.assertEqual( info.get('passwd'), 'c@t') self.assertEqual( info.get('host'), 'woot.com')
def chopurl(url, **opts): if isinstance(url, str): if url.find('://') == -1: newurl = alias(url) if newurl is None: raise s_exc.BadUrl(mesg=f':// not found in [{url}] and no alias found!', url=url) url = newurl info = s_urlhelp.chopurl(url) # flatten query params into info query = info.pop('query', None) if query is not None: info.update(query) elif isinstance(url, dict): info = dict(url) else: mesg = 'telepath.chopurl() requires a str or dict.' raise s_exc.BadArg(mesg) info.update(opts) return info
async def listen(self, url, **opts): ''' Bind and listen on the given host/port with possible SSL. Args: host (str): A hostname or IP address. port (int): The TCP port to bind. ''' info = s_urlhelp.chopurl(url, **opts) info.update(opts) host = info.get('host') port = info.get('port') sslctx = None if info.get('scheme') == 'ssl': sslctx = self.certdir.getServerSSLContext(hostname=host) server = await s_link.listen(host, port, self._onLinkInit, ssl=sslctx) self.listenservers.append(server) ret = server.sockets[0].getsockname() if self.addr is None: self.addr = ret return ret
async def listen(self, url, **opts): ''' Bind and listen on the given host/port with possible SSL. Args: host (str): A hostname or IP address. port (int): The TCP port to bind. ''' info = s_urlhelp.chopurl(url, **opts) info.update(opts) scheme = info.get('scheme') if scheme == 'unix': path = info.get('path') try: server = await s_link.unixlisten(path, self._onLinkInit) except Exception as e: if 'path too long' in str(e): logger.error( 'unix:// exceeds OS supported UNIX socket path length: %s', path) raise else: host = info.get('host') port = info.get('port') sslctx = None if scheme == 'ssl': caname = None hostname = None query = info.get('query') if query is not None: hostname = query.get('hostname', host) caname = query.get('ca') sslctx = self.certdir.getServerSSLContext(hostname=hostname, caname=caname) server = await s_link.listen(host, port, self._onLinkInit, ssl=sslctx) self.listenservers.append(server) ret = server.sockets[0].getsockname() if self.addr is None: self.addr = ret return ret
def chopLinkUrl(url): ''' Parse a link tuple from a url. Example: link = chopLinkUrl('tcp://1.2.3.4:80/') Notes: * url parameters become link properties * user:passwd@host syntax is used for authdata ''' urlinfo = s_urlhelp.chopurl(url) scheme = urlinfo.get('scheme') link = (scheme, {}) link[1]['url'] = url link[1]['host'] = urlinfo.get('host') link[1]['port'] = urlinfo.get('port') link[1]['path'] = urlinfo.get('path') link[1]['user'] = urlinfo.get('user') link[1]['passwd'] = urlinfo.get('passwd') query = urlinfo.get('query', {}) timeout = query.pop('timeout', None) if timeout is not None: link[1]['timeout'] = float(timeout) poolmax = query.pop('poolmax', None) if poolmax is not None: link[1]['poolmax'] = int(poolmax) poolsize = query.pop('poolsize', None) if poolsize is not None: link[1]['poolsize'] = int(poolsize) rc4key = query.pop('rc4key', None) if rc4key is not None: link[1]['rc4key'] = rc4key.encode('utf8') zerosig = query.pop('zerosig', None) if zerosig is not None: link[1]['zerosig'] = True retry = query.pop('retry', None) if retry is not None: link[1]['retry'] = int(retry, 0) link[1].update(query) return link
def chopLinkUrl(url): ''' Parse a link tuple from a url. Example: link = chopLinkUrl('tcp://1.2.3.4:80/') Notes: * url parameters become link properties * user:passwd@host syntax is used for authdata ''' urlinfo = s_urlhelp.chopurl(url) scheme = urlinfo.get('scheme') link = (scheme,{}) link[1]['url'] = url link[1]['host'] = urlinfo.get('host') link[1]['port'] = urlinfo.get('port') link[1]['path'] = urlinfo.get('path') link[1]['user'] = urlinfo.get('user') link[1]['passwd'] = urlinfo.get('passwd') query = urlinfo.get('query',{}) timeout = query.pop('timeout',None) if timeout != None: link[1]['timeout'] = float(timeout) poolmax = query.pop('poolmax',None) if poolmax != None: link[1]['poolmax'] = int(poolmax) poolsize = query.pop('poolsize',None) if poolsize != None: link[1]['poolsize'] = int(poolsize) rc4key = query.pop('rc4key',None) if rc4key != None: link[1]['rc4key'] = rc4key.encode('utf8') zerosig = query.pop('zerosig',None) if zerosig != None: link[1]['zerosig'] = True retry = query.pop('retry',None) if retry != None: link[1]['retry'] = int(retry,0) link[1].update(query) return link
async def listen(self, url, **opts): ''' Bind and listen on the given host/port with possible SSL. Args: host (str): A hostname or IP address. port (int): The TCP port to bind. ''' info = s_urlhelp.chopurl(url, **opts) info.update(opts) scheme = info.get('scheme') if scheme == 'unix': path = info.get('path') try: server = await s_link.unixlisten(path, self._onLinkInit) except Exception as e: if 'path too long' in str(e): logger.error(f'unix:// exceeds OS supported UNIX socket path length: {path}') raise else: host = info.get('host') port = info.get('port') sslctx = None if scheme == 'ssl': sslctx = self.certdir.getServerSSLContext(hostname=host) server = await s_link.listen(host, port, self._onLinkInit, ssl=sslctx) self.listenservers.append(server) ret = server.sockets[0].getsockname() if self.addr is None: self.addr = ret return ret
async def openurl(url, **opts): ''' Open a URL to a remote telepath object. Args: url (str): A telepath URL. **opts (dict): Telepath connect options. Returns: (synapse.telepath.Proxy): A telepath proxy object. The telepath proxy may then be used for sync or async calls: proxy = openurl(url) value = proxy.getFooThing() ... or ... proxy = await openurl(url) valu = await proxy.getFooThing() ... or ... async with await openurl(url) as proxy: valu = await proxy.getFooThing() ''' if url.find('://') == -1: newurl = alias(url) if newurl is None: raise s_exc.BadUrl(f':// not found in [{url}] and no alias found!') url = newurl info = s_urlhelp.chopurl(url) info.update(opts) host = info.get('host') port = info.get('port') auth = None user = info.get('user') if user is not None: passwd = info.get('passwd') auth = (user, {'passwd': passwd}) scheme = info.get('scheme') if scheme == 'cell': # cell:///path/to/celldir:share # cell://rel/path/to/celldir:share path = info.get('path') name = info.get('name', '*') # support cell://<relpath>/<to>/<cell> # by detecting host... host = info.get('host') if host: path = path.strip('/') path = os.path.join(host, path) if ':' in path: path, name = path.split(':') full = os.path.join(path, 'sock') link = await s_link.unixconnect(full) elif scheme == 'unix': # unix:///path/to/sock:share path, name = info.get('path').split(':') link = await s_link.unixconnect(path) else: path = info.get('path') name = info.get('name', path[1:]) sslctx = None if scheme == 'ssl': certpath = info.get('certdir') certdir = s_certdir.CertDir(certpath) sslctx = certdir.getClientSSLContext() link = await s_link.connect(host, port, ssl=sslctx) prox = await Proxy.anit(link, name) prox.onfini(link) try: await prox.handshake(auth=auth) except Exception: await prox.fini() raise return prox
async def test_discovery_consul(self): # Example data from https://www.consul.io/api/catalog.html consul_data = [{ "ID": "40e4a748-2192-161a-0510-9bf59fe950b5", "Node": "foobar", "Address": "192.168.10.10", "Datacenter": "dc1", "TaggedAddresses": { "lan": "192.168.10.10", "wan": "10.0.10.10" }, "NodeMeta": { "somekey": "somevalue" }, "CreateIndex": 51, "ModifyIndex": 51, "ServiceAddress": "172.17.0.3", "ServiceEnableTagOverride": False, "ServiceID": "32a2a47f7992:nodea:5000", "ServiceName": "foobar", "ServicePort": 5000, "ServiceMeta": { "foobar_meta_value": "baz" }, "ServiceTaggedAddresses": { "lan": { "address": "172.17.0.3", "port": 5000 }, "wan": { "address": "198.18.0.1", "port": 512 } }, "ServiceTags": ["tacos"], "ServiceProxy": { "DestinationServiceName": "", "DestinationServiceID": "", "LocalServiceAddress": "", "LocalServicePort": 0, "Config": None, "Upstreams": None }, "ServiceConnect": { "Native": False, "Proxy": None }, "Namespace": "default" }] with self.getTestDir() as dirn: async with await s_cell.Cell.anit(dirn) as cell: root = await cell.auth.getUserByName('root') await root.setPasswd('root') cell.consul_data = consul_data hhost, hport = await cell.addHttpsPort(0) thost, tport = await cell.dmon.listen('tcp://127.0.0.1:0') cell.addHttpApi('/v1/catalog/service/(.*)', ConsulV1Handler, {'cell': cell}) consul = f'https://127.0.0.1:{hport}' burl = f'tcp+consul://root:root@foobar/*?consul_nosslverify=1&consul={consul}' info = s_urlhelp.chopurl(burl) self.eq(info.get('host'), 'foobar') self.none(info.get('port')) # Discovery functions update info dict in place. await s_telepath.disc_consul(info) self.eq(info.get('host'), '192.168.10.10') self.eq(info.get('port'), 5000) self.eq(info.get('original_host'), 'foobar') # Support tag_address and service_tag_address url = burl + '&consul_tag_address=wan' info = s_urlhelp.chopurl(url) await s_telepath.disc_consul(info) self.eq(info.get('host'), '10.0.10.10') self.eq(info.get('port'), 5000) self.eq(info.get('original_host'), 'foobar') url = burl + '&consul_service_tag_address=wan' info = s_urlhelp.chopurl(url) await s_telepath.disc_consul(info) self.eq(info.get('host'), '198.18.0.1') self.eq(info.get('port'), 512) self.eq(info.get('original_host'), 'foobar') # Support servicetag based port finding url = burl + '&consul_tag=tacos' info = s_urlhelp.chopurl(url) await s_telepath.disc_consul(info) self.eq(info.get('host'), '192.168.10.10') self.eq(info.get('port'), 5000) self.eq(info.get('original_host'), 'foobar') # Punch in our host/port info into consul_data so we can do a end to end test consul_data[0]['Address'] = thost consul_data[0]['ServicePort'] = tport async with await s_telepath.openurl(burl) as proxy: self.isin('synapse.lib.cell.CellApi', proxy._getClasses()) # Sad path - non-existing service name. with self.raises(s_exc.BadUrl) as cm: url = f'tcp+consul://root:root@newp/*?consul_nosslverify=1&consul={consul}' await s_telepath.disc_consul(s_urlhelp.chopurl(url)) self.eq(cm.exception.get('name'), 'newp') # Sad path - multiple tag resolution. with self.raises(s_exc.BadUrl) as cm: url = burl + '&consul_tag_address=wan' url = url + '&consul_service_tag_address=lan' await s_telepath.disc_consul(s_urlhelp.chopurl(url)) self.eq(cm.exception.get('consul_tag_address'), 'wan') self.eq(cm.exception.get('consul_service_tag_address'), 'lan') # Sad path - ssl verification failure with self.raises(s_exc.BadUrl) as cm: url = f'tcp+consul://root:root@foobar/*?consul={consul}' await s_telepath.disc_consul(s_urlhelp.chopurl(url)) self.isin('certificate verify failed', cm.exception.get('mesg')) self.isinstance(cm.exception.__context__, ssl.SSLCertVerificationError) # Sad path - iterating through results and having a # non-existent consul_tag value. new_data = consul_data[0].copy() new_data['ServiceTags'] = [] cell.consul_data.insert(0, new_data) with self.raises(s_exc.BadUrl) as cm: url = burl + '&consul_tag=burritos' await s_telepath.disc_consul(s_urlhelp.chopurl(url)) self.eq('burritos', cm.exception.get('tag'))
def test_urlchop(self): url = 'http://vertex.link:8080/hehe.html' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'http', 'port': 8080, 'host': 'vertex.link', 'path': '/hehe.html', }, info ) url = 'tcp://*****:*****@vertex.link/' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'tcp', 'user': '******', 'host': 'vertex.link', 'path': '/', 'passwd': 'candy', }, info ) url = 'tcp://[email protected]' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'tcp', 'user': '******', 'host': 'vertex.link', 'path': '', }, info ) url = 'tcp://1.2.3.4:8080/api/v1/wow?key=valu&foo=bar' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'tcp', 'host': '1.2.3.4', 'port': 8080, 'path': '/api/v1/wow', 'query': {'key': 'valu', 'foo': 'bar', } }, info ) url = 'http://[1fff:0:a88:85a3::ac1f]:8001/index.html' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'http', 'host': '1fff:0:a88:85a3::ac1f', 'port': 8001, 'path': '/index.html', }, info ) url = 'http://::1/index.html' info = s_urlhelp.chopurl(url) self.eq({'scheme': 'http', 'host': '::1', 'path': '/index.html', }, info ) self.raises(s_exc.BadUrl, s_urlhelp.chopurl, 'www.vertex.link')
async def openurl(url, **opts): ''' Open a URL to a remote telepath object. Args: url (str): A telepath URL. **opts (dict): Telepath connect options. Returns: (synapse.telepath.Proxy): A telepath proxy object. The telepath proxy may then be used for sync or async calls: proxy = openurl(url) value = proxy.getFooThing() ... or ... proxy = await openurl(url) valu = await proxy.getFooThing() ... or ... async with await openurl(url) as proxy: valu = await proxy.getFooThing() ''' if url.find('://') == -1: newurl = alias(url) if newurl is None: raise s_exc.BadUrl( mesg=f':// not found in [{url}] and no alias found!', url=url) url = newurl info = s_urlhelp.chopurl(url) info.update(opts) scheme = info.get('scheme') if '+' in scheme: scheme, disc = scheme.split('+', 1) # Discovery protocols modify info dict inband? if disc == 'consul': await disc_consul(info) else: raise s_exc.BadUrl(mesg=f'Unknown discovery protocol [{disc}].', disc=disc) host = info.get('host') port = info.get('port') auth = None user = info.get('user') if user is not None: passwd = info.get('passwd') auth = (user, {'passwd': passwd}) if scheme == 'cell': # cell:///path/to/celldir:share # cell://rel/path/to/celldir:share path = info.get('path') name = info.get('name', '*') # support cell://<relpath>/<to>/<cell> # by detecting host... host = info.get('host') if host: path = path.strip('/') path = os.path.join(host, path) if ':' in path: path, name = path.split(':') full = os.path.join(path, 'sock') link = await s_link.unixconnect(full) elif scheme == 'unix': # unix:///path/to/sock:share name = '*' path = info.get('path') if ':' in path: path, name = path.split(':') link = await s_link.unixconnect(path) else: path = info.get('path') name = info.get('name', path[1:]) sslctx = None if scheme == 'ssl': certname = None certpath = None certdir = opts.get('certdir') query = info.get('query') if query is not None: certpath = query.get('certdir') certname = query.get('certname') if certdir is None: certdir = s_certdir.CertDir(certpath) sslctx = certdir.getClientSSLContext(certname=certname) link = await s_link.connect(host, port, ssl=sslctx) prox = await Proxy.anit(link, name) prox.onfini(link) try: await prox.handshake(auth=auth) except Exception: await prox.fini() raise return prox
async def openurl(url, **opts): ''' Open a URL to a remote telepath object. Args: url (str): A telepath URL. **opts (dict): Telepath connect options. Returns: (synapse.telepath.Proxy): A telepath proxy object. The telepath proxy may then be used for sync or async calls: proxy = openurl(url) value = proxy.getFooThing() ... or ... proxy = await openurl(url) valu = await proxy.getFooThing() ... or ... async with await openurl(url) as proxy: valu = await proxy.getFooThing() ''' if url.find('://') == -1: newurl = alias(url) if newurl is None: raise s_exc.BadUrl(f':// not found in [{url}] and no alias found!') url = newurl info = s_urlhelp.chopurl(url) info.update(opts) host = info.get('host') port = info.get('port') name = info.get('path')[1:] auth = None user = info.get('user') if user is not None: passwd = info.get('passwd') auth = (user, {'passwd': passwd}) sslctx = None if info.get('scheme') == 'ssl': certpath = info.get('certdir') certdir = s_certdir.CertDir(certpath) sslctx = certdir.getClientSSLContext() link = await s_link.connect(host, port, ssl=sslctx) prox = await Proxy.anit(link, name) prox.onfini(link) try: await prox.handshake(auth=auth) except Exception as e: await prox.fini() raise return prox