def __init__(self, allowcookies=False, cookiejar=None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
def __init__(self, allowcookies = False, cookiejar = None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
class WebClient(Configurable):
"Convenient HTTP request processing. Proxy is not supported in current version."
# When a cleanup task is created, the task releases dead connections by this interval
_default_cleanupinterval = 60
# Persist this number of connections at most for each host. If all connections are in
# use, new requests will wait until there are free connections.
_default_samehostlimit = 20
# Do not allow multiple requests to the same URL at the same time. If sameurllimit=True,
# requests to the same URL will always be done sequential.
_default_sameurllimit = False
# CA file used to verify HTTPS certificates. To be compatible with older Python versions,
# the new SSLContext is not enabled currently, so with the default configuration, the
# certificates are NOT verified. You may configure this to a .pem file in your system,
# usually /etc/pki/tls/cert.pem in Linux.
_default_cafile = None
# When following redirects and the server redirects too many times, raises an exception
# and end the process
_default_redirectlimit = 10
# Verify the host with the host in certificate
_default_verifyhost = True
def __init__(self, allowcookies=False, cookiejar=None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
async def open(self,
container,
request,
ignorewebexception=False,
timeout=None,
datagen=None,
cafile=None,
key=None,
certificate=None,
followredirect=True,
autodecompress=False,
allowcookies=None):
'''
Open http request with a Request object
:param container: a routine container hosting this routine
:param request: vlcp.utils.webclient.Request object
:param ignorewebexception: Do not raise exception on Web errors (4xx, 5xx), return a response normally
:param timeout: timeout on connection and single http request. When following redirect, new request
does not share the old timeout, which means if timeout=2:
connect to host: (2s)
wait for response: (2s)
response is 302, redirect
connect to redirected host: (2s)
wait for response: (2s)
...
:param datagen: if the request use a stream as the data parameter, you may provide a routine to generate
data for the stream. If the request failed early, this routine is automatically terminated.
:param cafile: provide a CA file for SSL certification check. If not provided, the SSL connection is NOT verified.
:param key: provide a key file, for client certification (usually not necessary)
:param certificate: provide a certificate file, for client certification (usually not necessary)
:param followredirect: if True (default), automatically follow 3xx redirections
:param autodecompress: if True, automatically detect Content-Encoding header and decode the body
:param allowcookies: override default settings to disable the cookies
'''
if cafile is None:
cafile = self.cafile
if allowcookies is None:
allowcookies = self.allowcookies
forcecreate = False
datagen_routine = None
if autodecompress:
if not request.has_header('Accept-Encoding'):
request.add_header('Accept-Encoding', 'gzip, deflate')
while True:
# Find or create a connection
conn, created = await self._getconnection(
container, request.host, request.path,
request.get_type() == 'https', forcecreate, cafile, key,
certificate, timeout)
# Send request on conn and wait for reply
try:
if allowcookies:
self.cookiejar.add_cookie_header(request)
if isinstance(request.data, bytes):
stream = MemoryStream(request.data)
else:
stream = request.data
if datagen and datagen_routine is None:
datagen_routine = container.subroutine(datagen)
else:
datagen_routine = None
timeout_, result = await container.execute_with_timeout(
timeout,
self._protocol.request_with_response(
container, conn, _bytes(request.host),
_bytes(request.path), _bytes(request.method),
[(_bytes(k), _bytes(v))
for k, v in request.header_items()], stream))
if timeout_:
if datagen_routine:
container.terminate(datagen_routine)
container.subroutine(
self._releaseconnection(conn, request.host,
request.path,
request.get_type() == 'https',
True), False)
raise WebException('HTTP request timeout')
finalresp, _ = result
resp = Response(request.get_full_url(), finalresp,
container.scheduler)
if allowcookies:
self.cookiejar.extract_cookies(resp, request)
if resp.iserror and not ignorewebexception:
try:
exc = WebException(resp.fullstatus)
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(
encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(
encoders.deflate_decoder())
data = await resp.stream.read(container, 4096)
exc.response = resp
exc.body = data
if datagen_routine:
container.terminate(datagen_routine)
await resp.shutdown()
container.subroutine(
self._releaseconnection(
conn, request.host, request.path,
request.get_type() == 'https', True), False)
raise exc
finally:
resp.close()
else:
try:
container.subroutine(
self._releaseconnection(
conn, request.host, request.path,
request.get_type() == 'https', False,
finalresp), False)
if followredirect and resp.status in (300, 301, 302,
303, 307, 308):
request.redirect(
resp,
ignorewebexception=ignorewebexception,
timeout=timeout,
cafile=cafile,
key=key,
certificate=certificate,
followredirect=followredirect,
autodecompress=autodecompress,
allowcookies=allowcookies)
resp.close()
continue
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(
encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(
encoders.deflate_decoder())
return resp
except:
resp.close()
raise
except HttpConnectionClosedException:
await self._releaseconnection(conn, request.host, request.path,
request.get_type() == 'https',
False)
if not created:
# Retry on a newly created connection
forcecreate = True
continue
else:
if datagen_routine:
container.terminate(datagen_routine)
raise
except Exception as exc:
await self._releaseconnection(conn, request.host, request.path,
request.get_type() == 'https',
True)
raise exc
break
async def _releaseconnection(self,
connection,
host,
path,
https=False,
forceclose=False,
respevent=None):
if not host:
raise ValueError
if forceclose:
await connection.shutdown(True)
if not forceclose and connection.connected and respevent:
async def releaseconn():
keepalive = await self._protocol.wait_for_response_end(
connection, connection, respevent.connmark, respevent.xid)
conns = self._connmap[host]
conns[2] -= 1
if keepalive:
connection.setdaemon(True)
conns[1 if https else 0].append(connection)
else:
await connection.shutdown()
connection.subroutine(releaseconn(), False)
else:
conns = self._connmap[host]
conns[2] -= 1
if self.sameurllimit:
self._requesting.remove((host, path, https))
if (host, path,
https) in self._pathwaiting or host in self._hostwaiting:
await connection.wait_for_send(
WebClientRequestDoneEvent(host, path, https))
if (host, path, https) in self._pathwaiting:
self._pathwaiting.remove((host, path, https))
if host in self._hostwaiting:
self._hostwaiting.remove(host)
async def _getconnection(self,
container,
host,
path,
https=False,
forcecreate=False,
cafile=None,
key=None,
certificate=None,
timeout=None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
await matcher
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[], [], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
conns[2] += 1
return (conn, False)
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(
conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
await conn.shutdown()
else:
# Wait for free connections
self._hostwaiting.add(host)
await matcher
conns = self._connmap.setdefault(host, [[], [], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
conns[2] += 1
return (conn, False)
# Create new connection
conns[2] += 1
conn = Client(
urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')),
self._protocol, container.scheduler, key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
_, m = await M_(connected, notconnected)
if m is notconnected:
conns[2] -= 1
await conn.shutdown(True)
raise IOError('Failed to connect to %r' % (conn.rawurl, ))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
await conn.shutdown(True)
raise CertificateException(
'Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
return (conn, True)
def cleanup(self, host=None):
"Cleaning disconnected connections"
if host is not None:
conns = self._connmap.get(host)
if conns is None:
return
# cleanup disconnected connections
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
if not conns[0] and not conns[1] and not conns[2]:
del self._connmap[host]
else:
hosts = list(self._connmap.keys())
for h in hosts:
self.cleanup(h)
def cleanup_task(self, container, interval=None):
'''
If this client object is persist for a long time, and you are worrying about memory leak,
create a routine with this method: myclient.cleanup_task(mycontainer, 60).
But remember that if you have created at lease one task, you must call myclient.endtask()
to completely release the webclient object.
'''
if interval is None:
interval = self.cleanupinterval
async def task():
th = container.scheduler.setTimer(interval, interval)
tm = TimerEvent.createMatcher(th)
try:
while True:
await tm
self.cleanup()
finally:
container.scheduler.cancelTimer(th)
t = container.subroutine(task(), False, daemon=True)
self._tasks.append(t)
return t
async def shutdown(self):
"Shutdown free connections to release resources"
for c0, c1, _ in list(self._connmap.values()):
c0bak = list(c0)
del c0[:]
for c in c0bak:
if c.connected:
await c.shutdown()
c1bak = list(c1)
del c1[:]
for c in c1bak:
if c.connected:
await c.shutdown()
def endtask(self):
for t in self._tasks:
t.close()
del self._tasks[:]
async def urlopen(self,
container,
url,
data=None,
method=None,
headers={},
rawurl=False,
*args,
**kwargs):
'''
Similar to urllib2.urlopen, but:
1. is a routine
2. data can be an instance of vlcp.event.stream.BaseStream, or str/bytes
3. can specify method
4. if datagen is not None, it is a routine which writes to <data>. It is automatically terminated if the connection is down.
5. can also specify key and certificate, for client certification
6. certificates are verified with CA if provided.
If there are keep-alived connections, they are automatically reused.
See open for available arguments
Extra argument:
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
return await self.open(
container, Request(url, data, method, headers, rawurl=rawurl),
*args, **kwargs)
async def manualredirect(self, container, exc, data, datagen=None):
"If data is a stream, it cannot be used again on redirect. Catch the ManualRedirectException and call a manual redirect with a new stream."
request = exc.request
request.data = data
return await self.open(container,
request,
datagen=datagen,
**exc.kwargs)
async def urlgetcontent(self,
container,
url,
data=None,
method=None,
headers={},
tostr=False,
encoding=None,
rawurl=False,
*args,
**kwargs):
'''
In Python2, bytes = str, so tostr and encoding has no effect.
In Python3, bytes are decoded into unicode str with encoding.
If encoding is not specified, charset in content-type is used if present, or default to utf-8 if not.
See open for available arguments
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
req = Request(url, data, method, headers, rawurl=rawurl)
with (await self.open(container, req, *args, **kwargs)) as resp:
encoding = 'utf-8'
if encoding is None:
m = Message()
m.add_header('Content-Type',
resp.get_header('Content-Type', 'text/html'))
encoding = m.get_content_charset('utf-8')
if not resp.stream:
content = b''
else:
content = await resp.stream.read(container)
if tostr:
content = _str(content, encoding)
return content
'''
Created on 2015/8/27
:author: hubo
'''
from __future__ import print_function
from vlcp.server import Server
from vlcp.event import RoutineContainer, Stream, TcpServer, MemoryStream, Client
from vlcp.protocol.http import Http, HttpRequestEvent, HttpConnectionStateEvent
from codecs import getincrementalencoder
import logging
http = Http(False)
class MainRoutine(RoutineContainer):
def __init__(self, scheduler=None, daemon=False):
RoutineContainer.__init__(self, scheduler=scheduler, daemon=daemon)
def main(self):
conn = Client('tcp://www.baidu.com/', http, self.scheduler)
conn.start()
connected = http.statematcher(conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = http.statematcher(conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
yield (connected, notconnected)
if self.matcher is notconnected:
print('Connect to server failed.')
else:
for m in http.requestwithresponse(self, conn, b'www.baidu.com', b'/', b'GET', []):
yield m
for r in self.http_responses:
print('Response received:')
print(r.status)
def run(self,
host=None,
skipovs=None,
skipiplink=None,
skiplogicalport=None):
skipovs = (skipovs is not None)
skipiplink = (skipiplink is not None)
skiplogicalport = (skiplogicalport is not None)
pool = TaskPool(self.scheduler)
pool.start()
if host is None:
host = os.environ.get('DOCKER_HOST', 'unix:///var/run/docker.sock')
enable_ssl = os.environ.get('DOCKER_TLS_VERIFY', '')
cert_root_path = os.environ.get('DOCKER_CERT_PATH', '~/.docker')
ca_path, cert_path, key_path = [
os.path.join(cert_root_path, f)
for f in ('ca.pem', 'cert.pem', 'key.pem')
]
if '/' not in host:
if enable_ssl:
host = 'ssl://' + host
else:
host = 'tcp://' + host
self._docker_conn = None
http_protocol = Http(False)
http_protocol.defaultport = 2375
http_protocol.ssldefaultport = 2375
http_protocol.persist = False
def _create_docker_conn():
self._docker_conn = Client(host, http_protocol, self.scheduler,
key_path, cert_path, ca_path)
self._docker_conn.start()
return self._docker_conn
def call_docker_api(path, data=None, method=None):
if self._docker_conn is None or not self._docker_conn.connected:
_create_docker_conn()
conn_up = HttpConnectionStateEvent.createMatcher(
HttpConnectionStateEvent.CLIENT_CONNECTED)
conn_noconn = HttpConnectionStateEvent.createMatcher(
HttpConnectionStateEvent.CLIENT_NOTCONNECTED)
yield (conn_up, conn_noconn)
if self.apiroutine.matcher is conn_noconn:
raise IOError('Cannot connect to docker API endpoint: ' +
repr(host))
if method is None:
if data is None:
method = b'GET'
else:
method = b'POST'
if data is None:
for m in http_protocol.requestwithresponse(
self.apiroutine, self._docker_conn, b'docker',
_bytes(path), method,
[(b'Accept-Encoding', b'gzip, deflate')]):
yield m
else:
for m in http_protocol.requestwithresponse(
self.apiroutine, self._docker_conn, b'docker',
_bytes(path), method,
[(b'Content-Type', b'application/json;charset=utf-8'),
(b'Accept-Encoding', b'gzip, deflate')],
MemoryStream(_bytes(json.dumps(data)))):
yield m
final_resp = self.apiroutine.http_finalresponse
output_stream = final_resp.stream
try:
if final_resp.statuscode >= 200 and final_resp.statuscode < 300:
if output_stream is not None and b'content-encoding' in final_resp.headerdict:
ce = final_resp.headerdict.get(b'content-encoding')
if ce.lower() == b'gzip' or ce.lower() == b'x-gzip':
output_stream.getEncoderList().append(
encoders.gzip_decoder())
elif ce.lower() == b'deflate':
output_stream.getEncoderList().append(
encoders.deflate_decoder())
if output_stream is None:
self.apiroutine.retvalue = {}
else:
for m in output_stream.read(self.apiroutine):
yield m
self.apiroutine.retvalue = json.loads(
self.apiroutine.data.decode('utf-8'))
else:
raise ValueError('Docker API returns error status: ' +
repr(final_resp.status))
finally:
if output_stream is not None:
output_stream.close(self.scheduler)
def execute_bash(script, ignoreerror=True):
def task():
try:
sp = subprocess.Popen(['bash'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outdata, errdata = sp.communicate(script)
sys.stderr.write(_str(errdata))
errno = sp.poll()
if errno != 0 and not ignoreerror:
print('Script failed, output:\n',
repr(outdata),
file=sys.stderr)
raise ValueError('Script returns %d' % (errno, ))
else:
return _str(outdata)
finally:
if sp.poll() is None:
try:
sp.terminate()
sleep(2)
if sp.poll() is None:
sp.kill()
except Exception:
pass
for m in pool.runTask(self.apiroutine, task):
yield m
ovsbridge = manager.get('module.dockerplugin.ovsbridge', 'dockerbr0')
vethprefix = manager.get('module.dockerplugin.vethprefix', 'vlcp')
ipcommand = manager.get('module.dockerplugin.ipcommand', 'ip')
ovscommand = manager.get('module.dockerplugin.ovscommand', 'ovs-vsctl')
find_invalid_ovs = _find_invalid_ovs % (shell_quote(ovscommand),
shell_quote(vethprefix))
find_unused_veth = _find_unused_veth % (shell_quote(ipcommand),
shell_quote(vethprefix))
print("docker API endpoint: ", host)
print("ovsbridge: ", ovsbridge)
print("vethprefix: ", vethprefix)
def invalid_ovs_ports():
for m in execute_bash(find_invalid_ovs):
yield m
first_invalid_ovs_list = self.apiroutine.retvalue.splitlines(False)
first_invalid_ovs_list = [
k.strip() for k in first_invalid_ovs_list if k.strip()
]
if first_invalid_ovs_list:
print(
"Detect %d invalid ports from OpenvSwitch, wait 5 seconds to detect again..."
% (len(first_invalid_ovs_list), ))
else:
self.apiroutine.retvalue = []
return
for m in self.apiroutine.waitWithTimeout(5):
yield m
for m in execute_bash(find_invalid_ovs):
yield m
second_invalid_ovs_list = self.apiroutine.retvalue.splitlines(
False)
second_invalid_ovs_list = [
k.strip() for k in second_invalid_ovs_list if k.strip()
]
invalid_ports = list(
set(first_invalid_ovs_list).intersection(
second_invalid_ovs_list))
if invalid_ports:
print(
'Detect %d invalid ports from intersection of two tries, removing...'
% (len(invalid_ports), ))
# Remove these ports
def _remove_ports():
for p in invalid_ports:
try:
_unplug_ovs(ovscommand, ovsbridge,
p[:-len('-tag')])
except Exception as exc:
print('Remove port %r failed: %s' % (p, exc))
for m in pool.runTask(self.apiroutine, _remove_ports):
yield m
self.apiroutine.retvalue = invalid_ports
return
def remove_unused_ports():
for m in execute_bash(find_unused_veth):
yield m
first_unused_ports = self.apiroutine.retvalue.splitlines(False)
first_unused_ports = [
k.strip() for k in first_unused_ports if k.strip()
]
if first_unused_ports:
print(
"Detect %d unused ports from ip-link, wait 5 seconds to detect again..."
% (len(first_unused_ports), ))
else:
self.apiroutine.retvalue = []
return
for m in self.apiroutine.waitWithTimeout(5):
yield m
for m in execute_bash(find_unused_veth):
yield m
second_unused_ports = self.apiroutine.retvalue.splitlines(False)
second_unused_ports = [
k.strip() for k in second_unused_ports if k.strip()
]
unused_ports = list(
set(first_unused_ports).intersection(second_unused_ports))
if unused_ports:
print(
'Detect %d unused ports from intersection of two tries, removing...'
% (len(unused_ports), ))
# Remove these ports
def _remove_ports():
for p in unused_ports:
try:
_unplug_ovs(ovscommand, ovsbridge,
p[:-len('-tag')])
except Exception as exc:
print(
'Remove port %r from OpenvSwitch failed: %s' %
(p, exc))
try:
_delete_veth(ipcommand, p[:-len('-tag')])
except Exception as exc:
print('Delete port %r with ip-link failed: %s' %
(p, exc))
for m in pool.runTask(self.apiroutine, _remove_ports):
yield m
self.apiroutine.retvalue = unused_ports
return
def detect_unused_logports():
# docker network ls
print("Check logical ports from docker API...")
for m in call_docker_api(
br'/v1.24/networks?filters={"driver":["vlcp"]}'):
yield m
network_ports = dict(
(n['Id'],
dict((p['EndpointID'], p['IPv4Address'])
for p in n['Containers'].values()))
for n in self.apiroutine.retvalue if n['Driver'] == 'vlcp'
) # Old version of docker API does not support filter by driver
print("Find %d networks and %d endpoints from docker API, recheck in 5 seconds..." % \
(len(network_ports), sum(len(ports) for ports in network_ports.values())))
def recheck_ports():
for m in self.apiroutine.waitWithTimeout(5):
yield m
# docker network inspect, use this for cross check
second_network_ports = {}
for nid in network_ports:
try:
for m in call_docker_api(br'/networks/' + _bytes(nid)):
yield m
except ValueError as exc:
print(
'WARNING: check network failed, the network may be removed. Message: ',
str(exc))
second_network_ports[nid] = {}
else:
second_network_ports[nid] = dict(
(p['EndpointID'], p['IPv4Address']) for p in
self.apiroutine.retvalue['Containers'].values())
print("Recheck find %d endpoints from docker API" % \
(sum(len(ports) for ports in second_network_ports.values()),))
self.apiroutine.retvalue = second_network_ports
def check_viperflow():
first_vp_ports = {}
for nid in network_ports:
for m in callAPI(
self.apiroutine, 'viperflow', 'listlogicalports',
{'logicalnetwork': 'docker-' + nid + '-lognet'}):
yield m
first_vp_ports[nid] = dict(
(p['id'], p.get('ip_address'))
for p in self.apiroutine.retvalue
if p['id'].startswith('docker-'))
print("Find %d endpoints from viperflow database, recheck in 5 seconds..." % \
(sum(len(ports) for ports in first_vp_ports.values()),))
for m in self.apiroutine.waitWithTimeout(5):
yield m
second_vp_ports = {}
for nid in network_ports:
for m in callAPI(
self.apiroutine, 'viperflow', 'listlogicalports',
{'logicalnetwork': 'docker-' + nid + '-lognet'}):
yield m
second_vp_ports[nid] = dict(
(p['id'], p.get('ip_address'))
for p in self.apiroutine.retvalue
if p['id'] in first_vp_ports[nid])
print("Find %d endpoints from viperflow database from the intersection of two tries" % \
(sum(len(ports) for ports in second_vp_ports.values()),))
second_vp_ports = dict((nid,
dict((pid[len('docker-'):], addr)
for pid, addr in v.items()))
for nid, v in second_vp_ports.items())
self.apiroutine.retvalue = second_vp_ports
for m in check_viperflow():
yield m
second_vp_ports = self.apiroutine.retvalue
for m in recheck_ports():
yield m
second_ports = self.apiroutine.retvalue
unused_logports = dict((nid, dict((pid, addr)
for pid, addr in v.items()
if pid not in network_ports[nid] and\
pid not in second_ports[nid]))
for nid, v in second_vp_ports.items())
self.apiroutine.retvalue = unused_logports
routines = []
if not skipovs:
routines.append(invalid_ovs_ports())
if not skipiplink:
routines.append(remove_unused_ports())
if not skiplogicalport:
routines.append(detect_unused_logports())
for m in self.apiroutine.executeAll(routines):
yield m
if skiplogicalport:
return
(unused_logports, ) = self.apiroutine.retvalue[-1]
if any(ports for ports in unused_logports.values()):
print("Find %d unused logical ports, first 20 ips:\n%r" % \
(sum(len(ports) for ports in unused_logports.values()),
[v for _,v in \
itertools.takewhile(lambda x: x[0] <= 20,
enumerate(addr for ports in unused_logports.values()
for addr in ports.values()))]))
print("Will remove them in 5 seconds, press Ctrl+C to cancel...")
for m in self.apiroutine.waitWithTimeout(5):
yield m
for ports in unused_logports.values():
for p, addr in ports.items():
try:
for m in callAPI(self.apiroutine, 'viperflow',
'deletelogicalport',
{'id': 'docker-' + p}):
yield m
except Exception as exc:
print("WARNING: remove logical port %r (IP: %s) failed, maybe it is already removed. Message: %s" % \
(p, addr, exc))
print("Done.")
'''
Created on 2015/8/27
:author: hubo
'''
from __future__ import print_function
from vlcp.server import Server
from vlcp.event import RoutineContainer, Stream, TcpServer, MemoryStream, Client
from vlcp.protocol.http import Http, HttpRequestEvent, HttpConnectionStateEvent
from codecs import getincrementalencoder
import logging
from vlcp.event.event import M_
http = Http(False)
class MainRoutine(RoutineContainer):
def __init__(self, scheduler=None, daemon=False):
RoutineContainer.__init__(self, scheduler=scheduler, daemon=daemon)
async def main(self):
conn = Client('tcp://www.baidu.com/', http, self.scheduler)
conn.start()
connected = http.statematcher(
conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = http.statematcher(
conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
_, m = await M_(connected, notconnected)
if m is notconnected:
print('Connect to server failed.')
else:
'''
Created on 2015/8/27
:author: hubo
'''
from vlcp.server import Server
from vlcp.event import RoutineContainer, Stream, TcpServer, MemoryStream
from vlcp.protocol.http import Http, HttpRequestEvent, escape_b, escape
from codecs import getincrementalencoder
import logging
http = Http(True)
class MainRoutine(RoutineContainer):
def __init__(self, scheduler=None, daemon=False):
RoutineContainer.__init__(self, scheduler=scheduler, daemon=daemon)
self.encoder = getincrementalencoder('utf-8')
async def main(self):
request = HttpRequestEvent.createMatcher()
while True:
ev = await request
ev.canignore = True
self.subroutine(self.handlehttp(ev))
document = '''
<!DOCTYPE html >
<html>
<head>
<title>Test Server Page</title>
class WebClient(Configurable):
"Convenient HTTP request processing. Proxy is not supported in current version."
_default_cleanupinterval = 60
_default_samehostlimit = 20
_default_sameurllimit = False
_default_cafile = None
_default_redirectlimit = 10
_default_verifyhost = True
def __init__(self, allowcookies = False, cookiejar = None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
def open(self, container, request, ignorewebexception = False, timeout = None, datagen = None, cafile = None, key = None, certificate = None,
followredirect = True, autodecompress = False, allowcookies = None):
'''
Open http request with a Request object
:param container: a routine container hosting this routine
:param request: vlcp.utils.webclient.Request object
:param ignorewebexception: Do not raise exception on Web errors (4xx, 5xx), return a response normally
:param timeout: timeout on connection and single http request. When following redirect, new request
does not share the old timeout, which means if timeout=2:
connect to host: (2s)
wait for response: (2s)
response is 302, redirect
connect to redirected host: (2s)
wait for response: (2s)
...
:param datagen: if the request use a stream as the data parameter, you may provide a routine to generate
data for the stream. If the request failed early, this routine is automatically terminated.
:param cafile: provide a CA file for SSL certification check. If not provided, the SSL connection is NOT verified.
:param key: provide a key file, for client certification (usually not necessary)
:param certificate: provide a certificate file, for client certification (usually not necessary)
:param followredirect: if True (default), automatically follow 3xx redirections
:param autodecompress: if True, automatically detect Content-Encoding header and decode the body
:param allowcookies: override default settings to disable the cookies
'''
with closing(container.delegateOther(self._open(container, request, ignorewebexception, timeout, datagen, cafile, key, certificate,
followredirect, autodecompress, allowcookies),
container)) as g:
for m in g:
yield m
def _open(self, container, request, ignorewebexception = False, timeout = None, datagen = None, cafile = None, key = None, certificate = None,
followredirect = True, autodecompress = False, allowcookies = None):
if cafile is None:
cafile = self.cafile
if allowcookies is None:
allowcookies = self.allowcookies
forcecreate = False
datagen_routine = None
if autodecompress:
if not request.has_header('Accept-Encoding'):
request.add_header('Accept-Encoding', 'gzip, deflate')
while True:
# Find or create a connection
for m in self._getconnection(container, request.host, request.path, request.get_type() == 'https',
forcecreate, cafile, key, certificate, timeout):
yield m
(conn, created) = container.retvalue
# Send request on conn and wait for reply
try:
if allowcookies:
self.cookiejar.add_cookie_header(request)
if isinstance(request.data, bytes):
stream = MemoryStream(request.data)
else:
stream = request.data
if datagen and datagen_routine is None:
datagen_routine = container.subroutine(datagen)
else:
datagen_routine = None
for m in container.executeWithTimeout(timeout, self._protocol.requestwithresponse(container, conn, _bytes(request.host), _bytes(request.path), _bytes(request.method),
[(_bytes(k), _bytes(v)) for k,v in request.header_items()], stream)):
yield m
if container.timeout:
if datagen_routine:
container.terminate(datagen_routine)
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True), False)
raise WebException('HTTP request timeout')
finalresp = container.http_finalresponse
resp = Response(request.get_full_url(), finalresp, container.scheduler)
if allowcookies:
self.cookiejar.extract_cookies(resp, request)
if resp.iserror and not ignorewebexception:
try:
exc = WebException(resp.fullstatus)
for m in resp.stream.read(container, 4096):
yield m
exc.response = resp
exc.body = container.data
if datagen_routine:
container.terminate(datagen_routine)
for m in resp.shutdown():
yield m
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True), False)
raise exc
finally:
resp.close()
else:
try:
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', False, finalresp), False)
if followredirect and resp.status in (300, 301, 302, 303, 307, 308):
request.redirect(resp, ignorewebexception = ignorewebexception, timeout = timeout, cafile = cafile, key = key,
certificate = certificate, followredirect = followredirect,
autodecompress = autodecompress, allowcookies = allowcookies)
resp.close()
continue
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(encoders.deflate_decoder())
container.retvalue = resp
except:
resp.close()
raise
except HttpConnectionClosedException:
for m in self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', False):
yield m
if not created:
# Retry on a newly created connection
forcecreate = True
continue
else:
if datagen_routine:
container.terminate(datagen_routine)
raise
except Exception as exc:
for m in self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True):
yield m
raise exc
break
def _releaseconnection(self, connection, host, path, https = False, forceclose = False, respevent = None):
if not host:
raise ValueError
if forceclose:
for m in connection.shutdown(True):
yield m
if not forceclose and connection.connected and respevent:
def releaseconn():
for m in self._protocol.waitForResponseEnd(connection, connection, respevent.connmark, respevent.xid):
yield m
keepalive = connection.retvalue
conns = self._connmap[host]
conns[2] -= 1
if keepalive:
connection.setdaemon(True)
conns[1 if https else 0].append(connection)
else:
for m in connection.shutdown():
yield m
connection.subroutine(releaseconn(), False)
else:
conns = self._connmap[host]
conns[2] -= 1
if self.sameurllimit:
self._requesting.remove((host, path, https))
if (host, path, https) in self._pathwaiting or host in self._hostwaiting:
for m in connection.waitForSend(WebClientRequestDoneEvent(host, path, https)):
yield m
if (host, path, https) in self._pathwaiting:
self._pathwaiting.remove((host, path, https))
if host in self._hostwaiting:
self._hostwaiting.remove(host)
def _getconnection(self, container, host, path, https = False, forcecreate = False, cafile = None, key = None, certificate = None,
timeout = None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
yield (matcher,)
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[],[], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
for m in conn.shutdown():
yield m
else:
# Wait for free connections
self._hostwaiting.add(host)
yield (matcher,)
conns = self._connmap.setdefault(host, [[],[], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
# Create new connection
conns[2] += 1
conn = Client(urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')), self._protocol, container.scheduler,
key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
yield (connected, notconnected)
if container.matcher is notconnected:
conns[2] -= 1
for m in conn.shutdown(True):
yield m
raise IOError('Failed to connect to %r' % (conn.rawurl,))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
for m in conn.shutdown(True):
yield m
raise CertificateException('Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
container.retvalue = (conn, True)
def cleanup(self, host = None):
"Cleaning disconnected connections"
if host is not None:
conns = self._connmap.get(host)
if conns is None:
return
# cleanup disconnected connections
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
if not conns[0] and not conns[1] and not conns[2]:
del self._connmap[host]
else:
hosts = list(self._connmap.keys())
for h in hosts:
self.cleanup(h)
def cleanup_task(self, container, interval = None):
'''
If this client object is persist for a long time, and you are worrying about memory leak,
create a routine with this method: myclient.cleanup_task(mycontainer, 60).
But remember that if you have created at lease one task, you must call myclient.endtask()
to completely release the webclient object.
'''
if interval is None:
interval = self.cleanupinterval
def task():
th = container.scheduler.setTimer(interval, interval)
tm = TimerEvent.createMatcher(th)
try:
while True:
yield (tm,)
self.cleanup()
finally:
container.scheduler.cancelTimer(th)
t = container.subroutine(task(), False, daemon = True)
self._tasks.append(t)
return t
def shutdown(self):
"Shutdown free connections to release resources"
for c0, c1, _ in list(self._connmap.values()):
c0bak = list(c0)
del c0[:]
for c in c0bak:
if c.connected:
for m in c.shutdown():
yield m
c1bak = list(c1)
del c1[:]
for c in c1bak:
if c.connected:
for m in c.shutdown():
yield m
def endtask(self):
for t in self._tasks:
t.close()
del self._tasks[:]
def urlopen(self, container, url, data = None, method = None, headers = {}, rawurl = False, *args, **kwargs):
'''
Similar to urllib2.urlopen, but:
1. is a routine
2. data can be an instance of vlcp.event.stream.BaseStream, or str/bytes
3. can specify method
4. if datagen is not None, it is a routine which writes to <data>. It is automatically terminated if the connection is down.
5. can also specify key and certificate, for client certification
6. certificates are verified with CA if provided.
If there are keep-alived connections, they are automatically reused.
See open for available arguments
Extra argument:
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
return self.open(container, Request(url, data, method, headers, rawurl=rawurl), *args, **kwargs)
def manualredirect(self, container, exc, data, datagen = None):
"If data is a stream, it cannot be used again on redirect. Catch the ManualRedirectException and call a manual redirect with a new stream."
request = exc.request
request.data = data
return self.open(container, request, datagen = datagen, **exc.kwargs)
def urlgetcontent(self, container, url, data = None, method = None, headers = {}, tostr = False, encoding = None, rawurl = False, *args, **kwargs):
'''
In Python2, bytes = str, so tostr and encoding has no effect.
In Python3, bytes are decoded into unicode str with encoding.
If encoding is not specified, charset in content-type is used if present, or default to utf-8 if not.
See open for available arguments
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
req = Request(url, data, method, headers, rawurl = rawurl)
for m in self.open(container, req, *args, **kwargs):
yield m
resp = container.retvalue
encoding = 'utf-8'
if encoding is None:
m = Message()
m.add_header('Content-Type', resp.get_header('Content-Type', 'text/html'))
encoding = m.get_content_charset('utf-8')
if not resp.stream:
content = b''
else:
for m in resp.stream.read(container):
yield m
content = container.data
if tostr:
content = _str(content, encoding)
container.retvalue = content