def __init__(self, server): Module.__init__(self, server) taskpool = TaskPool(self.scheduler) self.taskpool = taskpool self.routines.append(self.taskpool) self.routines.append(NetworkPlugin(self)) self.apiroutine = RoutineContainer(self.scheduler) self._reqid = 0 self.createAPI(api(self.getdockerinfo, self.apiroutine))
def run(self, host=None, skipovs=None, skipiplink=None, skiplogicalport=None): skipovs = (skipovs is not None) skipiplink = (skipiplink is not None) skiplogicalport = (skiplogicalport is not None) pool = TaskPool(self.scheduler) pool.start() if host is None: host = os.environ.get('DOCKER_HOST', 'unix:///var/run/docker.sock') enable_ssl = os.environ.get('DOCKER_TLS_VERIFY', '') cert_root_path = os.environ.get('DOCKER_CERT_PATH', '~/.docker') ca_path, cert_path, key_path = [ os.path.join(cert_root_path, f) for f in ('ca.pem', 'cert.pem', 'key.pem') ] if '/' not in host: if enable_ssl: host = 'ssl://' + host else: host = 'tcp://' + host self._docker_conn = None http_protocol = Http(False) http_protocol.defaultport = 2375 http_protocol.ssldefaultport = 2375 http_protocol.persist = False def _create_docker_conn(): self._docker_conn = Client(host, http_protocol, self.scheduler, key_path, cert_path, ca_path) self._docker_conn.start() return self._docker_conn def call_docker_api(path, data=None, method=None): if self._docker_conn is None or not self._docker_conn.connected: _create_docker_conn() conn_up = HttpConnectionStateEvent.createMatcher( HttpConnectionStateEvent.CLIENT_CONNECTED) conn_noconn = HttpConnectionStateEvent.createMatcher( HttpConnectionStateEvent.CLIENT_NOTCONNECTED) yield (conn_up, conn_noconn) if self.apiroutine.matcher is conn_noconn: raise IOError('Cannot connect to docker API endpoint: ' + repr(host)) if method is None: if data is None: method = b'GET' else: method = b'POST' if data is None: for m in http_protocol.requestwithresponse( self.apiroutine, self._docker_conn, b'docker', _bytes(path), method, [(b'Accept-Encoding', b'gzip, deflate')]): yield m else: for m in http_protocol.requestwithresponse( self.apiroutine, self._docker_conn, b'docker', _bytes(path), method, [(b'Content-Type', b'application/json;charset=utf-8'), (b'Accept-Encoding', b'gzip, deflate')], MemoryStream(_bytes(json.dumps(data)))): yield m final_resp = self.apiroutine.http_finalresponse output_stream = final_resp.stream try: if final_resp.statuscode >= 200 and final_resp.statuscode < 300: if output_stream is not None and b'content-encoding' in final_resp.headerdict: ce = final_resp.headerdict.get(b'content-encoding') if ce.lower() == b'gzip' or ce.lower() == b'x-gzip': output_stream.getEncoderList().append( encoders.gzip_decoder()) elif ce.lower() == b'deflate': output_stream.getEncoderList().append( encoders.deflate_decoder()) if output_stream is None: self.apiroutine.retvalue = {} else: for m in output_stream.read(self.apiroutine): yield m self.apiroutine.retvalue = json.loads( self.apiroutine.data.decode('utf-8')) else: raise ValueError('Docker API returns error status: ' + repr(final_resp.status)) finally: if output_stream is not None: output_stream.close(self.scheduler) def execute_bash(script, ignoreerror=True): def task(): try: sp = subprocess.Popen(['bash'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) outdata, errdata = sp.communicate(script) sys.stderr.write(_str(errdata)) errno = sp.poll() if errno != 0 and not ignoreerror: print('Script failed, output:\n', repr(outdata), file=sys.stderr) raise ValueError('Script returns %d' % (errno, )) else: return _str(outdata) finally: if sp.poll() is None: try: sp.terminate() sleep(2) if sp.poll() is None: sp.kill() except Exception: pass for m in pool.runTask(self.apiroutine, task): yield m ovsbridge = manager.get('module.dockerplugin.ovsbridge', 'dockerbr0') vethprefix = manager.get('module.dockerplugin.vethprefix', 'vlcp') ipcommand = manager.get('module.dockerplugin.ipcommand', 'ip') ovscommand = manager.get('module.dockerplugin.ovscommand', 'ovs-vsctl') find_invalid_ovs = _find_invalid_ovs % (shell_quote(ovscommand), shell_quote(vethprefix)) find_unused_veth = _find_unused_veth % (shell_quote(ipcommand), shell_quote(vethprefix)) print("docker API endpoint: ", host) print("ovsbridge: ", ovsbridge) print("vethprefix: ", vethprefix) def invalid_ovs_ports(): for m in execute_bash(find_invalid_ovs): yield m first_invalid_ovs_list = self.apiroutine.retvalue.splitlines(False) first_invalid_ovs_list = [ k.strip() for k in first_invalid_ovs_list if k.strip() ] if first_invalid_ovs_list: print( "Detect %d invalid ports from OpenvSwitch, wait 5 seconds to detect again..." % (len(first_invalid_ovs_list), )) else: self.apiroutine.retvalue = [] return for m in self.apiroutine.waitWithTimeout(5): yield m for m in execute_bash(find_invalid_ovs): yield m second_invalid_ovs_list = self.apiroutine.retvalue.splitlines( False) second_invalid_ovs_list = [ k.strip() for k in second_invalid_ovs_list if k.strip() ] invalid_ports = list( set(first_invalid_ovs_list).intersection( second_invalid_ovs_list)) if invalid_ports: print( 'Detect %d invalid ports from intersection of two tries, removing...' % (len(invalid_ports), )) # Remove these ports def _remove_ports(): for p in invalid_ports: try: _unplug_ovs(ovscommand, ovsbridge, p[:-len('-tag')]) except Exception as exc: print('Remove port %r failed: %s' % (p, exc)) for m in pool.runTask(self.apiroutine, _remove_ports): yield m self.apiroutine.retvalue = invalid_ports return def remove_unused_ports(): for m in execute_bash(find_unused_veth): yield m first_unused_ports = self.apiroutine.retvalue.splitlines(False) first_unused_ports = [ k.strip() for k in first_unused_ports if k.strip() ] if first_unused_ports: print( "Detect %d unused ports from ip-link, wait 5 seconds to detect again..." % (len(first_unused_ports), )) else: self.apiroutine.retvalue = [] return for m in self.apiroutine.waitWithTimeout(5): yield m for m in execute_bash(find_unused_veth): yield m second_unused_ports = self.apiroutine.retvalue.splitlines(False) second_unused_ports = [ k.strip() for k in second_unused_ports if k.strip() ] unused_ports = list( set(first_unused_ports).intersection(second_unused_ports)) if unused_ports: print( 'Detect %d unused ports from intersection of two tries, removing...' % (len(unused_ports), )) # Remove these ports def _remove_ports(): for p in unused_ports: try: _unplug_ovs(ovscommand, ovsbridge, p[:-len('-tag')]) except Exception as exc: print( 'Remove port %r from OpenvSwitch failed: %s' % (p, exc)) try: _delete_veth(ipcommand, p[:-len('-tag')]) except Exception as exc: print('Delete port %r with ip-link failed: %s' % (p, exc)) for m in pool.runTask(self.apiroutine, _remove_ports): yield m self.apiroutine.retvalue = unused_ports return def detect_unused_logports(): # docker network ls print("Check logical ports from docker API...") for m in call_docker_api( br'/v1.24/networks?filters={"driver":["vlcp"]}'): yield m network_ports = dict( (n['Id'], dict((p['EndpointID'], p['IPv4Address']) for p in n['Containers'].values())) for n in self.apiroutine.retvalue if n['Driver'] == 'vlcp' ) # Old version of docker API does not support filter by driver print("Find %d networks and %d endpoints from docker API, recheck in 5 seconds..." % \ (len(network_ports), sum(len(ports) for ports in network_ports.values()))) def recheck_ports(): for m in self.apiroutine.waitWithTimeout(5): yield m # docker network inspect, use this for cross check second_network_ports = {} for nid in network_ports: try: for m in call_docker_api(br'/networks/' + _bytes(nid)): yield m except ValueError as exc: print( 'WARNING: check network failed, the network may be removed. Message: ', str(exc)) second_network_ports[nid] = {} else: second_network_ports[nid] = dict( (p['EndpointID'], p['IPv4Address']) for p in self.apiroutine.retvalue['Containers'].values()) print("Recheck find %d endpoints from docker API" % \ (sum(len(ports) for ports in second_network_ports.values()),)) self.apiroutine.retvalue = second_network_ports def check_viperflow(): first_vp_ports = {} for nid in network_ports: for m in callAPI( self.apiroutine, 'viperflow', 'listlogicalports', {'logicalnetwork': 'docker-' + nid + '-lognet'}): yield m first_vp_ports[nid] = dict( (p['id'], p.get('ip_address')) for p in self.apiroutine.retvalue if p['id'].startswith('docker-')) print("Find %d endpoints from viperflow database, recheck in 5 seconds..." % \ (sum(len(ports) for ports in first_vp_ports.values()),)) for m in self.apiroutine.waitWithTimeout(5): yield m second_vp_ports = {} for nid in network_ports: for m in callAPI( self.apiroutine, 'viperflow', 'listlogicalports', {'logicalnetwork': 'docker-' + nid + '-lognet'}): yield m second_vp_ports[nid] = dict( (p['id'], p.get('ip_address')) for p in self.apiroutine.retvalue if p['id'] in first_vp_ports[nid]) print("Find %d endpoints from viperflow database from the intersection of two tries" % \ (sum(len(ports) for ports in second_vp_ports.values()),)) second_vp_ports = dict((nid, dict((pid[len('docker-'):], addr) for pid, addr in v.items())) for nid, v in second_vp_ports.items()) self.apiroutine.retvalue = second_vp_ports for m in check_viperflow(): yield m second_vp_ports = self.apiroutine.retvalue for m in recheck_ports(): yield m second_ports = self.apiroutine.retvalue unused_logports = dict((nid, dict((pid, addr) for pid, addr in v.items() if pid not in network_ports[nid] and\ pid not in second_ports[nid])) for nid, v in second_vp_ports.items()) self.apiroutine.retvalue = unused_logports routines = [] if not skipovs: routines.append(invalid_ovs_ports()) if not skipiplink: routines.append(remove_unused_ports()) if not skiplogicalport: routines.append(detect_unused_logports()) for m in self.apiroutine.executeAll(routines): yield m if skiplogicalport: return (unused_logports, ) = self.apiroutine.retvalue[-1] if any(ports for ports in unused_logports.values()): print("Find %d unused logical ports, first 20 ips:\n%r" % \ (sum(len(ports) for ports in unused_logports.values()), [v for _,v in \ itertools.takewhile(lambda x: x[0] <= 20, enumerate(addr for ports in unused_logports.values() for addr in ports.values()))])) print("Will remove them in 5 seconds, press Ctrl+C to cancel...") for m in self.apiroutine.waitWithTimeout(5): yield m for ports in unused_logports.values(): for p, addr in ports.items(): try: for m in callAPI(self.apiroutine, 'viperflow', 'deletelogicalport', {'id': 'docker-' + p}): yield m except Exception as exc: print("WARNING: remove logical port %r (IP: %s) failed, maybe it is already removed. Message: %s" % \ (p, addr, exc)) print("Done.")
def setUp(self): self.server = Server() self.container = RoutineContainer(self.server.scheduler) self.taskpool = TaskPool(self.server.scheduler) self.taskpool.start()
class Test(unittest.TestCase): def setUp(self): self.server = Server() self.container = RoutineContainer(self.server.scheduler) self.taskpool = TaskPool(self.server.scheduler) self.taskpool.start() def tearDown(self): pass def testTask(self): result = [] def routine(): def t(): return 1 for m in self.taskpool.runTask(self.container, t): yield m result.append(self.container.retvalue) self.container.subroutine(routine()) self.server.serve() self.assertEqual(result, [1]) def testGenTask(self): result = [] def routine(): def t(): for i in range(0,10): yield (TaskTestEvent(result = i, eof = False),) yield (TaskTestEvent(eof = True),) for m in self.taskpool.runGenTask(self.container, t): yield m def routine2(): tte = TaskTestEvent.createMatcher() while True: yield (tte,) if self.container.event.eof: break else: result.append(self.container.event.result) self.container.subroutine(routine()) self.container.subroutine(routine2()) self.server.serve() self.assertEqual(result, [0,1,2,3,4,5,6,7,8,9]) def testAsyncTask(self): result = [] result2 = [] def routine(): def t(sender): for i in range(0,10): sender((TaskTestEvent(result = i, eof = False),)) sender((TaskTestEvent(eof = True),)) return 1 for m in self.taskpool.runAsyncTask(self.container, t): yield m result2.append(self.container.retvalue) def routine2(): tte = TaskTestEvent.createMatcher() while True: yield (tte,) if self.container.event.eof: break else: result.append(self.container.event.result) self.container.subroutine(routine()) self.container.subroutine(routine2()) self.server.serve() self.assertEqual(result, [0,1,2,3,4,5,6,7,8,9]) self.assertEqual(result2, [1])
class Test(unittest.TestCase): def setUp(self): self.server = Server() self.container = RoutineContainer(self.server.scheduler) self.taskpool = TaskPool(self.server.scheduler) self.taskpool.start() def tearDown(self): pass def testTask(self): result = [] def routine(): def t(): return 1 for m in self.taskpool.runTask(self.container, t): yield m result.append(self.container.retvalue) self.container.subroutine(routine()) self.server.serve() self.assertEqual(result, [1]) def testGenTask(self): result = [] def routine(): def t(): for i in range(0, 10): yield (TaskTestEvent(result=i, eof=False), ) yield (TaskTestEvent(eof=True), ) for m in self.taskpool.runGenTask(self.container, t): yield m def routine2(): tte = TaskTestEvent.createMatcher() while True: yield (tte, ) if self.container.event.eof: break else: result.append(self.container.event.result) self.container.subroutine(routine()) self.container.subroutine(routine2()) self.server.serve() self.assertEqual(result, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) def testAsyncTask(self): result = [] result2 = [] def routine(): def t(sender): for i in range(0, 10): sender((TaskTestEvent(result=i, eof=False), )) sender((TaskTestEvent(eof=True), )) return 1 for m in self.taskpool.runAsyncTask(self.container, t): yield m result2.append(self.container.retvalue) def routine2(): tte = TaskTestEvent.createMatcher() while True: yield (tte, ) if self.container.event.eof: break else: result.append(self.container.event.result) self.container.subroutine(routine()) self.container.subroutine(routine2()) self.server.serve() self.assertEqual(result, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) self.assertEqual(result2, [1])