class ConnectionManager(threading.Thread): """Supplies helpers and maintains state for managing connections and tunnels.""" cm_count = 0 def __init__(self, options): self.options = options self.bound = [] # List of bound sockets NOTE: need a lock? self.opened = [] # List of "opened" sockets NOTE: need a lock? self.peers = {} # Peer.id ---> Peer self.peer_lock = threading.Lock() self.pipes = [] self.tunnels = {} # Tunnel.id ---> Tunnel self.tunnel_lock = threading.Lock() self.listeners = [] # List of Listeners self.connectors = [] # List of Connectors self.dispatchers = [] # List of Dispatchers self.handlers = [] # List of Handlers self.callbacks = {} self.identifier = self.options.id self.running = False # We are not running until we are started # Listeners bind to addresses for l_address in self.options.bind_addresses: l = Listener(self, (l_address["hostname"], int(l_address["port"])), l_address["scheme"] == "tls") self.listeners.append(l) self.routing_map = {} # The routing map is per example organized as: # # {'8000': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }, # {'8001': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }} # Determine dispatchers: prot_dispatch_map = { "tcp": TCPDispatcher, "tcps": TCPDispatcher, #'http': WSGI, #'https': WSGI "http": HTTPDispatcher, "https": HTTPDispatcher, } handler_map = { # This mapping should be automaticly loaded # and mapped in a "safe" way... "HobsHandler": HobsHandler, "WebsocketHandler": WebsocketHandler, "ManagementHandler": ManagementHandler, "PeerHandler": PeerHandler, "StaticWebHandler": StaticWebHandler, "TCPTunnelingHandler": TCPTunnelingHandler, "MiGISH": MiGISH, } for o in options.orchestration: port = int(o["port"]) protocol = o["protocol"] handler = o["handler"] forward_to_port = o["forward_to_port"] criteria = o["dispatch_arg"] params = (o["forward_to_host"], int(forward_to_port) if forward_to_port else None, o["forward_via"]) existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])] if existing_instance: # Reuse existing handler handler_i = existing_instance[0] else: # Instantiate new handler handler_params = options.handler_params[handler] if handler in options.handler_params else {} handler_i = handler_map[handler](self, **handler_params) self.handlers.append(handler_i) existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])] if existing_dispatcher: dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher else: dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher self.dispatchers.append(dispatcher_i) if port in self.routing_map: # Setup routing # TODO: check if dispatcher is of differnt type => misconfiguration self.routing_map[port]["handlers"].append( {"instance": handler_i, "criteria": criteria, "params": params} ) else: self.routing_map[port] = { "dispatcher": dispatcher_i, "handlers": [{"instance": handler_i, "criteria": criteria, "params": params}], } for p in self.options.peers: # Instanciate PeerConnectors # Note the constructor should also utilize the "path" logging.debug("Instantiating connector for %s." % repr(p)) peer_connector = PeerConnector(self, peer=p, identifier=self.identifier) self.connectors.append(peer_connector) self.performance_collector = PerformanceCollector(1) count = ConnectionManager.cm_count ConnectionManager.cm_count += 1 threading.Thread.__init__(self, name="CM-%d" % count) def run(self): """Start the connectionmanager.""" self.running = True # Now we are running! logging.debug("Starting...") self.performance_collector.start() # Start performance collector for t in self.handlers + self.connectors + self.listeners: # Start threads t.start() logging.debug("Started!") for t in ( [self.performance_collector] + self.handlers + self.connectors + self.listeners ): # Wait for them to exit t.join() logging.debug("Stopped.") def connect(self, address, peer_id=None, use_tls=False): """Create a connection to address. Possible via another mifcho instance.""" logging.debug( "Connecting to: %s %s %s." % (pprint.pformat(address), pprint.pformat(peer_id), pprint.pformat(use_tls)) ) conn = None peer = self.get_peer(peer_id) if not peer and not peer_id: # Connect "directly" to address try: conn = Connection.fromaddress(address, use_tls) if conn: # Add to opened connections self.opened.append(conn) except socket.error: logging.error("%s when connecting to %s." % (sys.exc_info()[1], address)) except: logging.error("Unexpected error", exc_info=3) elif peer and peer.interface: # Connect via peer interface # Extract address and parameters of peer interface. peer_address = (peer.interface[0], peer.interface[1]) try: peer_conn = self.connect(peer_address) # Socket Connect messages.send_request( # Send TunnelReq peer_conn, type="POST", url="/mifcho/tunnel", headers=[ ("X-Mifcho-Id", self.identifier), ("X-Mifcho-Tunnel-EndpointHost", address[0]), ("X-Mifcho-Tunnel-EndpointPort", address[1]), ], ) resp = (version, status, reason, headers) = messages.get_response(peer_conn) # Wait for response except: logging.error("Failed connecting to socket!") logging.debug("Failed receiving response!", exc_info=3) peer_conn = None resp = (version, status, reason, headers) = ("", 404, "Not Found", []) conn = peer_conn elif peer and not peer.interface: # Connect via peer control-line # and "callback". peer.lock.acquire() # CRITICAL ZONE START... tunnel = Tunnel.fromconnections(None, None) self.add_tunnel(tunnel) try: # Send request for tunnel messages.send_request( peer.connection, type="POST", url="/mifcho/tunnel_request", headers=[ ("X-Mifcho-Id", self.identifier), ("X-Mifcho-Tunnel-Id", tunnel.id), ("X-Mifcho-Tunnel-EndpointHost", address[0]), ("X-Mifcho-Tunnel-EndpointPort", address[1]), ], ) except: logging.error("Error sending request to peer!", exc_info=3) try: # Wait for response resp = (version, status, reason, headers) = messages.get_response(peer.connection) except: logging.error("Failed receiving response!") logging.debug("Failed receiving response!", exc_info=3) resp = (version, status, reason, headers) = ("", 404, "Not Found", []) peer.lock.release() # CRITICAL ZONE END.... if int(status) == 200: # All is good # Wait for PeerHandler to set tunnel-event wait_for_tunnel = True while status == 200 and self.running and wait_for_tunnel: tunnel.event.wait(1) wait_for_tunnel = not tunnel.event.is_set() conn = tunnel.peer_connection else: conn = None elif peer_id and not peer: logging.error("Could not find peer %s." % repr(peer_id)) logging.debug("Peers=[%s]" % repr(self.peers)) else: logging.error("Invalid params.") return conn def teardown(self, conn): """Tear down a socket properly and remove it from the connection-manager.""" if conn in self.opened: try: self.opened.remove(conn) except: pass if conn in self.bound: try: self.bound.remove(conn) except: pass try: conn.shutdown() except: pass try: conn.close() except: pass def register_callback(self, event, peer): """ Execute function when peer connects. """ if peer in self.callbacks: self.callbacks[peer].append(event) else: self.callbacks[peer] = [event] def add_peer(self, peer): success = False self.peer_lock.acquire() logging.debug("Adding peer %s, %s." % (pprint.pformat(peer), pprint.pformat(peer.id))) self.peers[peer.id] = peer if peer.id in self.callbacks: for e in self.callbacks[peer.id]: e.set() self.peer_lock.release() return success def get_peer(self, id): peer = None self.peer_lock.acquire() if id in self.peers: peer = self.peers[id] self.peer_lock.release() return peer def get_any_peer(self): peer = None self.peer_lock.acquire() if len(self.peers) > 0: peer = self.peers.values()[0] self.peer_lock.release() return peer def add_tunnel(self, tunnel): """Add a tunnel.""" success = False self.tunnel_lock.acquire() if tunnel.id not in self.tunnels: self.tunnels[tunnel.id] = tunnel self.tunnel_lock.release() return success def stop(self): """Shut down connection-manager.""" self.running = False # Tell listeners to stop self.performance_collector.stop() # Tell performance collector to stop for w in self.listeners + self.connectors + self.handlers: # Tell threads to stop w.stop() for opened_socket in self.bound + self.opened: # Tear down sockets self.teardown(opened_socket)
def __init__(self, options): self.options = options self.bound = [] # List of bound sockets NOTE: need a lock? self.opened = [] # List of "opened" sockets NOTE: need a lock? self.peers = {} # Peer.id ---> Peer self.peer_lock = threading.Lock() self.pipes = [] self.tunnels = {} # Tunnel.id ---> Tunnel self.tunnel_lock = threading.Lock() self.listeners = [] # List of Listeners self.connectors = [] # List of Connectors self.dispatchers = [] # List of Dispatchers self.handlers = [] # List of Handlers self.callbacks = {} self.identifier = self.options.id self.running = False # We are not running until we are started # Listeners bind to addresses for l_address in self.options.bind_addresses: l = Listener(self, (l_address["hostname"], int(l_address["port"])), l_address["scheme"] == "tls") self.listeners.append(l) self.routing_map = {} # The routing map is per example organized as: # # {'8000': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }, # {'8001': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }} # Determine dispatchers: prot_dispatch_map = { "tcp": TCPDispatcher, "tcps": TCPDispatcher, #'http': WSGI, #'https': WSGI "http": HTTPDispatcher, "https": HTTPDispatcher, } handler_map = { # This mapping should be automaticly loaded # and mapped in a "safe" way... "HobsHandler": HobsHandler, "WebsocketHandler": WebsocketHandler, "ManagementHandler": ManagementHandler, "PeerHandler": PeerHandler, "StaticWebHandler": StaticWebHandler, "TCPTunnelingHandler": TCPTunnelingHandler, "MiGISH": MiGISH, } for o in options.orchestration: port = int(o["port"]) protocol = o["protocol"] handler = o["handler"] forward_to_port = o["forward_to_port"] criteria = o["dispatch_arg"] params = (o["forward_to_host"], int(forward_to_port) if forward_to_port else None, o["forward_via"]) existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])] if existing_instance: # Reuse existing handler handler_i = existing_instance[0] else: # Instantiate new handler handler_params = options.handler_params[handler] if handler in options.handler_params else {} handler_i = handler_map[handler](self, **handler_params) self.handlers.append(handler_i) existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])] if existing_dispatcher: dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher else: dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher self.dispatchers.append(dispatcher_i) if port in self.routing_map: # Setup routing # TODO: check if dispatcher is of differnt type => misconfiguration self.routing_map[port]["handlers"].append( {"instance": handler_i, "criteria": criteria, "params": params} ) else: self.routing_map[port] = { "dispatcher": dispatcher_i, "handlers": [{"instance": handler_i, "criteria": criteria, "params": params}], } for p in self.options.peers: # Instanciate PeerConnectors # Note the constructor should also utilize the "path" logging.debug("Instantiating connector for %s." % repr(p)) peer_connector = PeerConnector(self, peer=p, identifier=self.identifier) self.connectors.append(peer_connector) self.performance_collector = PerformanceCollector(1) count = ConnectionManager.cm_count ConnectionManager.cm_count += 1 threading.Thread.__init__(self, name="CM-%d" % count)
def __init__(self, options): self.options = options self.bound = [] # List of bound sockets NOTE: need a lock? self.opened = [] # List of "opened" sockets NOTE: need a lock? self.peers = {} # Peer.id ---> Peer self.peer_lock = threading.Lock() self.pipes = [] self.tunnels = {} # Tunnel.id ---> Tunnel self.tunnel_lock = threading.Lock() self.listeners = [] # List of Listeners self.connectors = [] # List of Connectors self.dispatchers = [] # List of Dispatchers self.handlers = [] # List of Handlers self.callbacks = {} self.identifier = self.options.id self.running = False # We are not running until we are started # Listeners bind to addresses for l_address in self.options.bind_addresses: l = Listener( self, (l_address['hostname'], int(l_address['port'])), l_address['scheme'] == 'tls' ) self.listeners.append(l) self.routing_map = {} # The routing map is per example organized as: # # {'8000': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }, # {'8001': { # 'dispatcher': dispatcher_instance, # 'handlers': [{ # 'criteria': SOMETHING, # 'instance': handler_instance, # 'params': job_parameters # }] # }} # Determine dispatchers: prot_dispatch_map = { 'tcp': TCPDispatcher, 'tcps': TCPDispatcher, 'wsgi': WSGI, 'wsgis': WSGI, 'http': HTTPDispatcher, 'https': HTTPDispatcher } handler_map = { # This mapping should be automaticly loaded # and mapped in a "safe" way... 'HobsHandler': HobsHandler, 'WebsocketHandler': WebsocketHandler, #'ManagementHandler': ManagementHandler, #'StaticWebHandler': StaticWebHandler, 'PeerHandler': PeerHandler, 'TCPTunnelingHandler': TCPTunnelingHandler, 'MiGISH': MiGISH } for o in options.orchestration: port = int(o['port']) protocol = o['protocol'] handler = o['handler'] forward_to_port = o['forward_to_port'] criteria = o['dispatch_arg'] params = (o['forward_to_host'], int(forward_to_port) if forward_to_port else None, o['forward_via']) existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])] if existing_instance: # Reuse existing handler handler_i = existing_instance[0] else: # Instantiate new handler handler_params = options.handler_params[handler] if handler in options.handler_params else {} handler_i = handler_map[handler](self, **handler_params) self.handlers.append(handler_i) existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])] if existing_dispatcher: dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher else: dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher self.dispatchers.append(dispatcher_i) if port in self.routing_map: # Setup routing # TODO: check if dispatcher is of differnt type => misconfiguration self.routing_map[port]['handlers'].append({ 'instance': handler_i, 'criteria': criteria, 'params': params }) else: self.routing_map[port] = { 'dispatcher': dispatcher_i, 'handlers': [{ 'instance': handler_i, 'criteria': criteria, 'params': params }] } for p in self.options.peers: # Instanciate PeerConnectors # Note the constructor should also utilize the "path" peer_connector = PeerConnector( self, peer = p, identifier = self.identifier ) self.connectors.append(peer_connector) self.performance_collector = PerformanceCollector(1) count = ConnectionManager.cm_count ConnectionManager.cm_count += 1 threading.Thread.__init__(self, name='CM-%d' % count)