class ConnectionManager(threading.Thread):
"""Supplies helpers and maintains state for managing connections and tunnels."""
cm_count = 0
def __init__(self, options):
self.options = options
self.bound = [] # List of bound sockets NOTE: need a lock?
self.opened = [] # List of "opened" sockets NOTE: need a lock?
self.peers = {} # Peer.id ---> Peer
self.peer_lock = threading.Lock()
self.pipes = []
self.tunnels = {} # Tunnel.id ---> Tunnel
self.tunnel_lock = threading.Lock()
self.listeners = [] # List of Listeners
self.connectors = [] # List of Connectors
self.dispatchers = [] # List of Dispatchers
self.handlers = [] # List of Handlers
self.callbacks = {}
self.identifier = self.options.id
self.running = False # We are not running until we are started
# Listeners bind to addresses
for l_address in self.options.bind_addresses:
l = Listener(self, (l_address["hostname"], int(l_address["port"])), l_address["scheme"] == "tls")
self.listeners.append(l)
self.routing_map = {} # The routing map is per example organized as:
#
# {'8000': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# },
# {'8001': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# }}
# Determine dispatchers:
prot_dispatch_map = {
"tcp": TCPDispatcher,
"tcps": TCPDispatcher,
#'http': WSGI,
#'https': WSGI
"http": HTTPDispatcher,
"https": HTTPDispatcher,
}
handler_map = { # This mapping should be automaticly loaded
# and mapped in a "safe" way...
"HobsHandler": HobsHandler,
"WebsocketHandler": WebsocketHandler,
"ManagementHandler": ManagementHandler,
"PeerHandler": PeerHandler,
"StaticWebHandler": StaticWebHandler,
"TCPTunnelingHandler": TCPTunnelingHandler,
"MiGISH": MiGISH,
}
for o in options.orchestration:
port = int(o["port"])
protocol = o["protocol"]
handler = o["handler"]
forward_to_port = o["forward_to_port"]
criteria = o["dispatch_arg"]
params = (o["forward_to_host"], int(forward_to_port) if forward_to_port else None, o["forward_via"])
existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])]
if existing_instance: # Reuse existing handler
handler_i = existing_instance[0]
else: # Instantiate new handler
handler_params = options.handler_params[handler] if handler in options.handler_params else {}
handler_i = handler_map[handler](self, **handler_params)
self.handlers.append(handler_i)
existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])]
if existing_dispatcher:
dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher
else:
dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher
self.dispatchers.append(dispatcher_i)
if port in self.routing_map: # Setup routing
# TODO: check if dispatcher is of differnt type => misconfiguration
self.routing_map[port]["handlers"].append(
{"instance": handler_i, "criteria": criteria, "params": params}
)
else:
self.routing_map[port] = {
"dispatcher": dispatcher_i,
"handlers": [{"instance": handler_i, "criteria": criteria, "params": params}],
}
for p in self.options.peers: # Instanciate PeerConnectors
# Note the constructor should also utilize the "path"
logging.debug("Instantiating connector for %s." % repr(p))
peer_connector = PeerConnector(self, peer=p, identifier=self.identifier)
self.connectors.append(peer_connector)
self.performance_collector = PerformanceCollector(1)
count = ConnectionManager.cm_count
ConnectionManager.cm_count += 1
threading.Thread.__init__(self, name="CM-%d" % count)
def run(self):
"""Start the connectionmanager."""
self.running = True # Now we are running!
logging.debug("Starting...")
self.performance_collector.start() # Start performance collector
for t in self.handlers + self.connectors + self.listeners: # Start threads
t.start()
logging.debug("Started!")
for t in (
[self.performance_collector] + self.handlers + self.connectors + self.listeners
): # Wait for them to exit
t.join()
logging.debug("Stopped.")
def connect(self, address, peer_id=None, use_tls=False):
"""Create a connection to address. Possible via another mifcho instance."""
logging.debug(
"Connecting to: %s %s %s." % (pprint.pformat(address), pprint.pformat(peer_id), pprint.pformat(use_tls))
)
conn = None
peer = self.get_peer(peer_id)
if not peer and not peer_id: # Connect "directly" to address
try:
conn = Connection.fromaddress(address, use_tls)
if conn: # Add to opened connections
self.opened.append(conn)
except socket.error:
logging.error("%s when connecting to %s." % (sys.exc_info()[1], address))
except:
logging.error("Unexpected error", exc_info=3)
elif peer and peer.interface: # Connect via peer interface
# Extract address and parameters of peer interface.
peer_address = (peer.interface[0], peer.interface[1])
try:
peer_conn = self.connect(peer_address) # Socket Connect
messages.send_request( # Send TunnelReq
peer_conn,
type="POST",
url="/mifcho/tunnel",
headers=[
("X-Mifcho-Id", self.identifier),
("X-Mifcho-Tunnel-EndpointHost", address[0]),
("X-Mifcho-Tunnel-EndpointPort", address[1]),
],
)
resp = (version, status, reason, headers) = messages.get_response(peer_conn) # Wait for response
except:
logging.error("Failed connecting to socket!")
logging.debug("Failed receiving response!", exc_info=3)
peer_conn = None
resp = (version, status, reason, headers) = ("", 404, "Not Found", [])
conn = peer_conn
elif peer and not peer.interface: # Connect via peer control-line
# and "callback".
peer.lock.acquire() # CRITICAL ZONE START...
tunnel = Tunnel.fromconnections(None, None)
self.add_tunnel(tunnel)
try: # Send request for tunnel
messages.send_request(
peer.connection,
type="POST",
url="/mifcho/tunnel_request",
headers=[
("X-Mifcho-Id", self.identifier),
("X-Mifcho-Tunnel-Id", tunnel.id),
("X-Mifcho-Tunnel-EndpointHost", address[0]),
("X-Mifcho-Tunnel-EndpointPort", address[1]),
],
)
except:
logging.error("Error sending request to peer!", exc_info=3)
try: # Wait for response
resp = (version, status, reason, headers) = messages.get_response(peer.connection)
except:
logging.error("Failed receiving response!")
logging.debug("Failed receiving response!", exc_info=3)
resp = (version, status, reason, headers) = ("", 404, "Not Found", [])
peer.lock.release() # CRITICAL ZONE END....
if int(status) == 200: # All is good
# Wait for PeerHandler to set tunnel-event
wait_for_tunnel = True
while status == 200 and self.running and wait_for_tunnel:
tunnel.event.wait(1)
wait_for_tunnel = not tunnel.event.is_set()
conn = tunnel.peer_connection
else:
conn = None
elif peer_id and not peer:
logging.error("Could not find peer %s." % repr(peer_id))
logging.debug("Peers=[%s]" % repr(self.peers))
else:
logging.error("Invalid params.")
return conn
def teardown(self, conn):
"""Tear down a socket properly and remove it from the connection-manager."""
if conn in self.opened:
try:
self.opened.remove(conn)
except:
pass
if conn in self.bound:
try:
self.bound.remove(conn)
except:
pass
try:
conn.shutdown()
except:
pass
try:
conn.close()
except:
pass
def register_callback(self, event, peer):
"""
Execute function when peer connects.
"""
if peer in self.callbacks:
self.callbacks[peer].append(event)
else:
self.callbacks[peer] = [event]
def add_peer(self, peer):
success = False
self.peer_lock.acquire()
logging.debug("Adding peer %s, %s." % (pprint.pformat(peer), pprint.pformat(peer.id)))
self.peers[peer.id] = peer
if peer.id in self.callbacks:
for e in self.callbacks[peer.id]:
e.set()
self.peer_lock.release()
return success
def get_peer(self, id):
peer = None
self.peer_lock.acquire()
if id in self.peers:
peer = self.peers[id]
self.peer_lock.release()
return peer
def get_any_peer(self):
peer = None
self.peer_lock.acquire()
if len(self.peers) > 0:
peer = self.peers.values()[0]
self.peer_lock.release()
return peer
def add_tunnel(self, tunnel):
"""Add a tunnel."""
success = False
self.tunnel_lock.acquire()
if tunnel.id not in self.tunnels:
self.tunnels[tunnel.id] = tunnel
self.tunnel_lock.release()
return success
def stop(self):
"""Shut down connection-manager."""
self.running = False # Tell listeners to stop
self.performance_collector.stop() # Tell performance collector to stop
for w in self.listeners + self.connectors + self.handlers: # Tell threads to stop
w.stop()
for opened_socket in self.bound + self.opened: # Tear down sockets
self.teardown(opened_socket)
def __init__(self, options):
self.options = options
self.bound = [] # List of bound sockets NOTE: need a lock?
self.opened = [] # List of "opened" sockets NOTE: need a lock?
self.peers = {} # Peer.id ---> Peer
self.peer_lock = threading.Lock()
self.pipes = []
self.tunnels = {} # Tunnel.id ---> Tunnel
self.tunnel_lock = threading.Lock()
self.listeners = [] # List of Listeners
self.connectors = [] # List of Connectors
self.dispatchers = [] # List of Dispatchers
self.handlers = [] # List of Handlers
self.callbacks = {}
self.identifier = self.options.id
self.running = False # We are not running until we are started
# Listeners bind to addresses
for l_address in self.options.bind_addresses:
l = Listener(self, (l_address["hostname"], int(l_address["port"])), l_address["scheme"] == "tls")
self.listeners.append(l)
self.routing_map = {} # The routing map is per example organized as:
#
# {'8000': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# },
# {'8001': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# }}
# Determine dispatchers:
prot_dispatch_map = {
"tcp": TCPDispatcher,
"tcps": TCPDispatcher,
#'http': WSGI,
#'https': WSGI
"http": HTTPDispatcher,
"https": HTTPDispatcher,
}
handler_map = { # This mapping should be automaticly loaded
# and mapped in a "safe" way...
"HobsHandler": HobsHandler,
"WebsocketHandler": WebsocketHandler,
"ManagementHandler": ManagementHandler,
"PeerHandler": PeerHandler,
"StaticWebHandler": StaticWebHandler,
"TCPTunnelingHandler": TCPTunnelingHandler,
"MiGISH": MiGISH,
}
for o in options.orchestration:
port = int(o["port"])
protocol = o["protocol"]
handler = o["handler"]
forward_to_port = o["forward_to_port"]
criteria = o["dispatch_arg"]
params = (o["forward_to_host"], int(forward_to_port) if forward_to_port else None, o["forward_via"])
existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])]
if existing_instance: # Reuse existing handler
handler_i = existing_instance[0]
else: # Instantiate new handler
handler_params = options.handler_params[handler] if handler in options.handler_params else {}
handler_i = handler_map[handler](self, **handler_params)
self.handlers.append(handler_i)
existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])]
if existing_dispatcher:
dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher
else:
dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher
self.dispatchers.append(dispatcher_i)
if port in self.routing_map: # Setup routing
# TODO: check if dispatcher is of differnt type => misconfiguration
self.routing_map[port]["handlers"].append(
{"instance": handler_i, "criteria": criteria, "params": params}
)
else:
self.routing_map[port] = {
"dispatcher": dispatcher_i,
"handlers": [{"instance": handler_i, "criteria": criteria, "params": params}],
}
for p in self.options.peers: # Instanciate PeerConnectors
# Note the constructor should also utilize the "path"
logging.debug("Instantiating connector for %s." % repr(p))
peer_connector = PeerConnector(self, peer=p, identifier=self.identifier)
self.connectors.append(peer_connector)
self.performance_collector = PerformanceCollector(1)
count = ConnectionManager.cm_count
ConnectionManager.cm_count += 1
threading.Thread.__init__(self, name="CM-%d" % count)
def __init__(self, options):
self.options = options
self.bound = [] # List of bound sockets NOTE: need a lock?
self.opened = [] # List of "opened" sockets NOTE: need a lock?
self.peers = {} # Peer.id ---> Peer
self.peer_lock = threading.Lock()
self.pipes = []
self.tunnels = {} # Tunnel.id ---> Tunnel
self.tunnel_lock = threading.Lock()
self.listeners = [] # List of Listeners
self.connectors = [] # List of Connectors
self.dispatchers = [] # List of Dispatchers
self.handlers = [] # List of Handlers
self.callbacks = {}
self.identifier = self.options.id
self.running = False # We are not running until we are started
# Listeners bind to addresses
for l_address in self.options.bind_addresses:
l = Listener(
self,
(l_address['hostname'], int(l_address['port'])),
l_address['scheme'] == 'tls'
)
self.listeners.append(l)
self.routing_map = {} # The routing map is per example organized as:
#
# {'8000': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# },
# {'8001': {
# 'dispatcher': dispatcher_instance,
# 'handlers': [{
# 'criteria': SOMETHING,
# 'instance': handler_instance,
# 'params': job_parameters
# }]
# }}
# Determine dispatchers:
prot_dispatch_map = {
'tcp': TCPDispatcher,
'tcps': TCPDispatcher,
'wsgi': WSGI,
'wsgis': WSGI,
'http': HTTPDispatcher,
'https': HTTPDispatcher
}
handler_map = { # This mapping should be automaticly loaded
# and mapped in a "safe" way...
'HobsHandler': HobsHandler,
'WebsocketHandler': WebsocketHandler,
#'ManagementHandler': ManagementHandler,
#'StaticWebHandler': StaticWebHandler,
'PeerHandler': PeerHandler,
'TCPTunnelingHandler': TCPTunnelingHandler,
'MiGISH': MiGISH
}
for o in options.orchestration:
port = int(o['port'])
protocol = o['protocol']
handler = o['handler']
forward_to_port = o['forward_to_port']
criteria = o['dispatch_arg']
params = (o['forward_to_host'], int(forward_to_port) if forward_to_port else None, o['forward_via'])
existing_instance = [h for h in self.handlers if isinstance(h, handler_map[handler])]
if existing_instance: # Reuse existing handler
handler_i = existing_instance[0]
else: # Instantiate new handler
handler_params = options.handler_params[handler] if handler in options.handler_params else {}
handler_i = handler_map[handler](self, **handler_params)
self.handlers.append(handler_i)
existing_dispatcher = [d for d in self.dispatchers if isinstance(d, prot_dispatch_map[protocol])]
if existing_dispatcher:
dispatcher_i = existing_dispatcher[0] # Reuse existing dispatcher
else:
dispatcher_i = prot_dispatch_map[protocol](self) # Instantiate new dispatcher
self.dispatchers.append(dispatcher_i)
if port in self.routing_map: # Setup routing
# TODO: check if dispatcher is of differnt type => misconfiguration
self.routing_map[port]['handlers'].append({
'instance': handler_i,
'criteria': criteria,
'params': params
})
else:
self.routing_map[port] = {
'dispatcher': dispatcher_i,
'handlers': [{
'instance': handler_i,
'criteria': criteria,
'params': params
}]
}
for p in self.options.peers: # Instanciate PeerConnectors
# Note the constructor should also utilize the "path"
peer_connector = PeerConnector(
self,
peer = p,
identifier = self.identifier
)
self.connectors.append(peer_connector)
self.performance_collector = PerformanceCollector(1)
count = ConnectionManager.cm_count
ConnectionManager.cm_count += 1
threading.Thread.__init__(self, name='CM-%d' % count)