def disconnection_request(self, payload):
""" A request from a peer to disconnect a port"""
if not ('peer_port_id' in payload or
('peer_actor_id' in payload and 'peer_port_name' in payload
and 'peer_port_dir' in payload)):
# Not enough info to find port
return response.CalvinResponse(response.BAD_REQUEST)
# Check if port actually is local
try:
port = self._get_local_port(
payload['peer_actor_id'] if 'peer_actor_id' in payload else
None, payload['peer_port_name'] if 'peer_port_name' in payload
else None, payload['peer_port_dir']
if 'peer_port_dir' in payload else None,
payload['peer_port_id'] if 'peer_port_id' in payload else None)
except:
# We don't have the port
return response.CalvinResponse(response.NOT_FOUND)
else:
# Disconnect and destroy endpoints
endpoints = port.disconnect()
for ep in endpoints:
if isinstance(ep, endpoint.TunnelOutEndpoint):
self.monitor.unregister_out_endpoint(ep)
ep.destroy()
return response.CalvinResponse(True)
def migrate(self, actor_id, node_id, callback=None):
""" Migrate an actor actor_id to peer node node_id """
if actor_id not in self.actors:
# Can only migrate actors from our node
if callback:
callback(status=response.CalvinResponse(False))
return
if node_id == self.node.id:
# No need to migrate to ourself
if callback:
callback(status=response.CalvinResponse(True))
return
actor = self.actors[actor_id]
actor._migrating_to = node_id
actor.will_migrate()
actor_type = actor._type
ports = actor.connections(self.node.id)
# Disconnect ports and continue in _migrate_disconnect
self.node.pm.disconnect(callback=CalvinCB(self._migrate_disconnected,
actor=actor,
actor_type=actor_type,
ports=ports,
node_id=node_id,
callback=callback),
actor_id=actor_id)
def _connect_by_peer_actor_id(self, key, value, **state):
""" Gets called when storage responds with peer actor information"""
_log.analyze(self.node.id,
"+",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
if not isinstance(value, dict):
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.BAD_REQUEST,
"Storage return invalid information"),
**state)
return
else:
raise Exception("Storage return invalid information")
if not state['peer_node_id'] and 'node_id' in value and value[
'node_id']:
state['peer_node_id'] = value['node_id']
else:
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.BAD_REQUEST,
"Storage return invalid information"),
**state)
return
else:
raise Exception("Storage return invalid information")
self._connect(**state)
def deployment_add_requirements(self, application_id, reqs, cb):
app = None
try:
app = self.applications[application_id]
except:
_log.debug("deployment_add_requirements did not find app %s" %
(application_id, ))
cb(status=response.CalvinResponse(False))
return
_log.debug("deployment_add_requirements(app=%s,\n reqs=%s)" %
(self.applications[application_id], reqs))
# TODO extract groups
if "requirements" not in reqs:
# No requirements then we are happy
cb(status=response.CalvinResponse(True))
return
if hasattr(app, '_org_cb'):
# application deployment requirements ongoing, abort
cb(status=response.CalvinResponse(True))
return
app._org_cb = cb
name_map = app.get_actor_name_map(ns=app.ns)
app._track_actor_cb = app.get_actors(
)[:] # take copy of app's actor list, to remove each when done
app.actor_placement = {} # Clean placement slate
_log.analyze(self._node.id,
"+ APP REQ", {'requirements': reqs["requirements"]},
tb=True)
rr = reqs["requirements"].copy()
for actor_name, req in rr.iteritems():
# Component returns list of actor ids, actors returns list with single id
actor_ids = name_map.get(
(app.ns + ":" if app.ns else "") + actor_name, None)
# Apply same rule to all actors in a component, rule get component information and can act accordingly
for actor_id in actor_ids:
if actor_id not in self._node.am.actors.keys():
_log.debug("Only apply requirements to local actors")
continue
actor = self._node.am.actors[actor_id]
actor.deployment_add_requirements(
req, component=(actor_ids if len(actor_ids) > 1 else None))
_log.analyze(self._node.id,
"+ ACTOR REQ", {
'actor_id': actor_id,
'actor_ids': actor_ids
},
tb=True)
self.actor_requirements(app, actor_id)
_log.analyze(self._node.id,
"+ ACTOR REQ DONE", {'actor_ids': actor_ids},
tb=True)
_log.analyze(self._node.id,
"+ DONE", {'application_id': application_id},
tb=True)
def _connect_via_tunnel(self, status=None, **state):
""" All information and hopefully (status OK) a tunnel to the peer is available for a port connect"""
port = self._get_local_port(state['actor_id'], state['port_name'],
state['port_dir'], state['port_id'])
_log.analyze(
self.node.id,
"+ " + str(status),
dict({k: state[k]
for k in state.keys() if k != 'callback'},
port_is_connected=port.is_connected_to(
state['peer_port_id'])),
peer_node_id=state['peer_node_id'])
if port.is_connected_to(state['peer_port_id']):
# The other end beat us to connecting the port, lets just report success and return
_log.analyze(
self.node.id,
"+ IS CONNECTED",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
if state['callback']:
state['callback'](status=response.CalvinResponse(True),
**state)
return None
if not status:
# Failed getting a tunnel, just inform the one wanting to connect
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.INTERNAL_ERROR),
**state)
return None
# Finally we have all information and a tunnel
# Lets ask the peer if it can connect our port.
tunnel = self.tunnels[state['peer_node_id']]
_log.analyze(
self.node.id,
"+ SENDING",
dict({k: state[k]
for k in state.keys() if k != 'callback'},
tunnel_status=self.tunnels[state['peer_node_id']].status),
peer_node_id=state['peer_node_id'])
if 'retries' not in state:
state['retries'] = 0
self.proto.port_connect(callback=CalvinCB(self._connected_via_tunnel,
**state),
port_id=state['port_id'],
peer_node_id=state['peer_node_id'],
peer_port_id=state['peer_port_id'],
peer_actor_id=state['peer_actor_id'],
peer_port_name=state['peer_port_name'],
peer_port_dir=state['peer_port_dir'],
tunnel=tunnel)
def tunnel_destroy_handler(self, payload):
""" Destroy tunnel (response side) """
try:
self.network.link_check(to_rt_uuid)
except:
raise Exception("ERROR_UNKNOWN_RUNTIME")
try:
tunnel = self.tunnels[payload['from_rt_uuid']][
payload['tunnel_id']]
except:
raise Exception("ERROR_UNKNOWN_TUNNEL")
_log.analyze(self.rt_id,
"+ ERROR_UNKNOWN_TUNNEL",
payload,
peer_node_id=payload['from_rt_uuid'])
# We have the tunnel so close it
tunnel.close(local_only=True)
ok = False
try:
# Hope the tunnel don't mind,
# TODO since the requester likely don't know what to do and we have already closed it
ok = tunnel.down_handler()
except:
pass
msg = {
'cmd': 'REPLY',
'msg_uuid': payload['msg_uuid'],
'value': response.CalvinResponse(ok).encode()
}
self.network.links[payload['from_rt_uuid']].send(msg)
def reply_handler(self, payload):
""" Gets called when a REPLY messages arrives on this link """
try:
# Call the registered callback,for the reply message id, with the reply data as argument
self.replies.pop(payload['msg_uuid'])(
response.CalvinResponse(encoded=payload['value']))
except:
# We ignore unknown replies
return
def new(self,
actor_type,
args,
state=None,
prev_connections=None,
connection_list=None,
callback=None,
signature=None):
"""
Instantiate an actor of type 'actor_type'. Parameters are passed in 'args',
'name' is an optional parameter in 'args', specifying a human readable name.
Returns actor id on success and raises an exception if anything goes wrong.
Optionally applies a serialized state to the actor, the supplied args are ignored and args from state
is used instead.
Optionally reconnecting the ports, using either
1) an unmodified connections structure obtained by the connections command supplied as
prev_connections or,
2) a mangled list of tuples with (in_node_id, in_port_id, out_node_id, out_port_id) supplied as
connection_list
"""
_log.debug("class: %s args: %s state: %s", actor_type, args, state)
_log.analyze(self.node.id, "+", {
'actor_type': actor_type,
'state': state
})
try:
if state:
a = self._new_from_state(actor_type, state)
else:
a = self._new(actor_type, args)
except Exception as e:
_log.exception("Actor creation failed")
raise (e)
# Store the actor signature to enable GlobalStore lookup
a._signature = signature
self.actors[a.id] = a
self.node.storage.add_actor(a, self.node.id)
if prev_connections:
# Convert prev_connections to connection_list format
connection_list = self._prev_connections_to_connection_list(
prev_connections)
if connection_list:
# Migrated actor
self.connect(a.id, connection_list, callback=callback)
else:
# Nothing to connect then we are OK
if callback:
callback(status=response.CalvinResponse(True), actor_id=a.id)
else:
return a.id
def destroy_request(self, application_id, actor_ids):
""" Request from peer of local application parts destruction and related actors """
_log.debug("Destroy request, app: %s, actors: %s" %
(application_id, actor_ids))
_log.analyze(self._node.id, "+", {
'application_id': application_id,
'actor_ids': actor_ids
})
reply = response.CalvinResponse(True)
for actor_id in actor_ids:
if actor_id in self._node.am.list_actors():
self._node.am.destroy(actor_id)
else:
reply = response.CalvinResponse(False)
if application_id in self.applications:
del self.applications[application_id]
_log.debug("Destroy request reply %s" % reply)
_log.analyze(self._node.id, "+ RESPONSE", {'reply': str(reply)})
return reply
def _disconnected_port(self, reply, **state):
""" Get called for each peer port when diconnecting but callback should only be called once"""
try:
# Remove this peer from the list of remote peer ports
self.disconnecting_ports[state['port_id']].remove(state['peer_id'])
except:
pass
if not reply:
# Got failed response do callback, but also remove port from dictionary indicating we have sent the callback
self.disconnecting_ports.pop(state['port_id'])
if state['callback']:
state['callback'](status=response.CalvinResponse(False),
**state)
if state['port_id'] in self.disconnecting_ports:
if not self.disconnecting_ports[state['port_id']]:
# We still have port in dictionary and now list is empty hence we should send OK
self.disconnecting_ports.pop(state['port_id'])
if state['callback']:
state['callback'](status=response.CalvinResponse(True),
**state)
def link_request_finished(self, key, value, callback):
""" Called by storage when the node is (not) found """
_log.analyze(self.node.id, "+", {'value': value}, peer_node_id=key)
# Test if value is None or False indicating node does not currently exist in storage
if not value:
# the peer_id did not exist in storage
callback(status=response.CalvinResponse(response.NOT_FOUND,
{'peer_node_id': key}))
return
# join the peer node
self.join([value['uri']], callback, [key])
def _disconnecting_actor_cb(self, status, _callback, port_ids, **state):
""" Get called for each of the actor's ports when disconnecting, but callback should only be called once
status: OK or not
_callback: original callback
port_ids: list of port ids kept in context between calls when *changed* by this function, do not replace it
state: dictionary keeping disconnect information
"""
# Send negative response if not already done it
if not status and port_ids:
if _callback:
del port_ids[:]
_callback(status=response.CalvinResponse(False),
actor_id=state['actor_id'])
if state['port_id'] in port_ids:
# Remove this port from list
port_ids.remove(state['port_id'])
# If all ports done send positive response
if not port_ids:
if _callback:
_callback(status=response.CalvinResponse(True),
actor_id=state['actor_id'])
def tunnel_up(self, tunnel):
""" Callback that the tunnel is working """
tunnel_peer_id = tunnel.peer_node_id
# If a port connect have ordered a tunnel then it have a callback in pending
# which want to continue with the connection
if tunnel_peer_id in self.pending_tunnels:
for cb in self.pending_tunnels[tunnel_peer_id]:
try:
cb(status=response.CalvinResponse(True))
except:
pass
self.pending_tunnels.pop(tunnel_peer_id)
def _actor_connected(self, status, peer_port_id, actor_id, peer_port_ids,
_callback, **kwargs):
""" Get called for each of the actor's ports when connecting, but callback should only be called once
status: success or not
_callback: original callback
peer_port_ids: list of port ids kept in context between calls when *changed* by this function,
do not replace it
"""
# Send negative response if not already done it
if not status and peer_port_ids:
if _callback:
del peer_port_ids[:]
_callback(status=response.CalvinResponse(False),
actor_id=actor_id)
if peer_port_id in peer_port_ids:
# Remove this port from list
peer_port_ids.remove(peer_port_id)
# If all ports done send OK
if not peer_port_ids:
if _callback:
_callback(status=response.CalvinResponse(True),
actor_id=actor_id)
def tunnel_down(self, tunnel):
""" Callback that the tunnel is not accepted or is going down """
tunnel_peer_id = tunnel.peer_node_id
try:
self.tunnels.pop(tunnel_peer_id)
except:
pass
# If a port connect have ordered a tunnel then it have a callback in pending
# which want information on the failure
if tunnel_peer_id in self.pending_tunnels:
for cb in self.pending_tunnels[tunnel_peer_id]:
try:
cb(status=response.CalvinResponse(False))
except:
pass
self.pending_tunnels.pop(tunnel_peer_id)
# We should always return True which sends an OK on the destruction of the tunnel
return True
def app_destroy(self, to_rt_uuid, callback, app_id, actor_ids):
""" Destroys an application with remote actors on to_rt_uuid node
callback: called when finished with the peer's respons as argument
app_id: the application to destroy
actor_ids: optional list of actors to destroy
"""
if self.network.link_request(to_rt_uuid):
# Already have link just continue in _app_destroy
self._app_destroy(to_rt_uuid,
callback,
app_id,
actor_ids,
status=response.CalvinResponse(True))
else:
# Request link before continue in _app_destroy
self.node.network.link_request(
to_rt_uuid,
CalvinCB(self._app_destroy,
to_rt_uuid=to_rt_uuid,
callback=callback,
app_id=app_id,
actor_ids=actor_ids))
def actor_new(self, to_rt_uuid, callback, actor_type, state,
prev_connections):
""" Creates a new actor on to_rt_uuid node, but is only intended for migrating actors
callback: called when finished with the peers respons as argument
actor_type: see actor manager
state: see actor manager
prev_connections: see actor manager
"""
if self.node.network.link_request(
to_rt_uuid,
CalvinCB(self._actor_new,
to_rt_uuid=to_rt_uuid,
callback=callback,
actor_type=actor_type,
state=state,
prev_connections=prev_connections)):
# Already have link just continue in _actor_new
self._actor_new(to_rt_uuid,
callback,
actor_type,
state,
prev_connections,
status=response.CalvinResponse(True))
def _app_requirements(self, app):
_log.debug("_app_requirements(app=%s)" % (app, ))
_log.analyze(
self._node.id,
"+ ACTOR PLACEMENT", {
'placement':
{k: list(v)
for k, v in app.actor_placement.iteritems()}
},
tb=True)
if any([not n for n in app.actor_placement.values()]):
# At least one actor have no possible placement
app._org_cb(status=response.CalvinResponse(False))
del app._org_cb
_log.analyze(self._node.id,
"+ NO PLACEMENT", {'app_id': app.id},
tb=True)
return
# Collect an actor by actor matrix stipulating a weighting 0.0 - 1.0 for their connectivity
actor_ids, actor_matrix = self._actor_connectivity(app)
# Get list of all possible nodes
node_ids = set([])
for possible_nodes in app.actor_placement.values():
node_ids |= possible_nodes
node_ids = list(node_ids)
_log.analyze(self._node.id,
"+ ACTOR MATRIX", {
'actor_ids': actor_ids,
'actor_matrix': actor_matrix,
'node_ids': node_ids
},
tb=True)
# Weight the actors possible placement with their connectivity matrix
weighted_actor_placement = {}
for actor_id in actor_ids:
# actor matrix is symmetric, so independent if read horizontal or vertical
actor_weights = actor_matrix[actor_ids.index(actor_id)]
# Sum the actor weights for each actors possible nodes, matrix mult AA * AN,
# AA actor weights, AN actor * node with 1 when possible
weights = [
sum([
actor_weights[actor_ids.index(_id)]
if node_id in app.actor_placement[actor_id] else 0
for _id in actor_ids
]) for node_id in node_ids
]
# Get first node with highest weight
# FIXME should verify that the node actually exist also
# TODO should select from a resource sharing perspective also, instead of picking first max
_log.analyze(self._node.id, "+ WEIGHTS", {
'actor_id': actor_id,
'weights': weights
})
weighted_actor_placement[actor_id] = node_ids[weights.index(
max(weights))]
for actor_id, node_id in weighted_actor_placement.iteritems():
# TODO could add callback to try another possible node if the migration fails
_log.debug("Actor deployment %s \t-> %s" %
(app.actors[actor_id], node_id))
self._node.am.migrate(actor_id, node_id)
app._org_cb(status=response.CalvinResponse(True),
placement=weighted_actor_placement)
del app._org_cb
_log.analyze(self._node.id, "+ DONE", {'app_id': app.id}, tb=True)
def join_finished(self, tp_link, peer_id, uri, is_orginator):
""" Peer join is (not) accepted, called by transport plugin.
This may be initiated by us (is_orginator=True) or by the peer,
i.e. both nodes get called.
When inititated by us pending_joins likely have a callback
tp_link: the transport plugins object for the link (have send etc)
peer_id: the node id we joined
uri: the uri used for the join
is_orginator: did this node request the join True/False
"""
# while a link is pending it is the responsibility of the transport layer, since
# higher layers don't have any use for it anyway
_log.analyze(self.node.id,
"+", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
if tp_link is None:
# This is a failed join lets send it upwards
if uri in self.pending_joins:
cbs = self.pending_joins.pop(uri)
if cbs:
for cb in cbs:
cb(status=response.CalvinResponse(
response.SERVICE_UNAVAILABLE),
uri=uri,
peer_node_id=peer_id)
return
# Only support for one RT to RT communication link per peer
if peer_id in self.links:
# Likely simultaneous join requests, use the one requested by the node with highest id
if is_orginator and self.node.id > peer_id:
# We requested it and we have highest node id, hence the one in links is the peer's and we replace it
_log.analyze(self.node.id,
"+ REPLACE ORGINATOR", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
self.links[peer_id] = CalvinLink(self.node.id, peer_id,
tp_link, self.links[peer_id])
elif is_orginator and self.node.id < peer_id:
# We requested it and peer have highest node id, hence the one in links is peer's and we close this new
_log.analyze(self.node.id,
"+ DROP ORGINATOR", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
tp_link.disconnect()
elif not is_orginator and self.node.id > peer_id:
# Peer requested it and we have highest node id, hence the one in links is ours and we close this new
_log.analyze(self.node.id,
"+ DROP", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
tp_link.disconnect()
elif not is_orginator and self.node.id < peer_id:
# Peer requested it and peer have highest node id, hence the one in links is ours and we replace it
_log.analyze(self.node.id,
"+ REPLACE", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
self.links[peer_id] = CalvinLink(self.node.id, peer_id,
tp_link, self.links[peer_id])
else:
# No simultaneous join detected, just add the link
_log.analyze(self.node.id,
"+ INSERT", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
self.links[peer_id] = CalvinLink(self.node.id, peer_id, tp_link)
# Find and call any callbacks registered for the uri or peer id
_log.debug(
"%s: peer_id: %s, uri: %s\npending_joins_by_id: %s\npending_joins: %s"
% (self.node.id, peer_id, uri, self.pending_joins_by_id,
self.pending_joins))
if peer_id in self.pending_joins_by_id:
peer_uri = self.pending_joins_by_id.pop(peer_id)
if peer_uri in self.pending_joins:
cbs = self.pending_joins.pop(peer_uri)
if cbs:
for cb in cbs:
cb(status=response.CalvinResponse(True),
uri=peer_uri,
peer_node_id=peer_id)
if uri in self.pending_joins:
cbs = self.pending_joins.pop(uri)
if cbs:
for cb in cbs:
cb(status=response.CalvinResponse(True),
uri=uri,
peer_node_id=peer_id)
return
def join(self, uris, callback=None, corresponding_peer_ids=None):
""" Join the peers accessable from list of URIs
It is possible to have a list of corresponding peer_ids,
which is used to filter the uris list for already connected
or pending peer's connections.
URI can't be used for matching since not neccessarily the same if peer connect to us
uris: list of uris
callback: will get called for each uri with arguments status, peer_node_id and uri
corresponding_peer_ids: list of node ids matching the list of uris
TODO: If corresponding_peer_ids is not specified it is possible that the callback is never called
when a simultaneous join happens due to that it is not possible to detect by URI only.
Should add a timeout that cleans out callbacks with failed status replies and let client retry.
"""
# For each URI and when available a peer id
for uri, peer_id in zip(
uris, corresponding_peer_ids if corresponding_peer_ids
and len(uris) == len(corresponding_peer_ids) else [None] *
len(uris)):
if not (uri in self.pending_joins or peer_id
in self.pending_joins_by_id or peer_id in self.links):
# No simultaneous join detected
schema = uri.split(":", 1)[0]
_log.analyze(self.node.id,
"+", {
'uri': uri,
'peer_id': peer_id,
'schema': schema,
'transports': self.transports.keys()
},
peer_node_id=peer_id)
if schema in self.transports.keys():
# store we have a pending join and its callback
if peer_id:
self.pending_joins_by_id[peer_id] = uri
if callback:
self.pending_joins[uri] = [callback]
# Ask the transport plugin to do the join
_log.analyze(self.node.id,
"+ TRANSPORT", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
self.transports[schema].join(uri)
else:
# We have simultaneous joins
_log.analyze(self.node.id,
"+ SIMULTANEOUS", {
'uri': uri,
'peer_id': peer_id
},
peer_node_id=peer_id)
if callback:
if peer_id in self.links:
# Link was already established, then need to call the callback now
callback(status=response.CalvinResponse(True), uri=uri)
continue
# Otherwise also want to be called when the ongoing link setup finishes
if uri in self.pending_joins:
self.pending_joins[uri].append(callback)
else:
self.pending_joins[uri] = [callback]
def connect(self,
callback=None,
actor_id=None,
port_name=None,
port_dir=None,
port_id=None,
peer_node_id=None,
peer_actor_id=None,
peer_port_name=None,
peer_port_dir=None,
peer_port_id=None):
""" Obtain any missing information to enable making a connection and make actual connect
callback: an optional callback that gets called with status when finished
local port identified by:
actor_id, port_name and port_dir='in'/'out' or
port_id
peer_node_id: an optional node id the peer port is locate on, will use storage to find it if not supplied
peer port (remote or local) identified by:
peer_actor_id, peer_port_name and peer_port_dir='in'/'out' or
peer_port_id
connect -----------------------------> _connect -> _connect_via_tunnel -> _connected_via_tunnel -!
\> _connect_by_peer_port_id / \-> _connect_via_local -!
\-> _connect_by_actor_id ---/
"""
# Collect all parameters into a state that we keep between the chain of callbacks needed to complete a connection
state = {
'callback': callback,
'actor_id': actor_id,
'port_name': port_name,
'port_dir': port_dir,
'port_id': port_id,
'peer_node_id': peer_node_id,
'peer_actor_id': peer_actor_id,
'peer_port_name': peer_port_name,
'peer_port_dir': peer_port_dir,
'peer_port_id': peer_port_id
}
_log.analyze(self.node.id,
"+",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
try:
port = self._get_local_port(actor_id, port_name, port_dir, port_id)
except:
# not local
if port_id:
status = response.CalvinResponse(
response.BAD_REQUEST,
"First port %s must be local" % (port_id))
else:
status = response.CalvinResponse(
response.BAD_REQUEST,
"First port %s on actor %s must be local" %
(port_name, actor_id))
if callback:
callback(status=status,
actor_id=actor_id,
port_name=port_name,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_id=peer_port_id)
return
else:
raise Exception(str(status))
else:
# Found locally
state['port_id'] = port.id
# Check if the peer port is local even if a missing peer_node_id
if not peer_node_id and peer_actor_id in self.node.am.actors.iterkeys(
):
state['peer_node_id'] = self.node.id
if not state['peer_node_id'] and state['peer_port_id']:
try:
self._get_local_port(None, None, None, peer_port_id)
except:
# not local
pass
else:
# Found locally
state['peer_node_id'] = self.node.id
# Still no peer node id? ...
if not state['peer_node_id']:
if state['peer_port_id']:
# ... but an id of a port lets ask for more info
self.node.storage.get_port(
state['peer_port_id'],
CalvinCB(self._connect_by_peer_port_id, **state))
return
elif state['peer_actor_id'] and state['peer_port_name']:
# ... but an id of an actor lets ask for more info
self.node.storage.get_actor(
state['peer_actor_id'],
CalvinCB(self._connect_by_peer_actor_id, **state))
return
else:
# ... and no info on how to get more info, abort
status = response.CalvinResponse(
response.BAD_REQUEST,
"Need peer_node_id (%s), peer_actor_id(%s) and/or peer_port_id(%s)"
% (peer_node_id, peer_actor_id, peer_port_id))
if callback:
callback(status=status,
actor_id=actor_id,
port_name=port_name,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_id=peer_port_id)
return
else:
raise Exception(str(status))
else:
if not ((peer_actor_id and peer_port_name) or peer_port_id):
# We miss information on to find the peer port
status = response.CalvinResponse(
response.BAD_REQUEST,
"Need peer_port_name (%s), peer_actor_id(%s) and/or peer_port_id(%s)"
% (peer_port_name, peer_actor_id, peer_port_id))
if callback:
callback(status=status,
actor_id=actor_id,
port_name=port_name,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_id=peer_port_id)
return
else:
raise Exception(str(status))
self._connect(**state)
def _disconnect_port(self, callback=None, port_id=None):
""" Obtain any missing information to enable disconnecting one port and make the disconnect"""
# Collect all parameters into a state that we keep for the sub functions and callback
state = {'callback': callback, 'port_id': port_id, 'peer_ids': None}
# Check if port actually is local
try:
port = self._get_local_port(None, None, None, port_id)
except:
# not local
status = response.CalvinResponse(
response.NOT_FOUND, "Port %s must be local" % (port_id))
if callback:
callback(status=status, port_id=port_id)
return
else:
raise Exception(str(status))
else:
# Found locally
state['port_name'] = port.name
state['port_dir'] = "in" if isinstance(port, InPort) else "out"
state['actor_id'] = port.owner.id if port.owner else None
port = self.ports[state['port_id']]
# Now check the peer port, peer_ids is list of (peer_node_id, peer_port_id) tuples
peer_ids = []
if isinstance(port, InPort):
# Inport only have one possible peer
peer_ids = [port.get_peer()]
else:
# Outport have several possible peers
peer_ids = port.get_peers()
# Disconnect and destroy the endpoints
endpoints = port.disconnect()
for ep in endpoints:
if isinstance(ep, endpoint.TunnelOutEndpoint):
self.monitor.unregister_out_endpoint(ep)
ep.destroy()
ok = True
for peer_node_id, peer_port_id in peer_ids:
if peer_node_id == 'local':
# Use the disconnect request function since does not matter if local or remote request
if not self.disconnection_request(
{'peer_port_id': peer_port_id}):
ok = False
# Inform all the remote ports of the disconnect
remote_peers = [pp for pp in peer_ids if pp[0] and pp[0] != 'local']
# Keep track of disconnection of remote peer ports
self.disconnecting_ports[state['port_id']] = remote_peers
for peer_node_id, peer_port_id in remote_peers:
self.proto.port_disconnect(callback=CalvinCB(
self._disconnected_port,
peer_id=(peer_node_id, peer_port_id),
**state),
port_id=state['port_id'],
peer_node_id=peer_node_id,
peer_port_id=peer_port_id)
# Done disconnecting the port
if not remote_peers or not ok:
self.disconnecting_ports.pop(state['port_id'])
if state['callback']:
_log.analyze(
self.node.id, "+ DONE",
{k: state[k]
for k in state.keys() if k != 'callback'})
state['callback'](status=response.CalvinResponse(ok), **state)
def connection_request(self, payload):
""" A request from a peer to connect a port"""
_log.analyze(self.node.id,
"+",
payload,
peer_node_id=payload['from_rt_uuid'])
if not ('peer_port_id' in payload or
('peer_actor_id' in payload and 'peer_port_name' in payload
and 'peer_port_dir' in payload)):
# Not enough info to find port
_log.analyze(self.node.id,
"+ NOT ENOUGH DATA",
payload,
peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.BAD_REQUEST)
try:
port = self._get_local_port(payload['peer_actor_id'],
payload['peer_port_name'],
payload['peer_port_dir'],
payload['peer_port_id'])
except:
# We don't have the port
_log.analyze(self.node.id,
"+ PORT NOT FOUND",
payload,
peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.NOT_FOUND)
else:
if not 'tunnel_id' in payload:
# TODO implement connection requests not via tunnel
raise NotImplementedError()
tunnel = self.tunnels[payload['from_rt_uuid']]
if tunnel.id != payload['tunnel_id']:
# For some reason does the tunnel id not match the one we have to connect to the peer
# Likely due to that we have not yet received a tunnel request from the peer that replace our tunnel id
# Can happen when race of simultaneous link setup and commands can be received out of order
_log.analyze(self.node.id,
"+ WRONG TUNNEL",
payload,
peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.GONE)
if isinstance(port, InPort):
endp = endpoint.TunnelInEndpoint(port, tunnel,
payload['from_rt_uuid'],
payload['port_id'],
self.node.sched.trigger_loop)
else:
endp = endpoint.TunnelOutEndpoint(port, tunnel,
payload['from_rt_uuid'],
payload['port_id'],
self.node.sched.trigger_loop)
self.monitor.register_out_endpoint(endp)
invalid_endpoint = port.attach_endpoint(endp)
# Remove previous endpoint
if invalid_endpoint:
if isinstance(invalid_endpoint, endpoint.TunnelOutEndpoint):
self.monitor.unregister_out_endpoint(invalid_endpoint)
invalid_endpoint.destroy()
# Update storage
if isinstance(port, InPort):
self.node.storage.add_port(port, self.node.id, port.owner.id,
"in")
else:
self.node.storage.add_port(port, self.node.id, port.owner.id,
"out")
_log.analyze(self.node.id,
"+ OK",
payload,
peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.OK, {'port_id': port.id})
def _connected_via_tunnel(self, reply, **state):
""" Gets called when remote responds to our request for port connection """
_log.analyze(self.node.id,
"+ " + str(reply),
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
if reply in [response.BAD_REQUEST, response.NOT_FOUND]:
# Other end did not accept our port connection request
if state['retries'] == 0 and state['peer_node_id']:
# Maybe it is on another node now lets retry and lookup the port
state['peer_node_id'] = None
state['retries'] += 1
self.node.storage.get_port(
state['peer_port_id'],
CalvinCB(self._connect_by_peer_port_id, **state))
return None
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.NOT_FOUND),
**state)
return None
if reply == response.GONE:
# Other end did not accept our port connection request, likely due to they have not got the message
# about the tunnel in time
_log.analyze(
self.node.id,
"+ RETRY",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
if state['retries'] < 2:
state['retries'] += 1
self._connect_via_tunnel(**state)
return None
# Set up the port's endpoint
tunnel = self.tunnels[state['peer_node_id']]
port = self.ports[state['port_id']]
if isinstance(port, InPort):
endp = endpoint.TunnelInEndpoint(port, tunnel,
state['peer_node_id'],
reply.data['port_id'],
self.node.sched.trigger_loop)
else:
endp = endpoint.TunnelOutEndpoint(port, tunnel,
state['peer_node_id'],
reply.data['port_id'],
self.node.sched.trigger_loop)
# register into main loop
self.monitor.register_out_endpoint(endp)
invalid_endpoint = port.attach_endpoint(endp)
# remove previous endpoint
if invalid_endpoint:
if isinstance(invalid_endpoint, endpoint.TunnelOutEndpoint):
self.monitor.unregister_out_endpoint(invalid_endpoint)
invalid_endpoint.destroy()
# Done connecting the port
if state['callback']:
state['callback'](status=response.CalvinResponse(True), **state)
# Update storage
if isinstance(port, InPort):
self.node.storage.add_port(port, self.node.id, port.owner.id, "in")
else:
self.node.storage.add_port(port, self.node.id, port.owner.id,
"out")
def tunnel_new_handler(self, payload):
""" Create a new tunnel (response side) """
tunnel = self._get_tunnel(payload['from_rt_uuid'], payload['type'])
ok = False
_log.analyze(self.rt_id,
"+",
payload,
peer_node_id=payload['from_rt_uuid'])
if tunnel:
_log.analyze(self.rt_id,
"+ PENDING",
payload,
peer_node_id=payload['from_rt_uuid'])
# Got tunnel new request while we already have one pending
# it is not allowed to send new request while a tunnel is working
if tunnel.status != CalvinTunnel.STATUS.WORKING:
ok = True
# The one with lowest tunnel id loose
if tunnel.id < payload['tunnel_id']:
# Our tunnel has lowest id, change our tunnels id
# update status and call proper callbacks
# but send tunnel reply first, to get everything in order
msg = {
'cmd':
'REPLY',
'msg_uuid':
payload['msg_uuid'],
'value':
response.CalvinResponse(ok,
data={
'tunnel_id':
payload['tunnel_id']
}).encode()
}
self.network.links[payload['from_rt_uuid']].send(msg)
tunnel._setup_ack(
response.CalvinResponse(
True, data={'tunnel_id': payload['tunnel_id']}))
_log.analyze(self.rt_id,
"+ CHANGE ID",
payload,
peer_node_id=payload['from_rt_uuid'])
else:
# Our tunnel has highest id, keep our id
# update status and call proper callbacks
# but send tunnel reply first, to get everything in order
msg = {
'cmd':
'REPLY',
'msg_uuid':
payload['msg_uuid'],
'value':
response.CalvinResponse(ok,
data={
'tunnel_id':
payload['tunnel_id']
}).encode()
}
self.network.links[payload['from_rt_uuid']].send(msg)
tunnel._setup_ack(
response.CalvinResponse(True,
data={'tunnel_id': tunnel.id}))
_log.analyze(self.rt_id,
"+ KEEP ID",
payload,
peer_node_id=payload['from_rt_uuid'])
else:
# FIXME if this happens need to decide what to do
_log.analyze(self.rt_id,
"+ DROP FIXME",
payload,
peer_node_id=payload['from_rt_uuid'])
return
else:
# No simultaneous tunnel requests, lets create it...
tunnel = CalvinTunnel(self.network.links,
self.tunnels,
payload['from_rt_uuid'],
payload['type'],
payload['policy'],
rt_id=self.node.id,
id=payload['tunnel_id'])
_log.analyze(self.rt_id,
"+ NO SMASH",
payload,
peer_node_id=payload['from_rt_uuid'])
try:
# ... and see if the handler wants it
ok = self.tunnel_handlers[payload['type']](tunnel)
except:
pass
# Send the response
msg = {
'cmd':
'REPLY',
'msg_uuid':
payload['msg_uuid'],
'value':
response.CalvinResponse(ok, data={
'tunnel_id': tunnel.id
}).encode()
}
self.network.links[payload['from_rt_uuid']].send(msg)
# If handler did not want it close it again
if not ok:
tunnel.close(local_only=True)
def disconnect(self,
callback=None,
actor_id=None,
port_name=None,
port_dir=None,
port_id=None):
""" Do disconnect for port(s)
callback: an optional callback that gets called with status when finished
ports identified by only local actor_id:
actor_id: the actor that all ports will be disconnected on
callback will be called once when all ports are diconnected or first failed
local port identified by:
actor_id, port_name and port_dir='in'/'out' or
port_id
callback will be called once when all peer ports (fanout) are disconnected or first failed
disconnect -*> _disconnect_port -*> _disconnected_port (-*> _disconnecting_actor_cb) -> !
"""
port_ids = []
if actor_id and not (port_id or port_name or port_dir):
# We disconnect all ports on an actor
try:
actor = self.node.am.actors[actor_id]
except:
# actor not found
status = response.CalvinResponse(
response.NOT_FOUND, "Actor %s must be local" % (actor_id))
if callback:
callback(status=status,
actor_id=actor_id,
port_name=port_name,
port_id=port_id)
return
else:
raise Exception(str(status))
else:
port_ids.extend([p.id for p in actor.inports.itervalues()])
port_ids.extend([p.id for p in actor.outports.itervalues()])
# Need to collect all callbacks into one
if callback:
callback = CalvinCB(self._disconnecting_actor_cb,
_callback=callback,
port_ids=port_ids)
else:
# Just one port to disconnect
if port_id:
port_ids.append(port_id)
else:
# Awkward but lets get the port id from name etc so that the rest can loop over port ids
try:
port = self._get_local_port(actor_id, port_name, port_dir,
port_id)
except:
# not local
status = response.CalvinResponse(
response.NOT_FOUND,
"Port %s on actor %s must be local" %
(port_name if port_name else port_id,
actor_id if actor_id else "some"))
if callback:
callback(status=status,
actor_id=actor_id,
port_name=port_name,
port_id=port_id)
return
else:
raise Exception(str(status))
else:
# Found locally
port_ids.append(port.id)
_log.analyze(self.node.id, "+", {'port_ids': port_ids})
# Run over copy of list of ports since modified inside the loop
for port_id in port_ids[:]:
self._disconnect_port(callback, port_id)
def _connect(self, **state):
""" Do the connection of ports, all neccessary information supplied but
maybe not all pre-requisites for remote connections.
"""
_log.analyze(self.node.id,
"+",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
# Local connect
if self.node.id == state['peer_node_id']:
_log.analyze(
self.node.id,
"+ LOCAL",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
port1 = self._get_local_port(state['actor_id'], state['port_name'],
state['port_dir'], state['port_id'])
port2 = self._get_local_port(state['peer_actor_id'],
state['peer_port_name'],
state['peer_port_dir'],
state['peer_port_id'])
# Local connect wants the first port to be an inport
inport, outport = (port1,
port2) if isinstance(port1, InPort) else (port2,
port1)
self._connect_via_local(inport, outport)
if state['callback']:
state['callback'](status=response.CalvinResponse(True),
**state)
return None
# Remote connection
# TODO Currently we only have support for setting up a remote connection via tunnel
tunnel = None
if not state['peer_node_id'] in self.tunnels.iterkeys():
# No tunnel to peer, get one first
_log.analyze(
self.node.id,
"+ GET TUNNEL",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
tunnel = self.proto.tunnel_new(state['peer_node_id'], 'token', {})
tunnel.register_tunnel_down(CalvinCB(self.tunnel_down, tunnel))
tunnel.register_tunnel_up(CalvinCB(self.tunnel_up, tunnel))
tunnel.register_recv(CalvinCB(self.tunnel_recv_handler, tunnel))
self.tunnels[state['peer_node_id']] = tunnel
else:
tunnel = self.tunnels[state['peer_node_id']]
if tunnel.status == CalvinTunnel.STATUS.PENDING:
if not state['peer_node_id'] in self.pending_tunnels:
self.pending_tunnels[state['peer_node_id']] = []
# call _connect_via_tunnel when we get the response of the tunnel
self.pending_tunnels[state['peer_node_id']].append(
CalvinCB(self._connect_via_tunnel, **state))
return
elif tunnel.status == CalvinTunnel.STATUS.TERMINATED:
# TODO should we retry at this level?
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.INTERNAL_ERROR),
**state)
return
_log.analyze(
self.node.id,
"+ HAD TUNNEL",
dict({k: state[k]
for k in state.keys() if k != 'callback'},
tunnel_status=self.tunnels[state['peer_node_id']].status),
peer_node_id=state['peer_node_id'])
self._connect_via_tunnel(status=response.CalvinResponse(True), **state)
