def supervise_actor(self, actor_id, requirements):
try:
actor = self.node.am.actors[actor_id]
except:
return calvinresponse.CalvinResponse(calvinresponse.NOT_FOUND)
if actor._replication_data.id is None:
actor._replication_data = ReplicationData(
actor_id=actor_id, master=actor_id, requirements=requirements)
actor._replication_data.init_requirements()
elif actor._replication_data.is_master(actor_id):
# If we already is master that is OK, update requirements
# FIXME should not update during a replication, fix when we get the
# requirements from the deployment requirements
actor._replication_data.init_requirements(requirements)
self.node.storage.add_replication(actor._replication_data, cb=None)
return calvinresponse.CalvinResponse(True, {'replication_id': actor._replication_data.id})
else:
return calvinresponse.CalvinResponse(calvinresponse.BAD_REQUEST)
actor._replication_data.status = REPLICATION_STATUS.READY
# TODO add a callback to make sure storing worked
self.node.storage.add_replication(actor._replication_data, cb=None)
self.node.storage.add_actor(actor, self.node.id, cb=None)
#TODO trigger replication loop
return calvinresponse.CalvinResponse(True, {'replication_id': actor._replication_data.id})
def _destroy_final_cb(self, application, node_id, status):
_log.analyze(self._node.id, "+", {
'node_id': node_id,
'status': status
})
application._destroy_node_ids[node_id] = status
if any([s is None for s in application._destroy_node_ids.values()]):
return
# Done
for replication_id, replica_ids in application._replicas_actor_final.items(
):
for replica_id in replica_ids:
self._node.storage.remove_replica(replication_id, replica_id)
for replication_id, replica_node_ids in application._replicas_node_final.items(
):
for replica_node_id in replica_node_ids:
self._node.storage.remove_replica_node_force(
replication_id, replica_node_id)
if all(application._destroy_node_ids.values()):
application.destroy_cb(status=response.CalvinResponse(True))
else:
# Missing is the actors that could not be found.
# FIXME retry? They could have moved
missing = []
for status in application._destroy_node_ids.values():
missing += [] if status.data is None else status.data
application.destroy_cb(
status=response.CalvinResponse(False, data=missing))
self._node.control.log_application_destroy(application.id)
def disconnection_request(self, payload):
""" A request from a peer to disconnect a port"""
if not ('peer_port_id' in payload or
('peer_actor_id' in payload and
'peer_port_name' in payload and
'peer_port_dir' in payload)):
# Not enough info to find port
return response.CalvinResponse(response.BAD_REQUEST)
# Check if port actually is local
local_port_meta = PortMeta(
self,
actor_id=payload['peer_actor_id'] if 'peer_actor_id' in payload else None,
port_id=payload['peer_port_id'] if 'peer_port_id' in payload else None,
port_name=payload['peer_port_name'] if 'peer_port_name' in payload else None,
properties={'direction': payload['peer_port_dir'] if 'peer_port_dir' in payload else None},
node_id=self.node.id)
peer_port_meta = PortMeta(self, port_id=payload['port_id'], node_id=payload['from_rt_uuid'])
_log.analyze(self.node.id, "+", {'local': local_port_meta, 'peer': peer_port_meta},
peer_node_id=payload['from_rt_uuid'], tb=True)
try:
port = local_port_meta.port
except response.CalvinResponseException as e:
# We don't have the port
return response.CalvinResponse(response.NOT_FOUND)
else:
# Disconnect and destroy endpoints
return ConnectionFactory(self.node, PURPOSE.DISCONNECT).get(
local_port_meta.port, peer_port_meta, payload=payload
).disconnection_request(payload.get('terminate', False), payload.get('remaining_tokens', {}))
def _destroy_final(self, application):
""" Final destruction of the application on this node and send request to peers to also destroy the app """
if hasattr(application, '_destroy_node_ids'):
# Already called
return
_log.analyze(
self._node.id, "+", {
'node_info': application.node_info,
'origin_node_id': application.origin_node_id
})
application._destroy_node_ids = {
n: None
for n in application.node_info.keys()
}
for node_id, actor_ids in application.node_info.iteritems():
if not node_id:
_log.analyze(self._node.id, "+ UNKNOWN NODE", {})
application._destroy_node_ids[None] = response.CalvinResponse(
False)
continue
# Inform peers to destroy their part of the application
self._node.proto.app_destroy(
node_id, CalvinCB(self._destroy_final_cb, application,
node_id), application.id, actor_ids)
if application.id in self.applications:
del self.applications[application.id]
elif application.origin_node_id not in application.node_info and application.origin_node_id != self._node.id:
# All actors migrated from the original node, inform it also
_log.analyze(self._node.id, "+ SEP APP NODE", {})
self._node.proto.app_destroy(application.origin_node_id, None,
application.id, [])
self.storage.delete_application(application.id)
self._destroy_final_cb(application, '', response.CalvinResponse(True))
def authorization_decision_handler(self, payload):
"""
Return a response to the authorization request in the payload.
Signed JSON Web Tokens (JWT) with timestamps and information about
sender and receiver are used for both the request and response.
A Policy Decision Point (PDP) is used to determine if access is permitted.
"""
if not _sec_conf['authorization']['accept_external_requests']:
reply = response.CalvinResponse(response.NOT_FOUND)
else:
try:
decoded = self.node.authorization.decode_request(payload)
self.node.authorization.pdp.authorize(
decoded["request"],
callback=CalvinCB(self._authorization_decision_handler,
payload, decoded))
return
except Exception:
reply = response.CalvinResponse(response.INTERNAL_ERROR)
# Send reply
msg = {
'cmd': 'REPLY',
'msg_uuid': payload['msg_uuid'],
'value': reply.encode()
}
self.network.links[payload['from_rt_uuid']].send(msg)
def _disconnecting_actor_cb(self,
status,
_callback,
port_ids,
port_id=None,
actor_id=None):
""" Get called for each of the actor's ports when disconnecting, but callback should only be called once
status: OK or not
_callback: original callback
port_ids: list of port ids kept in context between calls when *changed* by this function, do not replace it
state: dictionary keeping disconnect information
"""
_log.analyze(self.node.id, "+", {
'port_ids': port_ids,
'port_id': port_id
})
# Send negative response if not already done it
if not status and port_ids:
if _callback:
del port_ids[:]
_callback(status=response.CalvinResponse(False),
actor_id=actor_id)
if port_id in port_ids:
# Remove this port from list
port_ids.remove(port_id)
# If all ports done send positive response
if not port_ids:
if _callback:
_callback(status=response.CalvinResponse(True),
actor_id=actor_id)
def handle_get_actor_module(self, tunnel, payload):
ok = False
actor_type = payload['actor_type']
data = None
path = _conf.get(None, 'compiled_actors_path')
if path is None:
_log.error("compiled_actors_path not set")
else:
if payload['compiler'] == 'mpy-cross':
try:
path = path + '/mpy-cross/' + actor_type.replace(
'.', '/') + '.mpy'
f = open(path, 'rb')
data = f.read()
f.close()
ok = True
except IOError as e:
_log.error("Failed to open '%s'" % path)
else:
_log.error("Unknown compiler '%s'" % payload['compiler'])
if ok:
self._proxy_send_reply(
tunnel, payload['msg_uuid'],
response.CalvinResponse(response.OK, {
'actor_type': actor_type,
'module': data
}).encode())
else:
self._proxy_send_reply(
tunnel, payload['msg_uuid'],
response.CalvinResponse(response.INTERNAL_ERROR, {
'actor_type': actor_type,
'module': None
}).encode())
def _update_requirements_placements(self,
actor_id,
possible_placements,
status=None,
move=False,
cb=None):
_log.analyze(self.node.id, "+ BEGIN", {}, tb=True)
if move and len(possible_placements) > 1:
possible_placements.discard(self.node.id)
actor = self.actors[actor_id]
if not possible_placements:
if cb:
cb(status=response.CalvinResponse(False))
return
if self.node.id in possible_placements:
# Actor could stay, then do that
if cb:
cb(status=response.CalvinResponse(True))
return
# TODO do a better selection between possible nodes
# TODO: should also ask authorization server before selecting node to migrate to.
# Try the possible placements in random order
pp = list(possible_placements)
random.shuffle(pp)
self.robust_migrate(actor_id, pp, callback=cb)
_log.analyze(self.node.id, "+ END", {})
def update_requirements(self,
actor_id,
requirements,
extend=False,
move=False,
authorization_check=False,
callback=None):
""" Update requirements and trigger a potential migration """
if actor_id not in self.actors:
# Can only migrate actors from our node
_log.analyze(self.node.id, "+ NO ACTOR", {'actor_id': actor_id})
if callback:
callback(status=response.CalvinResponse(False))
return
if not isinstance(requirements, (list, tuple)):
# Requirements need to be list
_log.analyze(self.node.id, "+ NO REQ LIST", {'actor_id': actor_id})
if callback:
callback(status=response.CalvinResponse(response.BAD_REQUEST))
return
actor = self.actors[actor_id]
actor.requirements_add(requirements, extend)
self.node.storage.add_actor(
actor, self.node.id) # Update requirements in registry
r = ReqMatch(self.node,
callback=CalvinCB(self._update_requirements_placements,
actor_id=actor_id,
move=move,
cb=callback))
r.match_for_actor(actor_id)
_log.analyze(self.node.id, "+ END", {'actor_id': actor_id})
def _update_requirements_placements(self, actor, possible_placements, status=None):
_log.analyze(self.node.id, "+ BEGIN", {}, tb=True)
# All possible actor placements derived
if not possible_placements:
if actor._replicate_callback:
actor._replicate_callback(status=calvinresponse.CalvinResponse(False))
return
# Select, always a list of node_ids, could be more than one
req = actor._replication_data.requirements
selected = req_operations[req['op']].select(self.node, actor, possible_placements, **req['kwargs'])
_log.analyze(self.node.id, "+", {'possible_placements': possible_placements, 'selected': selected})
if selected is None:
# When None - selection will never succeed
if actor._replicate_callback:
actor._replicate_callback(status=calvinresponse.CalvinResponse(False))
return
if actor._migrating_to is not None:
# If actor started migration skip replication
if actor._replicate_callback:
actor._replicate_callback(status=calvinresponse.CalvinResponse(False))
return
if not selected:
if actor._replicate_callback:
actor._replicate_callback(status=calvinresponse.CalvinResponse(False))
return
# FIXME create as many replicas as nodes in list (would need to serialize)
self.replicate(actor.id, selected[0], callback=actor._replicate_callback)
def _actor_connected(self, status, peer_port_id, actor_id, peer_port_ids,
_callback, **kwargs):
""" Get called for each of the actor's ports when connecting, but callback should only be called once
status: success or not
_callback: original callback
peer_port_ids: list of port ids kept in context between calls when *changed* by this function,
do not replace it
"""
_log.debug("_actor_connected %s %s" % (actor_id, status))
# Send negative response if not already done it
if not status and peer_port_ids:
self.actors[actor_id]._migration_connected = True
if _callback:
del peer_port_ids[:]
_callback(status=response.CalvinResponse(False),
actor_id=actor_id)
if peer_port_id in peer_port_ids:
# Remove this port from list
peer_port_ids.remove(peer_port_id)
# If all ports done send OK
if not peer_port_ids:
self.actors[actor_id]._migration_connected = True
if _callback:
_callback(status=response.CalvinResponse(True),
actor_id=actor_id)
def connection_request(self, payload):
""" A request from a peer to connect a port"""
_log.analyze(self.node.id, "+", payload, peer_node_id=payload['from_rt_uuid'])
if not ('peer_port_id' in payload or
('peer_actor_id' in payload and
'peer_port_name' in payload and
'peer_port_properties' in payload)):
# Not enough info to find port
_log.analyze(self.node.id, "+ NOT ENOUGH DATA", payload, peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.BAD_REQUEST)
our_port_meta = PortMeta(self,
actor_id=payload['peer_actor_id'],
port_id=payload['peer_port_id'],
port_name=payload['peer_port_name'],
properties=payload['peer_port_properties'],
node_id=self.node.id)
try:
port = our_port_meta.port
except:
# We don't have the port
_log.analyze(self.node.id, "+ PORT NOT FOUND", payload, peer_node_id=payload['from_rt_uuid'])
return response.CalvinResponse(response.NOT_FOUND)
else:
# Let a specific connection handler take care of the request
peer_port_meta = PortMeta(self,
port_id=payload['port_id'],
node_id=payload['from_rt_uuid'],
properties=payload['port_properties'])
return ConnectionFactory(self.node, PURPOSE.CONNECT).get(
port, peer_port_meta, payload=payload).connection_request()
def set_proxy_config_cb(key, value, callback):
if not value:
# the peer_id did not exist in storage
callback(status=response.CalvinResponse(response.NOT_FOUND, {'peer_node_id': key}))
return
callback(status=response.CalvinResponse(response.OK))
def append(self, key, value, cb=None):
cb = cb or self._dummy_cb
if key in self._data and isinstance(self._data[key], set):
self._data[key] |= set(value)
async.DelayedCall(0, cb, key, calvinresponse.CalvinResponse(True))
return
self._data[key] = set(value)
async.DelayedCall(0, cb, key, calvinresponse.CalvinResponse(True))
def connect(self, status=None, port_meta=None):
# TODO: status & port_meta unused
tunnel = None
if self.peer_port_meta.node_id not in self.token_tunnel.tunnels.iterkeys(
):
# No tunnel to peer, get one first
_log.analyze(self.node.id,
"+ GET TUNNEL",
self.peer_port_meta,
peer_node_id=self.peer_port_meta.node_id)
tunnel = self.node.proto.tunnel_new(self.peer_port_meta.node_id,
'token', {})
tunnel.register_tunnel_down(
CalvinCB(self.token_tunnel.tunnel_down, tunnel))
tunnel.register_tunnel_up(
CalvinCB(self.token_tunnel.tunnel_up, tunnel))
tunnel.register_recv(
CalvinCB(self.token_tunnel.tunnel_recv_handler, tunnel))
self.token_tunnel.tunnels[self.peer_port_meta.node_id] = tunnel
else:
tunnel = self.token_tunnel.tunnels[self.peer_port_meta.node_id]
if tunnel.status == CalvinTunnel.STATUS.PENDING:
if self.peer_port_meta.node_id not in self.token_tunnel.pending_tunnels:
self.token_tunnel.pending_tunnels[
self.peer_port_meta.node_id] = []
# call _connect_via_tunnel when we get the response of the tunnel
self.token_tunnel.pending_tunnels[
self.peer_port_meta.node_id].append(
CalvinCB(self._connect_via_tunnel))
return
elif tunnel.status == CalvinTunnel.STATUS.TERMINATED:
# TODO should we retry at this level?
if self.callback:
self.callback(status=response.CalvinResponse(
response.INTERNAL_ERROR),
actor_id=self.port.owner.id,
port_name=self.port.name,
port_id=self.port.id,
peer_node_id=self.peer_port_meta.node_id,
peer_actor_id=self.peer_port_meta.actor_id,
peer_port_name=self.peer_port_meta.port_name,
peer_port_id=self.peer_port_meta.port_id)
return
_log.analyze(
self.node.id,
"+ HAD TUNNEL", {
'local_port':
self.port,
'peer_port':
self.peer_port_meta,
'tunnel_status':
self.token_tunnel.tunnels[self.peer_port_meta.node_id].status
},
peer_node_id=self.peer_port_meta.node_id)
self._connect_via_tunnel(status=response.CalvinResponse(True))
def set_port_properties(self, port_id=None, actor_id=None, port_dir=None, port_name=None,
**port_properties):
_log.analyze(self.node.id, "+", port_properties)
port = self._get_local_port(actor_id=actor_id, port_name=port_name, port_dir=port_dir, port_id=port_id)
success = []
for port_property, value in port_properties.items():
success.append(self._set_port_property(port, port_property, value))
ok = all(success)
return response.CalvinResponse(True) if ok else response.CalvinResponse(response.BAD_REQUEST)
def _remove_cb(self, result, **kwargs):
_log.debug("SQL remove OK")
cb = kwargs.pop('cb', None)
key = kwargs.pop('key', None)
_log.debug("SQL remove OK %s" % str(result))
if cb is not None:
if key is None:
async.DelayedCall(0, CalvinCB(cb, calvinresponse.CalvinResponse(status=True)))
else:
async.DelayedCall(0, CalvinCB(cb, key, calvinresponse.CalvinResponse(status=True)))
def remove_index(self, prefix, indexes, value, cb=None):
cb = cb or self._dummy_cb
indexes = [prefix] + indexes
key = tuple(indexes)
if key in self._data and isinstance(self._data[key], set):
self._data[key] -= set(value)
async.DelayedCall(0, cb, calvinresponse.CalvinResponse(True))
return
del self._data[key]
async.DelayedCall(0, cb, calvinresponse.CalvinResponse(True))
def connection_request(self):
""" A request from a peer to connect a port"""
_log.analyze(self.node.id,
"+",
self.kwargs,
peer_node_id=self.peer_port_meta.node_id)
try:
payload = self.kwargs['payload']
except:
return response.CalvinResponse(False)
if 'tunnel_id' not in payload:
raise NotImplementedError()
tunnel = self.token_tunnel.tunnels[self.peer_port_meta.node_id]
if tunnel.id != payload['tunnel_id']:
# For some reason does the tunnel id not match the one we have to connect to the peer
# Likely due to that we have not yet received a tunnel request from the peer that replace our tunnel id
# Can happen when race of simultaneous link setup and commands can be received out of order
_log.analyze(self.node.id,
"+ WRONG TUNNEL",
payload,
peer_node_id=self.peer_port_meta.node_id)
return response.CalvinResponse(response.GONE)
self.port.set_queue(
queue.get(self.port, peer_port_meta=self.peer_port_meta))
if self.port.direction == "in":
endp = endpoint.TunnelInEndpoint(self.port, tunnel,
self.peer_port_meta.node_id,
self.peer_port_meta.port_id,
self.node.sched.trigger_loop)
else:
endp = endpoint.TunnelOutEndpoint(self.port, tunnel,
self.peer_port_meta.node_id,
self.peer_port_meta.port_id,
self.node.sched.trigger_loop)
if endp.use_monitor():
self.node.monitor.register_endpoint(endp)
invalid_endpoint = self.port.attach_endpoint(endp)
# Remove previous endpoint
if invalid_endpoint:
if invalid_endpoint.use_monitor():
self.node.monitor.unregister_endpoint(invalid_endpoint)
invalid_endpoint.destroy()
# Update storage
self.node.storage.add_port(self.port, self.node.id, self.port.owner.id)
_log.analyze(self.node.id,
"+ OK",
payload,
peer_node_id=self.peer_port_meta.node_id)
return response.CalvinResponse(response.OK, {'port_id': self.port.id})
def replicate(self, actor_id, dst_node_id, callback):
try:
actor = self.node.am.actors[actor_id]
except:
if callback:
callback(calvinresponse.CalvinResponse(calvinresponse.BAD_REQUEST))
return
if not actor._replication_data.is_master(actor.id):
# Only replicate master actor
if callback:
callback(calvinresponse.CalvinResponse(calvinresponse.BAD_REQUEST))
return
if actor._replication_data.status != REPLICATION_STATUS.READY:
if callback:
callback(calvinresponse.CalvinResponse(calvinresponse.SERVICE_UNAVAILABLE))
return
_log.analyze(self.node.id, "+", {'actor_id': actor_id, 'dst_node_id': dst_node_id})
actor._replication_data.status = REPLICATION_STATUS.REPLICATING
cb_status = CalvinCB(self._replication_status_cb, replication_data=actor._replication_data, cb=callback)
# TODO make name a property that combine name and counter in actor
new_id = uuid("ACTOR")
actor._replication_data.check_instances = time.time()
actor._replication_data.add_replica(new_id)
new_name = actor.name + "/{}".format(actor._replication_data.counter)
actor_type = actor._type
ports = actor.connections(self.node.id)
ports['actor_name'] = new_name
ports['actor_id'] = new_id
remap_ports = {pid: uuid("PORT") for pid in ports['inports'].keys() + ports['outports'].keys()}
actor._replication_data.set_remaped_ports(new_id, remap_ports, ports)
ports['inports'] = {remap_ports[pid]: v for pid, v in ports['inports'].items()}
ports['outports'] = {remap_ports[pid]: v for pid, v in ports['outports'].items()}
_log.analyze(self.node.id, "+ GET STATE", remap_ports)
state = actor.state(remap_ports)
state['_name'] = new_name
state['_id'] = new_id
# Make copy to make sure no objects are shared between actors or master actor state is changed
state = copy.deepcopy(state)
actor.will_replicate(ActorState(state, actor._replication_data))
if dst_node_id == self.node.id:
# Make copy to make sure no objects are shared between actors
ports = copy.deepcopy(ports)
self.node.am.new_from_migration(
actor_type, state=state, prev_connections=ports, callback=CalvinCB(
self._replicated,
replication_id=actor._replication_data.id,
actor_id=new_id, callback=cb_status, master_id=actor.id, dst_node_id=dst_node_id))
else:
self.node.proto.actor_new(
dst_node_id, CalvinCB(self._replicated, replication_id=actor._replication_data.id,
actor_id=new_id, callback=cb_status, master_id=actor.id,
dst_node_id=dst_node_id),
actor_type, state, ports)
def handle_config_cb(self, key, value, tunnel, msgid):
if not value:
self._proxy_send_reply(
tunnel, msgid,
response.CalvinResponse(response.INTERNAL_ERROR, {
'peer_node_id': key
}).encode())
return
self._proxy_send_reply(
tunnel, msgid,
response.CalvinResponse(response.OK, {
'time': time.time()
}).encode())
def execute_requirements(self, application_id, cb):
""" Build dynops iterator to collect all possible placements,
then trigger migration.
For initial deployment (all actors on the current node)
"""
app = None
try:
app = self.applications[application_id]
except:
_log.debug("execute_requirements did not find app %s" %
(application_id, ))
cb(status=response.CalvinResponse(False))
return
_log.debug("execute_requirements(app=%s)" %
(self.applications[application_id], ))
# TODO extract groups
if hasattr(app, '_org_cb'):
# application deployment requirements ongoing, abort
cb(status=response.CalvinResponse(False))
return
app._org_cb = cb
app.done_final = False
app._collect_placement_counter = 0
app._collect_placement_last_value = 0
actor_placement_it = dynops.List()
app.actor_placement = {} # Clean placement slate
_log.analyze(self._node.id, "+ APP REQ", {}, tb=True)
for actor_id in app.get_actors():
if actor_id not in self._node.am.actors.keys():
_log.debug("Only apply requirements to local actors")
continue
_log.analyze(self._node.id,
"+ ACTOR REQ", {'actor_id': actor_id},
tb=True)
actor_req = self.actor_requirements(
app, actor_id).set_name("Actor" + actor_id)
actor_placement_it.append((actor_id, actor_req),
trigger_iter=actor_req)
_log.analyze(self._node.id,
"+ ACTOR REQ DONE", {'actor_id': actor_id},
tb=True)
actor_placement_it.final()
collect_iter = dynops.Collect(actor_placement_it)
collect_iter.set_cb(self.collect_placement, collect_iter, app)
self.collect_placement(collect_iter, app)
_log.analyze(self._node.id,
"+ DONE", {'application_id': application_id},
tb=True)
def _connect_via_tunnel(self, status=None):
""" All information and hopefully (status OK) a tunnel to the peer is available for a port connect"""
_log.analyze(self.node.id,
"+ " + str(status), {
'local_port':
self.port,
'peer_port':
self.peer_port_meta,
'port_is_connected':
self.port.is_connected_to(self.peer_port_meta.port_id)
},
peer_node_id=self.peer_port_meta.node_id,
tb=True)
if self.port.is_connected_to(self.peer_port_meta.port_id):
# The other end beat us to connecting the port, lets just report success and return
_log.analyze(self.node.id,
"+ IS CONNECTED", {
'local_port': self.port,
'peer_port': self.peer_port_meta
},
peer_node_id=self.peer_port_meta.node_id)
self.async_reply(status=response.CalvinResponse(True))
return None
if not status:
# Failed getting a tunnel, just inform the one wanting to connect
self.async_reply(
status=response.CalvinResponse(response.INTERNAL_ERROR))
return None
# Finally we have all information and a tunnel
# Lets ask the peer if it can connect our port.
tunnel = self.token_tunnel.tunnels[self.peer_port_meta.node_id]
_log.analyze(
self.node.id,
"+ SENDING", {
'local_port':
self.port,
'peer_port':
self.peer_port_meta,
'tunnel_status':
self.token_tunnel.tunnels[self.peer_port_meta.node_id].status
},
peer_node_id=self.peer_port_meta.node_id)
self.node.proto.port_connect(callback=CalvinCB(
self._connected_via_tunnel),
port_id=self.port.id,
port_properties=self.port.properties,
peer_port_meta=self.peer_port_meta,
tunnel_id=tunnel.id)
def _validate_port_properties(self, port_properties, direction):
# TODO break out the validation and consolidate it with codegen.py:ConsolidatePortProperty
issues = []
for key, values in port_properties.items():
if not isinstance(values, (list, tuple)):
values = [values]
for value in values:
if key not in port_property_data.keys():
reason = "Port property {} is unknown".format(key)
issues.append(
calvinresponse.CalvinResponse(
calvinresponse.BAD_REQUEST, {'reason': reason}))
continue
ppdata = port_property_data[key]
if ppdata['type'] == "category":
if value not in ppdata['values']:
reason = "Port property {} can only have values {}".format(
key, ", ".join(ppdata['values'].keys()))
issues.append(
calvinresponse.CalvinResponse(
calvinresponse.BAD_REQUEST,
{'reason': reason}))
continue
if direction not in ppdata['values'][value]['direction']:
reason = "Port property {}={} is only for {} ports".format(
key, value, ppdata['values'][value]['direction'])
issues.append(
calvinresponse.CalvinResponse(
calvinresponse.BAD_REQUEST,
{'reason': reason}))
continue
if ppdata['type'] == 'scalar':
if not isinstance(value, numbers.Number):
reason = "Port property {} can only have scalar values".format(
key)
issues.append(
calvinresponse.CalvinResponse(
calvinresponse.BAD_REQUEST,
{'reason': reason}))
continue
if ppdata['type'] == 'string':
if not isinstance(value, basestring):
reason = "Port property {} can only have string values".format(
key)
issues.append(
calvinresponse.CalvinResponse(
calvinresponse.BAD_REQUEST,
{'reason': reason}))
continue
return issues
def _destroy_final_cb(self, application, node_id, status):
_log.analyze(self._node.id, "+", {
'node_id': node_id,
'status': status
})
application._destroy_node_ids[node_id] = status
if any([s is None for s in application._destroy_node_ids.values()]):
return
# Done
if all(application._destroy_node_ids.values()):
application.destroy_cb(status=response.CalvinResponse(True))
else:
application.destroy_cb(status=response.CalvinResponse(False))
self._node.control.log_application_destroy(application.id)
def _migrate_got_app(self, key, value, app_id, deploy_info, move, cb):
if response.isfailresponse(value):
if cb:
cb(status=response.CalvinResponse(response.NOT_FOUND))
return
app = Application(app_id,
value['name'],
value['origin_node_id'],
self._node.am,
actors=value['actors_name_map'],
deploy_info=deploy_info)
app.group_components()
app._migrated_actors = {a: None for a in app.actors}
for actor_id, actor_name in app.actors.iteritems():
req = app.get_req(actor_name)
if not req:
_log.analyze(self._node.id, "+ NO REQ", {
'actor_id': actor_id,
'actor_name': actor_name
})
# No requirement then leave as is.
self._migrated_cb(response.CalvinResponse(True), app, actor_id,
cb)
continue
if actor_id in self._node.am.actors:
_log.analyze(self._node.id, "+ OWN ACTOR", {
'actor_id': actor_id,
'actor_name': actor_name
})
self._node.am.update_requirements(actor_id,
req,
False,
move,
callback=CalvinCB(
self._migrated_cb,
app=app,
actor_id=actor_id,
cb=cb))
else:
_log.analyze(self._node.id, "+ OTHER NODE", {
'actor_id': actor_id,
'actor_name': actor_name
})
self.storage.get_actor(actor_id,
cb=CalvinCB(self._migrate_from_rt,
app=app,
actor_id=actor_id,
req=req,
move=move,
cb=cb))
def _change_index_cb(self, key, value, org_cb, index_items):
"""
Collect all the index levels operations into one callback
"""
_log.debug("index cb key:%s, value:%s, index_items:%s" % (key, value, index_items))
# cb False if not already done it at first False value
if not value and index_items:
org_cb(value=calvinresponse.CalvinResponse(False))
del index_items[:]
if key in index_items:
# remove this index level from list
index_items.remove(key)
# If all done send True
if not index_items:
org_cb(value=calvinresponse.CalvinResponse(True))
def _disconnected_peer(self, reply, terminate=DISCONNECT.TEMPORARY):
""" Get called for each peer port when diconnecting but callback should only be called once"""
try:
# Remove this peer from the list of peer connections
self._parallel_connections.remove(self)
except:
pass
if not reply:
# Got failed response do callback, but also inform
# parallel connections that we have sent the callback
self.parallel_set('sent_callback', True)
if self.callback:
_log.warning("Disconnect peer tunnel failed %s", str(reply))
self.callback(status=reply, port_id=self.port.id)
#self.callback(status=response.CalvinResponse(False), port_id=self.port.id)
return
try:
remaining_tokens = reply.data['remaining_tokens']
self._deserialize_remaining_tokens(remaining_tokens)
except:
_log.exception("Did not have remaining_tokens")
remaining_tokens = {}
self.port.exhausted_tokens(remaining_tokens)
if terminate:
self.node.storage.add_port(
self.port,
self.node.id,
self.port.owner.id,
exhausting_peers=remaining_tokens.keys())
if not getattr(self, 'sent_callback',
False) and not self._parallel_connections:
# Last peer connection we should send OK
if self.callback:
self.callback(status=response.CalvinResponse(True),
port_id=self.port.id)
def handle_req_match_cb(self, status, possible_placements, actor_id,
max_placements, tunnel, msgid):
if not possible_placements:
self._proxy_send_reply(
tunnel, msgid,
response.CalvinResponse(response.NOT_FOUND, {
'actor_id': actor_id
}).encode())
return
pp = list(possible_placements)
self._proxy_send_reply(
tunnel, msgid,
response.CalvinResponse(response.OK, {
'actor_id': actor_id,
'possible_placements': pp[:max_placements]
}).encode())
def reply_handler(self, payload):
""" Gets called when a REPLY messages arrives on this link """
try:
# Cancel timeout
self.replies_timeout.pop(payload['msg_uuid']).cancel()
except KeyError:
_log.warning("Tried to handle reply for unknown message msgid %s",
payload['msg_uuid'])
# We ignore any errors in cancelling timeout
pass
try:
# Call the registered callback,for the reply message id, with the reply data as argument
reply = self.replies.pop(payload['msg_uuid'])
# RTT here also inlcudes delay(actors running...) times in remote runtime
self._rtt = (self._rtt * 2 +
(time.time() - reply['send_time'])) / 3
reply['callback'](
response.CalvinResponse(encoded=payload['value']))
except KeyError:
_log.warning("Tried to handle reply for unknown message msgid %s",
payload['msg_uuid'])
except: # Dangerous but needed
_log.exception("Unknown exception in response handler")
# We ignore unknown replies
return
