def connect(self, callback=None, actor_id=None, port_name=None, port_properties=None, port_id=None, peer_node_id=None, peer_actor_id=None, peer_port_name=None, peer_port_properties=None, peer_port_id=None): """ Obtain any missing information to enable making a connection and make actual connect callback: an optional callback that gets called with status when finished local port identified by: actor_id, port_name and port_dir='in'/'out' or port_id peer_node_id: an optional node id the peer port is locate on, will use storage to find it if not supplied peer port (remote or local) identified by: peer_actor_id, peer_port_name and peer_port_dir='in'/'out' or peer_port_id """ local_port_meta = PortMeta(self, actor_id=actor_id, port_id=port_id, port_name=port_name, properties=port_properties, node_id=self.node.id) peer_port_meta = PortMeta(self, actor_id=peer_actor_id, port_id=peer_port_id, port_name=peer_port_name, properties=peer_port_properties, node_id=peer_node_id) _log.analyze(self.node.id, "+", {'local': local_port_meta, 'peer': peer_port_meta}, peer_node_id=peer_node_id, tb=True) try: port = local_port_meta.port except response.CalvinResponseException as e: if callback: callback(status=e.response, actor_id=actor_id, port_name=port_name, port_id=port_id, peer_node_id=peer_node_id, peer_actor_id=peer_actor_id, peer_port_name=peer_port_name, peer_port_id=peer_port_id) return else: raise e.response # Retrieve node id etc, raise exception if not possible, continue in _connect otherwise try: peer_port_meta.retrieve(callback=CalvinCB(self._connect, local_port=port, callback=callback)) except response.CalvinResponseException as e: if callback: callback(status=e.response, actor_id=actor_id, port_name=port_name, port_id=port_id, peer_node_id=peer_node_id, peer_actor_id=peer_actor_id, peer_port_name=peer_port_name, peer_port_id=peer_port_id) return else: raise e.response
def handle_peer_setup(self, handle, connection, match, data, hdr): """ POST /peer_setup Add calvin nodes to network Body: {"peers: ["calvinip://<address>:<port>", ...] } Response status code: OK or SERVICE_UNAVAILABLE Response: {<peer control uri>: [<peer node id>, <per peer status>], ...} """ _log.analyze(self.node.id, "+", data) self.node.peersetup(data['peers'], cb=CalvinCB(self.handle_peer_setup_cb, handle, connection))
def actor_requirements(self, app, actor_id): if actor_id not in self._node.am.list_actors(): _log.error( "Currently we ignore deployment requirements for actor not local to the node, %s" % actor_id) return actor = self._node.am.actors[actor_id] _log.debug("actor_requirements(actor_id=%s), reqs=%s" % (actor_id, actor.get_deployment_requirements())) possible_nodes = set([None]) # None to mark no real response impossible_nodes = set([None]) # None to mark no real response reqs = actor.get_deployment_requirements()[:] for req in actor.get_deployment_requirements(): if req['op'] == 'union_group': # Special operation that first forms a union of a requirement's list response set # To allow alternative requirements options self._union_requirements(req=req, app=app, actor_id=actor_id, possible_nodes=possible_nodes, impossible_nodes=impossible_nodes, reqs=reqs) else: try: req_operations[req['op']].req_op( self._node, CalvinCB(self._actor_requirements_cb, req=req, app=app, actor_id=actor_id, possible_nodes=possible_nodes, impossible_nodes=impossible_nodes, reqs=reqs), actor_id=actor_id, component=req['component'], **req['kwargs']) except: _log.error("actor_requirements one req failed for %s!!!" % actor_id, exc_info=True) reqs.remove(req) if not reqs and actor_id in app._track_actor_cb: _log.analyze(self._node.id, "+ LOOP DONE", { 'actor_id': actor_id, '_track_actor_cb': app._track_actor_cb }, tb=True) app._track_actor_cb.remove(actor_id) self._actor_requirements_combined(app, actor_id, possible_nodes, impossible_nodes) _log.analyze(self._node.id, "+ DONE", {'actor_id': actor_id}, tb=True)
def handle_get_index(self, handle, connection, match, data, hdr): """ GET /index/{key}?root_prefix_level={level} Fetch values under index key Response status code: OK or NOT_FOUND Response: {"result": <list of strings>} """ kwargs = {} if match.group(3) is not None: kwargs['root_prefix_level'] = int(match.group(3)) self.node.storage.get_index( match.group(1), cb=CalvinCB(self.get_index_cb, handle, connection), **kwargs)
def remove_index(self, prefix, indexes, value, cb=None): indexstrs = [ prefix + '/' + '/'.join(indexes[:l]) for l in range(1, len(indexes) + 1) ] for i in indexstrs[:]: self.remove(key=i, value=value, cb=CalvinCB(self._change_index_cb, org_cb=cb, index_items=indexstrs) if cb else None)
def handle_post_storage(self, handle, connection, match, data, hdr): """ POST /storage/{prefix-key} Store value under prefix-key Body: { "value": <string> } Response status code: OK or INTERNAL_ERROR Response: none """ self.node.storage.set("", match.group(1), data['value'], cb=CalvinCB(self.index_cb, handle, connection))
def handle_actor_migrate_lookup_peer_cb(self, key, value, handle, connection, actor_id, peer_node_id): if calvinresponse.isnotfailresponse(value): self.node.proto.actor_migrate_direct( value['node_id'], CalvinCB(self.handle_actor_migrate_proto_cb, handle, connection), actor_id, peer_node_id) else: self.send_response(handle, connection, None, status=calvinresponse.NOT_FOUND)
def authorization_runtime_search(self, actor_id, actorstore_signature, callback): """Search for runtime where the authorization decision for the actor is 'permit'.""" # _log.debug("authorization_runtime_search:\n\tactor_id={}\n\tactorstore_signature={}\n\tcallback={}".format(actor_id, actorstore_signature, callback)) #Search for runtimes supporting the actor with appropriate actorstore_signature r = ReqMatch(self.node, callback=CalvinCB( self._authorization_server_search, actor_id=actor_id, actorstore_signature=actorstore_signature, callback=callback)) r.match_for_actor(actor_id)
def _destroy_final(self, application): """ Final destruction of the application on this node and send request to peers to also destroy the app """ _log.analyze( self._node.id, "+ BEGIN 1", { 'node_info': application.node_info, 'origin_node_id': application.origin_node_id }) if hasattr(application, '_destroy_node_ids'): # Already called return _log.analyze( self._node.id, "+ BEGIN 2", { 'node_info': application.node_info, 'origin_node_id': application.origin_node_id }) application._destroy_node_ids = { n: None for n in application.node_info.keys() } for node_id, actor_ids in application.node_info.iteritems(): if not node_id: _log.analyze(self._node.id, "+ UNKNOWN NODE", {}) application._destroy_node_ids[None] = response.CalvinResponse( False, data=actor_ids) continue if node_id == self._node.id: ok = True for actor_id in actor_ids: if actor_id in self._node.am.list_actors(): _log.analyze(self._node.id, "+ LOCAL ACTOR", {'actor_id': actor_id}) try: self._node.am.destroy(actor_id) except: ok = False application._destroy_node_ids[ node_id] = response.CalvinResponse(ok) continue # Inform peers to destroy their part of the application self._node.proto.app_destroy( node_id, CalvinCB(self._destroy_final_cb, application, node_id), application.id, actor_ids) if application.id in self.applications: del self.applications[application.id] elif application.origin_node_id not in application.node_info and application.origin_node_id != self._node.id: # All actors migrated from the original node, inform it also _log.analyze(self._node.id, "+ SEP APP NODE", {}) self._node.proto.app_destroy(application.origin_node_id, None, application.id, []) self.storage.delete_application(application.id) self._destroy_final_cb(application, '', response.CalvinResponse(True))
def _global_lookup_cb(self, key, value, signature, org_cb): if value: nbr = [len(value)] actors = [] for a in value: self.node.storage.get( 'actor_type-', a, CalvinCB(self._global_lookup_collect, nbr, actors, signature=signature, org_cb=org_cb))
def _connect_via_tunnel(self, status=None, **state): """ All information and hopefully (status OK) a tunnel to the peer is available for a port connect""" port = self._get_local_port(state['actor_id'], state['port_name'], state['port_dir'], state['port_id']) _log.analyze( self.node.id, "+ " + str(status), dict({k: state[k] for k in state.keys() if k != 'callback'}, port_is_connected=port.is_connected_to( state['peer_port_id'])), peer_node_id=state['peer_node_id']) if port.is_connected_to(state['peer_port_id']): # The other end beat us to connecting the port, lets just report success and return _log.analyze( self.node.id, "+ IS CONNECTED", {k: state[k] for k in state.keys() if k != 'callback'}, peer_node_id=state['peer_node_id']) if state['callback']: state['callback'](status=response.CalvinResponse(True), **state) return None if not status: # Failed getting a tunnel, just inform the one wanting to connect if state['callback']: state['callback'](status=response.CalvinResponse( response.INTERNAL_ERROR), **state) return None # Finally we have all information and a tunnel # Lets ask the peer if it can connect our port. tunnel = self.tunnels[state['peer_node_id']] _log.analyze( self.node.id, "+ SENDING", dict({k: state[k] for k in state.keys() if k != 'callback'}, tunnel_status=self.tunnels[state['peer_node_id']].status), peer_node_id=state['peer_node_id']) if 'retries' not in state: state['retries'] = 0 self.proto.port_connect(callback=CalvinCB(self._connected_via_tunnel, **state), port_id=state['port_id'], peer_node_id=state['peer_node_id'], peer_port_id=state['peer_port_id'], peer_actor_id=state['peer_actor_id'], peer_port_name=state['peer_port_name'], peer_port_dir=state['peer_port_dir'], tunnel=tunnel)
def _get_cb(self, result, **kwargs): _log.debug("SQL get OK") cb = kwargs.pop('cb', None) key = kwargs.pop('key', None) try: r = json.loads(result[0][0]) except: # Empty hence deleted or don't exist r = calvinresponse.CalvinResponse(status=calvinresponse.NOT_FOUND) _log.debug("SQL get OK %s" % r) if cb is not None: async.DelayedCall(0, CalvinCB(cb, key, r))
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) yield threads.defer_to_thread(time.sleep, 2) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) yield threads.defer_to_thread(self.storage.add_application, application, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"]["name"] == application.name yield threads.defer_to_thread(self.storage.delete_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is None
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort( "in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == actor.name self.storage.delete_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.NOT_FOUND
def get_authorization_decision(self, callback, actor_id=None, requires=None, signer=None, decision_from_migration=None): """Get authorization decision using the authorization procedure specified in config.""" if decision_from_migration: try: _log.info("Security: Authorization decision from migration") # Decode JSON Web Token, which contains the authorization response. decoded = decode_jwt(decision_from_migration["jwt"], decision_from_migration["cert_name"], self.node.node_name, self.node.id, actor_id) authorization_response = decoded['response'] self._return_authorization_decision( authorization_response['decision'], authorization_response.get("obligations", []), callback) return except Exception as e: _log.error("Security: JWT decoding error - %s" % str(e)) self._return_authorization_decision("indeterminate", [], callback) return else: request = {} request["subject"] = self.get_subject_attributes() if signer is not None: request["subject"].update(signer) request["resource"] = {"node_id": self.node.id} if requires is not None: request["action"] = {"requires": requires} _log.debug("Security: authorization request: %s" % request) # Check if the authorization server is local (the runtime itself) or external. if self.sec_conf['authorization']['procedure'] == "external": if not HAS_JWT: _log.error( "Security: Install JWT to use external server as authorization method.\n" + "Note: NO AUTHORIZATION USED") return False _log.debug("Security: external authorization method chosen") self.authorize_using_external_server(request, callback, actor_id) else: _log.debug("Security: local authorization method chosen") # Authorize access using a local Policy Decision Point (PDP). self.node.authorization.pdp.authorize( request, CalvinCB(self._handle_local_authorization_response, callback=callback))
def __init__(self, node, pm): super(TunnelConnection.TokenTunnel, self).__init__() self.node = node self.pm = pm self.proto = node.proto # Register that we are interested in peer's requests for token transport tunnels self.proto.register_tunnel_handler( 'token', CalvinCB(self.tunnel_request_handles)) self.tunnels = {} # key: peer_node_id, value: tunnel instances self.pending_tunnels = { } # key: peer_node_id, value: list of CalvinCB instances # Alias to port manager's port lookup self._get_local_port = self.pm._get_local_port
def _maintenance_loop(self): # Migrate denied actors for actor in self.actor_mgr.migratable_actors(): self.actor_mgr.migrate(actor.id, actor.migration_info["node_id"], callback=CalvinCB(actor.remove_migration_info)) # Enable denied actors again if access is permitted. Will try to migrate if access still denied. for actor in self.actor_mgr.denied_actors(): actor.enable_or_migrate() # TODO: try to migrate shadow actors as well. # Since we may have moved stuff around, schedule strategy self.insert_task(self.strategy, 0) # Schedule next maintenance self.insert_task(self._maintenance_loop, self._maintenance_delay)
def robust_migrate(self, actor_id, node_ids, callback, **kwargs): """ Will try to migrate the actor to each of the suggested node_ids (which is modified), Optionally kwargs can contain state, actor_type and ports If all else fails the state, actor_type and ports will be returned in the callback. These could be used by the callback to either try another list of node_ids, recreate the actor locally or just ignore. """ if kwargs.get('status', False): # Success if callback: callback(status=kwargs['status']) return if node_ids: node_id = node_ids.pop(0) else: if callback: callback(status=kwargs.get('status', response.CalvinResponse(False)), state=kwargs.get('state', None), actor_type=kwargs.get('actor_type', None), ports=kwargs.get('ports', None)) return if 'state' in kwargs: # Retry another node self.node.proto.actor_new( node_id, CalvinCB(self._robust_migrate_cb, actor_id=actor_id, node_ids=node_ids, callback=callback), kwargs.get('actor_type', None), kwargs.get('state', None), kwargs.get('ports', None)) else: # Start with standard migration self.migrate( actor_id, node_id, CalvinCB(self._robust_migrate_cb, actor_id=actor_id, node_ids=node_ids, callback=callback))
def disconnect(self): """ Obtain any missing information to enable disconnecting one port peer and make the disconnect""" _log.analyze(self.node.id, "+", {'port_id': self.port.id}) # Disconnect and destroy the endpoints self._destroy_endpoints() # Inform peer port of disconnection self.node.proto.port_disconnect( callback=CalvinCB(self._disconnected_peer), port_id=self.port.id, peer_node_id=self.peer_port_meta.node_id, peer_port_id=self.peer_port_meta.port_id)
def _dereplicate_actor_cb(self, key, value, replication_data, terminate, cb): """ Get actor callback """ _log.analyze(self.node.id, "+", {'actor_id': key, 'value': value}) if value and 'node_id' in value: # Use app destroy since it can remotely destroy actors self.node.proto.app_destroy(value['node_id'], CalvinCB(self._dereplicated, replication_data=replication_data, last_replica_id=key, node_id=value['node_id'], cb=cb), None, [key], disconnect=terminate, replication_id=replication_data.id) else: # FIXME Should do retries if cb: cb(calvinresponse.CalvinResponse(False))
def _destroy_with_disconnect_exhausted(self, status, actor_id, terminate, callback=None): # FIXME handled failed exhaust when we do return anything but OK _log.debug("Disconnected and exhausted all inports %s %s" % (actor_id, str(status))) self.node.pm.disconnect(callback=CalvinCB( self._destroy_with_disconnect_cb, callback=callback), actor_id=actor_id, port_dir="out", terminate=terminate)
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) self.storage.add_application(application) value = self.storage.get_application(application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == application.id and value["value"][ "name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert application.id not in self.storage.localstore
def _delete_node_index(self, node, cb=None): indexes = node.attributes.get_indexed_public() _log.analyze(self.node.id, "+", {'indexes': indexes}) try: counter = [len(indexes)] # counter value by reference used in callback for index in indexes: self.remove_index(index, node.id, cb=CalvinCB(self._delete_node_cb, counter=counter, org_cb=cb)) # The remove index gets 1 second otherwise we call the callback anyway, i.e. stop the node async.DelayedCall(1.0, self._delete_node_timeout_cb, counter=counter, org_cb=cb) except: _log.debug("Remove node index failed", exc_info=True) if cb: cb()
def handle_post_node_attribute_indexed_public_cb(self, key, value, handle, connection, attributes): try: indexed_public = [] for attr in attributes.items(): indexed_string = format_index_string(attr) indexed_public.append(indexed_string) self.node.storage.add_index(indexed_string, key) value['attributes']['indexed_public'] = indexed_public self.node.storage.set(prefix="node-", key=key, value=value, cb=CalvinCB(self.index_cb, handle, connection)) except Exception as e: _log.error("Failed to update node %s", e) self.send_response(handle, connection, None, status=calvinresponse.INTERNAL_ERROR)
def maintenance_loop(self): # Migrate denied actors for actor in self.actor_mgr.migratable_actors(): self.actor_mgr.migrate(actor.id, actor.migration_info["node_id"], callback=CalvinCB( actor.remove_migration_info)) # Enable denied actors again if access is permitted. Will try to migrate if access still denied. for actor in self.actor_mgr.denied_actors(): actor.enable_or_migrate() # TODO: try to migrate shadow actors as well. self._maintenance_loop = None self.trigger_maintenance_loop(delay=True)
def _delete_fail_cb(self, failure, **kwargs): _log.debug("SQL delete FAIL") ok = False cb = kwargs.pop('cb', None) key = kwargs.pop('key', None) # TODO handle errors try: err = int(str(failure.value)[1:5]) except: err = 9999 _log.debug("SQL delete %s %i %s" % ("OK" if ok else "FAIL", err, str(failure))) if cb is not None: async.DelayedCall(0, CalvinCB(cb, key, calvinresponse.CalvinResponse(status=ok)))
def connect(self, actor_id=None, port_name=None, port_dir=None, port_id=None, peer_node_id=None, peer_actor_id=None, peer_port_name=None, peer_port_dir=None, peer_port_id=None, cb=None): self.pm.connect(actor_id=actor_id, port_name=port_name, port_dir=port_dir, port_id=port_id, peer_node_id=peer_node_id, peer_actor_id=peer_actor_id, peer_port_name=peer_port_name, peer_port_dir=peer_port_dir, peer_port_id=peer_port_id, callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def req_op(node, cb, signature, shadow_params, actor_id=None, component=None): """ Based on signature find actors' requires in global storage, filter actors based on params that are supplied and find any nodes with those capabilities """ # Lookup signature to get a list of ids of the actor types node.storage.get_index(['actor', 'signature', signature], CalvinCB(_signature_cb, node=node, shadow_params=shadow_params, actor_id=actor_id, cb=cb, component=component))
def _update_cache_request_finished(self, key, value, callback, force=False): """ Called by storage when the node is (not) found """ _log.debug( "Got response from storage key = %s, value = %s, callback = %s, force = %s", key, value, callback, force) _log.analyze(self.node.id, "+", {'value': value}, peer_node_id=key, tb=True) if response.isfailresponse(value): # the peer_id did not exist in storage _log.info("Failed to get node %s info from storage", key) if key in self._peer_cache: self._peer_cache.pop(key) if callback: callback(status=response.CalvinResponse( response.NOT_FOUND, {'peer_node_id': key})) return matching = [ s for s in value['attributes']['indexed_public'] if "node_name" in s ] if matching: first_match = matching[0].split("node_name/")[1] server_node_name_as_str = first_match.replace("/", "-") else: server_node_name_as_str = None # Set values from storage self._peer_cache[key]['uris'] = value['uris'] self._peer_cache[key]['timestamp'] = time.time() self._peer_cache[key]['server_name'] = server_node_name_as_str if 'proxy' not in value: # join the peer node self._link_request(key, callback=callback, force=True) else: if value['proxy'] == self.node.id: _log.error("No link to proxy client '%s'", key) else: self.link_request( value['proxy'], CalvinCB(self._routing_link_finished, dest_peer_id=key, callback=callback))
def _link_request_finished(self, status=None, peer_node_id=None, uri=None, peer_id=None): """ Will be called when join works or fails also on connection fails. Can be called multiple times one for each uri, but will stop on one success. This function will try all uris for a node and after that invalidate the cache if its too old. If all fails it will give up and make the callbacks, if join fails then it also stops. It means that the other end dont want us. """ # TODO: Should save if we are the originator, for reconnect reasons _log.debug("link request finished %s, %s, %s", status, peer_node_id, uri) if status: # It worked! _log.debug("Join success on peer %s with uri %s", peer_id, uri) self._execute_cached_callbacks(peer_id) # trigger routed links on link link = self._links[peer_id] for route in link.routes: self._execute_cached_callbacks(peer_id) else: _ = self._peer_cache[peer_id]['uris'].pop(0) _log.debug("Failed to connect to uri %s", _) if self._peer_cache[peer_id]['uris']: _log.debug("Trying next %s", self._peer_cache[peer_id]['uris'][0]) self._link_request(peer_id, force=True) elif self._peer_cache[peer_id]['timestamp'] + 5 * 60 < time.time(): _log.debug( "Cache old %s sec, updateing cache", time.time() - self._peer_cache[peer_id]['timestamp']) # Invalidate cache and try one more time # WARNING: here if one iteration takes more then 5 minutes then we are in a retry loop :/ self.node.storage.get_node( peer_id, CalvinCB(self._update_cache_request_finished, callback=None)) else: cbs = self._peer_cache[peer_id].pop('callbacks') self._peer_cache[peer_id]['callbacks'] = [] if cbs: for cb in cbs: cb(peer_id, None, status=response.CalvinResponse(False)) _log.warning("Join failed on peer %s on all uris", peer_id)