def connect(self, callback=None, actor_id=None, port_name=None, port_properties=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None, peer_port_properties=None,
peer_port_id=None):
""" Obtain any missing information to enable making a connection and make actual connect
callback: an optional callback that gets called with status when finished
local port identified by:
actor_id, port_name and port_dir='in'/'out' or
port_id
peer_node_id: an optional node id the peer port is locate on, will use storage to find it if not supplied
peer port (remote or local) identified by:
peer_actor_id, peer_port_name and peer_port_dir='in'/'out' or
peer_port_id
"""
local_port_meta = PortMeta(self, actor_id=actor_id, port_id=port_id, port_name=port_name,
properties=port_properties, node_id=self.node.id)
peer_port_meta = PortMeta(self, actor_id=peer_actor_id, port_id=peer_port_id, port_name=peer_port_name,
properties=peer_port_properties, node_id=peer_node_id)
_log.analyze(self.node.id, "+", {'local': local_port_meta, 'peer': peer_port_meta},
peer_node_id=peer_node_id, tb=True)
try:
port = local_port_meta.port
except response.CalvinResponseException as e:
if callback:
callback(status=e.response,
actor_id=actor_id,
port_name=port_name,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_id=peer_port_id)
return
else:
raise e.response
# Retrieve node id etc, raise exception if not possible, continue in _connect otherwise
try:
peer_port_meta.retrieve(callback=CalvinCB(self._connect, local_port=port, callback=callback))
except response.CalvinResponseException as e:
if callback:
callback(status=e.response,
actor_id=actor_id,
port_name=port_name,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_id=peer_port_id)
return
else:
raise e.response
def handle_peer_setup(self, handle, connection, match, data, hdr):
"""
POST /peer_setup
Add calvin nodes to network
Body: {"peers: ["calvinip://<address>:<port>", ...] }
Response status code: OK or SERVICE_UNAVAILABLE
Response: {<peer control uri>: [<peer node id>, <per peer status>], ...}
"""
_log.analyze(self.node.id, "+", data)
self.node.peersetup(data['peers'],
cb=CalvinCB(self.handle_peer_setup_cb, handle,
connection))
def actor_requirements(self, app, actor_id):
if actor_id not in self._node.am.list_actors():
_log.error(
"Currently we ignore deployment requirements for actor not local to the node, %s"
% actor_id)
return
actor = self._node.am.actors[actor_id]
_log.debug("actor_requirements(actor_id=%s), reqs=%s" %
(actor_id, actor.get_deployment_requirements()))
possible_nodes = set([None]) # None to mark no real response
impossible_nodes = set([None]) # None to mark no real response
reqs = actor.get_deployment_requirements()[:]
for req in actor.get_deployment_requirements():
if req['op'] == 'union_group':
# Special operation that first forms a union of a requirement's list response set
# To allow alternative requirements options
self._union_requirements(req=req,
app=app,
actor_id=actor_id,
possible_nodes=possible_nodes,
impossible_nodes=impossible_nodes,
reqs=reqs)
else:
try:
req_operations[req['op']].req_op(
self._node,
CalvinCB(self._actor_requirements_cb,
req=req,
app=app,
actor_id=actor_id,
possible_nodes=possible_nodes,
impossible_nodes=impossible_nodes,
reqs=reqs),
actor_id=actor_id,
component=req['component'],
**req['kwargs'])
except:
_log.error("actor_requirements one req failed for %s!!!" %
actor_id,
exc_info=True)
reqs.remove(req)
if not reqs and actor_id in app._track_actor_cb:
_log.analyze(self._node.id,
"+ LOOP DONE", {
'actor_id': actor_id,
'_track_actor_cb': app._track_actor_cb
},
tb=True)
app._track_actor_cb.remove(actor_id)
self._actor_requirements_combined(app, actor_id, possible_nodes,
impossible_nodes)
_log.analyze(self._node.id, "+ DONE", {'actor_id': actor_id}, tb=True)
def handle_get_index(self, handle, connection, match, data, hdr):
"""
GET /index/{key}?root_prefix_level={level}
Fetch values under index key
Response status code: OK or NOT_FOUND
Response: {"result": <list of strings>}
"""
kwargs = {}
if match.group(3) is not None:
kwargs['root_prefix_level'] = int(match.group(3))
self.node.storage.get_index(
match.group(1), cb=CalvinCB(self.get_index_cb, handle, connection), **kwargs)
def remove_index(self, prefix, indexes, value, cb=None):
indexstrs = [
prefix + '/' + '/'.join(indexes[:l])
for l in range(1,
len(indexes) + 1)
]
for i in indexstrs[:]:
self.remove(key=i,
value=value,
cb=CalvinCB(self._change_index_cb,
org_cb=cb,
index_items=indexstrs) if cb else None)
def handle_post_storage(self, handle, connection, match, data, hdr):
"""
POST /storage/{prefix-key}
Store value under prefix-key
Body:
{
"value": <string>
}
Response status code: OK or INTERNAL_ERROR
Response: none
"""
self.node.storage.set("", match.group(1), data['value'], cb=CalvinCB(self.index_cb, handle, connection))
def handle_actor_migrate_lookup_peer_cb(self, key, value, handle, connection,
actor_id, peer_node_id):
if calvinresponse.isnotfailresponse(value):
self.node.proto.actor_migrate_direct(
value['node_id'],
CalvinCB(self.handle_actor_migrate_proto_cb, handle, connection),
actor_id, peer_node_id)
else:
self.send_response(handle,
connection,
None,
status=calvinresponse.NOT_FOUND)
def authorization_runtime_search(self, actor_id, actorstore_signature,
callback):
"""Search for runtime where the authorization decision for the actor is 'permit'."""
# _log.debug("authorization_runtime_search:\n\tactor_id={}\n\tactorstore_signature={}\n\tcallback={}".format(actor_id, actorstore_signature, callback))
#Search for runtimes supporting the actor with appropriate actorstore_signature
r = ReqMatch(self.node,
callback=CalvinCB(
self._authorization_server_search,
actor_id=actor_id,
actorstore_signature=actorstore_signature,
callback=callback))
r.match_for_actor(actor_id)
def _destroy_final(self, application):
""" Final destruction of the application on this node and send request to peers to also destroy the app """
_log.analyze(
self._node.id, "+ BEGIN 1", {
'node_info': application.node_info,
'origin_node_id': application.origin_node_id
})
if hasattr(application, '_destroy_node_ids'):
# Already called
return
_log.analyze(
self._node.id, "+ BEGIN 2", {
'node_info': application.node_info,
'origin_node_id': application.origin_node_id
})
application._destroy_node_ids = {
n: None
for n in application.node_info.keys()
}
for node_id, actor_ids in application.node_info.iteritems():
if not node_id:
_log.analyze(self._node.id, "+ UNKNOWN NODE", {})
application._destroy_node_ids[None] = response.CalvinResponse(
False, data=actor_ids)
continue
if node_id == self._node.id:
ok = True
for actor_id in actor_ids:
if actor_id in self._node.am.list_actors():
_log.analyze(self._node.id, "+ LOCAL ACTOR",
{'actor_id': actor_id})
try:
self._node.am.destroy(actor_id)
except:
ok = False
application._destroy_node_ids[
node_id] = response.CalvinResponse(ok)
continue
# Inform peers to destroy their part of the application
self._node.proto.app_destroy(
node_id, CalvinCB(self._destroy_final_cb, application,
node_id), application.id, actor_ids)
if application.id in self.applications:
del self.applications[application.id]
elif application.origin_node_id not in application.node_info and application.origin_node_id != self._node.id:
# All actors migrated from the original node, inform it also
_log.analyze(self._node.id, "+ SEP APP NODE", {})
self._node.proto.app_destroy(application.origin_node_id, None,
application.id, [])
self.storage.delete_application(application.id)
self._destroy_final_cb(application, '', response.CalvinResponse(True))
def _global_lookup_cb(self, key, value, signature, org_cb):
if value:
nbr = [len(value)]
actors = []
for a in value:
self.node.storage.get(
'actor_type-', a,
CalvinCB(self._global_lookup_collect,
nbr,
actors,
signature=signature,
org_cb=org_cb))
def _connect_via_tunnel(self, status=None, **state):
""" All information and hopefully (status OK) a tunnel to the peer is available for a port connect"""
port = self._get_local_port(state['actor_id'], state['port_name'],
state['port_dir'], state['port_id'])
_log.analyze(
self.node.id,
"+ " + str(status),
dict({k: state[k]
for k in state.keys() if k != 'callback'},
port_is_connected=port.is_connected_to(
state['peer_port_id'])),
peer_node_id=state['peer_node_id'])
if port.is_connected_to(state['peer_port_id']):
# The other end beat us to connecting the port, lets just report success and return
_log.analyze(
self.node.id,
"+ IS CONNECTED",
{k: state[k]
for k in state.keys() if k != 'callback'},
peer_node_id=state['peer_node_id'])
if state['callback']:
state['callback'](status=response.CalvinResponse(True),
**state)
return None
if not status:
# Failed getting a tunnel, just inform the one wanting to connect
if state['callback']:
state['callback'](status=response.CalvinResponse(
response.INTERNAL_ERROR),
**state)
return None
# Finally we have all information and a tunnel
# Lets ask the peer if it can connect our port.
tunnel = self.tunnels[state['peer_node_id']]
_log.analyze(
self.node.id,
"+ SENDING",
dict({k: state[k]
for k in state.keys() if k != 'callback'},
tunnel_status=self.tunnels[state['peer_node_id']].status),
peer_node_id=state['peer_node_id'])
if 'retries' not in state:
state['retries'] = 0
self.proto.port_connect(callback=CalvinCB(self._connected_via_tunnel,
**state),
port_id=state['port_id'],
peer_node_id=state['peer_node_id'],
peer_port_id=state['peer_port_id'],
peer_actor_id=state['peer_actor_id'],
peer_port_name=state['peer_port_name'],
peer_port_dir=state['peer_port_dir'],
tunnel=tunnel)
def _get_cb(self, result, **kwargs):
_log.debug("SQL get OK")
cb = kwargs.pop('cb', None)
key = kwargs.pop('key', None)
try:
r = json.loads(result[0][0])
except:
# Empty hence deleted or don't exist
r = calvinresponse.CalvinResponse(status=calvinresponse.NOT_FOUND)
_log.debug("SQL get OK %s" % r)
if cb is not None:
async.DelayedCall(0, CalvinCB(cb, key, r))
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
yield threads.defer_to_thread(time.sleep, 2)
application = appmanager.Application(
calvinuuid.uuid('APP'), "test_app",
[calvinuuid.uuid('ACTOR'),
calvinuuid.uuid('ACTOR')])
yield threads.defer_to_thread(self.storage.add_application,
application,
cb=CalvinCB(cb))
yield threads.defer_to_thread(time.sleep, 2)
value = self.q.get(timeout=0.2)
assert value["value"] is True
yield threads.defer_to_thread(self.storage.get_application,
application.id,
cb=CalvinCB(cb))
yield threads.defer_to_thread(time.sleep, 2)
value = self.q.get(timeout=0.2)
assert value["value"]["name"] == application.name
yield threads.defer_to_thread(self.storage.delete_application,
application.id,
cb=CalvinCB(cb))
yield threads.defer_to_thread(time.sleep, 2)
value = self.q.get(timeout=0.2)
assert value["value"] is True
yield threads.defer_to_thread(self.storage.get_application,
application.id,
cb=CalvinCB(cb))
yield threads.defer_to_thread(time.sleep, 2)
value = self.q.get(timeout=0.2)
assert value["value"] is None
def test_actor_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
port1 = calvin.tests.TestPort("out", "out")
port2 = calvin.tests.TestPort(
"in",
"in",
)
port1.peers = [("local", port2.id)]
port2.peers = [("local", port1.id)]
actor = calvin.tests.TestActor("actor1", "type1", {},
{port1.name: port1})
self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert value["value"]["name"] == actor.name
self.storage.delete_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.OK
self.storage.get_actor(actor.id, cb=CalvinCB(cb))
yield wait_for(self.q.empty, condition=lambda x: not x())
value = self.q.get(timeout=.001)
assert isinstance(value["value"], calvinresponse.CalvinResponse
) and value["value"] == calvinresponse.NOT_FOUND
def get_authorization_decision(self,
callback,
actor_id=None,
requires=None,
signer=None,
decision_from_migration=None):
"""Get authorization decision using the authorization procedure specified in config."""
if decision_from_migration:
try:
_log.info("Security: Authorization decision from migration")
# Decode JSON Web Token, which contains the authorization response.
decoded = decode_jwt(decision_from_migration["jwt"],
decision_from_migration["cert_name"],
self.node.node_name, self.node.id,
actor_id)
authorization_response = decoded['response']
self._return_authorization_decision(
authorization_response['decision'],
authorization_response.get("obligations", []), callback)
return
except Exception as e:
_log.error("Security: JWT decoding error - %s" % str(e))
self._return_authorization_decision("indeterminate", [],
callback)
return
else:
request = {}
request["subject"] = self.get_subject_attributes()
if signer is not None:
request["subject"].update(signer)
request["resource"] = {"node_id": self.node.id}
if requires is not None:
request["action"] = {"requires": requires}
_log.debug("Security: authorization request: %s" % request)
# Check if the authorization server is local (the runtime itself) or external.
if self.sec_conf['authorization']['procedure'] == "external":
if not HAS_JWT:
_log.error(
"Security: Install JWT to use external server as authorization method.\n"
+ "Note: NO AUTHORIZATION USED")
return False
_log.debug("Security: external authorization method chosen")
self.authorize_using_external_server(request, callback,
actor_id)
else:
_log.debug("Security: local authorization method chosen")
# Authorize access using a local Policy Decision Point (PDP).
self.node.authorization.pdp.authorize(
request,
CalvinCB(self._handle_local_authorization_response,
callback=callback))
def __init__(self, node, pm):
super(TunnelConnection.TokenTunnel, self).__init__()
self.node = node
self.pm = pm
self.proto = node.proto
# Register that we are interested in peer's requests for token transport tunnels
self.proto.register_tunnel_handler(
'token', CalvinCB(self.tunnel_request_handles))
self.tunnels = {} # key: peer_node_id, value: tunnel instances
self.pending_tunnels = {
} # key: peer_node_id, value: list of CalvinCB instances
# Alias to port manager's port lookup
self._get_local_port = self.pm._get_local_port
def _maintenance_loop(self):
# Migrate denied actors
for actor in self.actor_mgr.migratable_actors():
self.actor_mgr.migrate(actor.id, actor.migration_info["node_id"],
callback=CalvinCB(actor.remove_migration_info))
# Enable denied actors again if access is permitted. Will try to migrate if access still denied.
for actor in self.actor_mgr.denied_actors():
actor.enable_or_migrate()
# TODO: try to migrate shadow actors as well.
# Since we may have moved stuff around, schedule strategy
self.insert_task(self.strategy, 0)
# Schedule next maintenance
self.insert_task(self._maintenance_loop, self._maintenance_delay)
def robust_migrate(self, actor_id, node_ids, callback, **kwargs):
""" Will try to migrate the actor to each of the suggested node_ids (which is modified),
Optionally kwargs can contain state, actor_type and ports
If all else fails the state, actor_type and ports will be returned in the callback.
These could be used by the callback to either try another list of node_ids,
recreate the actor locally or just ignore.
"""
if kwargs.get('status', False):
# Success
if callback:
callback(status=kwargs['status'])
return
if node_ids:
node_id = node_ids.pop(0)
else:
if callback:
callback(status=kwargs.get('status',
response.CalvinResponse(False)),
state=kwargs.get('state', None),
actor_type=kwargs.get('actor_type', None),
ports=kwargs.get('ports', None))
return
if 'state' in kwargs:
# Retry another node
self.node.proto.actor_new(
node_id,
CalvinCB(self._robust_migrate_cb,
actor_id=actor_id,
node_ids=node_ids,
callback=callback), kwargs.get('actor_type', None),
kwargs.get('state', None), kwargs.get('ports', None))
else:
# Start with standard migration
self.migrate(
actor_id, node_id,
CalvinCB(self._robust_migrate_cb,
actor_id=actor_id,
node_ids=node_ids,
callback=callback))
def disconnect(self):
""" Obtain any missing information to enable disconnecting one port peer and make the disconnect"""
_log.analyze(self.node.id, "+", {'port_id': self.port.id})
# Disconnect and destroy the endpoints
self._destroy_endpoints()
# Inform peer port of disconnection
self.node.proto.port_disconnect(
callback=CalvinCB(self._disconnected_peer),
port_id=self.port.id,
peer_node_id=self.peer_port_meta.node_id,
peer_port_id=self.peer_port_meta.port_id)
def _dereplicate_actor_cb(self, key, value, replication_data, terminate, cb):
""" Get actor callback """
_log.analyze(self.node.id, "+", {'actor_id': key, 'value': value})
if value and 'node_id' in value:
# Use app destroy since it can remotely destroy actors
self.node.proto.app_destroy(value['node_id'],
CalvinCB(self._dereplicated, replication_data=replication_data, last_replica_id=key,
node_id=value['node_id'], cb=cb),
None, [key], disconnect=terminate, replication_id=replication_data.id)
else:
# FIXME Should do retries
if cb:
cb(calvinresponse.CalvinResponse(False))
def _destroy_with_disconnect_exhausted(self,
status,
actor_id,
terminate,
callback=None):
# FIXME handled failed exhaust when we do return anything but OK
_log.debug("Disconnected and exhausted all inports %s %s" %
(actor_id, str(status)))
self.node.pm.disconnect(callback=CalvinCB(
self._destroy_with_disconnect_cb, callback=callback),
actor_id=actor_id,
port_dir="out",
terminate=terminate)
def test_application_functions(self):
self.q = Queue.Queue()
def cb(key, value):
self.q.put({"key": key, "value": value})
application = appmanager.Application(
calvinuuid.uuid('APP'), "test_app",
[calvinuuid.uuid('ACTOR'),
calvinuuid.uuid('ACTOR')])
self.storage.add_application(application)
value = self.storage.get_application(application.id,
cb=CalvinCB(func=cb))
value = self.q.get(timeout=0.2)
assert value["key"] == application.id and value["value"][
"name"] == application.name
self.storage.delete_application(application.id, cb=CalvinCB(func=cb))
value = self.q.get(timeout=0.2)
assert value
assert application.id not in self.storage.localstore
def _delete_node_index(self, node, cb=None):
indexes = node.attributes.get_indexed_public()
_log.analyze(self.node.id, "+", {'indexes': indexes})
try:
counter = [len(indexes)] # counter value by reference used in callback
for index in indexes:
self.remove_index(index, node.id, cb=CalvinCB(self._delete_node_cb, counter=counter, org_cb=cb))
# The remove index gets 1 second otherwise we call the callback anyway, i.e. stop the node
async.DelayedCall(1.0, self._delete_node_timeout_cb, counter=counter, org_cb=cb)
except:
_log.debug("Remove node index failed", exc_info=True)
if cb:
cb()
def handle_post_node_attribute_indexed_public_cb(self, key, value, handle, connection, attributes):
try:
indexed_public = []
for attr in attributes.items():
indexed_string = format_index_string(attr)
indexed_public.append(indexed_string)
self.node.storage.add_index(indexed_string, key)
value['attributes']['indexed_public'] = indexed_public
self.node.storage.set(prefix="node-", key=key, value=value,
cb=CalvinCB(self.index_cb, handle, connection))
except Exception as e:
_log.error("Failed to update node %s", e)
self.send_response(handle, connection, None, status=calvinresponse.INTERNAL_ERROR)
def maintenance_loop(self):
# Migrate denied actors
for actor in self.actor_mgr.migratable_actors():
self.actor_mgr.migrate(actor.id,
actor.migration_info["node_id"],
callback=CalvinCB(
actor.remove_migration_info))
# Enable denied actors again if access is permitted. Will try to migrate if access still denied.
for actor in self.actor_mgr.denied_actors():
actor.enable_or_migrate()
# TODO: try to migrate shadow actors as well.
self._maintenance_loop = None
self.trigger_maintenance_loop(delay=True)
def _delete_fail_cb(self, failure, **kwargs):
_log.debug("SQL delete FAIL")
ok = False
cb = kwargs.pop('cb', None)
key = kwargs.pop('key', None)
# TODO handle errors
try:
err = int(str(failure.value)[1:5])
except:
err = 9999
_log.debug("SQL delete %s %i %s" % ("OK" if ok else "FAIL", err, str(failure)))
if cb is not None:
async.DelayedCall(0, CalvinCB(cb, key, calvinresponse.CalvinResponse(status=ok)))
def connect(self, actor_id=None, port_name=None, port_dir=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_id=None, cb=None):
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_dir=port_dir,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_dir=peer_port_dir,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def req_op(node, cb, signature, shadow_params, actor_id=None, component=None):
""" Based on signature find actors' requires in global storage,
filter actors based on params that are supplied
and find any nodes with those capabilities
"""
# Lookup signature to get a list of ids of the actor types
node.storage.get_index(['actor', 'signature', signature],
CalvinCB(_signature_cb,
node=node,
shadow_params=shadow_params,
actor_id=actor_id,
cb=cb,
component=component))
def _update_cache_request_finished(self,
key,
value,
callback,
force=False):
""" Called by storage when the node is (not) found """
_log.debug(
"Got response from storage key = %s, value = %s, callback = %s, force = %s",
key, value, callback, force)
_log.analyze(self.node.id,
"+", {'value': value},
peer_node_id=key,
tb=True)
if response.isfailresponse(value):
# the peer_id did not exist in storage
_log.info("Failed to get node %s info from storage", key)
if key in self._peer_cache:
self._peer_cache.pop(key)
if callback:
callback(status=response.CalvinResponse(
response.NOT_FOUND, {'peer_node_id': key}))
return
matching = [
s for s in value['attributes']['indexed_public']
if "node_name" in s
]
if matching:
first_match = matching[0].split("node_name/")[1]
server_node_name_as_str = first_match.replace("/", "-")
else:
server_node_name_as_str = None
# Set values from storage
self._peer_cache[key]['uris'] = value['uris']
self._peer_cache[key]['timestamp'] = time.time()
self._peer_cache[key]['server_name'] = server_node_name_as_str
if 'proxy' not in value:
# join the peer node
self._link_request(key, callback=callback, force=True)
else:
if value['proxy'] == self.node.id:
_log.error("No link to proxy client '%s'", key)
else:
self.link_request(
value['proxy'],
CalvinCB(self._routing_link_finished,
dest_peer_id=key,
callback=callback))
def _link_request_finished(self,
status=None,
peer_node_id=None,
uri=None,
peer_id=None):
"""
Will be called when join works or fails also on connection fails.
Can be called multiple times one for each uri, but will stop on one success.
This function will try all uris for a node and after that invalidate the cache if its too old.
If all fails it will give up and make the callbacks, if join fails then it also stops.
It means that the other end dont want us.
"""
# TODO: Should save if we are the originator, for reconnect reasons
_log.debug("link request finished %s, %s, %s", status, peer_node_id,
uri)
if status:
# It worked!
_log.debug("Join success on peer %s with uri %s", peer_id, uri)
self._execute_cached_callbacks(peer_id)
# trigger routed links on link
link = self._links[peer_id]
for route in link.routes:
self._execute_cached_callbacks(peer_id)
else:
_ = self._peer_cache[peer_id]['uris'].pop(0)
_log.debug("Failed to connect to uri %s", _)
if self._peer_cache[peer_id]['uris']:
_log.debug("Trying next %s",
self._peer_cache[peer_id]['uris'][0])
self._link_request(peer_id, force=True)
elif self._peer_cache[peer_id]['timestamp'] + 5 * 60 < time.time():
_log.debug(
"Cache old %s sec, updateing cache",
time.time() - self._peer_cache[peer_id]['timestamp'])
# Invalidate cache and try one more time
# WARNING: here if one iteration takes more then 5 minutes then we are in a retry loop :/
self.node.storage.get_node(
peer_id,
CalvinCB(self._update_cache_request_finished,
callback=None))
else:
cbs = self._peer_cache[peer_id].pop('callbacks')
self._peer_cache[peer_id]['callbacks'] = []
if cbs:
for cb in cbs:
cb(peer_id,
None,
status=response.CalvinResponse(False))
_log.warning("Join failed on peer %s on all uris", peer_id)
