def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse ) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_application(application.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application( calvinuuid.uuid('APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value self.storage.get_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value assert value["key"] == application.id and value["value"][ "name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert application.id not in self.storage.localstore
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) yield threads.defer_to_thread(time.sleep, 2) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) yield threads.defer_to_thread(self.storage.add_application, application, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"]["name"] == application.name yield threads.defer_to_thread(self.storage.delete_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is None
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')], calvinuuid.uuid("NODE"), None) self.storage.add_application(application, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value self.storage.get_application( application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) # print value assert value["key"] == application.id and value[ "value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert application.id not in self.storage.localstore
def obtain_cert_node_info(name): """ Obtain node id based on name and domain from config Return dict with domain, node name and node id """ _log.debug("obtain_cert_node_info: node_name={}".format(name)) domain = _conf.get("security", "security_domain_name") if domain is None or name is None: # No security or name specified just use standard node UUID _log.debug("OBTAINING no security domain={}, name={}".format(domain, name)) return {'domain': None, 'name': name, 'id': calvinuuid.uuid("NODE")} runtime_dir = get_own_credentials_path(name) # Does existing signed runtime certificate exist, return info try: filenames = os.listdir(os.path.join(runtime_dir, "mine")) content = open(os.path.join(runtime_dir, "mine", filenames[0]), 'rt').read() cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content) subject = cert.get_subject() if subject.commonName != name or subject.organizationName != domain: raise Exception("names of cert incorrect") _log.debug("OBTAINING existing security domain={}, name={}, id={}".format(domain, name, subject.dnQualifier)) return {'domain': domain, 'name': name, 'id': subject.dnQualifier} except: pass #_log.exception("OBTAINING fail existing security domain={}, name={}".format(domain, name)) # No valid signed cert available, create new node id and let user create certificate later nodeid = calvinuuid.uuid("NODE") return {'domain': domain, 'name': name, 'id': nodeid}
def obtain_cert_node_info(name, security_dir=None): """ Obtain node id based on name and domain from config Return dict with domain, node name and node id """ _log.debug("obtain_cert_node_info: node_name={}".format(name)) domain = _conf.get("security", "domain_name") if domain is None or name is None: # No security or name specified just use standard node UUID _log.debug("OBTAINING no security domain={}, name={}".format( domain, name)) return {'domain': None, 'name': name, 'id': calvinuuid.uuid("NODE")} runtime_dir = get_own_credentials_path(name, security_dir=security_dir) # Does existing signed runtime certificate exist, return info try: filenames = os.listdir(os.path.join(runtime_dir, "mine")) content = open(os.path.join(runtime_dir, "mine", filenames[0]), 'rt').read() cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content) subject = cert.get_subject() if subject.commonName != name or subject.organizationName != domain: raise Exception("names of cert incorrect") _log.debug( "OBTAINING existing security domain={}, name={}, id={}".format( domain, name, subject.dnQualifier)) return {'domain': domain, 'name': name, 'id': subject.dnQualifier} except: pass #_log.exception("OBTAINING fail existing security domain={}, name={}".format(domain, name)) # No valid signed cert available, create new node id and let user create certificate later nodeid = calvinuuid.uuid("NODE") return {'domain': domain, 'name': name, 'id': nodeid}
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) yield threads.defer_to_thread(time.sleep, 2) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) yield threads.defer_to_thread(self.storage.add_application, application, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"]["name"] == application.name yield threads.defer_to_thread(self.storage.delete_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_application, application.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is None
def test_actor_and_port_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = TestPort("out", "out") port2 = TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peer = ("local", port1.id) actor1 = TestActor("actor1", "type1", {}, {port1.name: port1}) actor2 = TestActor("actor2", "type2", {port2.name: port2}, {}) self.storage.add_actor(actor1, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == actor1.id and value[ "value"]["name"] == actor1.name assert value["value"]["name"] == actor1.name assert value["value"]["type"] == actor1._type assert value["value"]["inports"] == [] assert value["value"]["outports"][0]["id"] == port1.id value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == port1.id assert value["value"]["name"] == port1.name assert value["value"]["direction"] == port1.direction assert value["value"]["peers"] == [["local", port2.id]] self.storage.add_actor(actor2, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == actor2.id assert value["value"]["name"] == actor2.name assert value["value"]["type"] == actor2._type assert value["value"]["inports"][0]["id"] == port2.id assert value["value"]["outports"] == [] value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == port2.id assert value["value"]["name"] == port2.name assert value["value"]["direction"] == port2.direction assert value["value"]["peer"] == ["local", port1.id] self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert actor1.id not in self.storage.localstore self.storage.delete_port(port1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert port1.id not in self.storage.localstore
def test_actor_and_port_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = TestPort("out", "out") port2 = TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor1 = TestActor("actor1", "type1", {}, {port1.name: port1}) actor2 = TestActor("actor2", "type2", {port2.name: port2}, {}) self.storage.add_actor(actor1, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == actor1.id and value[ "value"]["name"] == actor1.name assert value["value"]["name"] == actor1.name assert value["value"]["type"] == actor1._type assert value["value"]["inports"] == [] assert value["value"]["outports"][0]["id"] == port1.id value = self.storage.get_port(port1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == port1.id assert value["value"]["name"] == port1.name assert value["value"]["properties"]["direction"] == port1.direction assert value["value"]["peers"] == [["local", port2.id]] self.storage.add_actor(actor2, calvinuuid.uuid("NODE")) value = self.storage.get_actor(actor2.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == actor2.id assert value["value"]["name"] == actor2.name assert value["value"]["type"] == actor2._type assert value["value"]["inports"][0]["id"] == port2.id assert value["value"]["outports"] == [] value = self.storage.get_port(port2.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == port2.id assert value["value"]["name"] == port2.name assert value["value"]["properties"]["direction"] == port2.direction assert value["value"]["peers"] == ["local", port1.id] self.storage.delete_actor(actor1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert actor1.id not in self.storage.localstore self.storage.delete_port(port1.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert port1.id not in self.storage.localstore
def replication_args(self): """Returns args with a name with a random postfix based. The name must not contain '-', this will break the web interface """ args = self._get_managed() args['name'] = calvinuuid.remove_uuid(self.name) + calvinuuid.uuid("REPLICA") args['id'] = calvinuuid.uuid("ACTOR") return args
def replicate(self, actor_id, dst_node_id, callback): try: actor = self.node.am.actors[actor_id] except: if callback: callback(calvinresponse.CalvinResponse(calvinresponse.BAD_REQUEST)) return if not actor._replication_data.is_master(actor.id): # Only replicate master actor if callback: callback(calvinresponse.CalvinResponse(calvinresponse.BAD_REQUEST)) return if actor._replication_data.status != REPLICATION_STATUS.READY: if callback: callback(calvinresponse.CalvinResponse(calvinresponse.SERVICE_UNAVAILABLE)) return _log.analyze(self.node.id, "+", {'actor_id': actor_id, 'dst_node_id': dst_node_id}) actor._replication_data.status = REPLICATION_STATUS.REPLICATING cb_status = CalvinCB(self._replication_status_cb, replication_data=actor._replication_data, cb=callback) # TODO make name a property that combine name and counter in actor new_id = uuid("ACTOR") actor._replication_data.check_instances = time.time() actor._replication_data.add_replica(new_id) new_name = actor.name + "/{}".format(actor._replication_data.counter) actor_type = actor._type ports = actor.connections(self.node.id) ports['actor_name'] = new_name ports['actor_id'] = new_id remap_ports = {pid: uuid("PORT") for pid in ports['inports'].keys() + ports['outports'].keys()} actor._replication_data.set_remaped_ports(new_id, remap_ports, ports) ports['inports'] = {remap_ports[pid]: v for pid, v in ports['inports'].items()} ports['outports'] = {remap_ports[pid]: v for pid, v in ports['outports'].items()} _log.analyze(self.node.id, "+ GET STATE", remap_ports) state = actor.state(remap_ports) state['_name'] = new_name state['_id'] = new_id # Make copy to make sure no objects are shared between actors or master actor state is changed state = copy.deepcopy(state) actor.will_replicate(ActorState(state, actor._replication_data)) if dst_node_id == self.node.id: # Make copy to make sure no objects are shared between actors ports = copy.deepcopy(ports) self.node.am.new_from_migration( actor_type, state=state, prev_connections=ports, callback=CalvinCB( self._replicated, replication_id=actor._replication_data.id, actor_id=new_id, callback=cb_status, master_id=actor.id, dst_node_id=dst_node_id)) else: self.node.proto.actor_new( dst_node_id, CalvinCB(self._replicated, replication_id=actor._replication_data.id, actor_id=new_id, callback=cb_status, master_id=actor.id, dst_node_id=dst_node_id), actor_type, state, ports)
def testGlobalIndex(self): time.sleep(2) lindex = {} lindex['Harald'] = [calvinuuid.uuid("NODE") for i in range(1,5)] lindex['Per'] = [calvinuuid.uuid("NODE") for i in range(1,5)] common = calvinuuid.uuid("NODE") for n, node_ids in lindex.items(): for id_ in node_ids: #print "ADD", n, id_ utils.add_index(self.rt1, "node/affiliation/owner/com.ericsson/" + n, id_) h = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert(set(h['result']) == set(lindex["Harald"])) assert(set(p['result']) == set(lindex["Per"])) assert(set(e['result']) == set(lindex["Per"] + lindex["Harald"])) for n, node_ids in lindex.items(): utils.remove_index(self.rt1, "node/affiliation/owner/com.ericsson/" + n, node_ids[0]) h = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert(set(h['result']) == set(lindex["Harald"][1:])) assert(set(p['result']) == set(lindex["Per"][1:])) assert(set(e['result']) == set(lindex["Per"][1:] + lindex["Harald"][1:])) utils.add_index(self.rt1, "node/affiliation/owner/com.ericsson/Harald", common) utils.add_index(self.rt1, "node/affiliation/owner/com.ericsson/Per", common) h = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert(set(h['result']) == set(lindex["Harald"][1:] + [common])) assert(set(p['result']) == set(lindex["Per"][1:] + [common])) assert(set(e['result']) == set(lindex["Per"][1:] + lindex["Harald"][1:] + [common])) for node_id in lindex['Harald']: utils.remove_index(self.rt1, "node/affiliation/owner/com.ericsson/Harald", node_id) h = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = utils.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert(set(h['result']) == set([common])) assert(set(p['result']) == set(lindex["Per"][1:] + [common])) assert(set(e['result']) == set(lindex["Per"][1:] + [common]))
def testGlobalIndex(self): time.sleep(2) lindex = {} lindex["Harald"] = [calvinuuid.uuid("NODE") for i in range(1, 5)] lindex["Per"] = [calvinuuid.uuid("NODE") for i in range(1, 5)] common = calvinuuid.uuid("NODE") for n, node_ids in lindex.items(): for id_ in node_ids: # print "ADD", n, id_ request_handler.add_index(self.rt1, "node/affiliation/owner/com.ericsson/" + n, id_) h = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert set(h["result"]) == set(lindex["Harald"]) assert set(p["result"]) == set(lindex["Per"]) assert set(e["result"]) == set(lindex["Per"] + lindex["Harald"]) for n, node_ids in lindex.items(): request_handler.remove_index(self.rt1, "node/affiliation/owner/com.ericsson/" + n, node_ids[0]) h = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert set(h["result"]) == set(lindex["Harald"][1:]) assert set(p["result"]) == set(lindex["Per"][1:]) assert set(e["result"]) == set(lindex["Per"][1:] + lindex["Harald"][1:]) request_handler.add_index(self.rt1, "node/affiliation/owner/com.ericsson/Harald", common) request_handler.add_index(self.rt1, "node/affiliation/owner/com.ericsson/Per", common) h = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert set(h["result"]) == set(lindex["Harald"][1:] + [common]) assert set(p["result"]) == set(lindex["Per"][1:] + [common]) assert set(e["result"]) == set(lindex["Per"][1:] + lindex["Harald"][1:] + [common]) for node_id in lindex["Harald"]: request_handler.remove_index(self.rt1, "node/affiliation/owner/com.ericsson/Harald", node_id) h = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Harald") p = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson/Per") e = request_handler.get_index(self.rt2, "node/affiliation/owner/com.ericsson") assert set(h["result"]) == set([common]) assert set(p["result"]) == set(lindex["Per"][1:] + [common]) assert set(e["result"]) == set(lindex["Per"][1:] + [common])
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) yield threads.defer_to_thread(time.sleep, 2) port1 = TestPort("out", "out") port2 = TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor = TestActor("actor1", "type1", {}, {port1.name: port1}) yield threads.defer_to_thread(self.storage.add_actor, actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"]["name"] == actor.name yield threads.defer_to_thread(self.storage.delete_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is None
def __init__(self, name, type, inports, outports): self.id = calvinuuid.uuid("ACTOR") self.name = name self._type = type self.inports = inports self.outports = outports self._replication_id = _DummyRepSet()
def __init__(self): self.id = calvinuuid.uuid("NODE") self.control_uri = "http://localhost:5001" self.pm = Mock() self.storage = Mock() self.control = Mock() self.attributes = attribute_resolver.AttributeResolver({})
def __init__(self, links, tunnels, peer_node_id, tunnel_type, policy, rt_id=None, id=None): """ links: the calvin networks dictionary of links peer_node_id: the id of the peer that we use tunnel_type: what is the usage of the tunnel policy: TODO not used currently id: Tunnel objects on both nodes will use the same id number hence only supply if provided from other side """ super(CalvinTunnel, self).__init__() # The tunnel only use one link (links[peer_node_id]) at a time but it can switch at any point self.links = links self.tunnels = tunnels self.peer_node_id = peer_node_id self.tunnel_type = tunnel_type self.policy = policy self.rt_id = rt_id # id may change while status is PENDING, but is fixed in WORKING self.id = id if id else calvinuuid.uuid("TUNNEL") # If id supplied then we must be the second end and hence working self.status = CalvinTunnel.STATUS.WORKING if id else CalvinTunnel.STATUS.PENDING # Add the tunnel to the dictionary if self.peer_node_id: if self.peer_node_id in self.tunnels: self.tunnels[self.peer_node_id][self.id]=self else: self.tunnels[self.peer_node_id] = {self.id: self} # The callbacks recv for incoming message, down for tunnel failed or died, up for tunnel working self.recv_handler = None self.down_handler = None self.up_handler = None
def __init__(self): self.id = calvinuuid.uuid("NODE") self.control_uri = "http://localhost:5001" self.pm = Mock() self.storage = Mock() self.control = Mock() self.attributes = attribute_resolver.AttributeResolver({})
def __init__(self, actor_type, name='', allow_invalid_transitions=True, disable_transition_checks=False, disable_state_checks=False, actor_id=None, app_id=""): """Should _not_ be overridden in subclasses.""" super(Actor, self).__init__() self._type = actor_type self.name = name # optional: human_readable_name self.id = actor_id or calvinuuid.uuid("ACTOR") self.master_nodes = [] _log.debug("New actor id: %s, supplied actor id %s" % (self.id, actor_id)) self.app_id = app_id self._deployment_requirements = [] self._signature = None self._component_members = set([self.id]) # We are only part of component if this is extended self._managed = set(('id', 'name', '_deployment_requirements', '_signature', 'master_nodes')) self._calvinsys = None self._using = {} self.control = calvincontrol.get_calvincontrol() self.metering = metering.get_metering() self._migrating_to = None # During migration while on the previous node set to the next node id self._last_time_warning = 0.0 self.inports = {p: actorport.InPort(p, self) for p in self.inport_names} self.outports = {p: actorport.OutPort(p, self) for p in self.outport_names} hooks = { (Actor.STATUS.PENDING, Actor.STATUS.ENABLED): self.will_start, (Actor.STATUS.ENABLED, Actor.STATUS.PENDING): self.will_stop, } self.fsm = Actor.FSM(Actor.STATUS, Actor.STATUS.LOADED, Actor.VALID_TRANSITIONS, hooks, allow_invalid_transitions=allow_invalid_transitions, disable_transition_checks=disable_transition_checks, disable_state_checks=disable_state_checks) self.metering.add_actor_info(self)
def __init__(self, uri, control_uri, attributes=None): super(Node, self).__init__() self.uri = uri self.control_uri = control_uri try: self.attributes = AttributeResolver(attributes) except: _log.exception("Attributes not correct, uses empty attribute!") self.attributes = AttributeResolver(None) self.id = calvinuuid.uuid("NODE") self.monitor = Event_Monitor() self.am = actormanager.ActorManager(self) self.control = calvincontrol.get_calvincontrol() _scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler self.sched = _scheduler(self, self.am, self.monitor) self.control.start(node=self, uri=control_uri) self.async_msg_ids = {} self._calvinsys = CalvinSys(self) # Default will multicast and listen on all interfaces # TODO: be able to specify the interfaces # @TODO: Store capabilities self.storage = storage.Storage(self) self.network = CalvinNetwork(self) self.proto = CalvinProto(self, self.network) self.pm = PortManager(self, self.proto) self.app_manager = appmanager.AppManager(self) # The initialization that requires the main loop operating is deferred to start function async.DelayedCall(0, self.start)
def _set_state(self, state): """Set port state.""" self.name = state.get('name') self.id = state.get('id', calvinuuid.uuid("PORT")) self.properties.update(state.get('properties', {})) if 'queue' in state: self.queue._set_state(state.get('queue'))
def __init__(self, actor_type, name='', allow_invalid_transitions=True, disable_transition_checks=False, disable_state_checks=False, actor_id=None): """Should _not_ be overridden in subclasses.""" super(Actor, self).__init__() self._type = actor_type self.name = name # optional: human_readable_name self.id = actor_id or calvinuuid.uuid("ACTOR") self._deployment_requirements = [] self._managed = set(('id', 'name', '_deployment_requirements')) self._calvinsys = None self._using = {} self.control = calvincontrol.get_calvincontrol() self._migrating_to = None # During migration while on the previous node set to the next node id self.inports = {p: actorport.InPort(p, self) for p in self.inport_names} self.outports = {p: actorport.OutPort(p, self) for p in self.outport_names} hooks = { (Actor.STATUS.PENDING, Actor.STATUS.ENABLED): self.will_start, (Actor.STATUS.ENABLED, Actor.STATUS.PENDING): self.will_stop, } self.fsm = Actor.FSM(Actor.STATUS, Actor.STATUS.LOADED, Actor.VALID_TRANSITIONS, hooks, allow_invalid_transitions=allow_invalid_transitions, disable_transition_checks=disable_transition_checks, disable_state_checks=disable_state_checks)
def __init__(self, name, direction): self.id = calvinuuid.uuid("PORT") self.name = name self.direction = direction self.peers = None self.properties = {} self.queue = FanoutFIFO({'queue_length': 4, 'direction': direction}, {})
def __init__(self, actor_type, name='', allow_invalid_transitions=True, disable_transition_checks=False, disable_state_checks=False, actor_id=None, security=None): """Should _not_ be overridden in subclasses.""" super(Actor, self).__init__() self._type = actor_type self._name = name # optional: human_readable_name self._id = actor_id or calvinuuid.uuid("ACTOR") _log.debug("New actor id: %s, supplied actor id %s" % (self._id, actor_id)) self._deployment_requirements = [] self._port_property_capabilities = None self._signature = None self._component_members = set( [self._id]) # We are only part of component if this is extended self._managed = set( ('_id', '_name', '_has_started', '_deployment_requirements', '_signature', '_subject_attributes', '_migration_info', "_port_property_capabilities", "_replication_data")) self._has_started = False self._calvinsys = None self._using = {} # self.control = calvincontrol.get_calvincontrol() # self.metering = metering.get_metering() self._migration_info = None self._migrating_to = None # During migration while on the previous node set to the next node id self._last_time_warning = 0.0 self.sec = security self._subject_attributes = self.sec.get_subject_attributes( ) if self.sec is not None else None self.authorization_checks = None self._replication_data = ReplicationData(initialize=False) self._exhaust_cb = None self.inports = { p: actorport.InPort(p, self, pp) for p, pp in self.inport_properties.items() } self.outports = { p: actorport.OutPort(p, self, pp) for p, pp in self.outport_properties.items() } hooks = { (Actor.STATUS.PENDING, Actor.STATUS.ENABLED): self._will_start, (Actor.STATUS.ENABLED, Actor.STATUS.PENDING): self.will_stop, } self.fsm = Actor.FSM( Actor.STATUS, Actor.STATUS.LOADED, Actor.VALID_TRANSITIONS, hooks, allow_invalid_transitions=allow_invalid_transitions, disable_transition_checks=disable_transition_checks, disable_state_checks=disable_state_checks)
def __init__(self, name, direction): self.id = calvinuuid.uuid("PORT") self.name = name self.direction = direction self.peers = None self.properties = {} self.queue = FanoutFIFO({'queue_length': 4, 'direction': direction}, {})
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) port1 = calvin.tests.TestPort("out", "out") port2 = calvin.tests.TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peers = [("local", port1.id)] actor = calvin.tests.TestActor("actor1", "type1", {}, {port1.name: port1}) self.storage.add_actor(actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert value["value"]["name"] == actor.name self.storage.delete_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.OK self.storage.get_actor(actor.id, cb=CalvinCB(cb)) yield wait_for(self.q.empty, condition=lambda x: not x()) value = self.q.get(timeout=.001) assert isinstance(value["value"], calvinresponse.CalvinResponse) and value["value"] == calvinresponse.NOT_FOUND
def __init__(self, name, direction): self.id = calvinuuid.uuid("PORT") self.name = name self.direction = direction self.peer = None self.peers = None self.fifo = FIFO(5)
def test_actor_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) yield threads.defer_to_thread(time.sleep, 2) port1 = TestPort("out", "out") port2 = TestPort("in", "in", ) port1.peers = [("local", port2.id)] port2.peer = ("local", port1.id) actor = TestActor("actor1", "type1", {}, {port1.name: port1}) yield threads.defer_to_thread(self.storage.add_actor, actor, calvinuuid.uuid("NODE"), cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"]["name"] == actor.name yield threads.defer_to_thread(self.storage.delete_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is True yield threads.defer_to_thread(self.storage.get_actor, actor.id, cb=CalvinCB(cb)) yield threads.defer_to_thread(time.sleep, 2) value = self.q.get(timeout=0.2) assert value["value"] is None
def __init__(self, uri, control_uri, attributes=None): super(Node, self).__init__() self.uri = uri self.control_uri = control_uri self.attributes = attributes self.id = calvinuuid.uuid("NODE") _log.debug("Calvin init 1") self.monitor = Event_Monitor() _log.debug("Calvin init 2") self.am = actormanager.ActorManager(self) _log.debug("Calvin init 3") self.control = calvincontrol.get_calvincontrol() _log.debug("Calvin init 4") self.sched = scheduler.Scheduler(self, self.am, self.monitor) _log.debug("Calvin init 5") self.control.start(node=self, uri=control_uri) self.async_msg_ids = {} _log.debug("Calvin init 6") self.storage = storage.Storage() self.storage.start() _log.debug("Calvin init 7") self.network = CalvinNetwork(self) _log.debug("Calvin init 8") self.proto = CalvinProto(self, self.network) self.pm = PortManager(self, self.proto) _log.debug("Calvin init 9") self.app_manager = appmanager.AppManager(self) _log.debug("Calvin init 10") # The initialization that requires the main loop operating is deferred to start function async.DelayedCall(0, self.start) _log.debug("Calvin init 11")
def _set_state(self, state): """Set port state.""" self.name = state.get('name') self.id = state.get('id', calvinuuid.uuid("PORT")) self.properties.update(state.get('properties', {})) if 'queue' in state: self.queue._set_state(state.get('queue'))
def __init__(self, name, type, inports, outports): self.id = calvinuuid.uuid("ACTOR") self.name = name self._type = type self.inports = inports self.outports = outports self._replication_id = _DummyRepSet()
def send_with_reply(self, callback, msg): """ Adds a message id to the message and send it, also registers the callback for the reply. """ msg_id = calvinuuid.uuid("MSGID") self.replies[msg_id] = callback msg['msg_uuid'] = msg_id self.send(msg)
def uuids(request): if request.param == "prefixed": return { "trace_id": calvinuuid.uuid("TRACE"), "app_id": calvinuuid.uuid("APP"), "port_id": calvinuuid.uuid("PORT"), "node_id": calvinuuid.uuid("NODE"), "actor_id": calvinuuid.uuid("ACTOR") } elif request.param == "non-prefixed": return { "trace_id": calvinuuid.uuid(""), "app_id": calvinuuid.uuid(""), "port_id": calvinuuid.uuid(""), "node_id": calvinuuid.uuid(""), "actor_id": calvinuuid.uuid("") }
def register(self, user_id=None): if not user_id: user_id = calvinuuid.uuid("METERING") if user_id in self.users: raise Exception("User id already in use") self.users[user_id] = time.time() self.active = True return user_id
def register(self, user_id=None): if not user_id: user_id = calvinuuid.uuid("METERING") if user_id in self.users: raise Exception("User id already in use") self.users[user_id] = time.time() self.active = True return user_id
def uuids(request): if request.param == "prefixed": return { "trace_id": calvinuuid.uuid("TRACE"), "app_id": calvinuuid.uuid("APP"), "port_id": calvinuuid.uuid("PORT"), "node_id": calvinuuid.uuid("NODE"), "actor_id": calvinuuid.uuid("ACTOR") } elif request.param == "non-prefixed": return { "trace_id": calvinuuid.uuid(""), "app_id": calvinuuid.uuid(""), "port_id": calvinuuid.uuid(""), "node_id": calvinuuid.uuid(""), "actor_id": calvinuuid.uuid("") }
def send_with_reply(self, callback, msg): """ Adds a message id to the message and send it, also registers the callback for the reply. """ msg_id = calvinuuid.uuid("MSGID") self.replies[msg_id] = callback msg['msg_uuid'] = msg_id self.send(msg)
def send_with_reply(self, callback, msg): """ Adds a message id to the message and send it, also registers the callback for the reply. """ msg_id = calvinuuid.uuid("MSGID") self.replies[msg_id] = callback self.replies_timeout[msg_id] = async.DelayedCall(10.0, CalvinCB(self.reply_timeout, msg_id)) msg['msg_uuid'] = msg_id self.send(msg)
def _get_node_names(runtimes): from calvin.utilities.attribute_resolver import AttributeResolver from calvin.utilities import calvinuuid from copy import deepcopy for i in range(len(runtimes)): rt_attribute = deepcopy(runtimes[i]["attributes"]) attributes = AttributeResolver(rt_attribute) runtimes[i]["node_name"] = attributes.get_node_name_as_str() runtimes[i]["id"] = calvinuuid.uuid("")
def __init__(self, uris, node_name=None, control_uri=None): self.id = calvinuuid.uuid("NODE") self.node_name = node_name or self.id self.uris = uris self.control_uri = control_uri or uris[0] self.external_control_uri = self.control_uri self.pm = Mock() self.storage = Mock() self.control = Mock() self.attributes = attribute_resolver.AttributeResolver({})
def obtain_cert_node_info(name): """ Obtain node id based on name and domain from config Return dict with domain, node name and node id """ cert_conffile = _conf.get("security", "certificate_conf") domain = _conf.get("security", "certificate_domain") if domain is None or name is None: # No security or name specified just use standard node UUID _log.debug("OBTAINING no security domain={}, name={}".format(domain, name)) return {'domain': None, 'name': name, 'id': calvinuuid.uuid("NODE")} cert_conf = Config(cert_conffile, domain) name_dir = os.path.join(cert_conf.configuration["CA_default"]["runtimes_dir"], name) # Does existing signed runtime certificate exist, return info try: filenames = os.listdir(os.path.join(name_dir, "mine")) content = open(os.path.join(name_dir, "mine", filenames[0]), 'rt').read() cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, content) subject = cert.get_subject() if subject.commonName != name or subject.organizationName != domain: raise _log.debug("OBTAINING existing security domain={}, name={}".format(domain, name)) return {'domain': domain, 'name': name, 'id': subject.dnQualifier} except: pass #_log.exception("OBTAINING fail existing security domain={}, name={}".format(domain, name)) # Create new CSR csrfile = new_runtime(cert_conf, name, nodeid=calvinuuid.uuid("NODE")) _log.debug("OBTAINING new security csr={}, domain={}, name={}".format(csrfile, domain, name)) try: content = open(csrfile, 'rt').read() cert = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, content) subject = cert.get_subject() # TODO multicast signing of CSR, now just sign it assuming local CA sign_req(cert_conf, os.path.basename(csrfile), name) _log.debug("OBTAINING new security domain={}, name={}".format(domain, name)) return {'domain': domain, 'name': name, 'id': subject.dnQualifier} except: #_log.exception("OBTAINING fail new security domain={}, name={}".format(domain, name)) return {'domain': None, 'name': name, 'id': calvinuuid.uuid("NODE")}
def __init__(self, control_uri): super(StorageNode, self).__init__() self.id = calvinuuid.uuid("NODE") self.control_uri = control_uri self.control = calvincontrol.get_calvincontrol() _scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler self.sched = _scheduler(self, FakeAM(), FakeMonitor()) self.control.start(node=self, uri=control_uri) self.storage = storage.Storage(self) async.DelayedCall(0, self.start)
def __init__(self, uris, node_name=None, control_uri=None): self.id = calvinuuid.uuid("NODE") self.node_name = node_name or self.id self.uris = uris self.control_uri = control_uri or uris[0] self.external_control_uri = self.control_uri self.pm = Mock() self.storage = Mock() self.control = Mock() self.attributes = attribute_resolver.AttributeResolver({})
def send_with_reply(self, callback, msg, dest_peer_id=None): """ Adds a message id to the message and send it, also registers the callback for the reply. """ msg_id = calvinuuid.uuid("MSGID") self.replies[msg_id] = {'callback': callback, 'send_time': time.time()} self.replies_timeout[msg_id] = async .DelayedCall( 10.0, CalvinCB(self.reply_timeout, msg_id)) msg['msg_uuid'] = msg_id self.send(msg, dest_peer_id)
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) self.storage.add_application(application) value = self.storage.get_application( application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == application.id and value[ "value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert application.id not in self.storage.localstore
def __init__(self, name, owner, fifo_size=5): super(Port, self).__init__() # Human readable port name self.name = name # Actor instance to which the port belongs (may change over time) self.owner = owner # Unique id to universally identify port (immutable) self.id = calvinuuid.uuid("PORT") # The token queue. Not all scenarios use it, # but needed when e.g. changing from local to remote connection. self.fifo = fifo.FIFO(fifo_size)
def __init__(self, control_uri): super(StorageNode, self).__init__() self.id = calvinuuid.uuid("NODE") self.control_uri = control_uri self.control = calvincontrol.get_calvincontrol() _scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel( ) <= logging.DEBUG else scheduler.Scheduler self.sched = _scheduler(self, FakeAM(), FakeMonitor()) self.control.start(node=self, uri=control_uri) self.storage = storage.Storage(self) async .DelayedCall(0, self.start)
def test_application_functions(self): self.q = Queue.Queue() def cb(key, value): self.q.put({"key": key, "value": value}) application = appmanager.Application(calvinuuid.uuid( 'APP'), "test_app", [calvinuuid.uuid('ACTOR'), calvinuuid.uuid('ACTOR')]) self.storage.add_application(application) value = self.storage.get_application( application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value["key"] == application.id and value[ "value"]["name"] == application.name self.storage.delete_application(application.id, cb=CalvinCB(func=cb)) value = self.q.get(timeout=0.2) assert value assert application.id not in self.storage.localstore
def __init__(self, func, *args, **kwargs): super(CalvinCB, self).__init__() self.id = calvinuuid.uuid("CB") self.func = func self.args = list(args) self.kwargs = kwargs # Ref a functions name if we wrap several CalvinCB and need to take __str__ try: self.name = self.func.__name__ except: self.name = self.func.name if hasattr(self.func, 'name') else "unknown"
def __init__(self, name, owner, fifo_size=5): super(Port, self).__init__() # Human readable port name self.name = name # Actor instance to which the port belongs (may change over time) self.owner = owner # Unique id to universally identify port (immutable) self.id = calvinuuid.uuid("PORT") # The token queue. Not all scenarios use it, # but needed when e.g. changing from local to remote connection. self.fifo = fifo.FIFO(fifo_size)
def __init__(self, func, *args, **kwargs): super(CalvinCB, self).__init__() self._debug_info = get_debug_info() self._id = calvinuuid.uuid("CB") self.func = func self.args = list(args) self.kwargs = kwargs # Ref a functions name if we wrap several CalvinCB and need to take __str__ try: self.name = self.func.__name__ except: self.name = self.func.name if hasattr(self.func, 'name') else "unknown"
def __init__(self, uris, node_name=None, control_uri=None): self.id = calvinuuid.uuid("NODE") self.node_name = node_name or self.id self.uris = uris self.control_uri = control_uri or uris[0] self.external_control_uri = self.control_uri self.pm = Mock() self._calvinsys = DummySys(self) self.storage = Mock() self.control = Mock() self.metering = metering.set_metering(metering.Metering(self)) self.attributes = attribute_resolver.AttributeResolver({})
def __init__(self, name, owner): super(Port, self).__init__() # Human readable port name self.name = name # Actor instance to which the port belongs (may change over time) self.owner = owner # Unique id to universally identify port (immutable) self.id = calvinuuid.uuid("PORT") # The token queue, will be set when connected. self.queue = queue.common.QueueNone() self.properties = {} self.properties['routing'] = 'default' self.properties['nbr_peers'] = 1
def __init__(self, runtime, deployable, node_info=None, node=None): super(Deployer, self).__init__() self.runtime = runtime self.deployable = deployable self.actor_map = {} self.node_info = node_info if node_info else {} self.actor_connections = {} self.app_id = calvinuuid.uuid("APP") self.node = node if "name" in self.deployable: self.name = self.deployable["name"] else: self.name = self.app_id
def manage_runtime_create(args): if args.domain: if not args.attr: raise Exception("No runtime attributes supplied") if not args.domain: raise Exception("No domain name supplied") attr = json.loads(args.attr) if not all (k in attr['indexed_public']['node_name'] for k in ("organization","name")): raise Exception("please supply name and organization of runtime") attributes=AttributeResolver(attr) node_name=attributes.get_node_name_as_str() nodeid = calvinuuid.uuid("NODE") print "CSR created at:" + certificate.new_runtime(node_name, args.domain, security_dir=args.dir, nodeid=nodeid)
def __init__(self, actor_id=None, master=None, requirements=None, initialize=True): super(ReplicationData, self).__init__() self.id = uuid("REPLICATION") if initialize else None self.master = master self.instances = [] if actor_id is None else [actor_id] # TODO requirements should be plugin operation, now just do target number self.requirements = requirements self.counter = 0 # {<actor_id>: {'known_peer_ports': [peer-ports id list], <org-port-id: <replicated-port-id>, ...}, ...} self.remaped_ports = {} self.status = REPLICATION_STATUS.UNUSED self._terminate_with_node = False self._one_per_runtime = False