def _get_node_names(runtimes):
from calvin.utilities.attribute_resolver import AttributeResolver
from copy import deepcopy
for i in range(len(runtimes)):
rt_attribute = deepcopy(runtimes[i]["attributes"])
attributes=AttributeResolver(rt_attribute)
runtimes[i]["node_name"] = attributes.get_node_name_as_str()
def set_proxy_config(peer_id, name, capabilities, port_property_capability, storage, callback):
"""
Store node
"""
try:
for c in list_port_property_capabilities(which=port_property_capability):
storage.add_index(['node', 'capabilities', c], peer_id, root_prefix_level=3)
for c in capabilities:
storage.add_index(['node', 'capabilities', c], peer_id, root_prefix_level=3)
except:
_log.error("Failed to set capabilities")
attributes = AttributeResolver({"indexed_public": {"node_name": {"name": name}}})
indexes = attributes.get_indexed_public()
try:
for index in indexes:
storage.add_index(index, peer_id)
except:
_log.error("Failed to add node index")
storage.set(prefix="node-", key=peer_id,
value={"proxy": storage.node.id,
"uris": None,
"control_uris": None,
"authz_server": None, # Set correct value
"attributes": {'public': attributes.get_public(),
'indexed_public': attributes.get_indexed_public(as_list=False)}},
cb=CalvinCB(set_proxy_config_cb, callback=callback))
def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.id = calvinuuid.uuid("NODE")
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.control.start(node=self, uri=control_uri)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def __init__(self, node, peer_id, attributes, capabilities,
port_property_capability):
self.node = node
self.id = peer_id
self.attributes = AttributeResolver(json.loads(attributes))
self.capabilities = capabilities
self.port_property_capability = port_property_capability
def _get_node_names(runtimes):
from calvin.utilities.attribute_resolver import AttributeResolver
from copy import deepcopy
for i in range(len(runtimes)):
rt_attribute = deepcopy(runtimes[i]["attributes"])
attributes = AttributeResolver(rt_attribute)
runtimes[i]["node_name"] = attributes.get_node_name_as_str()
class PeerNode(object):
def __init__(self, node, peer_id, attributes, capabilities,
port_property_capability):
self.node = node
self.id = peer_id
self.attributes = AttributeResolver(json.loads(attributes))
self.capabilities = capabilities
self.port_property_capability = port_property_capability
def add_node(self, cb):
try:
for c in list_port_property_capabilities(
which=self.port_property_capability):
self.node.storage.add_index(['node', 'capabilities', c],
self.id,
root_prefix_level=3)
for c in self.capabilities:
self.node.storage.add_index(['node', 'capabilities', c],
self.id,
root_prefix_level=3)
except Exception as e:
_log.error("Failed to set capabilities %s" % e)
public = None
indexed_public = None
indexes = self.attributes.get_indexed_public()
for index in indexes:
self.node.storage.add_index(index, self.id)
public = self.attributes.get_public()
indexed_public = self.attributes.get_indexed_public(as_list=False)
self.node.storage.set(prefix="node-",
key=self.id,
value={
"proxy": self.node.id,
"uris": None,
"control_uris": None,
"attributes": {
'public': public,
'indexed_public': indexed_public
}
},
cb=cb)
def remove_node(self):
self.node.storage.delete(prefix="node-", key=self.id, cb=None)
try:
for c in list_port_property_capabilities(
which=self.port_property_capability):
self.node.storage.remove_index(['node', 'capabilities', c],
self.id)
for c in self.capabilities:
self.node.storage.remove_index(['node', 'capabilities', c],
self.id)
for index in self.attributes.get_indexed_public():
self.node.storage.remove_index(index, self.id)
except Exception as e:
_log.error("Failed to remove index %s" % e)
def test_cpu_total_invalid_value(self):
"""
Tests invalid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "2"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att.get_indexed_public()[0], '/node/attribute/cpuTotal')
def test_mem_avail_invalid_value(self):
"""
Tests invalid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memAvail": "1"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memAvail')
self.assertEqual(att.get_indexed_public()[0], '/node/resource/memAvail')
def test_cpu_affinity(self):
"""
Tests cpu affinity parameter in indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuAffinity": "dedicated"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAffinity')
self.assertEqual(att_list[0][3], 'dedicated')
self.assertEqual(att.get_indexed_public()[0], '/node/attribute/cpuAffinity/dedicated')
def test_cpu_total_invalid_value(self):
"""
Tests invalid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "2"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/cpuTotal')
def test_mem_avail_invalid_value(self):
"""
Tests invalid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memAvail": "1"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memAvail')
self.assertEqual(att.get_indexed_public()[0],
'/node/resource/memAvail')
def manage_runtime_create(args):
if args.domain:
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
attr = json.loads(args.attr)
if not all (k in attr['indexed_public']['node_name'] for k in ("organization","name")):
raise Exception("please supply name and organization of runtime")
attributes=AttributeResolver(attr)
node_name=attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
print "CSR created at:" + certificate.new_runtime(node_name, args.domain, security_dir=args.dir, nodeid=nodeid)
def test_cpu_resources(self):
"""
Tests valid cpu resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuAvail": "100"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAvail')
self.assertEqual(att_list[0][3], '0')
self.assertEqual(att_list[0][4], '25')
self.assertEqual(att_list[0][5], '50')
self.assertEqual(att_list[0][6], '75')
self.assertEqual(att_list[0][7], '100')
self.assertEqual(att.get_indexed_public()[0], '/node/resource/cpuAvail/0/25/50/75/100')
def test_cpu_affinity(self):
"""
Tests cpu affinity parameter in indexed_public field
"""
att = AttributeResolver(
{"indexed_public": {
"cpuAffinity": "dedicated"
}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAffinity')
self.assertEqual(att_list[0][3], 'dedicated')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/cpuAffinity/dedicated')
def test_cpu_total(self):
"""
Tests valid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "10000000"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att_list[0][3], '1')
self.assertEqual(att_list[0][4], '1000')
self.assertEqual(att_list[0][5], '100000')
self.assertEqual(att_list[0][6], '1000000')
self.assertEqual(att_list[0][7], '10000000')
self.assertEqual(att.get_indexed_public()[0], '/node/attribute/cpuTotal/1/1000/100000/1000000/10000000')
def manage_runtime_create(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
if args.hostnames and len(args.hostnames)>4:
raise Exception("At most 3 hostnames can be supplied")
attr = json.loads(args.attr)
if not all (k in attr['indexed_public']['node_name'] for k in ("organization","name")):
raise Exception("please supply name and organization of runtime")
attributes=AttributeResolver(attr)
node_name=attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
rt_cred = runtime_credentials.RuntimeCredentials(node_name, domain=args.domain, security_dir=args.dir, nodeid=nodeid, hostnames=args.hostnames)
print "node_name_start<{}>node_name_stop".format(rt_cred.get_node_name())
def test_mem_total(self):
"""
Tests valid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memTotal": "10G"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memTotal')
self.assertEqual(att_list[0][3], '1K')
self.assertEqual(att_list[0][4], '100K')
self.assertEqual(att_list[0][5], '1M')
self.assertEqual(att_list[0][6], '100M')
self.assertEqual(att_list[0][7], '1G')
self.assertEqual(att_list[0][8], '10G')
self.assertEqual(att.get_indexed_public()[0], '/node/attribute/memTotal/1K/100K/1M/100M/1G/10G')
def test_cpu_resources(self):
"""
Tests valid cpu resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuAvail": "100"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAvail')
self.assertEqual(att_list[0][3], '0')
self.assertEqual(att_list[0][4], '25')
self.assertEqual(att_list[0][5], '50')
self.assertEqual(att_list[0][6], '75')
self.assertEqual(att_list[0][7], '100')
self.assertEqual(att.get_indexed_public()[0],
'/node/resource/cpuAvail/0/25/50/75/100')
def __init__(self, uri, control_uri, attributes=None, self_start=True):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.id = calvinuuid.uuid("NODE")
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
hb_timeout = _conf.get('global', 'heartbeat_timeout') or DEFAULT_HEARTBEAT_TIMEOUT
self.heartbeat_timeout = float(hb_timeout)
hb_delay = _conf.get('global', 'heartbeat_delay') or DEFAULT_HEARTBEAT_DELAY
self.heartbeat_delay = float(hb_delay)
self.heartbeat_addr = self._clean_addr()
self.heartbeat_port = _conf.get('global', 'heartbeat_port') or int(self._clean_uri().split(":")[1]) + DEFAULT_HEARTBEAT_PORT_DIFF
rr_delay = _conf.get('global', 'resource_reporter_delay') or 0.25
self.resource_reporter_delay = float(rr_delay)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
self.resource_manager = ResourceManager()
self.app_monitor = AppMonitor(self, self.app_manager, self.storage)
self.lost_node_handler = LostNodeHandler(self, self.resource_manager, self.pm, self.am, self.storage)
self.heartbeat_actor = None
self.outgoing_heartbeats = defaultdict(list)
# The initialization that requires the main loop operating is deferred to start function
if self_start:
async.DelayedCall(0, self.start)
def test_cpu_total(self):
"""
Tests valid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "10000000"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att_list[0][3], '1')
self.assertEqual(att_list[0][4], '1000')
self.assertEqual(att_list[0][5], '100000')
self.assertEqual(att_list[0][6], '1000000')
self.assertEqual(att_list[0][7], '10000000')
self.assertEqual(
att.get_indexed_public()[0],
'/node/attribute/cpuTotal/1/1000/100000/1000000/10000000')
def test_mem_total(self):
"""
Tests valid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memTotal": "10G"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memTotal')
self.assertEqual(att_list[0][3], '1K')
self.assertEqual(att_list[0][4], '100K')
self.assertEqual(att_list[0][5], '1M')
self.assertEqual(att_list[0][6], '100M')
self.assertEqual(att_list[0][7], '1G')
self.assertEqual(att_list[0][8], '10G')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/memTotal/1K/100K/1M/100M/1G/10G')
def test_total_valid(self):
"""
Test valid values for total RAM.
Verify if storage is as expected
"""
values = ["1K", "100K", "1M", "100M", "1G", "10G"]
for i in values:
# verify set return
self.done = False
self.node.attributes = AttributeResolver(
{"indexed_public": {
"memTotal": i
}})
self.storage.add_node(self.node, cb=self.cb)
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE +
map(str, values[:values.index(i) + 1]),
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
self.external_uri = attributes.pop('external_uri', self.uri) \
if attributes else self.uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
self.id = certificate.obtain_cert_node_info(self.node_name)['id']
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
try:
self.domain = _conf.get("security", "security_domain_name")
# cert_name is the node's certificate filename (without file extension)
self.cert_name = certificate.get_own_cert_name(self.node_name)
except:
self.domain = None
self.cert_name = None
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
class PeerNode(object):
def __init__(self, node, peer_id, attributes, capabilities, port_property_capability):
self.node = node
self.id = peer_id
self.attributes = AttributeResolver(json.loads(attributes))
self.capabilities = capabilities
self.port_property_capability = port_property_capability
def add_node(self, cb):
try:
for c in list_port_property_capabilities(which=self.port_property_capability):
self.node.storage.add_index(['node', 'capabilities', c], self.id, root_prefix_level=3)
for c in self.capabilities:
self.node.storage.add_index(['node', 'capabilities', c], self.id, root_prefix_level=3)
except Exception as e:
_log.error("Failed to set capabilities %s" % e)
public = None
indexed_public = None
indexes = self.attributes.get_indexed_public()
for index in indexes:
self.node.storage.add_index(index, self.id)
public = self.attributes.get_public()
indexed_public = self.attributes.get_indexed_public(as_list=False)
self.node.storage.set(prefix="node-", key=self.id,
value={"proxy": self.node.id,
"uris": None,
"control_uris": None,
"attributes": {'public': public,
'indexed_public': indexed_public}},
cb=cb)
def remove_node(self):
self.node.storage.delete(prefix="node-", key=self.id, cb=None)
try:
for c in list_port_property_capabilities(which=self.port_property_capability):
self.node.storage.remove_index(['node', 'capabilities', c], self.id)
for c in self.capabilities:
self.node.storage.remove_index(['node', 'capabilities', c], self.id)
for index in self.attributes.get_indexed_public():
self.node.storage.remove_index(index, self.id)
except Exception as e:
_log.error("Failed to remove index %s" % e)
def manage_runtime_create(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
if args.hostnames and len(args.hostnames) > 4:
raise Exception("At most 3 hostnames can be supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
rt_cred = runtime_credentials.RuntimeCredentials(node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
hostnames=args.hostnames)
print "node_name_start<{}>node_name_stop".format(rt_cred.get_node_name())
def setup(self):
_conf.set('global', 'storage_type', 'local')
self.node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.node.attributes = AttributeResolver(
{"indexed_public": {
"cpuTotal": "1"
}})
self.storage = storage.Storage(self.node)
self.cpu = CpuMonitor(self.node.id, self.storage)
self.done = False
self.storage.add_node(self.node)
yield threads.defer_to_thread(time.sleep, .01)
def manage_runtime_do_it_all(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
ca = certificate_authority.CA(domain=args.domain,
commonName=args.domain + " CA",
security_dir=args.dir)
ca_cert_path = ca.export_ca_cert("/tmp")
certificate.store_trusted_root_cert(ca_cert_path,
certificate.TRUSTSTORE_TRANSPORT,
security_dir=args.dir)
os.remove(ca_cert_path)
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
rt_cred = runtime_credentials.RuntimeCredentials(
node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
ca_cert = rt_cred.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr_path = rt_cred.get_encrypted_csr_path()
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request_path=rsa_encrypted_csr_path)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
print "\ncertificate stored at: {}\n".format(
rt_cred.store_own_cert(certpath=cert_path, security_dir=args.dir))
def manage_runtime_get_name(args):
from calvin.utilities.attribute_resolver import AttributeResolver
# Attributes
runtime_attr = {}
if args.attr_file:
try:
runtime_attr = json.load(open(args.attr_file))
except Exception as e:
print "Attribute file not JSON:\n", e
return -1
elif args.attr:
try:
runtime_attr = json.loads(args.attr)
except Exception as e:
print "Attributes not JSON:\n", e
return -1
else:
print "Error, either supply the attributes of the runtime, or the path to the file containg the attributes"
return -1
attributes = AttributeResolver(runtime_attr)
print "node_name_start<{}>node_name_stop\n".format(attributes.get_node_name_as_str())
def _set_aux(self, key, value, prefix_index, new_value=None):
"""
Auxiliary method to set indexes .
Removes old indexes before adding the new ones. Triggered by a get in the database
"""
# if new value is exactly the same, we don't need to change anything..
if value is new_value:
_log.debug("%s, value: %s. Nothing changed, just return.." % (prefix_index, value))
return
# erase indexes related to old value
if value is not None:
old_data = AttributeResolver({"indexed_public": {prefix_index: str(value)}})
_log.debug("Removing " + str(key) + " for " + prefix_index + ": " + str(value))
for index in old_data.get_indexed_public():
self.storage.remove_index(index=index, value=key, root_prefix_level=2)
# insert the new ones
if new_value is not None:
new_data = AttributeResolver({"indexed_public": {prefix_index: str(new_value)}})
_log.debug("After possible removal, adding new node " + str(self.node_id) + " for " + prefix_index + ": " + str(new_value))
for index in new_data.get_indexed_public():
self.storage.add_index(index=index, value=self.node_id, root_prefix_level=2, cb=None)
def manage_runtime_get_name(args):
from calvin.utilities.attribute_resolver import AttributeResolver
# Attributes
runtime_attr = {}
if args.attr_file:
try:
runtime_attr = json.load(open(args.attr_file))
except Exception as e:
print "Attribute file not JSON:\n", e
return -1
elif args.attr:
try:
runtime_attr = json.loads(args.attr)
except Exception as e:
print "Attributes not JSON:\n", e
return -1
else:
print "Error, either supply the attributes of the runtime, or the path to the file containg the attributes"
return -1
attributes = AttributeResolver(runtime_attr)
print "node_name_start<{}>node_name_stop\n".format(
attributes.get_node_name_as_str())
def _set_aux(self, key, value, prefix_index, new_value=None):
"""
Auxiliary method to set indexes .
Removes old indexes before adding the new ones. Triggered by a get in the database
"""
# if new value is exactly the same, we don't need to change anything..
if value is new_value:
_log.debug("%s, value: %s. Nothing changed, just return.." %
(prefix_index, value))
return
# erase indexes related to old value
if value is not None:
old_data = AttributeResolver(
{"indexed_public": {
prefix_index: str(value)
}})
_log.debug("Removing " + str(key) + " for " + prefix_index + ": " +
str(value))
for index in old_data.get_indexed_public():
self.storage.remove_index(index=index,
value=key,
root_prefix_level=2)
# insert the new ones
if new_value is not None:
new_data = AttributeResolver(
{"indexed_public": {
prefix_index: str(new_value)
}})
_log.debug("After possible removal, adding new node " +
str(self.node_id) + " for " + prefix_index + ": " +
str(new_value))
for index in new_data.get_indexed_public():
self.storage.add_index(index=index,
value=self.node_id,
root_prefix_level=2,
cb=None)
def __init__(self, uris, control_uri, attributes=None):
super(Node, self).__init__()
self.quitting = False
# Warn if its not a uri
if not isinstance(uris, list):
_log.error("Calvin uris must be a list %s" % uris)
raise TypeError("Calvin uris must be a list!")
# Uris
self.uris = uris
if attributes:
ext_uris = attributes.pop('external_uri', None)
if ext_uris is not None:
self.uris += ext_uris
# Control uri
self.control_uri = control_uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
try:
security_dir = _conf.get("security", "security_dir")
self.runtime_credentials = RuntimeCredentials(self.node_name, node=self, security_dir=security_dir)
self.id = self.runtime_credentials.get_node_id()
except Exception as err:
_log.debug("No runtime credentials, err={}".format(err))
self.runtime_credentials = None
self.id = calvinuuid.uuid("Node")
self.certificate_authority = certificate_authority.CertificateAuthority(self)
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.rm = replicationmanager.ReplicationManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
calvinsys = get_calvinsys()
calvinsys.init(self)
calvinlib = get_calvinlib()
calvinlib.init(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
"""
def __init__(self, uris, control_uri, attributes=None):
super(Node, self).__init__()
self.quitting = False
# Warn if its not a uri
if not isinstance(uris, list):
_log.error("Calvin uris must be a list %s" % uris)
raise TypeError("Calvin uris must be a list!")
# Uris
self.uris = uris
if attributes:
ext_uris = attributes.pop('external_uri', None)
if ext_uris is not None:
self.uris += ext_uris
# Control uri
self.control_uri = control_uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
try:
security_dir = _conf.get("security", "security_dir")
self.runtime_credentials = RuntimeCredentials(self.node_name, node=self, security_dir=security_dir)
self.id = self.runtime_credentials.get_node_id()
except Exception as err:
_log.debug("No runtime credentials, err={}".format(err))
self.runtime_credentials = None
self.id = calvinuuid.uuid("Node")
self.certificate_authority = certificate_authority.CertificateAuthority(self)
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.rm = replicationmanager.ReplicationManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
calvinsys = get_calvinsys()
calvinsys.init(self)
calvinlib = get_calvinlib()
calvinlib.init(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def insert_local_reply(self):
msg_id = calvinuuid.uuid("LMSG")
self.async_msg_ids[msg_id] = None
return msg_id
def set_local_reply(self, msg_id, reply):
if msg_id in self.async_msg_ids:
self.async_msg_ids[msg_id] = reply
def connect(self, actor_id=None, port_name=None, port_dir=None, port_properties=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_properties=None, peer_port_id=None, cb=None):
if port_properties is None and port_dir is not None:
port_properties = {'direction': port_dir}
if peer_port_properties is None and peer_port_dir is not None:
peer_port_properties = {'direction': peer_port_dir}
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_properties=port_properties,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_properties=peer_port_properties,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def peersetup(self, peers, cb=None):
""" Sets up a RT to RT communication channel, only needed if the peer can't be found in storage.
peers: a list of peer uris, e.g. ["calvinip://127.0.0.1:5001"]
"""
_log.debug("peersetup(%s)" % (peers))
peers_copy = peers[:]
peer_node_ids = {}
if not cb:
callback = CalvinCB(self.logging_callback, preamble="peersetup cb")
else:
callback = CalvinCB(self.peersetup_collect_cb, peers=peers_copy, peer_node_ids=peer_node_ids, org_cb=cb)
self.network.join(peers, callback=callback)
def peersetup_collect_cb(self, status, uri, peer_node_id, peer_node_ids, peers, org_cb):
if uri in peers:
peers.remove(uri)
peer_node_ids[uri] = (peer_node_id, status)
if not peers:
# Get highest status, i.e. any error
comb_status = max([s for _, s in peer_node_ids.values()])
org_cb(peer_node_ids=peer_node_ids, status=comb_status)
def logging_callback(self, preamble=None, *args, **kwargs):
_log.debug("\n%s# NODE: %s \n# %s %s %s \n%s" %
('#' * 40, self.id, preamble if preamble else "*", args, kwargs, '#' * 40))
def new(self, actor_type, args, deploy_args=None, state=None, prev_connections=None, connection_list=None):
# TODO requirements should be input to am.new
# TODO: make it possible to use security/credentials here.
actor_def, signer = self.am.lookup_and_verify(actor_type)
actor_id = self.am.new(actor_type, args, state, prev_connections, connection_list,
signature=deploy_args['signature'] if deploy_args and 'signature' in deploy_args else None,
actor_def=actor_def)
if deploy_args:
app_id = deploy_args['app_id']
if 'app_name' not in deploy_args:
app_name = app_id
else:
app_name = deploy_args['app_name']
self.app_manager.add(app_id, actor_id,
deploy_info = deploy_args['deploy_info'] if 'deploy_info' in deploy_args else None)
return actor_id
def calvinsys(self):
"""Return a CalvinSys instance"""
# FIXME: We still need to sort out actor requirements vs. node capabilities and user permissions.
# @TODO: Write node capabilities to storage
return self._calvinsys
#
# Event loop
#
def run(self):
"""main loop on node"""
_log.debug("Node %s is running" % self.id)
self.sched.run()
def start(self):
""" Run once when main loop is started """
interfaces = _conf.get(None, 'transports')
self.network.register(interfaces, ['json'])
self.network.start_listeners(self.uris)
# Start storage after network, proto etc since storage proxy expects them
self.storage.start(cb=CalvinCB(self._storage_started_cb))
self.storage.add_node(self)
# Start control API
proxy_control_uri = _conf.get(None, 'control_proxy')
_log.debug("Start control API on %s with uri: %s and proxy: %s" % (self.id, self.control_uri, proxy_control_uri))
if proxy_control_uri is not None:
self.control.start(node=self, uri=proxy_control_uri, tunnel=True)
else:
if self.control_uri is not None:
self.control.start(node=self, uri=self.control_uri, external_uri=self.external_control_uri)
def stop(self, callback=None):
# TODO: Handle blocking in poorly implemented calvinsys/runtime south.
self.quitting = True
def stopped(*args):
_log.analyze(self.id, "+", {'args': args})
_log.debug(args)
self.sched.stop()
_log.analyze(self.id, "+ SCHED STOPPED", {'args': args})
self.control.stop()
_log.analyze(self.id, "+ CONTROL STOPPED", {'args': args})
def deleted_node(*args, **kwargs):
_log.analyze(self.id, "+", {'args': args, 'kwargs': kwargs})
self.storage.stop(stopped)
_log.analyze(self.id, "+", {})
self.storage.delete_node(self, cb=deleted_node)
for link in self.network.list_direct_links():
self.network.link_get(link).close()
def stop_with_cleanup(self):
# Set timeout in case some actor is refusing to stop (or leave if already migrating)
timeout = async.DelayedCall(50, self.stop)
self.quitting = True
# get all actors
if not self.am.actors:
# No actors, we're basically done
return self.stop()
actors = []
for actor in self.am.actors.values():
# Do not delete migrating actors (for now)
if actor._migrating_to is None:
actors.append(actor)
# delete all actors
for actor in actors:
self.am.destroy(actor.id)
# and die - hopefully, things should clean up nicely within reasonable time
def poll_deleted(retry):
if self.am.actors:
_log.info("{} actors remaining, rechecking in {} secs".format(len(self.am.actors)))
async.DelayedCall(1*retry, poll_deleted)
else :
_log.info("All done, exiting")
timeout.cancel()
self.stop()
async.DelayedCall(0.5, poll_deleted, retry=1)
def stop_with_migration(self, callback=None):
# Set timeout if we are still failing after 50 seconds
timeout_stop = async.DelayedCall(50, self.stop)
self.quitting = True
actors = []
already_migrating = []
if not self.am.actors:
return self.stop(callback)
for actor in self.am.actors.values():
if actor._migrating_to is None:
actors.append(actor)
else:
already_migrating.append(actor.id)
def poll_migrated():
# When already migrating, we can only poll, since we don't get the callback
if self.am.actors:
# Check again in a sec
async.DelayedCall(1, poll_migrated)
return
timeout_stop.cancel()
self.stop(callback)
def migrated(actor_id, **kwargs):
actor = self.am.actors.get(actor_id, None)
status = kwargs['status']
if actor is not None:
# Failed to migrate according to requirements, try the current known peers
peer_ids = self.network.list_direct_links()
if peer_ids:
# This will remove the actor from the list of actors
self.am.robust_migrate(actor_id, peer_ids, callback=CalvinCB(migrated, actor_id=actor_id))
return
else:
# Ok, we have failed migrate actor according to requirements and to any known peer
# FIXME find unknown peers and try migrate to them, now just destroy actor, so storage is cleaned
_log.error("Failed to evict actor %s before quitting" % actor_id)
self.am.destroy(actor_id)
if self.am.actors:
return
timeout_stop.cancel()
self.stop(callback)
if already_migrating:
async.DelayedCall(1, poll_migrated)
if not actors:
return
elif not actors:
# No actors
return self.stop(callback)
# Migrate the actors according to their requirements
# (even actors without explicit requirements will migrate based on e.g. requires and port property needs)
for actor in actors:
if actor._replication_data.terminate_with_node(actor.id):
_log.info("TERMINATE REPLICA")
self.rm.terminate(actor.id, callback=CalvinCB(migrated, actor_id=actor.id))
else:
_log.info("TERMINATE MIGRATE ACTOR")
self.am.update_requirements(actor.id, [], extend=True, move=True,
authorization_check=False, callback=CalvinCB(migrated, actor_id=actor.id))
def _storage_started_cb(self, *args, **kwargs):
self.authentication.find_authentication_server()
self.authorization.register_node()
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug("Some security features are enabled, let's make sure certificates are in place")
_ca_conf = _conf.get("security","certificate_authority")
security_dir = _conf.get("security","security_dir")
storage_type = _conf.get("global","storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf["ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf["domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf["enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error("runtime_certificate: Failed to parse security configuration in calvin.conf, err={}".format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(node_name, domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
if is_ca:
_log.debug("No runtime certificate, but node is a CA, just sign csr, domain={}".format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path)
else:
_log.debug("No runtime certicificate can be found, send CSR to CA")
truststore_dir = certificate.get_truststore_path(type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug("CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht","securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers['location'].split('/node/')
ca_control_uris.append(ca_control_uri)
_log.debug("CA control_uri={}, node_id={}".format(ca_control_uri, ca_node_id))
else:
_log.error("There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration ")
raise Exception("There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration ")
cert_available=False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i=0
csr = json.dumps(runtime.get_csr_and_enrollment_password())
while not cert_available and i<len(ca_control_uris):
certstr=None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
j=0
while not certstr and j<10:
try:
certstr = request_handler.sign_csr_request(ca_control_uris[i], csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j*j*j
_log.debug("RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}".format(time_to_sleep, err))
time.sleep(time_to_sleep)
j=j+1
pass
else:
cert_available = True
i = i+1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr)
else:
_log.debug("Runtime certificate available")
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
"""
def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
self.external_uri = attributes.pop('external_uri', self.uri) \
if attributes else self.uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
# Obtain node id, when using security also handle runtime certificate
self.id = certificate.obtain_cert_node_info(self.attributes.get_node_name_as_str())['id']
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def insert_local_reply(self):
msg_id = calvinuuid.uuid("LMSG")
self.async_msg_ids[msg_id] = None
return msg_id
def set_local_reply(self, msg_id, reply):
if msg_id in self.async_msg_ids:
self.async_msg_ids[msg_id] = reply
def connect(self, actor_id=None, port_name=None, port_dir=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_id=None, cb=None):
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_dir=port_dir,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_dir=peer_port_dir,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def disconnect(self, actor_id=None, port_name=None, port_dir=None, port_id=None, cb=None):
_log.debug("disconnect(actor_id=%s, port_name=%s, port_dir=%s, port_id=%s)" %
(actor_id if actor_id else "", port_name if port_name else "",
port_dir if port_dir else "", port_id if port_id else ""))
self.pm.disconnect(actor_id=actor_id, port_name=port_name,
port_dir=port_dir, port_id=port_id,
callback=CalvinCB(self.logging_callback, preamble="disconnect cb") if cb is None else cb)
def peersetup(self, peers, cb=None):
""" Sets up a RT to RT communication channel, only needed if the peer can't be found in storage.
peers: a list of peer uris, e.g. ["calvinip://127.0.0.1:5001"]
"""
_log.debug("peersetup(%s)" % (peers))
peers_copy = peers[:]
peer_node_ids = {}
if not cb:
callback = CalvinCB(self.logging_callback, preamble="peersetup cb")
else:
callback = CalvinCB(self.peersetup_collect_cb, peers=peers_copy, peer_node_ids=peer_node_ids, org_cb=cb)
self.network.join(peers, callback=callback)
def peersetup_collect_cb(self, status, uri, peer_node_id, peer_node_ids, peers, org_cb):
if uri in peers:
peers.remove(uri)
peer_node_ids[uri] = (peer_node_id, status)
if not peers:
# Get highest status, i.e. any error
comb_status = max([s for _, s in peer_node_ids.values()])
org_cb(peer_node_ids=peer_node_ids, status=comb_status)
def logging_callback(self, preamble=None, *args, **kwargs):
_log.debug("\n%s# NODE: %s \n# %s %s %s \n%s" %
('#' * 40, self.id, preamble if preamble else "*", args, kwargs, '#' * 40))
def new(self, actor_type, args, deploy_args=None, state=None, prev_connections=None, connection_list=None):
# TODO requirements should be input to am.new
actor_id = self.am.new(actor_type, args, state, prev_connections, connection_list,
signature=deploy_args['signature'] if deploy_args and 'signature' in deploy_args else None,
credentials=deploy_args['credentials'] if deploy_args and 'credentials' in deploy_args else None)
if deploy_args:
app_id = deploy_args['app_id']
if 'app_name' not in deploy_args:
app_name = app_id
else:
app_name = deploy_args['app_name']
self.app_manager.add(app_id, actor_id,
deploy_info = deploy_args['deploy_info'] if 'deploy_info' in deploy_args else None)
return actor_id
def calvinsys(self):
"""Return a CalvinSys instance"""
# FIXME: We still need to sort out actor requirements vs. node capabilities and user permissions.
# @TODO: Write node capabilities to storage
return self._calvinsys
#
# Event loop
#
def run(self):
"""main loop on node"""
_log.debug("Node %s is running" % self.id)
self.sched.run()
def start(self):
""" Run once when main loop is started """
interfaces = _conf.get(None, 'transports')
self.network.register(interfaces, ['json'])
self.network.start_listeners(self.uri)
# Start storage after network, proto etc since storage proxy expects them
self.storage.start()
self.storage.add_node(self)
# Start control api
proxy_control_uri = _conf.get(None, 'control_proxy')
_log.debug("Start control API on %s with uri: %s and proxy: %s" % (self.id, self.control_uri, proxy_control_uri))
if proxy_control_uri is not None:
self.control.start(node=self, uri=proxy_control_uri, tunnel=True)
else:
if self.control_uri is not None:
self.control.start(node=self, uri=self.control_uri)
def stop(self, callback=None):
def stopped(*args):
_log.analyze(self.id, "+", {'args': args})
_log.debug(args)
self.sched.stop()
_log.analyze(self.id, "+ SCHED STOPPED", {'args': args})
self.control.stop()
_log.analyze(self.id, "+ CONTROL STOPPED", {'args': args})
def deleted_node(*args, **kwargs):
_log.analyze(self.id, "+", {'args': args, 'kwargs': kwargs})
self.storage.stop(stopped)
_log.analyze(self.id, "+", {})
self.storage.delete_node(self, cb=deleted_node)
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
"""
def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
self.external_uri = attributes.pop('external_uri', self.uri) \
if attributes else self.uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
self.id = certificate.obtain_cert_node_info(self.node_name)['id']
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
try:
self.domain = _conf.get("security", "security_domain_name")
# cert_name is the node's certificate filename (without file extension)
self.cert_name = certificate.get_own_cert_name(self.node_name)
except:
self.domain = None
self.cert_name = None
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def insert_local_reply(self):
msg_id = calvinuuid.uuid("LMSG")
self.async_msg_ids[msg_id] = None
return msg_id
def set_local_reply(self, msg_id, reply):
if msg_id in self.async_msg_ids:
self.async_msg_ids[msg_id] = reply
def connect(self, actor_id=None, port_name=None, port_dir=None, port_properties=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_properties=None, peer_port_id=None, cb=None):
if port_properties is None and port_dir is not None:
port_properties = {'direction': port_dir}
if peer_port_properties is None and peer_port_dir is not None:
peer_port_properties = {'direction': peer_port_dir}
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_properties=port_properties,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_properties=peer_port_properties,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def disconnect(self, actor_id=None, port_name=None, port_dir=None, port_id=None, cb=None):
_log.debug("disconnect(actor_id=%s, port_name=%s, port_dir=%s, port_id=%s)" %
(actor_id if actor_id else "", port_name if port_name else "",
port_dir if port_dir else "", port_id if port_id else ""))
self.pm.disconnect(actor_id=actor_id, port_name=port_name,
port_dir=port_dir, port_id=port_id,
callback=CalvinCB(self.logging_callback, preamble="disconnect cb") if cb is None else cb)
def peersetup(self, peers, cb=None):
""" Sets up a RT to RT communication channel, only needed if the peer can't be found in storage.
peers: a list of peer uris, e.g. ["calvinip://127.0.0.1:5001"]
"""
_log.debug("peersetup(%s)" % (peers))
peers_copy = peers[:]
peer_node_ids = {}
if not cb:
callback = CalvinCB(self.logging_callback, preamble="peersetup cb")
else:
callback = CalvinCB(self.peersetup_collect_cb, peers=peers_copy, peer_node_ids=peer_node_ids, org_cb=cb)
self.network.join(peers, callback=callback)
def peersetup_collect_cb(self, status, uri, peer_node_id, peer_node_ids, peers, org_cb):
if uri in peers:
peers.remove(uri)
peer_node_ids[uri] = (peer_node_id, status)
if not peers:
# Get highest status, i.e. any error
comb_status = max([s for _, s in peer_node_ids.values()])
org_cb(peer_node_ids=peer_node_ids, status=comb_status)
def logging_callback(self, preamble=None, *args, **kwargs):
_log.debug("\n%s# NODE: %s \n# %s %s %s \n%s" %
('#' * 40, self.id, preamble if preamble else "*", args, kwargs, '#' * 40))
def new(self, actor_type, args, deploy_args=None, state=None, prev_connections=None, connection_list=None):
# TODO requirements should be input to am.new
# TODO: make it possible to use security/credentials here.
actor_def, signer = self.am.lookup_and_verify(actor_type)
actor_id = self.am.new(actor_type, args, state, prev_connections, connection_list,
signature=deploy_args['signature'] if deploy_args and 'signature' in deploy_args else None,
actor_def=actor_def)
if deploy_args:
app_id = deploy_args['app_id']
if 'app_name' not in deploy_args:
app_name = app_id
else:
app_name = deploy_args['app_name']
self.app_manager.add(app_id, actor_id,
deploy_info = deploy_args['deploy_info'] if 'deploy_info' in deploy_args else None)
return actor_id
def calvinsys(self):
"""Return a CalvinSys instance"""
# FIXME: We still need to sort out actor requirements vs. node capabilities and user permissions.
# @TODO: Write node capabilities to storage
return self._calvinsys
#
# Event loop
#
def run(self):
"""main loop on node"""
_log.debug("Node %s is running" % self.id)
self.sched.run()
def start(self):
""" Run once when main loop is started """
interfaces = _conf.get(None, 'transports')
self.network.register(interfaces, ['json'])
self.network.start_listeners(self.uri)
# Start storage after network, proto etc since storage proxy expects them
self.storage.start(cb=CalvinCB(self._storage_started_cb))
self.storage.add_node(self)
# Start control API
proxy_control_uri = _conf.get(None, 'control_proxy')
_log.debug("Start control API on %s with uri: %s and proxy: %s" % (self.id, self.control_uri, proxy_control_uri))
if proxy_control_uri is not None:
self.control.start(node=self, uri=proxy_control_uri, tunnel=True)
else:
if self.control_uri is not None:
self.control.start(node=self, uri=self.control_uri, external_uri=self.external_control_uri)
def stop(self, callback=None):
def stopped(*args):
_log.analyze(self.id, "+", {'args': args})
_log.debug(args)
self.sched.stop()
_log.analyze(self.id, "+ SCHED STOPPED", {'args': args})
self.control.stop()
_log.analyze(self.id, "+ CONTROL STOPPED", {'args': args})
def deleted_node(*args, **kwargs):
_log.analyze(self.id, "+", {'args': args, 'kwargs': kwargs})
self.storage.stop(stopped)
_log.analyze(self.id, "+", {})
self.storage.delete_node(self, cb=deleted_node)
def _storage_started_cb(self, *args, **kwargs):
self.authorization.register_node()
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
"""
def __init__(self, uri, control_uri, attributes=None, self_start=True):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.id = calvinuuid.uuid("NODE")
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
hb_timeout = _conf.get('global', 'heartbeat_timeout') or DEFAULT_HEARTBEAT_TIMEOUT
self.heartbeat_timeout = float(hb_timeout)
hb_delay = _conf.get('global', 'heartbeat_delay') or DEFAULT_HEARTBEAT_DELAY
self.heartbeat_delay = float(hb_delay)
self.heartbeat_addr = self._clean_addr()
self.heartbeat_port = _conf.get('global', 'heartbeat_port') or int(self._clean_uri().split(":")[1]) + DEFAULT_HEARTBEAT_PORT_DIFF
rr_delay = _conf.get('global', 'resource_reporter_delay') or 0.25
self.resource_reporter_delay = float(rr_delay)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
self.resource_manager = ResourceManager()
self.app_monitor = AppMonitor(self, self.app_manager, self.storage)
self.lost_node_handler = LostNodeHandler(self, self.resource_manager, self.pm, self.am, self.storage)
self.heartbeat_actor = None
self.outgoing_heartbeats = defaultdict(list)
# The initialization that requires the main loop operating is deferred to start function
if self_start:
async.DelayedCall(0, self.start)
@property
def storage_node(self):
return False
def is_storage_node(self, node_id):
if node_id == self.id:
return self.storage_node
#if node_id not in self.network.links:
# return False
return self.storage.proxy == self.network.links[node_id].transport.get_uri()
def _clean_uri(self):
return self.control_uri.replace("http://", "")
def _clean_addr(self):
uri = self._clean_uri()
if uri == "localhost":
addr = socket.gethostbyname(uri.split(":")[0])
else:
addr = uri.split(":")[0]
return addr
def insert_local_reply(self):
msg_id = calvinuuid.uuid("LMSG")
self.async_msg_ids[msg_id] = None
return msg_id
def set_local_reply(self, msg_id, reply):
if msg_id in self.async_msg_ids:
self.async_msg_ids[msg_id] = reply
def connect(self, actor_id=None, port_name=None, port_dir=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_id=None, cb=None):
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_dir=port_dir,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_dir=peer_port_dir,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def disconnect(self, actor_id=None, port_name=None, port_dir=None, port_id=None, cb=None):
_log.debug("disconnect(actor_id=%s, port_name=%s, port_dir=%s, port_id=%s)" %
(actor_id if actor_id else "", port_name if port_name else "",
port_dir if port_dir else "", port_id if port_id else ""))
self.pm.disconnect(actor_id=actor_id, port_name=port_name,
port_dir=port_dir, port_id=port_id,
callback=CalvinCB(self.logging_callback, preamble="disconnect cb") if cb is None else cb)
def peersetup(self, peers, cb=None):
""" Sets up a RT to RT communication channel, only needed if the peer can't be found in storage.
peers: a list of peer uris, e.g. ["calvinip://127.0.0.1:5001"]
"""
_log.info("peersetup(%s)" % (peers))
peers_copy = peers[:]
peer_node_ids = {}
if not cb:
callback = CalvinCB(self.logging_callback, preamble="peersetup cb")
else:
callback = CalvinCB(self.peersetup_collect_cb, peers=peers_copy, peer_node_ids=peer_node_ids, org_cb=cb)
peers = filter(None, peers)
self.network.join(peers, callback=callback)
def peersetup_collect_cb(self, status, uri, peer_node_id, peer_node_ids, peers, org_cb):
_log.debug("Peersetup collect cb: {} - {} - {} - {} - {}".format(status, uri, peer_node_id, peer_node_ids, peers))
self.resource_manager.register_uri(peer_node_id, uri)
if status:
self.resource_manager.register_uri(peer_node_id, uri)
self._register_heartbeat_receiver(peer_node_id)
if uri in peers:
peers.remove(uri)
peer_node_ids[uri] = (peer_node_id, status)
if not peers:
# Get highest status, i.e. any error
comb_status = max([s for _, s in peer_node_ids.values()])
org_cb(peer_node_ids=peer_node_ids, status=comb_status)
if peer_node_id and status:
self._send_rm_info(peer_node_id)
def _send_rm_info(self, peer_node_id):
# Send own info to new peers and retreive there info
callback = CalvinCB(self._send_rm_info_cb)
usages = self.resource_manager.sync_info()
self.proto.send_rm_info(peer_node_id, usages, callback)
def _send_rm_info_cb(self, status, *args, **kwargs):
# Receives other peers rm info
if status.data:
self.resource_manager.sync_info(status.data[0])
def sync_rm_info(self, usages, callback):
# sync received info
usages = self.resource_manager.sync_info(usages)
callback(usages, status=response.CalvinResponse(True))
def logging_callback(self, preamble=None, *args, **kwargs):
_log.debug("\n%s# NODE: %s \n# %s %s %s \n%s" %
('#' * 40, self.id, preamble if preamble else "*", args, kwargs, '#' * 40))
def new(self, actor_type, args, deploy_args=None, state=None, prev_connections=None, connection_list=None, callback=None):
# TODO requirements should be input to am.new
callback = CalvinCB(self._new, args=args, deploy_args=deploy_args, state=state, prev_connections=prev_connections,
connection_list=connection_list, callback=callback)
self.am.new(actor_type, args, state, prev_connections, connection_list,
app_id=deploy_args['app_id'] if deploy_args else None,
signature=deploy_args['signature'] if deploy_args and 'signature' in deploy_args else None, callback=callback)
def _new(self, actor_id, status, args, deploy_args, state, prev_connections, connection_list, callback):
if not status:
if callback:
callback(status=status, actor_id=actor_id)
return
if deploy_args:
app_id = deploy_args['app_id']
if 'app_name' not in deploy_args:
app_name = app_id
else:
app_name = deploy_args['app_name']
self.app_manager.add(app_id, actor_id,
deploy_info = deploy_args['deploy_info'] if 'deploy_info' in deploy_args else None)
if callback:
callback(status=status, actor_id=actor_id)
def deployment_control(self, app_id, actor_id, deploy_args):
""" Updates an actor's deployment """
self.am.deployment_control(app_id, actor_id, deploy_args)
def calvinsys(self):
"""Return a CalvinSys instance"""
# FIXME: We still need to sort out actor requirements vs. node capabilities and user permissions.
# @TODO: Write node capabilities to storage
return self._calvinsys
@property
def hostname(self):
return socket.gethostname()
@property
def testing(self):
return "CALVIN_TESTING" in os.environ and os.environ["CALVIN_TESTING"]
def report_resource_usage(self, usage):
_log.debug("Reporting resource usage for node {}: {}".format(self.id, usage))
self.resource_manager.register(self.id, usage, self.uri)
usage['uri'] = self.uri
for peer_id in self.network.list_links():
callback = CalvinCB(self._report_resource_usage_cb, peer_id)
self.proto.report_usage(peer_id, self.id, usage, callback=callback)
self.app_monitor.check_reliabilities()
def _report_resource_usage_cb(self, peer_id, status):
if not status:
_log.error("Failed to report resource usage to: {} - {} - {}".format(peer_id, self.resource_manager.node_uris.get(peer_id), status))
else:
_log.debug("Report resource usage callback received status {} for {}".format(status, peer_id))
def register_resource_usage(self, node_id, usage, callback):
if self.storage_node:
callback(status=response.CalvinResponse(True))
return
_log.debug("Registering resource usage for node {}: {}".format(node_id, usage))
uri = usage.get('uri')
self.resource_manager.register(node_id, usage, uri)
self._register_heartbeat_receiver(node_id)
callback(status=response.CalvinResponse(True))
def report_replication_time(self, actor_type, replication_time, node_id):
_log.info('New replication time: {} when handling lost node {}'.format(replication_time, node_id))
self.storage.new_replication_time(actor_type, replication_time)
def lost_node(self, node_id):
_log.debug("Lost node: {}".format(node_id))
_log.analyze(self.id, "+", "Lost node {}".format(node_id))
if self.storage_node:
_log.debug("{} Is storage node, ignoring lost node".format(self.id))
return
result = self.heartbeat_actor.receive()
nodes = set()
if result and result.production:
if node_id in result.production[0]:
return
if node_id in self.network.links:
link = self.network.links[node_id]
self.network.peer_disconnected(link, node_id, "Heartbeat timeout")
self.lost_node_handler.handle_lost_node(node_id)
def lost_node_request(self, node_id, cb):
_log.debug("Lost node: {}".format(node_id))
_log.analyze(self.id, "+", "Lost node {}".format(node_id))
if self.storage_node:
_log.debug("{} Is storage node, ignoring lost node".format(self.id))
if cb:
cb(status=response.CalvinResponse(False))
return
self.lost_node_handler.handle_lost_node_request(node_id, cb)
def lost_actor(self, lost_actor_id, lost_actor_info, required_reliability, cb):
_log.analyze(self.id, "+", "Lost actor {}".format(lost_actor_id))
self.am.delete_actor(lost_actor_id)
replicator = Replicator(self, lost_actor_id, lost_actor_info, required_reliability)
replicator.replicate_lost_actor(cb, time.time())
def _heartbeat_timeout(self, node_id):
self._clear_heartbeat_timeouts(node_id)
self.heartbeat_actor.deregister(node_id)
self.lost_node(node_id)
def clear_outgoing_heartbeat(self, data):
if "node_id" in data:
node_id = data['node_id']
self._clear_heartbeat_timeouts(node_id)
timeout_call = async.DelayedCall(self.heartbeat_timeout, CalvinCB(self._heartbeat_timeout, node_id=node_id))
self.outgoing_heartbeats[node_id].append(timeout_call)
self.resource_manager.register_uri(data['node_id'], data['uri'])
def _clear_heartbeat_timeouts(self, node_id):
for timeout_call in self.outgoing_heartbeats[node_id]:
try:
timeout_call.cancel()
except Exception as e:
pass
#
# Event loop
#
def run(self):
"""main loop on node"""
_log.debug("Node %s is running" % self.id)
self.sched.run()
def start(self):
""" Run once when main loop is started """
if not self.storage_node:
self._start_heartbeat_system()
interfaces = _conf.get(None, 'transports')
self.network.register(interfaces, ['json'])
self.network.start_listeners(self.uri)
# Start storage after network, proto etc since storage proxy expects them
self.storage.start()
self.storage.add_node(self)
# Start control api
proxy_control_uri = _conf.get(None, 'control_proxy')
_log.debug("Start control API on %s with uri: %s and proxy: %s" % (self.id, self.control_uri, proxy_control_uri))
if proxy_control_uri is not None:
self.control.start(node=self, uri=proxy_control_uri, tunnel=True)
else:
if self.control_uri is not None:
self.control.start(node=self, uri=self.control_uri)
if not self.storage_node:
self._start_resource_reporter()
def _start_resource_reporter(self):
self.new("sys.NodeResourceReporter", {'node': self, 'delay': self.resource_reporter_delay}, callback=self._start_rr)
def _start_rr(self, status, actor_id):
if not status:
_log.error("Failed to start resource reporter")
return
_log.info("Successfully started resource reporter with delay {}".format(self.resource_reporter_delay))
actor = self.am.actors[actor_id]
if not actor.inports or not actor.outports:
_log.warning("Could not set up ResourceReporter: {}".format(actor))
return
in_port = actor.inports['in']
out_port = actor.outports['out']
self.connect(actor_id, port_name=in_port.name, port_dir='in', port_id=in_port.id,
peer_node_id=self.id, peer_actor_id=actor_id, peer_port_name=out_port.name,
peer_port_dir='out', peer_port_id=out_port.id)
def _start_heartbeat_system(self):
if self.testing:
return
port = self.heartbeat_port
addr = self.heartbeat_addr
self.new("net.Heartbeat", {'node': self, 'address': addr, 'port': port, 'delay': self.heartbeat_delay}, callback=self._start_hb)
def _start_hb(self, status, actor_id):
if not status:
_log.error("Failed to start heartbeat system: ".format(status))
return
_log.info("Successfully started heartbeat actor with timeout {} and delay {}".format(self.heartbeat_timeout, self.heartbeat_delay))
actor = self.am.actors[actor_id]
in_port = actor.inports['in']
out_port = actor.outports['out']
self.connect(actor_id, port_name=in_port.name, port_dir='in', port_id=in_port.id,
peer_node_id=self.id, peer_actor_id=actor_id, peer_port_name=out_port.name,
peer_port_dir='out', peer_port_id=out_port.id)
self.heartbeat_actor = actor
def _register_heartbeat_receiver(self, node_id):
if self.storage_node or self.is_storage_node(node_id):
return
if not self.heartbeat_actor:
self._start_heartbeat_system()
if self.testing or node_id in self.heartbeat_actor.nodes:
return
_log.info("Registering receiver: {}".format(node_id))
self.heartbeat_actor.register(node_id)
def stop(self, callback=None):
def stopped(*args):
_log.analyze(self.id, "+", {'args': args})
_log.debug(args)
self.sched.stop()
_log.analyze(self.id, "+ SCHED STOPPED", {'args': args})
self.control.stop()
_log.analyze(self.id, "+ CONTROL STOPPED", {'args': args})
def deleted_node(*args, **kwargs):
_log.analyze(self.id, "+", {'args': args, 'kwargs': kwargs})
self.storage.stop(stopped)
_log.analyze(self.id, "+", {})
self.storage.delete_node(self.id, self.attributes.get_indexed_public(), cb=deleted_node)
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global rt
global request_handler
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.makedirs(credentials_testdir)
os.makedirs(runtimesdir)
os.makedirs(runtimes_truststore)
os.makedirs(runtimes_truststore_signing_path)
os.makedirs(actor_store_path)
os.makedirs(os.path.join(actor_store_path, "test"))
shutil.copy(
os.path.join(orig_actor_store_path, "test", "__init__.py"),
os.path.join(actor_store_path, "test", "__init__.py"))
os.makedirs(os.path.join(actor_store_path, "std"))
shutil.copy(
os.path.join(orig_actor_store_path, "std", "__init__.py"),
os.path.join(actor_store_path, "std", "__init__.py"))
shutil.copytree(orig_application_store_path,
application_store_path)
filelist = [
f for f in os.listdir(application_store_path)
if f.endswith(".sign.93d58fef")
]
for f in filelist:
os.remove(os.path.join(application_store_path, f))
shutil.copytree(
os.path.join(security_testdir, "identity_provider"),
identity_provider_path)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
print "Trying to create a new test application/actor signer."
cs = code_signer.CS(organization="testsigner",
commonName="signer",
security_dir=credentials_testdir)
#Create signed version of CountTimer actor
orig_actor_CountTimer_path = os.path.join(orig_actor_store_path, "std",
"CountTimer.py")
actor_CountTimer_path = os.path.join(actor_store_path, "std",
"CountTimer.py")
shutil.copy(orig_actor_CountTimer_path, actor_CountTimer_path)
# cs.sign_file(actor_CountTimer_path)
#Create unsigned version of CountTimer actor
actor_CountTimerUnsigned_path = actor_CountTimer_path.replace(
".py", "Unsigned.py")
shutil.copy(actor_CountTimer_path, actor_CountTimerUnsigned_path)
replace_text_in_file(actor_CountTimerUnsigned_path, "CountTimer",
"CountTimerUnsigned")
#Create signed version of Sum actor
orig_actor_Sum_path = os.path.join(orig_actor_store_path, "std",
"Sum.py")
actor_Sum_path = os.path.join(actor_store_path, "std", "Sum.py")
shutil.copy(orig_actor_Sum_path, actor_Sum_path)
# cs.sign_file(actor_Sum_path)
#Create unsigned version of Sum actor
actor_SumUnsigned_path = actor_Sum_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sum_path, actor_SumUnsigned_path)
replace_text_in_file(actor_SumUnsigned_path, "Sum", "SumUnsigned")
#Create incorrectly signed version of Sum actor
# actor_SumFake_path = actor_Sum_path.replace(".py", "Fake.py")
# shutil.copy(actor_Sum_path, actor_SumFake_path)
# #Change the class name to SumFake
# replace_text_in_file(actor_SumFake_path, "Sum", "SumFake")
# cs.sign_file(actor_SumFake_path)
# #Now append to the signed file so the signature verification fails
# with open(actor_SumFake_path, "a") as fd:
# fd.write(" ")
#Create signed version of Sink actor
orig_actor_Sink_path = os.path.join(orig_actor_store_path, "test",
"Sink.py")
actor_Sink_path = os.path.join(actor_store_path, "test", "Sink.py")
shutil.copy(orig_actor_Sink_path, actor_Sink_path)
# cs.sign_file(actor_Sink_path)
#Create unsigned version of Sink actor
actor_SinkUnsigned_path = actor_Sink_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sink_path, actor_SinkUnsigned_path)
replace_text_in_file(actor_SinkUnsigned_path, "Sink", "SinkUnsigned")
#Sign applications
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctly_signed.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctlySignedApp_incorrectlySignedActor.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"))
# #Now append to the signed file so the signature verification fails
# with open(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"), "a") as fd:
# fd.write(" ")
# print "Export Code Signers certificate to the truststore for code signing"
# out_file = cs.export_cs_cert(runtimes_truststore_signing_path)
print "Trying to create a new test domain configuration."
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
#
print "Copy CA cert into truststore of runtimes folder"
ca.export_ca_cert(runtimes_truststore)
#Define the runtime attributes
rt0_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'CA'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt1_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode1'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt2_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode2'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'otherStreet',
'streetNumber': 1
}
}
}
rt3_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode3'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt4_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode4'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt5_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode5'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt_attributes = []
rt_attributes.append(deepcopy(rt0_attributes))
rt_attributes.append(deepcopy(rt1_attributes))
rt_attributes.append(deepcopy(rt2_attributes))
rt_attributes.append(deepcopy(rt3_attributes))
rt_attributes.append(deepcopy(rt4_attributes))
rt_attributes.append(deepcopy(rt5_attributes))
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes:
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
#Let's hash passwords in users.json file (the runtimes will try to do this
# but they will all try to do it at the same time, so it will be overwritten
# multiple times and the first test will always fail)
# self.arp = FileAuthenticationRetrievalPoint(identity_provider_path)
# self.arp.check_stored_users_db_for_unhashed_passwords()
#The policy allows access to control interface for everyone, for more advanced rules
# it might be appropriate to run set_credentials for request_handler, e.g.,
# request_handler.set_credentials({domain_name:{"user": "******", "password": "******"}})
rt_conf = copy.deepcopy(_conf)
# rt_conf.set('security', 'runtime_to_runtime_security', "tls")
# rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
# rt_conf.set('security', 'certificate_authority_control_uri',"https://%s:5020" % hostname )
rt_conf.set('security', 'security_dir', credentials_testdir)
rt_conf.set('global', 'actor_paths', [actor_store_path])
rt_conf.set('global', 'storage_type', "securedht")
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf = copy.deepcopy(rt_conf)
# rt0_conf.set('security','enrollment_password',enrollment_passwords[0])
#The csruntime certificate requests assumes TLS for the control interface
# rt0_conf.set('security', 'control_interface_security', "tls")
# rt0_conf.set('security','certificate_authority','True')
# rt0_conf.set("security", "security_conf", {
# "comment": "Certificate Authority",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
# rt1_conf.set('security','enrollment_password',enrollment_passwords[1])
# rt1_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
# rt2_conf.set('security','enrollment_password',enrollment_passwords[2])
# rt2_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path,
# "accept_external_requests": True
# }
# })
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
# rt3_conf.set('security','enrollment_password',enrollment_passwords[3])
# rt3_conf.set("security", "security_conf", {
# "comment": "RADIUS authentication, local authorization",
# "authentication": {
# "procedure": "radius",
# "server_ip": "localhost",
# "secret": "elxghyc5lz1_passwd"
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
# rt4_conf.set('security','enrollment_password',enrollment_passwords[4])
# rt4_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "external"
# }
# })
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
# rt5_conf.set('global','storage_type','proxy')
# rt5_conf.set('global','storage_proxy',"calvinip://%s:5000" % ip_addr )
# rt5_conf.set('security','enrollment_password',enrollment_passwords[5])
# rt5_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "external",
# "server_uuid": runtimes[1].node_id
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes_cpy)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes_cpy[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
# rt.append(RT("https://{}:502{}".format(hostname, i)))
rt.append(RT("http://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(10)
request.addfinalizer(self.teardown)
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
"""
def __init__(self, uris, control_uri, attributes=None):
super(Node, self).__init__()
self.quitting = False
self.super_node_class = None
# Warn if its not a uri
if not isinstance(uris, list):
_log.error("Calvin uris must be a list %s" % uris)
raise TypeError("Calvin uris must be a list!")
# Uris
self.uris = uris
if attributes:
ext_uris = attributes.pop('external_uri', None)
if ext_uris is not None:
self.uris += ext_uris
# Control uri
self.control_uri = control_uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
try:
security_dir = _conf.get("security", "security_dir")
self.runtime_credentials = RuntimeCredentials(self.node_name, node=self, security_dir=security_dir)
self.id = self.runtime_credentials.get_node_id()
except Exception as err:
_log.debug("No runtime credentials, err={}".format(err))
self.runtime_credentials = None
self.id = calvinuuid.uuid("Node")
self.certificate_authority = certificate_authority.CertificateAuthority(self)
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
self.am = actormanager.ActorManager(self)
self.rm = replicationmanager.ReplicationManager(self)
self.control = calvincontrol.get_calvincontrol()
# _scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
# _scheduler = scheduler.NonPreemptiveScheduler
# _scheduler = scheduler.RoundRobinScheduler
_scheduler = scheduler.SimpleScheduler
# _scheduler = scheduler.BaselineScheduler
self.sched = _scheduler(self, self.am)
self.async_msg_ids = {}
calvinsys = get_calvinsys()
calvinsys.init(self)
calvinlib = get_calvinlib()
calvinlib.init()
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
self.cpu_monitor = CpuMonitor(self.id, self.storage)
self.mem_monitor = MemMonitor(self.id, self.storage)
self.proxy_handler = ProxyHandler(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def insert_local_reply(self):
msg_id = calvinuuid.uuid("LMSG")
self.async_msg_ids[msg_id] = None
return msg_id
def set_local_reply(self, msg_id, reply):
if msg_id in self.async_msg_ids:
self.async_msg_ids[msg_id] = reply
def connect(self, actor_id=None, port_name=None, port_dir=None, port_properties=None, port_id=None,
peer_node_id=None, peer_actor_id=None, peer_port_name=None,
peer_port_dir=None, peer_port_properties=None, peer_port_id=None, cb=None):
if port_properties is None and port_dir is not None:
port_properties = {'direction': port_dir}
if peer_port_properties is None and peer_port_dir is not None:
peer_port_properties = {'direction': peer_port_dir}
self.pm.connect(actor_id=actor_id,
port_name=port_name,
port_properties=port_properties,
port_id=port_id,
peer_node_id=peer_node_id,
peer_actor_id=peer_actor_id,
peer_port_name=peer_port_name,
peer_port_properties=peer_port_properties,
peer_port_id=peer_port_id,
callback=CalvinCB(self.logging_callback, preamble="connect cb") if cb is None else cb)
def peersetup(self, peers, cb=None):
""" Sets up a RT to RT communication channel, only needed if the peer can't be found in storage.
peers: a list of peer uris, e.g. ["calvinip://127.0.0.1:5001"]
"""
_log.debug("peersetup(%s)" % (peers))
peers_copy = peers[:]
peer_node_ids = {}
if not cb:
callback = CalvinCB(self.logging_callback, preamble="peersetup cb")
else:
callback = CalvinCB(self.peersetup_collect_cb, peers=peers_copy, peer_node_ids=peer_node_ids, org_cb=cb)
self.network.join(peers, callback=callback)
def peersetup_collect_cb(self, status, uri, peer_node_id, peer_node_ids, peers, org_cb):
if uri in peers:
peers.remove(uri)
peer_node_ids[uri] = (peer_node_id, status)
if not peers:
# Get highest status, i.e. any error
comb_status = max([s for _, s in peer_node_ids.values()])
org_cb(peer_node_ids=peer_node_ids, status=comb_status)
def logging_callback(self, preamble=None, *args, **kwargs):
_log.debug("\n%s# NODE: %s \n# %s %s %s \n%s" %
('#' * 40, self.id, preamble if preamble else "*", args, kwargs, '#' * 40))
def new(self, actor_type, args, deploy_args=None, state=None, prev_connections=None, connection_list=None, security=None, access_decision=None):
# TODO requirements should be input to am.new
# TODO: make it possible to use security/credentials here.
actor_def, signer = self.am.lookup_and_verify(actor_type)
actor_id = self.am.new(actor_type, args, state, prev_connections, connection_list,
signature=deploy_args['signature'] if deploy_args and 'signature' in deploy_args else None,
actor_def=actor_def,
security=security,
access_decision=access_decision)
if deploy_args:
app_id = deploy_args['app_id']
if 'app_name' not in deploy_args:
app_name = app_id
else:
app_name = deploy_args['app_name']
self.app_manager.add(app_id, actor_id,
deploy_info = deploy_args['deploy_info'] if 'deploy_info' in deploy_args else None)
return actor_id
#
# Event loop
#
def run(self):
"""main loop on node"""
_log.debug("Node %s is running" % self.id)
self.sched.run()
def start(self):
""" Run once when main loop is started """
interfaces = _conf.get(None, 'transports')
self.network.register(interfaces, ['json'])
self.network.start_listeners(self.uris)
# Start storage after network, proto etc since storage proxy expects them
self.storage.start(cb=CalvinCB(self._storage_started_cb))
self.storage.add_node(self)
# Start control API
proxy_control_uri = _conf.get(None, 'control_proxy')
_log.debug("Start control API on %s with uri: %s and proxy: %s" % (self.id, self.control_uri, proxy_control_uri))
if proxy_control_uri is not None:
self.control.start(node=self, uri=proxy_control_uri, tunnel=True)
else:
if self.control_uri is not None:
self.control.start(node=self, uri=self.control_uri, external_uri=self.external_control_uri)
def stop(self, callback=None):
# TODO: Handle blocking in poorly implemented calvinsys/runtime south.
self.quitting = True
def stopped(*args):
_log.analyze(self.id, "+", {'args': args})
_log.debug(args)
self.sched.stop()
_log.analyze(self.id, "+ SCHED STOPPED", {'args': args})
self.control.stop()
_log.analyze(self.id, "+ CONTROL STOPPED", {'args': args})
def deleted_node(*args, **kwargs):
_log.analyze(self.id, "+", {'args': args, 'kwargs': kwargs})
self.storage.stop(stopped)
_log.analyze(self.id, "+", {})
self.storage.delete_node(self, cb=deleted_node)
self.cpu_monitor.stop()
self.mem_monitor.stop()
for link in self.network.list_direct_links():
self.network.link_get(link).close()
def stop_with_cleanup(self):
# Set timeout in case some actor is refusing to stop (or leave if already migrating)
timeout = async.DelayedCall(50, self.stop)
self.quitting = True
# get all actors
if not self.am.actors:
# No actors, we're basically done
return self.stop()
actors = []
for actor in self.am.actors.values():
# Do not delete migrating actors (for now)
if actor._migrating_to is None:
actors.append(actor)
# delete all actors
for actor in actors:
self.am.destroy(actor.id)
# and die - hopefully, things should clean up nicely within reasonable time
def poll_deleted(retry):
if self.am.actors:
_log.info("{} actors remaining, rechecking in {} secs".format(len(self.am.actors)))
async.DelayedCall(1*retry, poll_deleted)
else :
_log.info("All done, exiting")
timeout.cancel()
self.stop()
async.DelayedCall(0.5, poll_deleted, retry=1)
def stop_with_migration(self, callback=None):
# Set timeout if we are still failing after 50 seconds
timeout_stop = async.DelayedCall(50, self.stop)
self.quitting = True
actors = []
already_migrating = []
if not self.am.actors:
return self.stop(callback)
for actor in self.am.actors.values():
if actor._migrating_to is None:
actors.append(actor)
else:
already_migrating.append(actor.id)
def poll_migrated():
# When already migrating, we can only poll, since we don't get the callback
if self.am.actors:
# Check again in a sec
async.DelayedCall(1, poll_migrated)
return
timeout_stop.cancel()
self.stop(callback)
def migrated(actor_id, **kwargs):
actor = self.am.actors.get(actor_id, None)
status = kwargs['status']
if actor is not None:
# Failed to migrate according to requirements, try the current known peers
peer_ids = self.network.list_direct_links()
if peer_ids:
# This will remove the actor from the list of actors
self.am.robust_migrate(actor_id, peer_ids, callback=CalvinCB(migrated, actor_id=actor_id))
return
else:
# Ok, we have failed migrate actor according to requirements and to any known peer
# FIXME find unknown peers and try migrate to them, now just destroy actor, so storage is cleaned
_log.error("Failed to evict actor %s before quitting" % actor_id)
self.am.destroy(actor_id)
if self.am.actors:
return
timeout_stop.cancel()
self.stop(callback)
if already_migrating:
async.DelayedCall(1, poll_migrated)
if not actors:
return
elif not actors:
# No actors
return self.stop(callback)
# Migrate the actors according to their requirements
# (even actors without explicit requirements will migrate based on e.g. requires and port property needs)
for actor in actors:
if actor._replication_id.terminate_with_node(actor.id):
_log.info("TERMINATE REPLICA")
self.rm.terminate(actor.id, callback=CalvinCB(migrated, actor_id=actor.id))
else:
_log.info("TERMINATE MIGRATE ACTOR")
self.am.update_requirements(actor.id, [], extend=True, move=True,
authorization_check=False, callback=CalvinCB(migrated, actor_id=actor.id))
def _storage_started_cb(self, *args, **kwargs):
self.authentication.find_authentication_server()
self.authorization.register_node()
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global hostname
global rt
global rt_attributes
global request_handler
try:
ipv6_hostname = socket.gethostbyaddr('::1')
except Exception as err:
print(
"Failed to resolve the IPv6 localhost hostname, please update the corresponding entry in the /etc/hosts file, e.g.,:\n"
"\t::1 <hostname>.localdomain <hostname>.local <hostname> localhost"
)
raise
try:
ipv6_hostname = socket.gethostbyaddr('::ffff:127.0.0.1')
except Exception as err:
print(
"Failed to resolve ::ffff:127.0.0.1, please add the following line (with your hostname) to /etc/hosts :\n"
"::ffff:127.0.0.1: <hostname>.localdomain <hostname>.local <hostname>"
)
raise
try:
hostname = socket.gethostname()
ip_addr = socket.gethostbyname(hostname)
fqdn = socket.getfqdn(hostname)
print("\n\tip_addr={}"
"\n\thostname={}"
"\n\tfqdn={}".format(ip_addr, hostname, fqdn))
except Exception as err:
print(
"Failed to resolve the hostname, ip_addr or the FQDN of the runtime, err={}"
.format(err))
raise
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.mkdir(credentials_testdir)
os.mkdir(runtimesdir)
os.mkdir(runtimes_truststore)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
_log.info("Trying to create a new test domain configuration.")
try:
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
except Exception as err:
_log.error("Failed to create CA, err={}".format(err))
_log.info("Copy CA cert into truststore of runtimes folder")
ca.export_ca_cert(runtimes_truststore)
node_names = []
rt_attributes = []
for i in range(6):
node_name = {
'organization': 'org.testexample',
'name': 'testNode{}'.format(i)
}
owner = {'organization': domain_name, 'personOrGroup': 'testOwner'}
address = {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
rt_attribute = {
'indexed_public': {
'owner': owner,
'node_name': node_name,
'address': address
}
}
rt_attributes.append(rt_attribute)
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes_cpy:
_log.info("rt_attribute={}".format(rt_attribute))
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
rt_conf = copy.deepcopy(_conf)
rt_conf.set('security', 'runtime_to_runtime_security', "tls")
rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
rt_conf.set('security', 'security_dir', credentials_testdir)
rt0_conf = copy.deepcopy(rt_conf)
rt_conf.set('global', 'storage_type', 'proxy')
rt_conf.set('global', 'storage_proxy', "calvinip://%s:5000" % hostname)
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf.set('global', 'storage_type', 'local')
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
rt.append(RT("https://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(2)
request.addfinalizer(self.teardown)
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug(
"Some security features are enabled, let's make sure certificates are in place"
)
_ca_conf = _conf.get("security", "certificate_authority")
security_dir = _conf.get("security", "security_dir")
storage_type = _conf.get("global", "storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf[
"ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf[
"domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf[
"enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error(
"runtime_certificate: Failed to parse security configuration in calvin.conf, err={}"
.format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir,
node_name + ".csr")
if is_ca == "True":
_log.debug(
"No runtime certificate, but node is a CA, just sign csr, domain={}"
.format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path,
security_dir=security_dir)
else:
_log.debug(
"No runtime certicificate can be found, send CSR to CA"
)
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug(
"CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht", "securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers[
'location'].split('/node/')
ca_control_uri = ca_control_uri.replace(
"http", "https")
ca_control_uris.append(ca_control_uri)
_log.debug(
"CA control_uri={}, node_id={}".format(
ca_control_uri, ca_node_id))
else:
_log.error(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
raise Exception(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
cert_available = False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i = 0
while not cert_available and i < len(ca_control_uris):
certstr = None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
rsa_encrypted_csr = runtime.get_encrypted_csr()
j = 0
while not certstr and j < 10:
try:
certstr = request_handler.sign_csr_request(
ca_control_uris[i],
rsa_encrypted_csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j * j * j
_log.debug(
"RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}"
.format(time_to_sleep, err))
time.sleep(time_to_sleep)
j = j + 1
pass
else:
cert_available = True
i = i + 1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr,
security_dir=security_dir)
else:
_log.debug("Runtime certificate available")
def __init__(self, node, peer_id, attributes, capabilities, port_property_capability):
self.node = node
self.id = peer_id
self.attributes = AttributeResolver(json.loads(attributes))
self.capabilities = capabilities
self.port_property_capability = port_property_capability
def __init__(self, uris, control_uri, attributes=None):
super(Node, self).__init__()
self.quitting = False
self.super_node_class = None
# Warn if its not a uri
if not isinstance(uris, list):
_log.error("Calvin uris must be a list %s" % uris)
raise TypeError("Calvin uris must be a list!")
# Uris
self.uris = uris
if attributes:
ext_uris = attributes.pop('external_uri', None)
if ext_uris is not None:
self.uris += ext_uris
# Control uri
self.control_uri = control_uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
try:
security_dir = _conf.get("security", "security_dir")
self.runtime_credentials = RuntimeCredentials(self.node_name, node=self, security_dir=security_dir)
self.id = self.runtime_credentials.get_node_id()
except Exception as err:
_log.debug("No runtime credentials, err={}".format(err))
self.runtime_credentials = None
self.id = calvinuuid.uuid("Node")
self.certificate_authority = certificate_authority.CertificateAuthority(self)
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
self.am = actormanager.ActorManager(self)
self.rm = replicationmanager.ReplicationManager(self)
self.control = calvincontrol.get_calvincontrol()
# _scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
# _scheduler = scheduler.NonPreemptiveScheduler
# _scheduler = scheduler.RoundRobinScheduler
_scheduler = scheduler.SimpleScheduler
# _scheduler = scheduler.BaselineScheduler
self.sched = _scheduler(self, self.am)
self.async_msg_ids = {}
calvinsys = get_calvinsys()
calvinsys.init(self)
calvinlib = get_calvinlib()
calvinlib.init()
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
self.cpu_monitor = CpuMonitor(self.id, self.storage)
self.mem_monitor = MemMonitor(self.id, self.storage)
self.proxy_handler = ProxyHandler(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
