def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.id = calvinuuid.uuid("NODE")
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.control.start(node=self, uri=control_uri)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def set_proxy_config(peer_id, name, capabilities, port_property_capability, storage, callback):
"""
Store node
"""
try:
for c in list_port_property_capabilities(which=port_property_capability):
storage.add_index(['node', 'capabilities', c], peer_id, root_prefix_level=3)
for c in capabilities:
storage.add_index(['node', 'capabilities', c], peer_id, root_prefix_level=3)
except:
_log.error("Failed to set capabilities")
attributes = AttributeResolver({"indexed_public": {"node_name": {"name": name}}})
indexes = attributes.get_indexed_public()
try:
for index in indexes:
storage.add_index(index, peer_id)
except:
_log.error("Failed to add node index")
storage.set(prefix="node-", key=peer_id,
value={"proxy": storage.node.id,
"uris": None,
"control_uris": None,
"authz_server": None, # Set correct value
"attributes": {'public': attributes.get_public(),
'indexed_public': attributes.get_indexed_public(as_list=False)}},
cb=CalvinCB(set_proxy_config_cb, callback=callback))
def __init__(self, node, peer_id, attributes, capabilities,
port_property_capability):
self.node = node
self.id = peer_id
self.attributes = AttributeResolver(json.loads(attributes))
self.capabilities = capabilities
self.port_property_capability = port_property_capability
def test_total_valid(self):
"""
Test valid values for total RAM.
Verify if storage is as expected
"""
values = ["1K", "100K", "1M", "100M", "1G", "10G"]
for i in values:
# verify set return
self.done = False
self.node.attributes = AttributeResolver(
{"indexed_public": {
"memTotal": i
}})
self.storage.add_node(self.node, cb=self.cb)
yield wait_for(self._test_done)
assert isinstance(self.get_ans, calvinresponse.CalvinResponse
) and self.get_ans == calvinresponse.OK
# verify index ok and present for level i
self.done = False
self.storage.get_index(index=self.MEMTOTAL_INDEX_BASE +
map(str, values[:values.index(i) + 1]),
root_prefix_level=2,
cb=CalvinCB(self.cb2))
yield wait_for(self._test_done)
assert self.node.id in self.get_ans
def _get_node_names(runtimes):
from calvin.utilities.attribute_resolver import AttributeResolver
from copy import deepcopy
for i in range(len(runtimes)):
rt_attribute = deepcopy(runtimes[i]["attributes"])
attributes = AttributeResolver(rt_attribute)
runtimes[i]["node_name"] = attributes.get_node_name_as_str()
def __init__(self, uri, control_uri, attributes=None):
super(Node, self).__init__()
self.uri = uri
self.control_uri = control_uri
self.external_uri = attributes.pop('external_uri', self.uri) \
if attributes else self.uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
self.id = certificate.obtain_cert_node_info(self.node_name)['id']
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
try:
self.domain = _conf.get("security", "security_domain_name")
# cert_name is the node's certificate filename (without file extension)
self.cert_name = certificate.get_own_cert_name(self.node_name)
except:
self.domain = None
self.cert_name = None
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def test_cpu_total_invalid_value(self):
"""
Tests invalid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "2"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/cpuTotal')
def test_mem_avail_invalid_value(self):
"""
Tests invalid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memAvail": "1"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memAvail')
self.assertEqual(att.get_indexed_public()[0],
'/node/resource/memAvail')
def setup(self):
_conf.set('global', 'storage_type', 'local')
self.node = calvin.tests.TestNode(["127.0.0.1:5000"])
self.node.attributes = AttributeResolver(
{"indexed_public": {
"cpuTotal": "1"
}})
self.storage = storage.Storage(self.node)
self.cpu = CpuMonitor(self.node.id, self.storage)
self.done = False
self.storage.add_node(self.node)
yield threads.defer_to_thread(time.sleep, .01)
def manage_runtime_create(args):
if args.domain:
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
attr = json.loads(args.attr)
if not all (k in attr['indexed_public']['node_name'] for k in ("organization","name")):
raise Exception("please supply name and organization of runtime")
attributes=AttributeResolver(attr)
node_name=attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
print "CSR created at:" + certificate.new_runtime(node_name, args.domain, security_dir=args.dir, nodeid=nodeid)
def _set_aux(self, key, value, prefix_index, new_value=None):
"""
Auxiliary method to set indexes .
Removes old indexes before adding the new ones. Triggered by a get in the database
"""
# if new value is exactly the same, we don't need to change anything..
if value is new_value:
_log.debug("%s, value: %s. Nothing changed, just return.." %
(prefix_index, value))
return
# erase indexes related to old value
if value is not None:
old_data = AttributeResolver(
{"indexed_public": {
prefix_index: str(value)
}})
_log.debug("Removing " + str(key) + " for " + prefix_index + ": " +
str(value))
for index in old_data.get_indexed_public():
self.storage.remove_index(index=index,
value=key,
root_prefix_level=2)
# insert the new ones
if new_value is not None:
new_data = AttributeResolver(
{"indexed_public": {
prefix_index: str(new_value)
}})
_log.debug("After possible removal, adding new node " +
str(self.node_id) + " for " + prefix_index + ": " +
str(new_value))
for index in new_data.get_indexed_public():
self.storage.add_index(index=index,
value=self.node_id,
root_prefix_level=2,
cb=None)
def test_cpu_affinity(self):
"""
Tests cpu affinity parameter in indexed_public field
"""
att = AttributeResolver(
{"indexed_public": {
"cpuAffinity": "dedicated"
}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAffinity')
self.assertEqual(att_list[0][3], 'dedicated')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/cpuAffinity/dedicated')
def test_cpu_resources(self):
"""
Tests valid cpu resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuAvail": "100"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuAvail')
self.assertEqual(att_list[0][3], '0')
self.assertEqual(att_list[0][4], '25')
self.assertEqual(att_list[0][5], '50')
self.assertEqual(att_list[0][6], '75')
self.assertEqual(att_list[0][7], '100')
self.assertEqual(att.get_indexed_public()[0],
'/node/resource/cpuAvail/0/25/50/75/100')
def test_cpu_total(self):
"""
Tests valid CPU power in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"cpuTotal": "10000000"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'cpuTotal')
self.assertEqual(att_list[0][3], '1')
self.assertEqual(att_list[0][4], '1000')
self.assertEqual(att_list[0][5], '100000')
self.assertEqual(att_list[0][6], '1000000')
self.assertEqual(att_list[0][7], '10000000')
self.assertEqual(
att.get_indexed_public()[0],
'/node/attribute/cpuTotal/1/1000/100000/1000000/10000000')
def test_mem_total(self):
"""
Tests valid RAM resources in the indexed_public field
"""
att = AttributeResolver({"indexed_public": {"memTotal": "10G"}})
att_list = att.get_indexed_public(as_list=True)
self.assertEqual(att_list[0][2], 'memTotal')
self.assertEqual(att_list[0][3], '1K')
self.assertEqual(att_list[0][4], '100K')
self.assertEqual(att_list[0][5], '1M')
self.assertEqual(att_list[0][6], '100M')
self.assertEqual(att_list[0][7], '1G')
self.assertEqual(att_list[0][8], '10G')
self.assertEqual(att.get_indexed_public()[0],
'/node/attribute/memTotal/1K/100K/1M/100M/1G/10G')
def manage_runtime_create(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
if args.hostnames and len(args.hostnames) > 4:
raise Exception("At most 3 hostnames can be supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
rt_cred = runtime_credentials.RuntimeCredentials(node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
hostnames=args.hostnames)
print "node_name_start<{}>node_name_stop".format(rt_cred.get_node_name())
def manage_runtime_do_it_all(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
ca = certificate_authority.CA(domain=args.domain,
commonName=args.domain + " CA",
security_dir=args.dir)
ca_cert_path = ca.export_ca_cert("/tmp")
certificate.store_trusted_root_cert(ca_cert_path,
certificate.TRUSTSTORE_TRANSPORT,
security_dir=args.dir)
os.remove(ca_cert_path)
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
rt_cred = runtime_credentials.RuntimeCredentials(
node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
ca_cert = rt_cred.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr_path = rt_cred.get_encrypted_csr_path()
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request_path=rsa_encrypted_csr_path)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
print "\ncertificate stored at: {}\n".format(
rt_cred.store_own_cert(certpath=cert_path, security_dir=args.dir))
def manage_runtime_get_name(args):
from calvin.utilities.attribute_resolver import AttributeResolver
# Attributes
runtime_attr = {}
if args.attr_file:
try:
runtime_attr = json.load(open(args.attr_file))
except Exception as e:
print "Attribute file not JSON:\n", e
return -1
elif args.attr:
try:
runtime_attr = json.loads(args.attr)
except Exception as e:
print "Attributes not JSON:\n", e
return -1
else:
print "Error, either supply the attributes of the runtime, or the path to the file containg the attributes"
return -1
attributes = AttributeResolver(runtime_attr)
print "node_name_start<{}>node_name_stop\n".format(
attributes.get_node_name_as_str())
def __init__(self, uris, control_uri, attributes=None):
super(Node, self).__init__()
self.quitting = False
# Warn if its not a uri
if not isinstance(uris, list):
_log.error("Calvin uris must be a list %s" % uris)
raise TypeError("Calvin uris must be a list!")
# Uris
self.uris = uris
if attributes:
ext_uris = attributes.pop('external_uri', None)
if ext_uris is not None:
self.uris += ext_uris
# Control uri
self.control_uri = control_uri
self.external_control_uri = attributes.pop('external_control_uri', self.control_uri) \
if attributes else self.control_uri
try:
self.attributes = AttributeResolver(attributes)
except:
_log.exception("Attributes not correct, uses empty attribute!")
self.attributes = AttributeResolver(None)
self.node_name = self.attributes.get_node_name_as_str()
# Obtain node id, when using security also handle runtime certificate
try:
security_dir = _conf.get("security", "security_dir")
self.runtime_credentials = RuntimeCredentials(self.node_name, node=self, security_dir=security_dir)
self.id = self.runtime_credentials.get_node_id()
except Exception as err:
_log.debug("No runtime credentials, err={}".format(err))
self.runtime_credentials = None
self.id = calvinuuid.uuid("Node")
self.certificate_authority = certificate_authority.CertificateAuthority(self)
self.authentication = authentication.Authentication(self)
self.authorization = authorization.Authorization(self)
self.metering = metering.set_metering(metering.Metering(self))
self.monitor = Event_Monitor()
self.am = actormanager.ActorManager(self)
self.rm = replicationmanager.ReplicationManager(self)
self.control = calvincontrol.get_calvincontrol()
_scheduler = scheduler.DebugScheduler if _log.getEffectiveLevel() <= logging.DEBUG else scheduler.Scheduler
self.sched = _scheduler(self, self.am, self.monitor)
self.async_msg_ids = {}
self._calvinsys = CalvinSys(self)
calvinsys = get_calvinsys()
calvinsys.init(self)
calvinlib = get_calvinlib()
calvinlib.init(self)
# Default will multicast and listen on all interfaces
# TODO: be able to specify the interfaces
# @TODO: Store capabilities
self.storage = storage.Storage(self)
self.network = CalvinNetwork(self)
self.proto = CalvinProto(self, self.network)
self.pm = PortManager(self, self.proto)
self.app_manager = appmanager.AppManager(self)
# The initialization that requires the main loop operating is deferred to start function
async.DelayedCall(0, self.start)
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global rt
global request_handler
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.makedirs(credentials_testdir)
os.makedirs(runtimesdir)
os.makedirs(runtimes_truststore)
os.makedirs(runtimes_truststore_signing_path)
os.makedirs(actor_store_path)
os.makedirs(os.path.join(actor_store_path, "test"))
shutil.copy(
os.path.join(orig_actor_store_path, "test", "__init__.py"),
os.path.join(actor_store_path, "test", "__init__.py"))
os.makedirs(os.path.join(actor_store_path, "std"))
shutil.copy(
os.path.join(orig_actor_store_path, "std", "__init__.py"),
os.path.join(actor_store_path, "std", "__init__.py"))
shutil.copytree(orig_application_store_path,
application_store_path)
filelist = [
f for f in os.listdir(application_store_path)
if f.endswith(".sign.93d58fef")
]
for f in filelist:
os.remove(os.path.join(application_store_path, f))
shutil.copytree(
os.path.join(security_testdir, "identity_provider"),
identity_provider_path)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
print "Trying to create a new test application/actor signer."
cs = code_signer.CS(organization="testsigner",
commonName="signer",
security_dir=credentials_testdir)
#Create signed version of CountTimer actor
orig_actor_CountTimer_path = os.path.join(orig_actor_store_path, "std",
"CountTimer.py")
actor_CountTimer_path = os.path.join(actor_store_path, "std",
"CountTimer.py")
shutil.copy(orig_actor_CountTimer_path, actor_CountTimer_path)
# cs.sign_file(actor_CountTimer_path)
#Create unsigned version of CountTimer actor
actor_CountTimerUnsigned_path = actor_CountTimer_path.replace(
".py", "Unsigned.py")
shutil.copy(actor_CountTimer_path, actor_CountTimerUnsigned_path)
replace_text_in_file(actor_CountTimerUnsigned_path, "CountTimer",
"CountTimerUnsigned")
#Create signed version of Sum actor
orig_actor_Sum_path = os.path.join(orig_actor_store_path, "std",
"Sum.py")
actor_Sum_path = os.path.join(actor_store_path, "std", "Sum.py")
shutil.copy(orig_actor_Sum_path, actor_Sum_path)
# cs.sign_file(actor_Sum_path)
#Create unsigned version of Sum actor
actor_SumUnsigned_path = actor_Sum_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sum_path, actor_SumUnsigned_path)
replace_text_in_file(actor_SumUnsigned_path, "Sum", "SumUnsigned")
#Create incorrectly signed version of Sum actor
# actor_SumFake_path = actor_Sum_path.replace(".py", "Fake.py")
# shutil.copy(actor_Sum_path, actor_SumFake_path)
# #Change the class name to SumFake
# replace_text_in_file(actor_SumFake_path, "Sum", "SumFake")
# cs.sign_file(actor_SumFake_path)
# #Now append to the signed file so the signature verification fails
# with open(actor_SumFake_path, "a") as fd:
# fd.write(" ")
#Create signed version of Sink actor
orig_actor_Sink_path = os.path.join(orig_actor_store_path, "test",
"Sink.py")
actor_Sink_path = os.path.join(actor_store_path, "test", "Sink.py")
shutil.copy(orig_actor_Sink_path, actor_Sink_path)
# cs.sign_file(actor_Sink_path)
#Create unsigned version of Sink actor
actor_SinkUnsigned_path = actor_Sink_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sink_path, actor_SinkUnsigned_path)
replace_text_in_file(actor_SinkUnsigned_path, "Sink", "SinkUnsigned")
#Sign applications
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctly_signed.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctlySignedApp_incorrectlySignedActor.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"))
# #Now append to the signed file so the signature verification fails
# with open(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"), "a") as fd:
# fd.write(" ")
# print "Export Code Signers certificate to the truststore for code signing"
# out_file = cs.export_cs_cert(runtimes_truststore_signing_path)
print "Trying to create a new test domain configuration."
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
#
print "Copy CA cert into truststore of runtimes folder"
ca.export_ca_cert(runtimes_truststore)
#Define the runtime attributes
rt0_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'CA'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt1_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode1'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt2_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode2'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'otherStreet',
'streetNumber': 1
}
}
}
rt3_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode3'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt4_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode4'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt5_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode5'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt_attributes = []
rt_attributes.append(deepcopy(rt0_attributes))
rt_attributes.append(deepcopy(rt1_attributes))
rt_attributes.append(deepcopy(rt2_attributes))
rt_attributes.append(deepcopy(rt3_attributes))
rt_attributes.append(deepcopy(rt4_attributes))
rt_attributes.append(deepcopy(rt5_attributes))
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes:
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
#Let's hash passwords in users.json file (the runtimes will try to do this
# but they will all try to do it at the same time, so it will be overwritten
# multiple times and the first test will always fail)
# self.arp = FileAuthenticationRetrievalPoint(identity_provider_path)
# self.arp.check_stored_users_db_for_unhashed_passwords()
#The policy allows access to control interface for everyone, for more advanced rules
# it might be appropriate to run set_credentials for request_handler, e.g.,
# request_handler.set_credentials({domain_name:{"user": "******", "password": "******"}})
rt_conf = copy.deepcopy(_conf)
# rt_conf.set('security', 'runtime_to_runtime_security', "tls")
# rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
# rt_conf.set('security', 'certificate_authority_control_uri',"https://%s:5020" % hostname )
rt_conf.set('security', 'security_dir', credentials_testdir)
rt_conf.set('global', 'actor_paths', [actor_store_path])
rt_conf.set('global', 'storage_type', "securedht")
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf = copy.deepcopy(rt_conf)
# rt0_conf.set('security','enrollment_password',enrollment_passwords[0])
#The csruntime certificate requests assumes TLS for the control interface
# rt0_conf.set('security', 'control_interface_security', "tls")
# rt0_conf.set('security','certificate_authority','True')
# rt0_conf.set("security", "security_conf", {
# "comment": "Certificate Authority",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
# rt1_conf.set('security','enrollment_password',enrollment_passwords[1])
# rt1_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
# rt2_conf.set('security','enrollment_password',enrollment_passwords[2])
# rt2_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path,
# "accept_external_requests": True
# }
# })
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
# rt3_conf.set('security','enrollment_password',enrollment_passwords[3])
# rt3_conf.set("security", "security_conf", {
# "comment": "RADIUS authentication, local authorization",
# "authentication": {
# "procedure": "radius",
# "server_ip": "localhost",
# "secret": "elxghyc5lz1_passwd"
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
# rt4_conf.set('security','enrollment_password',enrollment_passwords[4])
# rt4_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "external"
# }
# })
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
# rt5_conf.set('global','storage_type','proxy')
# rt5_conf.set('global','storage_proxy',"calvinip://%s:5000" % ip_addr )
# rt5_conf.set('security','enrollment_password',enrollment_passwords[5])
# rt5_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "external",
# "server_uuid": runtimes[1].node_id
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes_cpy)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes_cpy[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
# rt.append(RT("https://{}:502{}".format(hostname, i)))
rt.append(RT("http://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(10)
request.addfinalizer(self.teardown)
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug(
"Some security features are enabled, let's make sure certificates are in place"
)
_ca_conf = _conf.get("security", "certificate_authority")
security_dir = _conf.get("security", "security_dir")
storage_type = _conf.get("global", "storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf[
"ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf[
"domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf[
"enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error(
"runtime_certificate: Failed to parse security configuration in calvin.conf, err={}"
.format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir,
node_name + ".csr")
if is_ca == "True":
_log.debug(
"No runtime certificate, but node is a CA, just sign csr, domain={}"
.format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path,
security_dir=security_dir)
else:
_log.debug(
"No runtime certicificate can be found, send CSR to CA"
)
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug(
"CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht", "securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers[
'location'].split('/node/')
ca_control_uri = ca_control_uri.replace(
"http", "https")
ca_control_uris.append(ca_control_uri)
_log.debug(
"CA control_uri={}, node_id={}".format(
ca_control_uri, ca_node_id))
else:
_log.error(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
raise Exception(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
cert_available = False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i = 0
while not cert_available and i < len(ca_control_uris):
certstr = None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
rsa_encrypted_csr = runtime.get_encrypted_csr()
j = 0
while not certstr and j < 10:
try:
certstr = request_handler.sign_csr_request(
ca_control_uris[i],
rsa_encrypted_csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j * j * j
_log.debug(
"RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}"
.format(time_to_sleep, err))
time.sleep(time_to_sleep)
j = j + 1
pass
else:
cert_available = True
i = i + 1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr,
security_dir=security_dir)
else:
_log.debug("Runtime certificate available")
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global hostname
global rt
global rt_attributes
global request_handler
try:
ipv6_hostname = socket.gethostbyaddr('::1')
except Exception as err:
print(
"Failed to resolve the IPv6 localhost hostname, please update the corresponding entry in the /etc/hosts file, e.g.,:\n"
"\t::1 <hostname>.localdomain <hostname>.local <hostname> localhost"
)
raise
try:
ipv6_hostname = socket.gethostbyaddr('::ffff:127.0.0.1')
except Exception as err:
print(
"Failed to resolve ::ffff:127.0.0.1, please add the following line (with your hostname) to /etc/hosts :\n"
"::ffff:127.0.0.1: <hostname>.localdomain <hostname>.local <hostname>"
)
raise
try:
hostname = socket.gethostname()
ip_addr = socket.gethostbyname(hostname)
fqdn = socket.getfqdn(hostname)
print("\n\tip_addr={}"
"\n\thostname={}"
"\n\tfqdn={}".format(ip_addr, hostname, fqdn))
except Exception as err:
print(
"Failed to resolve the hostname, ip_addr or the FQDN of the runtime, err={}"
.format(err))
raise
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.mkdir(credentials_testdir)
os.mkdir(runtimesdir)
os.mkdir(runtimes_truststore)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
_log.info("Trying to create a new test domain configuration.")
try:
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
except Exception as err:
_log.error("Failed to create CA, err={}".format(err))
_log.info("Copy CA cert into truststore of runtimes folder")
ca.export_ca_cert(runtimes_truststore)
node_names = []
rt_attributes = []
for i in range(6):
node_name = {
'organization': 'org.testexample',
'name': 'testNode{}'.format(i)
}
owner = {'organization': domain_name, 'personOrGroup': 'testOwner'}
address = {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
rt_attribute = {
'indexed_public': {
'owner': owner,
'node_name': node_name,
'address': address
}
}
rt_attributes.append(rt_attribute)
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes_cpy:
_log.info("rt_attribute={}".format(rt_attribute))
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
rt_conf = copy.deepcopy(_conf)
rt_conf.set('security', 'runtime_to_runtime_security', "tls")
rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
rt_conf.set('security', 'security_dir', credentials_testdir)
rt0_conf = copy.deepcopy(rt_conf)
rt_conf.set('global', 'storage_type', 'proxy')
rt_conf.set('global', 'storage_proxy', "calvinip://%s:5000" % hostname)
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf.set('global', 'storage_type', 'local')
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
rt.append(RT("https://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(2)
request.addfinalizer(self.teardown)
