def manage_runtime_remove(args):
if not args.domain:
raise Exception("No domain supplied")
domaindir = os.path.join(args.dir, args.domain) if args.dir else None
runtime = runtime_credentials.RuntimeCredentials(args.node_name,
security_dir=args.dir)
runtime.remove_runtime(args.node_name, domaindir)
def start(self):
callbacks = {'connected': [CalvinCB(self._connected)]}
tcp_f = TCPServerFactory(callbacks)
runtime_to_runtime_security = _conf.get("security","runtime_to_runtime_security")
trusted_ca_certs = []
if runtime_to_runtime_security=="tls":
_log.debug("TwistedCalvinServer with TLS chosen")
try:
self._runtime_credentials = runtime_credentials.RuntimeCredentials(self._node_name)
ca_cert_list_str, ca_cert_list_x509, truststore =certificate.get_truststore(certificate.TRUSTSTORE_TRANSPORT)
for ca_cert in ca_cert_list_str:
trusted_ca_certs.append(ssl.Certificate.loadPEM(ca_cert))
server_credentials_data = self._runtime_credentials.get_runtime_credentials()
server_credentials = ssl.PrivateCertificate.loadPEM(server_credentials_data)
except Exception as err:
_log.exception("Server failed to load credentials, err={}".format(err))
try:
self._tcp_server = reactor.listenSSL(self._port, tcp_f, server_credentials.options(*trusted_ca_certs), interface=self._iface)
except Exception as err:
_log.exception("Server failed listenSSL, err={}".format(err))
else:
_log.debug("TwistedCalvinServer without TLS chosen")
try:
self._tcp_server = reactor.listenTCP(self._port, tcp_f, interface=self._iface)
except error.CannotListenError:
_log.exception("Could not listen on port %s:%s", self._iface, self._port)
raise
except Exception as exc:
_log.exception("Failed when trying listening on port %s:%s", self._iface, self._port)
raise
self._port = self._tcp_server.getHost().port
self._callback_execute('server_started', self._port)
return self._port
def reactor_listen(node_name, factory, host, port):
listener = None
control_interface_security = _conf.get("security", "control_interface_security")
if control_interface_security == "tls":
_log.debug("ServerProtocolFactory with TLS enabled chosen")
try:
# TODO: figure out how to set more than one root cert in twisted truststore
runtime_cred = runtime_credentials.RuntimeCredentials(node_name)
server_credentials_data = runtime_cred.get_runtime_credentials()
server_credentials = ssl.PrivateCertificate.loadPEM(server_credentials_data)
except Exception as err:
_log.error("Failed to fetch server credentials, err={}".format(err))
raise
try:
listener = reactor.listenSSL(port, factory, server_credentials.options(), interface=host)
except Exception as err:
_log.error("Server failed listenSSL, err={}".format(err))
else:
listener = reactor.listenTCP(port, factory, interface=host)
# @tif, check with Harald how to verify that workaround is still ok.
# WORKAROUND This is here due to an obscure error in twisted trying to write to a listening port
# on some architectures/OSes. The default is to raise a RuntimeError.
listener.doWrite = lambda: None
return listener
def manage_runtime_import(args):
if not args.node_name:
raise Exception("No node name supplied")
if not args.certificate:
raise Exception("No certificate supplied")
runtime = runtime_credentials.RuntimeCredentials(args.node_name,
security_dir=args.dir)
cert_path = runtime.store_own_cert(certpath=args.certificate)
print "cert_path_start<{}>cert_path_stop".format(cert_path)
def join(self):
from twisted.internet._sslverify import OpenSSLCertificateAuthorities
from OpenSSL import crypto
if self._proto:
raise Exception("Already connected")
# Own callbacks
callbacks = {'connected': [CalvinCB(self._connected)],
'disconnected': [CalvinCB(self._disconnected)],
'connection_failed': [CalvinCB(self._connection_failed)],
'data': [CalvinCB(self._data)],
'set_proto': [CalvinCB(self._set_proto)]}
self._factory = TCPClientFactory(callbacks) # addr="%s:%s" % (self._host_ip, self._host_port))
runtime_to_runtime_security = _conf.get("security","runtime_to_runtime_security")
if runtime_to_runtime_security=="tls":
_log.debug("TwistedCalvinTransport with TLS chosen")
trusted_ca_certs = []
try:
self._runtime_credentials = runtime_credentials.RuntimeCredentials(self._node_name)
ca_cert_list_str, ca_cert_list_x509, truststore = certificate.get_truststore(certificate.TRUSTSTORE_TRANSPORT)
for ca_cert in ca_cert_list_str:
trusted_ca_certs.append(crypto.load_certificate(crypto.FILETYPE_PEM, ca_cert))
ca_certs = OpenSSLCertificateAuthorities(trusted_ca_certs)
client_credentials_data =self._runtime_credentials.get_runtime_credentials()
client_credentials = ssl.PrivateCertificate.loadPEM(client_credentials_data)
except Exception as err:
_log.error("TwistedCalvinTransport: Failed to load client credentials, err={}".format(err))
raise
try:
options = ssl.optionsForClientTLS(self._server_node_name,
trustRoot=ca_certs,
clientCertificate=client_credentials)
except Exception as err:
_log.error("TwistedCalvinTransport: Failed to create optionsForClientTLS "
"\n\terr={}"
"\n\tself._server_node_name={}".format(err,
self._server_node_name))
raise
try:
endpoint = endpoints.SSL4ClientEndpoint(reactor,
self._host_ip,
int(self._host_port),
options)
except:
_log.error("TwistedCalvinTransport: Client failed connectSSL")
raise
try:
endpoint.connect(self._factory)
except Exception as e:
_log.error("TwistedCalvinTransport: Failed endpoint.connect, e={}".format(e))
raise
else:
reactor.connectTCP(self._host_ip, int(self._host_port), self._factory)
def manage_runtime_encrypt_csr_with_enrollment_password(args):
if not args.node_name:
raise Exception("No node name supplied")
if not args.enrollment_password:
raise Exception("No enrollment password supplied")
rt_cred = runtime_credentials.RuntimeCredentials(
args.node_name,
security_dir=args.dir,
enrollment_password=args.enrollment_password)
try:
encr_csr = rt_cred.cert_enrollment_encrypt_csr()
except Exception as err:
print "Failed to encrypt CSR, err={}".format(err)
raise
encr_csr_path = rt_cred.get_encrypted_csr_path()
print "encr_csr_path_start<{}>encr_csr_path_stop".format(encr_csr_path)
def _generate_certiticates(ca, runtimes, domain_name, credentials_testdir):
from calvin.utilities import runtime_credentials
from calvin.utilities import calvinuuid
for i in range(len(runtimes)):
runtimes[i]["id"] = calvinuuid.uuid("")
node_name = runtimes[i]["node_name"]
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
# hostnames=["elxahyc5lz1","elxahyc5lz1.localdomain"],
security_dir=credentials_testdir,
nodeid=runtimes[i]["id"],
enrollment_password=runtimes[i]["enrollment_password"])
runtimes[i]["credentials"] = runtime
csr_path = runtime.get_csr_path()
#CAs generated certificate for runtime
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path)
def manage_runtime_create(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
if args.hostnames and len(args.hostnames) > 4:
raise Exception("At most 3 hostnames can be supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
rt_cred = runtime_credentials.RuntimeCredentials(node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
hostnames=args.hostnames)
print "node_name_start<{}>node_name_stop".format(rt_cred.get_node_name())
def _generate_certiticates(ca, runtimes, domain_name, credentials_testdir):
from calvin.utilities import runtime_credentials
for i in range(len(runtimes)):
node_name = runtimes[i]["node_name"]
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
# hostnames=["elxahyc5lz1","elxahyc5lz1.localdomain"],
security_dir=credentials_testdir,
nodeid=runtimes[i]["id"],
enrollment_password=runtimes[i]["enrollment_password"])
runtimes[i]["credentials"] = runtime
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Decrypt encrypted CSR with CAs private key
rsa_encrypted_csr = runtime.get_encrypted_csr()
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
def start(self, host, port):
control_interface_security = _conf.get("security","control_interface_security")
if control_interface_security=="tls":
_log.debug("ServerProtocolFactory with TLS enabled chosen")
try:
#TODO: figure out how to set more than one root cert in twisted truststore
runtime_cred = runtime_credentials.RuntimeCredentials(self._node_name)
server_credentials_data = runtime_cred.get_runtime_credentials()
server_credentials = ssl.PrivateCertificate.loadPEM(server_credentials_data)
except Exception as err:
_log.error("Failed to fetch server credentials, err={}".format(err))
raise
try:
self._tls_server = reactor.listenSSL(port, self, server_credentials.options(), interface=host)
except Exception as err:
_log.error("Server failed listenSSL, err={}".format(err))
self._port = self._tls_server.getHost().port
else:
self._port = reactor.listenTCP(port, self, interface=host)
# WORKAROUND This is here due to an obscure error in twisted trying to write to a listening port
# on some architectures/OSes. The default is to raise a RuntimeError.
self._port.doWrite = lambda : None
def manage_runtime_do_it_all(args):
if not args.attr:
raise Exception("No runtime attributes supplied")
if not args.domain:
raise Exception("No domain name supplied")
attr = json.loads(args.attr)
if not all(k in attr['indexed_public']['node_name']
for k in ("organization", "name")):
raise Exception("please supply name and organization of runtime")
ca = certificate_authority.CA(domain=args.domain,
commonName=args.domain + " CA",
security_dir=args.dir)
ca_cert_path = ca.export_ca_cert("/tmp")
certificate.store_trusted_root_cert(ca_cert_path,
certificate.TRUSTSTORE_TRANSPORT,
security_dir=args.dir)
os.remove(ca_cert_path)
attributes = AttributeResolver(attr)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("NODE")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
rt_cred = runtime_credentials.RuntimeCredentials(
node_name,
domain=args.domain,
security_dir=args.dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
ca_cert = rt_cred.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr_path = rt_cred.get_encrypted_csr_path()
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request_path=rsa_encrypted_csr_path)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
print "\ncertificate stored at: {}\n".format(
rt_cred.store_own_cert(certpath=cert_path, security_dir=args.dir))
def test_dht_multi(self, monkeypatch):
iface = "0.0.0.0"
a = None
b = None
q = Queue.Queue()
rt_credentials = []
def server_started(aa, *args):
for b in args:
if isinstance(b, twisted.python.failure.Failure):
b.printTraceback()
else:
_log.debug("** %s" % b)
q.put([aa, args])
try:
amount_of_servers = 5
# Twisted is using 20 threads so having > 20 server
# causes threadlocks really easily.
servers = []
callbacks = []
for servno in range(0, amount_of_servers):
uri = "http://{}:500{}".format(ip_addr, servno)
control_uri = "http://{}:502{}".format(ip_addr, servno)
rt_cred = runtime_credentials.RuntimeCredentials(
name=name + str(servno),
domain=domain,
security_dir=testdir)
rt_credentials.append(rt_cred)
node_id = rt_cred.node_id
a = AutoDHTServer(node_id, control_uri, rt_cred)
servers.append(a)
callback = CalvinCB(server_started, str(servno))
servers[servno].start(iface,
network="Niklas",
cb=callback,
name=name + "{}".format(servno))
callbacks.append(callback)
# Wait for start
started = []
while len(started) < amount_of_servers:
try:
server = yield threads.defer_to_thread(q.get)
except Queue.Empty:
_log.debug("Queue empty!")
#raise
if server not in started:
started.append(server)
#print("DHT Servers added: {}".format(started))
callbacks[int(
server[0][0])].func = lambda *args, **kvargs: None
else:
print("Server: {} already started." \
" {} out of {}".format(started,
len(started),
amount_of_servers))
print("All {} out of {} started".format(started, len(started),
amount_of_servers))
for servno in range(0, amount_of_servers):
assert [str(servno), self._sucess_start] in started
yield threads.defer_to_thread(q.queue.clear)
yield threads.defer_to_thread(time.sleep, 8)
key = "HARE"
value = json.dumps(["morot"])
set_def = servers[0].append(key=key, value=value)
set_value = yield threads.defer_to_thread(set_def.wait, 10)
assert set_value
print("Node with port {} posted append key={}, value={}".format(
servers[0].dht_server.port.getHost().port, key, value))
value = json.dumps(["selleri"])
set_def = servers[0].append(key=key, value=value)
set_value = yield threads.defer_to_thread(set_def.wait, 10)
assert set_value
print("Node with port {} posted append key={}, value={}".format(
servers[0].dht_server.port.getHost().port, key, value))
get_def = servers[0].get_concat(key=key)
get_value = yield threads.defer_to_thread(get_def.wait, 10)
assert set(json.loads(get_value)) == set(["morot", "selleri"])
print("Node with port {} confirmed key={}, value={} was reachable".
format(servers[0].dht_server.port.getHost().port, key,
get_value))
drawNetworkState("1nice_graph.png", servers, amount_of_servers)
yield threads.defer_to_thread(time.sleep, 7)
drawNetworkState("1middle_graph.png", servers, amount_of_servers)
yield threads.defer_to_thread(time.sleep, 7)
drawNetworkState("1end_graph.png", servers, amount_of_servers)
get_def = servers[0].get_concat(key=key)
get_value = yield threads.defer_to_thread(get_def.wait, 10)
assert set(json.loads(get_value)) == set(["morot", "selleri"])
print("Node with port {} got right value: {}".format(
servers[0].dht_server.port.getHost().port, get_value))
value = json.dumps(["morot"])
set_def = servers[0].remove(key=key, value=value)
set_value = yield threads.defer_to_thread(set_def.wait, 10)
assert set_value
print("Node with port {} posted remove key={}, value={}".format(
servers[0].dht_server.port.getHost().port, key, value))
get_def = servers[1].get_concat(key=key)
get_value = yield threads.defer_to_thread(get_def.wait, 10)
assert set(json.loads(get_value)) == set(["selleri"])
print("Node with port {} got right value: {}".format(
servers[0].dht_server.port.getHost().port, get_value))
for i in range(0, amount_of_servers):
name_dir = rt_credentials[i].get_credentials_path()
filenames = os.listdir(os.path.join(name_dir, "others"))
print("Node with port {} has {} certificates in store".format(
servers[i].dht_server.port.getHost().port, len(filenames)))
except AssertionError as e:
print(
"Node with port {} got wrong value: {}, should have been {}, e={}"
.format(servers[0].dht_server.port.getHost().port, get_value,
value, e))
pytest.fail(traceback.format_exc())
except Exception as e:
traceback.print_exc()
pytest.fail(traceback.format_exc())
finally:
yield threads.defer_to_thread(time.sleep, 10)
for server in servers:
server.stop()
def test_dht_multi(self, monkeypatch):
iface = "0.0.0.0"
a = None
b = None
q = Queue.Queue()
def server_started(aa, *args):
for b in args:
if isinstance(b, twisted.python.failure.Failure):
b.printTraceback()
else:
_log.debug("** %s" % b)
q.put([aa, args])
try:
amount_of_servers = 1
# Twisted is using 20 threads so having > 20 server
# causes threadlocks really easily.
servers = []
callbacks = []
for servno in range(0, amount_of_servers):
uri = "http://{}:500{}".format(ip_addr, servno)
control_uri = "http://{}:502{}".format(ip_addr, servno)
rt_cred = runtime_credentials.RuntimeCredentials(
name=name, domain=domain, security_dir=testdir)
node_id = rt_cred.node_id
a = evilAutoDHTServer(node_id, control_uri, rt_cred)
servers.append(a)
callback = CalvinCB(server_started, str(servno))
servers[servno].start(iface,
network="Niklas",
cb=callback,
type="poison",
name=name)
# servers[servno].start(iface, network="Hej", cb=callback, type="eclipse", name=name)
# servers[servno].start(iface, network="Hej", cb=callback, type="sybil")
# servers[servno].start(iface, network="Hej", cb=callback, type="insert")
callbacks.append(callback)
print("Starting {}".format(
servers[servno].dht_server.port.getHost().port))
# Wait for start
started = []
while len(started) < amount_of_servers:
try:
server = yield threads.defer_to_thread(q.get)
except Queue.Empty:
_log.debug("Queue empty!")
#raise
if server not in started:
started.append(server)
_log.debug("DHT Servers added: {}".format(started))
callbacks[int(
server[0][0])].func = lambda *args, **kvargs: None
else:
_log.debug("Server: {} already started." \
" {} out of {}".format(started,
len(started),
amount_of_servers))
_log.debug("All {} out of {} started".format(
started, len(started), amount_of_servers))
for servno in range(0, amount_of_servers):
assert [str(servno), self._sucess_start] in started
yield threads.defer_to_thread(q.queue.clear)
yield threads.defer_to_thread(time.sleep, 15)
evilPort = servers[0].dht_server.port.getHost().port
drawNetworkState("start_graph.png", servers, amount_of_servers)
servers[0].dht_server.kserver.protocol.turn_evil(evilPort)
print("Node with port {} turned evil".format(
servers[servno].dht_server.port.getHost().port))
yield threads.defer_to_thread(time.sleep, 12)
# assert get_value[0] == "banan"
# assert get_value[1] == "bambu"
# assert get_value[2] == "morot"
yield threads.defer_to_thread(time.sleep, 5)
print("Attacking node exiting")
except AssertionError as e:
print("Server {} with port {} got wrong value".format(
servno, servers[servno].dht_server.port.getHost().port))
pytest.fail(traceback.format_exc())
except Exception as e:
traceback.print_exc()
pytest.fail(traceback.format_exc())
finally:
for server in servers:
server.stop()
yield threads.defer_to_thread(time.sleep, 5)
commonName="sec-dht-test-security-CA",
security_dir=testdir)
print "Created new domain."
print "Import CA cert into truststore."
testca.export_ca_cert(runtimes_truststore)
print "Generate runtime credentials and sign their certificates"
for i in range(1, 5):
for j in range(0, 6):
name = "node{}:{}".format(i, j)
enrollment_password = testca.cert_enrollment_add_new_runtime(name)
nodeid = calvinuuid.uuid("NODE")
rt_cred = runtime_credentials.RuntimeCredentials(
name,
domain="test",
security_dir=testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
csr_path = os.path.join(rt_cred.runtime_dir, name + ".csr")
try:
with open(csr_path + ".challenge_password", 'w') as csr_fd:
csr_fd.write(enrollment_password)
except Exception as err:
_log.exception(
"Failed to write challenge password to file, err={}".format(
err))
raise
certpath = testca.sign_csr(csr_path)
rt_cred.store_own_cert(certpath=certpath)
print "Generate evil node runtime credentials and sign certificate"
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global rt
global request_handler
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.makedirs(credentials_testdir)
os.makedirs(runtimesdir)
os.makedirs(runtimes_truststore)
os.makedirs(runtimes_truststore_signing_path)
os.makedirs(actor_store_path)
os.makedirs(os.path.join(actor_store_path, "test"))
shutil.copy(
os.path.join(orig_actor_store_path, "test", "__init__.py"),
os.path.join(actor_store_path, "test", "__init__.py"))
os.makedirs(os.path.join(actor_store_path, "std"))
shutil.copy(
os.path.join(orig_actor_store_path, "std", "__init__.py"),
os.path.join(actor_store_path, "std", "__init__.py"))
shutil.copytree(orig_application_store_path,
application_store_path)
filelist = [
f for f in os.listdir(application_store_path)
if f.endswith(".sign.93d58fef")
]
for f in filelist:
os.remove(os.path.join(application_store_path, f))
shutil.copytree(
os.path.join(security_testdir, "identity_provider"),
identity_provider_path)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
print "Trying to create a new test application/actor signer."
cs = code_signer.CS(organization="testsigner",
commonName="signer",
security_dir=credentials_testdir)
#Create signed version of CountTimer actor
orig_actor_CountTimer_path = os.path.join(orig_actor_store_path, "std",
"CountTimer.py")
actor_CountTimer_path = os.path.join(actor_store_path, "std",
"CountTimer.py")
shutil.copy(orig_actor_CountTimer_path, actor_CountTimer_path)
# cs.sign_file(actor_CountTimer_path)
#Create unsigned version of CountTimer actor
actor_CountTimerUnsigned_path = actor_CountTimer_path.replace(
".py", "Unsigned.py")
shutil.copy(actor_CountTimer_path, actor_CountTimerUnsigned_path)
replace_text_in_file(actor_CountTimerUnsigned_path, "CountTimer",
"CountTimerUnsigned")
#Create signed version of Sum actor
orig_actor_Sum_path = os.path.join(orig_actor_store_path, "std",
"Sum.py")
actor_Sum_path = os.path.join(actor_store_path, "std", "Sum.py")
shutil.copy(orig_actor_Sum_path, actor_Sum_path)
# cs.sign_file(actor_Sum_path)
#Create unsigned version of Sum actor
actor_SumUnsigned_path = actor_Sum_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sum_path, actor_SumUnsigned_path)
replace_text_in_file(actor_SumUnsigned_path, "Sum", "SumUnsigned")
#Create incorrectly signed version of Sum actor
# actor_SumFake_path = actor_Sum_path.replace(".py", "Fake.py")
# shutil.copy(actor_Sum_path, actor_SumFake_path)
# #Change the class name to SumFake
# replace_text_in_file(actor_SumFake_path, "Sum", "SumFake")
# cs.sign_file(actor_SumFake_path)
# #Now append to the signed file so the signature verification fails
# with open(actor_SumFake_path, "a") as fd:
# fd.write(" ")
#Create signed version of Sink actor
orig_actor_Sink_path = os.path.join(orig_actor_store_path, "test",
"Sink.py")
actor_Sink_path = os.path.join(actor_store_path, "test", "Sink.py")
shutil.copy(orig_actor_Sink_path, actor_Sink_path)
# cs.sign_file(actor_Sink_path)
#Create unsigned version of Sink actor
actor_SinkUnsigned_path = actor_Sink_path.replace(".py", "Unsigned.py")
shutil.copy(actor_Sink_path, actor_SinkUnsigned_path)
replace_text_in_file(actor_SinkUnsigned_path, "Sink", "SinkUnsigned")
#Sign applications
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctly_signed.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_correctlySignedApp_incorrectlySignedActor.calvin"))
# cs.sign_file(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"))
# #Now append to the signed file so the signature verification fails
# with open(os.path.join(application_store_path, "test_security1_incorrectly_signed.calvin"), "a") as fd:
# fd.write(" ")
# print "Export Code Signers certificate to the truststore for code signing"
# out_file = cs.export_cs_cert(runtimes_truststore_signing_path)
print "Trying to create a new test domain configuration."
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
#
print "Copy CA cert into truststore of runtimes folder"
ca.export_ca_cert(runtimes_truststore)
#Define the runtime attributes
rt0_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'CA'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt1_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode1'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt2_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode2'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'otherStreet',
'streetNumber': 1
}
}
}
rt3_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode3'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt4_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode4'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt5_attributes = {
'indexed_public': {
'owner': {
'organization': domain_name,
'personOrGroup': 'testOwner1'
},
'node_name': {
'organization': 'org.testexample',
'name': 'testNode5'
},
'address': {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
}
}
rt_attributes = []
rt_attributes.append(deepcopy(rt0_attributes))
rt_attributes.append(deepcopy(rt1_attributes))
rt_attributes.append(deepcopy(rt2_attributes))
rt_attributes.append(deepcopy(rt3_attributes))
rt_attributes.append(deepcopy(rt4_attributes))
rt_attributes.append(deepcopy(rt5_attributes))
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes:
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
#Let's hash passwords in users.json file (the runtimes will try to do this
# but they will all try to do it at the same time, so it will be overwritten
# multiple times and the first test will always fail)
# self.arp = FileAuthenticationRetrievalPoint(identity_provider_path)
# self.arp.check_stored_users_db_for_unhashed_passwords()
#The policy allows access to control interface for everyone, for more advanced rules
# it might be appropriate to run set_credentials for request_handler, e.g.,
# request_handler.set_credentials({domain_name:{"user": "******", "password": "******"}})
rt_conf = copy.deepcopy(_conf)
# rt_conf.set('security', 'runtime_to_runtime_security', "tls")
# rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
# rt_conf.set('security', 'certificate_authority_control_uri',"https://%s:5020" % hostname )
rt_conf.set('security', 'security_dir', credentials_testdir)
rt_conf.set('global', 'actor_paths', [actor_store_path])
rt_conf.set('global', 'storage_type', "securedht")
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf = copy.deepcopy(rt_conf)
# rt0_conf.set('security','enrollment_password',enrollment_passwords[0])
#The csruntime certificate requests assumes TLS for the control interface
# rt0_conf.set('security', 'control_interface_security', "tls")
# rt0_conf.set('security','certificate_authority','True')
# rt0_conf.set("security", "security_conf", {
# "comment": "Certificate Authority",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
# rt1_conf.set('security','enrollment_password',enrollment_passwords[1])
# rt1_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
# rt2_conf.set('security','enrollment_password',enrollment_passwords[2])
# rt2_conf.set("security", "security_conf", {
# "comment": "Local authentication, local authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path,
# "accept_external_requests": True
# }
# })
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
# rt3_conf.set('security','enrollment_password',enrollment_passwords[3])
# rt3_conf.set("security", "security_conf", {
# "comment": "RADIUS authentication, local authorization",
# "authentication": {
# "procedure": "radius",
# "server_ip": "localhost",
# "secret": "elxghyc5lz1_passwd"
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
# rt4_conf.set('security','enrollment_password',enrollment_passwords[4])
# rt4_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "local",
# "identity_provider_path": identity_provider_path
# },
# "authorization": {
# "procedure": "external"
# }
# })
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
# rt5_conf.set('global','storage_type','proxy')
# rt5_conf.set('global','storage_proxy',"calvinip://%s:5000" % ip_addr )
# rt5_conf.set('security','enrollment_password',enrollment_passwords[5])
# rt5_conf.set("security", "security_conf", {
# "comment": "Local authentication, external authorization",
# "authentication": {
# "procedure": "external",
# "server_uuid": runtimes[1].node_id
# },
# "authorization": {
# "procedure": "local",
# "policy_storage_path": policy_storage_path
# }
# })
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes_cpy)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes_cpy[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
# rt.append(RT("https://{}:502{}".format(hostname, i)))
rt.append(RT("http://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(10)
request.addfinalizer(self.teardown)
def join(self):
if self._proto:
raise Exception("Already connected")
# Own callbacks
callbacks = {
'connected': [CalvinCB(self._connected)],
'disconnected': [CalvinCB(self._disconnected)],
'connection_failed': [CalvinCB(self._connection_failed)],
'data': [CalvinCB(self._data)],
'set_proto': [CalvinCB(self._set_proto)]
}
self._factory = TCPClientFactory(
callbacks) # addr="%s:%s" % (self._host_ip, self._host_port))
runtime_to_runtime_security = _conf.get("security",
"runtime_to_runtime_security")
if runtime_to_runtime_security == "tls":
_log.debug("TwistedCalvinTransport with TLS chosen")
try:
self._runtime_credentials = runtime_credentials.RuntimeCredentials(
self._node_name)
ca_cert_list_str, ca_cert_list_x509, truststore = certificate.get_truststore(
certificate.TRUSTSTORE_TRANSPORT)
#TODO: figure out how to set more than one root cert in twisted truststore
twisted_trusted_ca_cert = ssl.Certificate.loadPEM(
ca_cert_list_str[0])
client_credentials_data = self._runtime_credentials.get_runtime_credentials(
)
client_credentials = ssl.PrivateCertificate.loadPEM(
client_credentials_data)
except Exception as err:
_log.error(
"TwistedCalvinTransport: Failed to load client credentials, err={}"
.format(err))
raise
try:
options = ssl.optionsForClientTLS(self._server_node_name,
twisted_trusted_ca_cert,
client_credentials)
except Exception as err:
_log.error(
"TwistedCalvinTransport: Failed to create optionsForClientTLS "
"\n\terr={}"
"\n\tself._server_node_name={}".format(
err, self._server_node_name))
raise
try:
endpoint = endpoints.SSL4ClientEndpoint(
reactor, self._host_ip, int(self._host_port), options)
except:
_log.error("TwistedCalvinTransport: Client failed connectSSL")
raise
try:
endpoint.connect(self._factory)
except Exception as e:
_log.error(
"TwistedCalvinTransport: Failed endpoint.connect, e={}".
format(e))
raise
else:
reactor.connectTCP(self._host_ip, int(self._host_port),
self._factory)
def setup(self, request):
from calvin.Tools.csruntime import csruntime
from conftest import _config_pytest
import fileinput
global hostname
global rt
global rt_attributes
global request_handler
try:
ipv6_hostname = socket.gethostbyaddr('::1')
except Exception as err:
print(
"Failed to resolve the IPv6 localhost hostname, please update the corresponding entry in the /etc/hosts file, e.g.,:\n"
"\t::1 <hostname>.localdomain <hostname>.local <hostname> localhost"
)
raise
try:
ipv6_hostname = socket.gethostbyaddr('::ffff:127.0.0.1')
except Exception as err:
print(
"Failed to resolve ::ffff:127.0.0.1, please add the following line (with your hostname) to /etc/hosts :\n"
"::ffff:127.0.0.1: <hostname>.localdomain <hostname>.local <hostname>"
)
raise
try:
hostname = socket.gethostname()
ip_addr = socket.gethostbyname(hostname)
fqdn = socket.getfqdn(hostname)
print("\n\tip_addr={}"
"\n\thostname={}"
"\n\tfqdn={}".format(ip_addr, hostname, fqdn))
except Exception as err:
print(
"Failed to resolve the hostname, ip_addr or the FQDN of the runtime, err={}"
.format(err))
raise
try:
shutil.rmtree(credentials_testdir)
except Exception as err:
print "Failed to remove old tesdir, err={}".format(err)
pass
try:
os.mkdir(credentials_testdir)
os.mkdir(runtimesdir)
os.mkdir(runtimes_truststore)
except Exception as err:
_log.error(
"Failed to create test folder structure, err={}".format(err))
print "Failed to create test folder structure, err={}".format(err)
raise
_log.info("Trying to create a new test domain configuration.")
try:
ca = certificate_authority.CA(domain=domain_name,
commonName="testdomain CA",
security_dir=credentials_testdir)
except Exception as err:
_log.error("Failed to create CA, err={}".format(err))
_log.info("Copy CA cert into truststore of runtimes folder")
ca.export_ca_cert(runtimes_truststore)
node_names = []
rt_attributes = []
for i in range(6):
node_name = {
'organization': 'org.testexample',
'name': 'testNode{}'.format(i)
}
owner = {'organization': domain_name, 'personOrGroup': 'testOwner'}
address = {
'country': 'SE',
'locality': 'testCity',
'street': 'testStreet',
'streetNumber': 1
}
rt_attribute = {
'indexed_public': {
'owner': owner,
'node_name': node_name,
'address': address
}
}
rt_attributes.append(rt_attribute)
rt_attributes_cpy = deepcopy(rt_attributes)
runtimes = []
#Initiate Requesthandler with trusted CA cert
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=credentials_testdir)
# The following is less than optimal if multiple CA certs exist
ca_cert_path = os.path.join(truststore_dir,
os.listdir(truststore_dir)[0])
request_handler = RequestHandler(verify=ca_cert_path)
#Generate credentials, create CSR, sign with CA and import cert for all runtimes
enrollment_passwords = []
for rt_attribute in rt_attributes_cpy:
_log.info("rt_attribute={}".format(rt_attribute))
attributes = AttributeResolver(rt_attribute)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
enrollment_password = ca.cert_enrollment_add_new_runtime(node_name)
enrollment_passwords.append(enrollment_password)
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain=domain_name,
security_dir=credentials_testdir,
nodeid=nodeid,
enrollment_password=enrollment_password)
runtimes.append(runtime)
ca_cert = runtime.get_truststore(
type=certificate.TRUSTSTORE_TRANSPORT)[0][0]
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
#Encrypt CSR with CAs public key (to protect enrollment password)
rsa_encrypted_csr = runtime.cert_enrollment_encrypt_csr(
csr_path, ca_cert)
#Decrypt encrypted CSR with CAs private key
csr = ca.decrypt_encrypted_csr(
encrypted_enrollment_request=rsa_encrypted_csr)
csr_path = ca.store_csr_with_enrollment_password(csr)
cert_path = ca.sign_csr(csr_path)
runtime.store_own_cert(certpath=cert_path,
security_dir=credentials_testdir)
rt_conf = copy.deepcopy(_conf)
rt_conf.set('security', 'runtime_to_runtime_security', "tls")
rt_conf.set('security', 'control_interface_security', "tls")
rt_conf.set('security', 'domain_name', domain_name)
rt_conf.set('security', 'security_dir', credentials_testdir)
rt0_conf = copy.deepcopy(rt_conf)
rt_conf.set('global', 'storage_type', 'proxy')
rt_conf.set('global', 'storage_proxy', "calvinip://%s:5000" % hostname)
# Runtime 0: local authentication, signature verification, local authorization.
# Primarily acts as Certificate Authority for the domain
rt0_conf.set('global', 'storage_type', 'local')
rt0_conf.save("/tmp/calvin5000.conf")
# Runtime 1: local authentication, signature verification, local authorization.
rt1_conf = copy.deepcopy(rt_conf)
rt1_conf.save("/tmp/calvin5001.conf")
# Runtime 2: local authentication, signature verification, local authorization.
# Can also act as authorization server for other runtimes.
# Other street compared to the other runtimes
rt2_conf = copy.deepcopy(rt_conf)
rt2_conf.save("/tmp/calvin5002.conf")
# Runtime 3: external authentication (RADIUS), signature verification, local authorization.
rt3_conf = copy.deepcopy(rt_conf)
rt3_conf.save("/tmp/calvin5003.conf")
# Runtime 4: local authentication, signature verification, external authorization (runtime 2).
rt4_conf = copy.deepcopy(rt_conf)
rt4_conf.save("/tmp/calvin5004.conf")
# Runtime 5: external authentication (runtime 1), signature verification, local authorization.
rt5_conf = copy.deepcopy(rt_conf)
rt5_conf.save("/tmp/calvin5005.conf")
#Start all runtimes
for i in range(len(rt_attributes)):
_log.info("Starting runtime {}".format(i))
try:
logfile = _config_pytest.getoption("logfile") + "500{}".format(
i)
outfile = os.path.join(
os.path.dirname(logfile),
os.path.basename(logfile).replace("log", "out"))
if outfile == logfile:
outfile = None
except:
logfile = None
outfile = None
csruntime(hostname,
port=5000 + i,
controlport=5020 + i,
attr=rt_attributes[i],
loglevel=_config_pytest.getoption("loglevel"),
logfile=logfile,
outfile=outfile,
configfile="/tmp/calvin500{}.conf".format(i))
rt.append(RT("https://{}:502{}".format(hostname, i)))
# Wait to be sure that all runtimes has started
time.sleep(1)
time.sleep(2)
request.addfinalizer(self.teardown)
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug(
"Some security features are enabled, let's make sure certificates are in place"
)
_ca_conf = _conf.get("security", "certificate_authority")
security_dir = _conf.get("security", "security_dir")
storage_type = _conf.get("global", "storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf[
"ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf[
"domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf[
"enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error(
"runtime_certificate: Failed to parse security configuration in calvin.conf, err={}"
.format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir,
node_name + ".csr")
if is_ca == "True":
_log.debug(
"No runtime certificate, but node is a CA, just sign csr, domain={}"
.format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path,
security_dir=security_dir)
else:
_log.debug(
"No runtime certicificate can be found, send CSR to CA"
)
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug(
"CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht", "securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers[
'location'].split('/node/')
ca_control_uri = ca_control_uri.replace(
"http", "https")
ca_control_uris.append(ca_control_uri)
_log.debug(
"CA control_uri={}, node_id={}".format(
ca_control_uri, ca_node_id))
else:
_log.error(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
raise Exception(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
cert_available = False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i = 0
while not cert_available and i < len(ca_control_uris):
certstr = None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
rsa_encrypted_csr = runtime.get_encrypted_csr()
j = 0
while not certstr and j < 10:
try:
certstr = request_handler.sign_csr_request(
ca_control_uris[i],
rsa_encrypted_csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j * j * j
_log.debug(
"RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}"
.format(time_to_sleep, err))
time.sleep(time_to_sleep)
j = j + 1
pass
else:
cert_available = True
i = i + 1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr,
security_dir=security_dir)
else:
_log.debug("Runtime certificate available")
