def pytest_configure(config):
global _config_pytest
_config_pytest = config
filename = config.getoption("logfile")
if filename:
calvinlogger.set_file(filename)
levels = config.getoption("loglevel")
for level in levels:
module = None
if ":" in level:
module, level = level.split(":")
print("Setting debuglevel %s on module %s" % (level, module))
if level == "CRITICAL":
calvinlogger.get_logger(module).setLevel(logging.CRITICAL)
elif level == "ERROR":
calvinlogger.get_logger(module).setLevel(logging.ERROR)
elif level == "WARNING":
calvinlogger.get_logger(module).setLevel(logging.WARNING)
elif level == "INFO":
calvinlogger.get_logger(module).setLevel(logging.INFO)
elif level == "DEBUG":
calvinlogger.get_logger(module).setLevel(logging.DEBUG)
elif level == "ANALYZE":
calvinlogger.get_logger(module).setLevel(5)
if not os.environ.get('CALVIN_GLOBAL_DHT_NETWORK_FILTER'):
# TODO: add func to set any argument from here also
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
_conf.add_section('ARGUMENTS')
_conf.set('ARGUMENTS', 'DHT_NETWORK_FILTER', str(uuid.uuid4()))
def pytest_configure(config):
filename = config.getoption("logfile")
if filename:
calvinlogger.set_file(filename)
levels = config.getoption("loglevel")
for level in levels:
module = None
if ":" in level:
module, level = level.split(":")
print("Setting debuglevel %s on module %s" % (level, module))
if level == "CRITICAL":
calvinlogger.get_logger(module).setLevel(logging.CRITICAL)
elif level == "ERROR":
calvinlogger.get_logger(module).setLevel(logging.ERROR)
elif level == "WARNING":
calvinlogger.get_logger(module).setLevel(logging.WARNING)
elif level == "INFO":
calvinlogger.get_logger(module).setLevel(logging.INFO)
elif level == "DEBUG":
calvinlogger.get_logger(module).setLevel(logging.DEBUG)
elif level == "ANALYZE":
calvinlogger.get_logger(module).setLevel(5)
# TODO: add func to set any argument from here also
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
_conf.add_section('ARGUMENTS')
_conf.set('ARGUMENTS', 'DHT_NETWORK_FILTER', str(uuid.uuid4()))
def start_gui(interface4, port, mockdevices):
import calvinextras
import inspect
import os.path
from twisted.web.server import Site
from twisted.web.static import File
from twisted.internet import endpoints, reactor
from calvin.utilities import calvinconfig
# find installation path of calvinextras package
extras_path = os.path.dirname(inspect.getfile(calvinextras))
# build path to gui files
gui_path = os.path.join(extras_path, "CalvinGUI", "Build", "GUI")
gui_config_path = os.path.join(extras_path, "CalvinGUI", "calvin.conf")
if mockdevices:
# Patch config
_conf = calvinconfig.get()
delta_config = _conf.config_at_path(gui_config_path)
_conf.update_config(delta_config)
# Add endpoint to twisted reactor
resource = File(gui_path)
factory = Site(resource)
endpoint = endpoints.TCP4ServerEndpoint(reactor, interface=interface4, port=port)
endpoint.listen(factory)
_log.info("Calvin GUI server listening on http://{}:{}".format(interface4, port))
def set_config_from_args(args):
from calvin.utilities import calvinconfig
global _conf
_conf = calvinconfig.get()
_conf.add_section("ARGUMENTS")
for arg in vars(args):
if getattr(args, arg) is not None:
_log.debug("Adding ARGUMENTS to config {}={}".format(arg, getattr(args, arg)))
_conf.set("ARGUMENTS", arg, getattr(args, arg))
def set_config_from_args(args):
from calvin.utilities import calvinconfig
global _conf
_conf = calvinconfig.get()
_conf.add_section("ARGUMENTS")
for arg in vars(args):
if getattr(args, arg) is not None:
_log.debug("Adding ARGUMENTS to config {}={}".format(arg, getattr(args, arg)))
_conf.set("ARGUMENTS", arg, getattr(args, arg))
def _client_connected(self, uri, protocol):
"""
Callback when the client connects still needs a join to be finnshed
before we can callback upper layers
"""
_log.debug("Client connected")
import socket
if uri in self._peers:
_log.info("Peer %s already connected" % uri)
# Disconnect client localy and remove callbacks
class ErrorMessage:
def __init__(self, str):
self._str = str
def getErrorMessage(self):
return self._str
self._peers[uri]._transport._proto.connectionLost(
ErrorMessage("Connection was closed cleanly."))
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
runtime_to_runtime_security = _conf.get("security",
"runtime_to_runtime_security")
if runtime_to_runtime_security == "tls":
_log.debug("TLS enabled, get FQDN of runtime")
try:
junk, ipv6 = uri.split("//")
ipv6_addr_list = ipv6.split(":")[:-1]
ipv6_addr = ":".join(ipv6_addr_list)
hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(
ipv6_addr)
fqdn = socket.getfqdn(hostname)
except Exception as err:
_log.error("Could not resolve ip address to hostname"
"\n\terr={}"
"\n\turi={}".format(err, uri))
raise
else:
fqdn = None
tp = CalvinTransport(self._rt_id,
uri,
self._callbacks,
self._client_transport,
proto=protocol,
node_name=self._node_name,
server_node_name=fqdn,
client_validator=self._client_validator)
self._callback_execute('peer_connected', tp, tp.get_uri())
self._peers[uri] = tp
def run_debug(script, timeout):
import logging
config = calvinconfig.get()
debug_modules = config.get('developer', 'debug_modules')
if not debug_modules:
print "No 'debug_modules' option in 'developer' section of config => output verbosity is DEBUG everywhere."
calvinlogger.get_logger().setLevel(logging.DEBUG)
else:
for m in debug_modules:
calvinlogger.get_logger(m).setLevel(logging.DEBUG)
print "\n{0}\n{1}\n{0}".format("-"*80, config)
run(script, timeout)
def __init__(self, name, nodeid, search, certificate_stored):
""" Maintain state of a Node and store `configuration`.
name is the runtime name,
nodeid is the runtime's id,
search is a function taking csr as kw arg to search for signing CA
certificate_stored is a function that is called when the sign certificate is stored
"""
self.state = STATE.NEW
self.conf = calvinconfig.get()
self.domain = self.conf.get("security", "security_domain_name")
self.nodeid = nodeid
self.name = name
self.search = search
self.certificate_stored = certificate_stored
self.discover()
def __init__(self, name, nodeid, search, certificate_stored):
""" Maintain state of a Node and store `configuration`.
name is the runtime name,
nodeid is the runtime's id,
search is a function taking csr as kw arg to search for signing CA
certificate_stored is a function that is called when the sign certificate is stored
"""
self.state = STATE.NEW
self.conf = calvinconfig.get()
self.domain = self.conf.get("security","security_domain_name")
self.nodeid = nodeid
self.name = name
self.search = search
self.certificate_stored = certificate_stored
self.discover()
def _client_connected(self, uri, protocol):
"""
Callback when the client connects still needs a join to be finnshed
before we can callback upper layers
"""
_log.debug("Client connected")
import socket
if uri in self._peers:
_log.info("Peer %s already connected" % uri)
# Disconnect client localy and remove callbacks
class ErrorMessage:
def __init__(self, str):
self._str = str
def getErrorMessage(self):
return self._str
self._peers[uri]._transport._proto.connectionLost(ErrorMessage("Connection was closed cleanly."))
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
runtime_to_runtime_security = _conf.get("security","runtime_to_runtime_security")
if runtime_to_runtime_security=="tls":
_log.debug("TLS enabled, get FQDN of runtime")
try:
junk, ipv6 = uri.split("//")
ipv6_addr_list = ipv6.split(":")[:-1]
ipv6_addr = ":".join(ipv6_addr_list)
hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipv6_addr)
fqdn = socket.getfqdn(hostname)
except Exception as err:
_log.error("Could not resolve ip address to hostname"
"\n\terr={}"
"\n\turi={}".format(err, uri))
raise
else:
fqdn=None
tp = CalvinTransport(self._rt_id, uri, self._callbacks,
self._client_transport, proto=protocol,
node_name=self._node_name,
server_node_name=fqdn,
client_validator=self._client_validator)
self._callback_execute('peer_connected', tp, tp.get_uri())
self._peers[uri] = tp
def __init__(self, name, csrdata=None):
"""Hold Ca state."""
self.name = name
self.state = STATE.NEW
self.conf = calvinconfig.get()
self.ca = None
self.cafile = ""
self.csrdata = "" if csrdata is None else csrdata
self.csrfile = ""
self.certfile = ""
self.signed_cert = None
self.is_ca = self.conf.get("security", "certificate_authority")
self.domain = self.conf.get("security", "security_domain_name")
if self.is_ca == "True":
try:
self.ca = certificate_authority.CA(self.domain)
except (Exception), err:
_log.error("CA could not be started, err={}".format(err))
raise Exception("CA could not be started")
def __init__(self, name, csrdata=None):
"""Hold Ca state."""
self.name = name
self.state = STATE.NEW
self.conf = calvinconfig.get()
self.ca = None
self.cafile = ""
self.csrdata = "" if csrdata is None else csrdata
self.csrfile = ""
self.certfile = ""
self.signed_cert = None
self.is_ca = self.conf.get("security","certificate_authority")
self.domain = self.conf.get("security","security_domain_name")
if self.is_ca=="True":
try:
self.ca=certificate_authority.CA(self.domain)
except (Exception), err:
_log.error("CA could not be started, err={}".format(err))
raise Exception("CA could not be started")
def _client_connected(self, uri, protocol):
"""
Callback when the client connects still needs a join to be finnshed
before we can callback upper layers
"""
import socket
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
runtime_to_runtime_security = _conf.get("security",
"runtime_to_runtime_security")
if runtime_to_runtime_security == "tls":
_log.debug("TLS enabled, get FQDN of runtime")
try:
junk, ipv6 = uri.split("//")
ipv6_addr_list = ipv6.split(":")[:-1]
ipv6_addr = ":".join(ipv6_addr_list)
hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(
ipv6_addr)
fqdn = socket.getfqdn(hostname)
except Exception as err:
_log.error("Could not resolve ip address to hostname"
"\n\terr={}"
"\n\turi={}".format(err, uri))
raise
else:
fqdn = None
tp = CalvinTransport(self._rt_id,
uri,
self._callbacks,
self._client_transport,
proto=protocol,
node_name=self._node_name,
server_node_name=fqdn,
client_validator=self._client_validator)
self._callback_execute('peer_connected', tp, tp.get_uri())
self._peers[uri] = tp
from calvin.runtime.north.calvin_network import CalvinNetwork
from calvin.runtime.north.calvin_proto import CalvinProto
from calvin.runtime.north.portmanager import PortManager
from calvin.runtime.south.monitor import Event_Monitor
from calvin.runtime.south.plugins.async import async
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities.calvin_callback import CalvinCB
from calvin.utilities.security import security_modules_check
from calvin.utilities.runtime_credentials import RuntimeCredentials
from calvin.utilities import calvinuuid
from calvin.utilities import certificate
from calvin.utilities.calvinlogger import get_logger, set_file
from calvin.utilities import calvinconfig
_log = get_logger(__name__)
_conf = calvinconfig.get()
def addr_from_uri(uri):
_, host = uri.split("://")
addr, _ = host.split(":")
return addr
class Node(object):
"""A node of calvin
the uri is a list of server connection points
the control_uri is the local console
attributes is a supplied list of external defined attributes that will be used as the key when storing index
such as name of node
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Abstraction for the diffrent frameworks that can be used by the system.
"""
import os
from calvin.utilities import calvinconfig
_CONF = calvinconfig.get()
_FW_PATH = _CONF.get(None, 'environmental_sensor_plugin')
if _FW_PATH is not None:
# Spec
_MODULES = {'environmental': ['Environmental']}
_FW_MODULES = []
__all__ = []
if not _FW_MODULES:
DIRNAME = os.path.dirname(__file__) + "/"
for fw_module, _, _ in os.walk(DIRNAME):
if "impl" in fw_module:
_FW_MODULES.append(fw_module.replace(DIRNAME, ""))
import twisted
import shutil
from calvin.utilities.calvin_callback import CalvinCB
from calvin.utilities import calvinlogger
from calvin.runtime.south.plugins.storage.twistedimpl.securedht.append_server import *
from calvin.runtime.south.plugins.storage.twistedimpl.securedht.dht_server import *
from calvin.runtime.south.plugins.storage.twistedimpl.securedht.service_discovery_ssdp import *
from calvin.runtime.south.plugins.storage.twistedimpl.securedht.dht_server_commons import drawNetworkState
from kademlia.node import Node
from kademlia.utils import deferredDict, digest
from calvin.runtime.south.plugins.async import threads
from calvin.utilities import calvinconfig
_conf = calvinconfig.get()
_conf.add_section("security")
_conf_file = os.path.join(os.getenv("HOME"), ".calvin/security/test/openssl.conf")
_conf.set("security", "certificate_conf", _conf_file)
_conf.set("security", "certificate_domain", "test")
_cert_conf = None
_log = calvinlogger.get_logger(__name__)
name = "node1:"
@pytest.fixture(scope="session", autouse=True)
def cleanup(request):
def fin():
reactor.callFromThread(reactor.stop)
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug("Some security features are enabled, let's make sure certificates are in place")
_ca_conf = _conf.get("security","certificate_authority")
security_dir = _conf.get("security","security_dir")
storage_type = _conf.get("global","storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf["ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf["domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf["enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error("runtime_certificate: Failed to parse security configuration in calvin.conf, err={}".format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(node_name, domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir, node_name + ".csr")
if is_ca:
_log.debug("No runtime certificate, but node is a CA, just sign csr, domain={}".format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path)
else:
_log.debug("No runtime certicificate can be found, send CSR to CA")
truststore_dir = certificate.get_truststore_path(type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug("CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht","securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers['location'].split('/node/')
ca_control_uris.append(ca_control_uri)
_log.debug("CA control_uri={}, node_id={}".format(ca_control_uri, ca_node_id))
else:
_log.error("There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration ")
raise Exception("There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration ")
cert_available=False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i=0
csr = json.dumps(runtime.get_csr_and_enrollment_password())
while not cert_available and i<len(ca_control_uris):
certstr=None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
j=0
while not certstr and j<10:
try:
certstr = request_handler.sign_csr_request(ca_control_uris[i], csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j*j*j
_log.debug("RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}".format(time_to_sleep, err))
time.sleep(time_to_sleep)
j=j+1
pass
else:
cert_available = True
i = i+1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr)
else:
_log.debug("Runtime certificate available")
['ServerProtocolFactory', 'LineProtocol', 'RawDataProtocol'],
'client_connection': ['ClientProtocolFactory'],
'http_client': ['HTTPClient']
}
_FW_MODULES = []
__all__ = []
if not _FW_MODULES:
DIRNAME = os.path.dirname(__file__)
DIRS = os.listdir(DIRNAME)
for i, fw_module in enumerate(DIRS):
if "impl" in fw_module:
_FW_MODULES.append(fw_module)
_CONF = calvinconfig.get()
_FW_PATH = _CONF.get(None, 'framework')
def get_framework():
"""
Get the framework used on the runtime
"""
return _FW_PATH
def get_frameworks():
"""
Get all frameworks in the system
"""
return _FW_MODULES
def runtime_certificate(rt_attributes):
import copy
import requests
import sys
from calvin.requests.request_handler import RequestHandler
from calvin.utilities.attribute_resolver import AttributeResolver
from calvin.utilities import calvinconfig
from calvin.utilities import calvinuuid
from calvin.utilities import runtime_credentials
from calvin.utilities import certificate
from calvin.utilities import certificate_authority
from calvin.runtime.south.plugins.storage.twistedimpl.dht.service_discovery_ssdp import parse_http_response
global _conf
global _log
_conf = calvinconfig.get()
if not _conf.get_section("security"):
#If the security section is empty, no securty features are enabled and certificates aren't needed
_log.debug("No runtime security enabled")
else:
_log.debug(
"Some security features are enabled, let's make sure certificates are in place"
)
_ca_conf = _conf.get("security", "certificate_authority")
security_dir = _conf.get("security", "security_dir")
storage_type = _conf.get("global", "storage_type")
if _ca_conf:
try:
ca_ctrl_uri = _ca_conf[
"ca_control_uri"] if "ca_control_uri" in _ca_conf else None
domain_name = _ca_conf[
"domain_name"] if "domain_name" in _ca_conf else None
is_ca = _ca_conf["is_ca"] if "is_ca" in _ca_conf else None
enrollment_password = _ca_conf[
"enrollment_password"] if "enrollment_password" in _ca_conf else None
except Exception as err:
_log.error(
"runtime_certificate: Failed to parse security configuration in calvin.conf, err={}"
.format(err))
raise
#AttributeResolver tranforms the attributes, so make a deepcopy instead
rt_attributes_cpy = copy.deepcopy(rt_attributes)
attributes = AttributeResolver(rt_attributes_cpy)
node_name = attributes.get_node_name_as_str()
nodeid = calvinuuid.uuid("")
runtime = runtime_credentials.RuntimeCredentials(
node_name,
domain_name,
security_dir=security_dir,
nodeid=nodeid,
enrollment_password=enrollment_password)
certpath, cert, certstr = runtime.get_own_cert()
if not cert:
csr_path = os.path.join(runtime.runtime_dir,
node_name + ".csr")
if is_ca == "True":
_log.debug(
"No runtime certificate, but node is a CA, just sign csr, domain={}"
.format(domain_name))
ca = certificate_authority.CA(domain=domain_name,
security_dir=security_dir)
cert_path = ca.sign_csr(csr_path, is_ca=True)
runtime.store_own_cert(certpath=cert_path,
security_dir=security_dir)
else:
_log.debug(
"No runtime certicificate can be found, send CSR to CA"
)
truststore_dir = certificate.get_truststore_path(
type=certificate.TRUSTSTORE_TRANSPORT,
security_dir=security_dir)
request_handler = RequestHandler(verify=truststore_dir)
ca_control_uris = []
#TODO: add support for multiple CA control uris
if ca_ctrl_uri:
_log.debug(
"CA control_uri in config={}".format(ca_ctrl_uri))
ca_control_uris.append(ca_ctrl_uri)
elif storage_type in ["dht", "securedht"]:
_log.debug("Find CA via SSDP")
responses = discover()
if not responses:
_log.error("No responses received")
for response in responses:
cmd, headers = parse_http_response(response)
if 'location' in headers:
ca_control_uri, ca_node_id = headers[
'location'].split('/node/')
ca_control_uri = ca_control_uri.replace(
"http", "https")
ca_control_uris.append(ca_control_uri)
_log.debug(
"CA control_uri={}, node_id={}".format(
ca_control_uri, ca_node_id))
else:
_log.error(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
raise Exception(
"There is no runtime certificate. For automatic certificate enrollment using proxy storage,"
"the CA control uri must be configured in the calvin configuration "
)
cert_available = False
# Loop through all CA:s that responded until hopefully one signs our CSR
# Potential improvement would be to have domain name in response and only try
# appropriate CAs
i = 0
while not cert_available and i < len(ca_control_uris):
certstr = None
#Repeatedly (maximum 10 attempts) send CSR to CA until a certificate is returned (this to remove the requirement of the CA
#node to be be the first node to start)
rsa_encrypted_csr = runtime.get_encrypted_csr()
j = 0
while not certstr and j < 10:
try:
certstr = request_handler.sign_csr_request(
ca_control_uris[i],
rsa_encrypted_csr)['certificate']
except requests.exceptions.RequestException as err:
time_to_sleep = 1 + j * j * j
_log.debug(
"RequestException, CSR not accepted or CA not up and running yet, sleep {} seconds and try again, err={}"
.format(time_to_sleep, err))
time.sleep(time_to_sleep)
j = j + 1
pass
else:
cert_available = True
i = i + 1
#TODO: check that everything is ok with signed cert, e.g., check that the CA domain
# matches the expected and that the CA cert is trusted
runtime.store_own_cert(certstring=certstr,
security_dir=security_dir)
else:
_log.debug("Runtime certificate available")
def setup_module(module):
calvinconfig.get().set('global', 'storage_type', 'dht')
def setup_module(module):
calvinconfig.get().set('global', 'storage_type', 'dht')
