def __init__(self, con): log.info("Logging into DCC") self._version = 20171023 self.comms = con if not self.comms.identity.username: log.error("Username not found") raise ValueError("Username not found") elif not self.comms.identity.password: log.error("Password not found") raise ValueError("Password not found") thread = threading.Thread(target=self.comms.receive) thread.daemon = True # This thread will continuously run in background to receive response or actions from DCC thread.start() # Wait for Subscription to be complete and then proceed to publish message time.sleep(0.5) self._iotcc_json = self._create_iotcc_json() self._iotcc_json_load_retry = int(read_liota_config('IOTCC_PATH', 'iotcc_load_retry')) self.enable_reboot_getprop = read_liota_config('IOTCC_PATH', 'enable_reboot_getprop') self.counter = 0 self.recv_msg_queue = self.comms.userdata self.boottime = boottime() self.dev_file_path = self._get_file_storage_path("dev_file_path") # Liota internal entity file system path special for iotcc self.entity_file_path = self._get_file_storage_path("entity_file_path") self.file_ops_lock = Lock()
def __init__(self, con): log.info("Logging into DCC") self._version = 20171023 self.comms = con if not self.comms.identity.username: log.error("Username not found") raise ValueError("Username not found") elif not self.comms.identity.password: log.error("Password not found") raise ValueError("Password not found") thread = threading.Thread(target=self.comms.receive) thread.daemon = True # This thread will continuously run in background to receive response or actions from DCC thread.start() # Wait for Subscription to be complete and then proceed to publish message time.sleep(0.5) self._iotcc_json = self._create_iotcc_json() self._iotcc_json_load_retry = int( read_liota_config('IOTCC_PATH', 'iotcc_load_retry')) self.enable_reboot_getprop = read_liota_config( 'IOTCC_PATH', 'enable_reboot_getprop') self.counter = 0 self.recv_msg_queue = self.comms.userdata self.boottime = boottime() self.dev_file_path = self._get_file_storage_path("dev_file_path") # Liota internal entity file system path special for iotcc self.entity_file_path = self._get_file_storage_path("entity_file_path") self.file_ops_lock = Lock()
def initialize(): """ Initialization for metric handling: create events priority queue, collect queue, and send queue; spawn event check thread and send thread; and create collection thread pool. :return: """ global is_initialization_done if is_initialization_done: log.debug("Initialization already done") pass else: log.debug("Initializing.............") global event_ds if event_ds is None: event_ds = EventsPriorityQueue() global event_checker_thread if event_checker_thread is None: event_checker_thread = EventCheckerThread( name="EventCheckerThread") global collect_queue if collect_queue is None: collect_queue = Queue() global send_queue if send_queue is None: send_queue = Queue() global send_thread if send_thread is None: send_thread = SendThread(name="SendThread") global collect_thread_pool collect_thread_pool_size = int(read_liota_config('CORE_CFG','collect_thread_pool_size')) collect_thread_pool = CollectionThreadPool(collect_thread_pool_size) is_initialization_done = True
def initialize(): global is_initialization_done if is_initialization_done: log.debug("Initialization already done") pass else: log.debug("Initializing.............") global event_ds if event_ds is None: event_ds = EventsPriorityQueue() global event_checker_thread if event_checker_thread is None: event_checker_thread = EventCheckerThread( name="EventCheckerThread") global collect_queue if collect_queue is None: collect_queue = Queue() global send_queue if send_queue is None: send_queue = Queue() global send_thread if send_thread is None: send_thread = SendThread(name="SendThread") global collect_thread_pool collect_thread_pool_size = int( read_liota_config('CORE_CFG', 'collect_thread_pool_size')) collect_thread_pool = CollectionThreadPool(collect_thread_pool_size) is_initialization_done = True
def _create_iotcc_json(self): msg = { "iotcc": { "EdgeSystem": { "SystemName": "", "EntityType": "", "uuid": "" }, "OGProperties": { "OrganizationGroup": "" }, "Devices": [] } } iotcc_path = read_liota_config('IOTCC_PATH', 'iotcc_path') path = os.path.dirname(iotcc_path) mkdir(path) try: with open(iotcc_path, 'w') as f: json.dump(msg, f, sort_keys=True, indent=4, ensure_ascii=False) log.debug('Initialized ' + iotcc_path) f.close() except IOError, err: log.error('Could not open {0} file '.format(iotcc_path) + err)
def initialize(): """ Initialization for metric handling: create events priority queue, collect queue, and send queue; spawn event check thread and send thread; and create collection thread pool. :return: """ global is_initialization_done if is_initialization_done: log.debug("Initialization already done") pass else: log.debug("Initializing.............") global event_ds if event_ds is None: event_ds = EventsPriorityQueue() global event_checker_thread if event_checker_thread is None: event_checker_thread = EventCheckerThread( name="EventCheckerThread") global collect_queue if collect_queue is None: collect_queue = Queue() global send_queue if send_queue is None: send_queue = Queue() global send_thread if send_thread is None: send_thread = SendThread(name="SendThread") global collect_thread_pool collect_thread_pool_size = int( read_liota_config('CORE_CFG', 'collect_thread_pool_size')) collect_thread_pool = CollectionThreadPool(collect_thread_pool_size) is_initialization_done = True
def initialize(): global is_initialization_done if is_initialization_done: log.debug("Initialization already done") pass else: log.debug("Initializing.............") global event_ds if event_ds is None: event_ds = EventsPriorityQueue() global event_checker_thread if event_checker_thread is None: event_checker_thread = EventCheckerThread( name="EventCheckerThread") global collect_queue if collect_queue is None: collect_queue = Queue() global send_queue if send_queue is None: send_queue = Queue() global send_thread if send_thread is None: send_thread = SendThread(name="SendThread") global collect_thread_pool collect_thread_pool_size = int(read_liota_config('CORE_CFG','collect_thread_pool_size')) collect_thread_pool = CollectionThreadPool(collect_thread_pool_size) is_initialization_done = True
def __init__(self, con): """ Initialization of IoT Pulse Center :param con: DCC Comms connection Object """ log.info("Logging into DCC") self._dcc_load_time = datetime.datetime.now() self._version = 20171118 self.comms = con if not self.comms.identity.username: log.error("Username not found") raise ValueError("Username not found") elif not self.comms.identity.password: log.error("Password not found") raise ValueError("Password not found") recv_thread = threading.Thread(target=self.comms.receive) recv_thread.daemon = True # This thread will continuously run in background to receive response or actions from DCC recv_thread.start() # Wait for Subscription to be complete and then proceed to publish message time.sleep(0.5) self._iotcc_json = self._create_iotcc_json() self._iotcc_json_load_retry = int( read_liota_config('IOTCC_PATH', 'iotcc_load_retry')) self.enable_reboot_getprop = read_liota_config( 'IOTCC_PATH', 'enable_reboot_getprop') self._sys_properties = read_liota_config('IOTCC_PATH', 'system_properties') self.counter = 0 self._recv_msg_queue = self.comms.userdata self._req_ops_lock = Lock() self._req_dict = {} dispatch_thread = threading.Thread(target=self._dispatch_recvd_msg) dispatch_thread.daemon = True # This thread will continuously run in background to check and dispatch received responses dispatch_thread.start() self.dev_file_path = self._get_file_storage_path("dev_file_path") # Liota internal entity file system path special for iotcc self.entity_file_path = self._get_file_storage_path("entity_file_path") self.file_ops_lock = Lock()
def store_edge_system_uuid(self, entity_name, reg_entity_id): try: uuid_path = read_liota_config('UUID_PATH', 'uuid_path') uuid_config = ConfigParser.RawConfigParser() uuid_config.optionxform = str uuid_config.add_section('GATEWAY') uuid_config.set('GATEWAY', 'uuid', reg_entity_id) uuid_config.set('GATEWAY', 'name', entity_name) with open(uuid_path, 'w') as configfile: uuid_config.write(configfile) except ConfigParser.ParsingError, err: log.error('Could not open config file ' + err)
def store_edge_system_uuid(self, entity_name, reg_entity_id): try: uuid_path = read_liota_config('UUID_PATH', 'uuid_path') uuid_config = ConfigParser.RawConfigParser() uuid_config.optionxform = str uuid_config.add_section('GATEWAY') uuid_config.set('GATEWAY', 'uuid', reg_entity_id) uuid_config.set('GATEWAY', 'name', entity_name) with open(uuid_path, 'w') as configfile: uuid_config.write(configfile) except ConfigParser.ParsingError, err: log.error('Could not open config file ' + err)
def __init__(self, con): """ Initialization of IoT Pulse Center :param con: DCC Comms connection Object """ log.info("Logging into DCC") self._dcc_load_time = datetime.datetime.now() self._version = 20171118 self.comms = con if not self.comms.identity.username: log.error("Username not found") raise ValueError("Username not found") elif not self.comms.identity.password: log.error("Password not found") raise ValueError("Password not found") recv_thread = threading.Thread(target=self.comms.receive) recv_thread.daemon = True # This thread will continuously run in background to receive response or actions from DCC recv_thread.start() # Wait for Subscription to be complete and then proceed to publish message time.sleep(0.5) self._iotcc_json = self._create_iotcc_json() self._iotcc_json_load_retry = int(read_liota_config('IOTCC_PATH', 'iotcc_load_retry')) self.enable_reboot_getprop = read_liota_config('IOTCC_PATH', 'enable_reboot_getprop') self._sys_properties = read_liota_config('IOTCC_PATH', 'system_properties') self.counter = 0 self._recv_msg_queue = self.comms.userdata self._req_ops_lock = Lock() self._req_dict = {} dispatch_thread = threading.Thread(target=self._dispatch_recvd_msg) dispatch_thread.daemon = True # This thread will continuously run in background to check and dispatch received responses dispatch_thread.start() self.dev_file_path = self._get_file_storage_path("dev_file_path") # Liota internal entity file system path special for iotcc self.entity_file_path = self._get_file_storage_path("entity_file_path") self.file_ops_lock = Lock()
def run(self, registry): # Acquire resources from registry iotcc = registry.get("iotcc") # Get values from configuration file iotcc_json_path = read_liota_config('IOTCC_PATH', 'iotcc_path') if iotcc_json_path == '': return try: with open(iotcc_json_path, 'r') as f: iotcc_details_json_obj = json.load(f)["iotcc"] f.close() except IOError, err: return
def run(self, registry): # Acquire resources from registry iotcc = registry.get("iotcc") # Get values from configuration file iotcc_json_path = read_liota_config('IOTCC_PATH', 'iotcc_path') if iotcc_json_path == '': return try: with open(iotcc_json_path, 'r') as f: iotcc_details_json_obj = json.load(f)["iotcc"] f.close() except IOError, err: return
def setUp(self): """ Setup all required parameters for AmqpDccComms tests :return: None """ # ConfigParser to parse init file self.config = ConfigParser() self.uuid_file = read_liota_config('UUID_PATH', 'uuid_path') # Broker details self.url = "Broker-IP" self.port = "Broker-Port" self.amqp_username = "******" self.amqp_password = "******" self.enable_authentication = True self.transport = "tcp" self.connection_disconnect_timeout_sec = 2 # EdgeSystem name self.edge_system = Dell5KEdgeSystem("TestGateway") # TLS configurations self.root_ca_cert = "/etc/liota/amqp/conf/ca.crt" self.client_cert_file = "/etc/liota/amqp/conf/client.crt" self.client_key_file = "/etc/liota/amqp/conf/client.key" self.cert_required = "CERT_REQUIRED" self.tls_version = "PROTOCOL_TLSv1" self.cipher = None # Encapsulate the authentication details self.identity = Identity(self.root_ca_cert, self.amqp_username, self.amqp_password, self.client_cert_file, self.client_key_file) # Encapsulate TLS parameters self.tls_conf = TLSConf(self.cert_required, self.tls_version, self.cipher) self.send_message = "test-message" # Creating messaging attributes self.amqp_msg_attr = AmqpPublishMessagingAttributes( exchange_name="test_exchange", routing_key=["test"]) with mock.patch.object(Amqp, '_init_or_re_init') as mocked_init_or_re_init, \ mock.patch.object(Amqp, "declare_publish_exchange") as mocked_declare_publish_exchange: # Instantiate client for DCC communication self.client = AmqpDccComms(edge_system_name=self.edge_system.name, url=self.url, port=self.port)
def __init__(self, url, port, identity=None, tls_conf=None, qos_details=None, client_id=None, clean_session=False, userdata=None, protocol="MQTTv311", transport="tcp", keep_alive=60, enable_authentication=False, conn_disconn_timeout=int(read_liota_config('MQTT_CFG', 'mqtt_conn_disconn_timeout'))): """ :param url: MQTT Broker URL or IP :param port: MQTT Broker Port :param identity: Identity Object :param tls_conf: TLSConf object :param qos_details: QoSDetails object :param client_id: Client ID :param clean_session: Connect with Clean session or not :param userdata: userdata is user defined data of any type that is passed as the "userdata" parameter to callbacks. :param protocol: allows explicit setting of the MQTT version to use for this client :param transport: Set transport to "websockets" to use WebSockets as the transport mechanism. Set to "tcp" to use raw TCP, which is the default. :param keep_alive: KeepAliveInterval :param enable_authentication: Enable user-name password authentication or not :param username: Username for authentication :param password: Password for authentication :param conn_disconn_timeout: Connect-Disconnect-Timeout """ self.url = url self.port = port self.identity = identity self.tls_conf = tls_conf self.qos_details = qos_details self.client_id = client_id self.clean_session = clean_session self.userdata = userdata self.protocol = protocol self.transport = transport self.keep_alive = keep_alive self.enable_authentication = enable_authentication self._conn_disconn_timeout = conn_disconn_timeout self._paho_client = paho.Client(self.client_id, self.clean_session, self.userdata, protocol=getattr(paho, self.protocol), transport=self.transport) self._connect_result_code = sys.maxsize self._disconnect_result_code = sys.maxsize self._paho_client.on_message = self.on_message self._paho_client.on_publish = self.on_publish self._paho_client.on_subscribe = self.on_subscribe self._paho_client.on_connect = self.on_connect self._paho_client.on_disconnect = self.on_disconnect self.sub_dict = {} self.connect_soc()
def __init__(self, con): log.info("Logging into DCC") self.comms = con if not self.comms.identity.username: log.error("Username not found") raise ValueError("Username not found") elif not self.comms.identity.password: log.error("Password not found") raise ValueError("Password not found") thread = threading.Thread(target=self.comms.receive) thread.daemon = True # This thread will continuously run in background to receive response or actions from DCC thread.start() # Wait for Subscription to be complete and then proceed to publish message time.sleep(0.5) self.proto = HelixProtocol(self.comms, self.comms.identity.username, self.comms.identity.password) self._iotcc_json = self._create_iotcc_json() self._iotcc_json_load_retry = int(read_liota_config('IOTCC', 'iotcc_load_retry')) self.counter = 0 self.recv_msg_queue = self.comms.userdata self.dev_file_path = self._get_file_storage_path("dev_file_path") # Liota internal entity file system path special for iotcc self.entity_file_path = self._get_file_storage_path("entity_file_path") self.file_ops_lock = Lock() try: def on_response(msg): log.debug("Received msg: {0}".format(msg)) json_msg = json.loads(msg) self.proto.on_receive(json_msg) if json_msg["type"] == "connection_response": if json_msg["body"]["result"] == "succeeded": log.info("Connection verified") else: raise Exception("Helix Protocol Version mismatch") else: log.debug("Processed msg: {0}".format(json_msg["type"])) on_response(self.recv_msg_queue.get(True, 300)) # Block on Queue for not more then 300 seconds else it will raise an exception on_response(self.recv_msg_queue.get(True, 300)) except Exception as error: self.comms.client.disconnect() log.error("HelixProtocolException: " + repr(error)) raise Exception("HelixProtocolException")
def _create_iotcc_json(self): msg = { "iotcc": { "EdgeSystem": {"SystemName": "", "EntityType": "", "uuid": "", "LocalUuid": ""}, "OGProperties": {"OrganizationGroup": ""}, "Devices": [] } } iotcc_path = read_liota_config('IOTCC_PATH', 'iotcc_path') path = os.path.dirname(iotcc_path) mkdir(path) try: with open(iotcc_path, 'w') as f: json.dump(msg, f, sort_keys=True, indent=4, ensure_ascii=False) log.debug('Initialized ' + iotcc_path) f.close() except IOError, err: log.error('Could not open {0} file '.format(iotcc_path) + err)
def connect_soc(self): """ Establishes connection with MQTT Broker :return: """ # Set up TLS support if self.tls_conf: if self.identity is None: raise ValueError("Identity required to be set") # Creating the tls context if ssl is None: raise ValueError("This platform has no SSL/TLS") # Validate CA certificate path if self.identity.root_ca_cert is None and not hasattr(ssl.SSLContext, 'load_default_certs'): raise ValueError("Error : CA certificate path is missing") else: if self.identity.root_ca_cert and not (os.path.exists(self.identity.root_ca_cert)): raise ValueError("Error : Wrong CA certificate path") if self.tls_conf.tls_version is None: tls_version = ssl.PROTOCOL_TLSv1_2 # If the python version supports it, use highest TLS version automatically if hasattr(ssl, "PROTOCOL_TLS"): tls_version = ssl.PROTOCOL_TLS else: tls_version = getattr(ssl, self.tls_conf.tls_version) context = ssl.SSLContext(tls_version) # Validate client certificate path if self.identity.cert_file: if os.path.exists(self.identity.cert_file): client_cert_available = True else: raise ValueError("Error : Wrong client certificate path") else: client_cert_available = False # Validate client key file path if self.identity.key_file: if os.path.exists(self.identity.key_file): client_key_available = True else: raise ValueError("Error : Wrong client key path.") else: client_key_available = False ''' Multiple conditions for certificate validations # 1. Both Client certificate and key file should be present # 2. If client certificate is not there throw an error # 3. If client key is not there throw an error # 4. If both are not there proceed without client certificate and key ''' if client_cert_available and client_key_available: context.load_cert_chain(self.identity.cert_file, self.identity.key_file) elif not client_cert_available and client_key_available: raise ValueError("Error : Client key found, but client certificate not found") elif client_cert_available and not client_key_available: raise ValueError("Error : Client certificate found, but client key not found") else: log.info("Client Certificate and Client Key are not provided") if getattr(ssl, self.tls_conf.cert_required) == ssl.CERT_NONE and hasattr(context, 'check_hostname'): context.check_hostname = False context.verify_mode = ssl.CERT_REQUIRED if self.tls_conf.cert_required is None else getattr(ssl, self.tls_conf.cert_required) if self.identity.root_ca_cert is not None: context.load_verify_locations(self.identity.root_ca_cert) else: context.load_default_certs() if self.tls_conf.cipher is not None: context.set_ciphers(self.tls_conf.ciphers) # Setting the verify_flags to VERIFY_CRL_CHECK_CHAIN in this mode # certificate revocation lists (CRLs) of all certificates in the # peer cert chain are checked if the path of CRLs in PEM or DER format # is specified crl_path = read_liota_config('CRL_PATH', 'crl_path') if crl_path and crl_path != "None" and crl_path != "": if os.path.exists(crl_path): context.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN context.load_verify_locations(cafile=crl_path) else: raise ValueError("Error : Wrong Client CRL path {0}".format(crl_path)) # Setting the tls context self._paho_client.tls_set_context(context) if getattr(ssl, self.tls_conf.cert_required) != ssl.CERT_NONE: # Default to secure, sets context.check_hostname attribute # if available self._paho_client.tls_insecure_set(False) else: # But with ssl.CERT_NONE, we can not check_hostname self._paho_client.tls_insecure_set(True) else: log.info("TLS configuration is not set") # Set up username-password if self.enable_authentication: if self.identity is None: raise ValueError("Identity required to be set") else: if self.identity.username is None: raise ValueError("Username not found") elif self.identity.password is None: raise ValueError("Password not found") else: self._paho_client.username_pw_set(self.identity.username, self.identity.password) else: log.info("Authentication is disabled") if self.qos_details: # Set QoS parameters self._paho_client.max_inflight_messages_set(self.qos_details.in_flight) self._paho_client.max_queued_messages_set(self.qos_details.queue_size) self._paho_client.message_retry_set(self.qos_details.retry) # Connect with MQTT Broker self._paho_client.connect(host=self.url, port=self.port, keepalive=self.keep_alive) # Start network loop to handle auto-reconnect self._paho_client.loop_start() ten_ms_count = 0 while (ten_ms_count != self._conn_disconn_timeout * 100) and (self._connect_result_code == sys.maxsize): ten_ms_count += 1 time.sleep(0.01) if self._connect_result_code == sys.maxsize: log.error("Connection timeout.") # Stopping background network loop as connection establishment failed. self._paho_client.loop_stop() raise Exception("Connection Timeout") elif self._connect_result_code == 0: log.info("Connected to MQTT Broker.") log.info("Connect time consumption: " + str(float(ten_ms_count) * 10) + "ms.") else: log.error("Connection error with result code : {0} : {1} ". format(str(self._connect_result_code), paho.connack_string(self._connect_result_code))) # Stopping background network loop as connection establishment failed. self._paho_client.loop_stop() raise Exception("Connection error with result code : {0} : {1} ". format(str(self._connect_result_code), paho.connack_string(self._connect_result_code)))
import os import Queue import datetime from time import gmtime, strftime from threading import Lock from re import match from liota.dccs.dcc import DataCenterComponent, RegistrationFailure from liota.entities.metrics.metric import Metric from liota.lib.utilities.utility import LiotaConfigPath, getUTCmillis, mkdir, read_liota_config from liota.lib.utilities.si_unit import parse_unit from liota.entities.metrics.registered_metric import RegisteredMetric from liota.entities.registered_entity import RegisteredEntity log = logging.getLogger(__name__) timeout = int(read_liota_config('IOTCC_PATH', 'iotcc_response_timeout')) class Request: def __init__(self, transaction_id, user_queue): # assume transaction_id will be unique in one process self.transaction_id = transaction_id self.user_queue = user_queue class IotControlCenter(DataCenterComponent): """ The implementation of IoTCC cloud provider solution """ def __init__(self, con):
import Queue import datetime from time import gmtime, strftime from threading import Lock import ast from re import match from liota.dccs.dcc import DataCenterComponent, RegistrationFailure from liota.entities.metrics.metric import Metric from liota.lib.utilities.utility import LiotaConfigPath, getUTCmillis, mkdir, read_liota_config from liota.lib.utilities.si_unit import parse_unit from liota.entities.metrics.registered_metric import RegisteredMetric from liota.entities.registered_entity import RegisteredEntity log = logging.getLogger(__name__) timeout = int(read_liota_config('IOTCC_PATH', 'iotcc_response_timeout')) class Request: def __init__(self, transaction_id, user_queue): # assume transaction_id will be unique in one process self.transaction_id = transaction_id self.user_queue = user_queue class IotControlCenter(DataCenterComponent): """ The implementation of IoTCC cloud provider solution """ def __init__(self, con):
actual_package_manager.package_message_queue.put(["terminate"]) log.info("Exiting Mainthread") exit() is_package_manager_initialized = False package_message_queue = None package_messenger_thread = None package_thread = None package_lock = None package_path = None package_messenger_pipe = None # Parse Liota configuration file package_path = os.path.abspath( read_liota_config('PKG_CFG', 'pkg_path') ) package_messenger_pipe = os.path.abspath( read_liota_config('PKG_CFG', 'pkg_msg_pipe') ) assert(isinstance(package_path, basestring)) assert(isinstance(package_messenger_pipe, basestring)) package_startup_list_path = None package_startup_list = [] # Parse packages to load at start-up package_startup_list_path = os.path.abspath( read_liota_config('PKG_CFG', 'pkg_list') )