class MqttGateway(Thread): running = False subscribed_event = ['gcp_state_changed', 'digital_twin'] def __init__(self, queue, thread_event: Event): Thread.__init__(self) self.config: dict = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self._observer_notify_queue: Queue = Queue(maxsize=100) self._observer_publish_queue: Queue = queue self._thread_ready: Event = thread_event config = self.get_mqtt_config() self.mqtt_client = MqttClient(config=config, connect_callback=self.on_connect, message_callback=self.on_message) if not self.mqtt_client.connect(): # todo: unscribcribe from subject self.log.critical("TODO: Unsubscribe itself form framework") def __del__(self): self.running = False def run(self): self._thread_ready.set() while self.running: queue_item = self._observer_notify_queue.get() if queue_item.event == "digital_twin": self._handle_digital_twin_event(msg=queue_item) def notify(self, msg: ObserverMessage): self._observer_notify_queue.put(item=msg) def get_mqtt_config(self) -> dict: return { 'broker': self.config['mqtt_gateway']['broker_address'], 'port': 1883, 'stay_alive': 60 } def on_connect(self): topics = ['iot/#'] self.mqtt_client.subscribe(topics=topics) self._thread_ready.set() self.running = True def on_message(self, topic: str, payload: str) -> None: self.log.info(f'Received {payload!r} on topic {topic!r}') self._log_mqtt_traffic(topic=topic, payload=payload) message = IotMessage(mqtt_topic=topic, data=payload) if message.is_valid(): handler = self._select_handler(event=message.event) handler(msg=message) else: self.log.warning('The MQTT message is not valid') def _log_mqtt_traffic(self, topic: str, payload: str) -> None: data = { 'timestamp': datetime.now(), 'source': type(self).__name__, 'topic': topic, 'payload': payload } msg = ObserverMessage(event="iot_traffic", data=data) self._observer_publish_queue.put(msg) def _select_handler(self, event: str) -> Callable: handler_map = { 'state': self._handle_state_change, 'telemetry': self._handle_telemetry, 'system': self._handle_system, 'verification': self._handle_verification } return handler_map.get(event, self._unknown_event) def _unknown_event(self, msg: IotMessage) -> None: self.log.warning(f'Unknown event {msg.event} - No action selected') def _handle_state_change(self, msg: IotMessage) -> None: self.log.debug("Handling state event") message = { 'device_id': msg.device_id, 'event_type': msg.event, 'state': msg.payload.get('state') } item = ObserverMessage(event="device_state_changed", data=message) self._observer_publish_queue.put(item) def _handle_telemetry(self, msg: IotMessage) -> None: self.log.debug("Handling telemetry event") message = {'timestamp': datetime.now(), 'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="device_sensor_data", data=message) self._observer_publish_queue.put(item) def _handle_system(self, msg: IotMessage) -> None: self.log.debug(f"Handling system message from device {msg.device_id}") if msg.device_id != "framework": message = {'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="digital_twin", data=message, subject="device_status") self._observer_publish_queue.put(item) def _handle_verification(self, msg: IotMessage) -> None: self.log.debug(f"Handling verification event {msg.event}") if msg.payload.get("action") == "ping": self.mqtt_client.publish(topic="iot/devices/system/verification", msg={"action": "pong"}) def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "poll_devices": self.mqtt_client.publish(topic="iot/devices/framework/system", msg={"event": "poll"})
class HealthMonitor(Thread): running = True update_time_sec = 600 subscribed_event = [] def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self.observer_publish_queue = queue self._thread_ready = thread_event self.observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: self.running = False def run(self) -> None: self.log.info(f'Updating system information every {self.update_time_sec} seconds.') self._thread_ready.set() while self.running: start_time = time() host_data = self._fetch_host_data() msg = ObserverMessage(event="host_health", data=host_data) self.observer_publish_queue.put(msg) sleep_time = self.update_time_sec - ((time() - start_time) % self.update_time_sec) sleep(sleep_time) def notify(self, msg: ObserverMessage) -> None: pass def _fetch_host_data(self) -> dict: data = { 'timestamp': self._get_timestamp(), 'temperature': self.poll_system_temp(), 'cpu_load': self.poll_cpu_load() } return data @staticmethod def _get_timestamp() -> datetime: return datetime.now() def poll_system_temp(self) -> float: temp_file = self._get_temperature_file() try: with open(temp_file) as file: return float(file.readline()) / 1000 except FileNotFoundError: self.log.critical(f'Temperature file {temp_file!r} does not exist') return 0 @staticmethod def _get_temperature_file() -> Path: return Path('/sys/class/thermal/thermal_zone0/temp') def poll_cpu_load(self) -> float: cpu_command = ["cat", "/proc/stat"] try: with subprocess.Popen(cpu_command, stdout=subprocess.PIPE) as process_result: proc_stat, _ = process_result.communicate() cpu_data = proc_stat.decode('utf-8').split('\n')[0].split()[1:-1] cpu_data = [int(field) for field in cpu_data] cpu_usage = ((cpu_data[0] + cpu_data[2]) * 100 / (cpu_data[0] + cpu_data[2] + cpu_data[3])) return round(cpu_usage, 3) except FileNotFoundError as error: self.log.critical(f'Command {" ".join(cpu_command)!r} was not found on the system: {error}') except ValueError as error: self.log.error(f'Parsing of the data went wrong: {error}') return 0
class GBridge(threading.Thread): subscribed_event = ['device_state_changed'] mqtt_client = None g_bridge_connected = False attached_devices = [] pending_messages = [] pending_subscribed_topics = [] def __init__(self, queue, thread_event: threading.Event): threading.Thread.__init__(self) self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug) gateway_configuration = MqttGatewayConfiguration() self.observer_notify_queue = Queue(maxsize=100) self.observer_publish_queue = queue self._thread_ready = thread_event keys_dir = get_keys_dir() gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file) gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs) self.gateway_id = gateway_configuration.gateway_id self.connect_to_iot_core_broker(gateway_configuration) def __del__(self): self.detach_all_devices() self.mqtt_client.disconnect() self.mqtt_client.loop_stop() def run(self): self.mqtt_client.loop_start() self.wait_for_connection(5) self._thread_ready.set() while self.g_bridge_connected: queue_item = self.observer_notify_queue.get() self.send_data(msg=queue_item) time.sleep(ONE_MILLISECOND_SECONDS) def notify(self, msg, _) -> None: self.observer_notify_queue.put(item=msg) @staticmethod def poll_events(): return [] def connect_to_iot_core_broker(self, conf): # Create the MQTT client and connect to Cloud IoT. gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \ f'{conf.registry_id}/devices/{conf.gateway_id}' self.mqtt_client = mqtt.Client(gateway_id) jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm) self.mqtt_client.username_pw_set(username='******', password=jwt_pwd) self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2) self.mqtt_client.on_connect = self.on_connect self.mqtt_client.on_publish = self.on_publish self.mqtt_client.on_disconnect = self.on_disconnect self.mqtt_client.on_subscribe = self.on_subscribe self.mqtt_client.on_message = self.on_message self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port) def wait_for_connection(self, timeout): total_time = 0 while not self.g_bridge_connected and total_time < timeout: time.sleep(1) total_time += 1 if not self.g_bridge_connected: self.log.critical('Could not connect to Iot Core MQTT bridge') raise RuntimeError() def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc): self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}') self.g_bridge_connected = True self.subscribe_to_topics(self.gateway_id, True) if self.attached_devices: # Not empty list, Previously already had connected devices self.log.warning('Re-connect occurred! Re-attaching all connected devices.') def subscribe_to_topics(self, dev_id, gateway): config_topic = f'/devices/{dev_id}/config' command_topic = f'/devices/{dev_id}/commands/#' subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}] if gateway: gateway_error_topic = f'/devices/{dev_id}/errors' subscriptions.append({'topic': gateway_error_topic, 'qos': 0}) for subscription in subscriptions: self.subscribe(subscription.get('topic'), subscription.get('qos')) def subscribe(self, topic, qos): _, mid = self.mqtt_client.subscribe(topic, qos) self.pending_subscribed_topics.append(mid) while topic in self.pending_subscribed_topics: time.sleep(0.01) self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.') def on_disconnect(self, _unused_client, _unused_userdata, rc): self.log.warning(f'Disconnected: {error_str(rc)!r}') self.g_bridge_connected = False def on_publish(self, _unused_client, _unused_userdata, mid): self.log.debug(f'ACK received for message {mid!r}') if mid in self.pending_messages: self.pending_messages.remove(mid) def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos): if granted_qos[0] == 128: self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed') else: if mid in self.pending_subscribed_topics: self.pending_subscribed_topics.remove(mid) def on_message(self, _unused_client, _unused_userdata, message): payload = message.payload.decode('utf-8') self.log.info(f'Received message {payload!r} on topic {message.topic!r}.') if not payload: return # todo: fix this so that is better if message.topic.split('/')[3] == "commands": device_id = GBridge.get_id_from_topic(message.topic) queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload} item = {'event': 'gcp_state_changed', 'message': queue_message} self.observer_publish_queue.put(item) def attach_device(self, device_id): self.log.debug(f'Attaching device {device_id!r}.') attach_topic = f'/devices/{device_id}/attach' if device_id not in self.attached_devices: self.attached_devices.append(device_id) self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY self.subscribe_to_topics(device_id, False) def detach_device(self, device_id): self.log.warning(f'Detaching device {device_id!r}.') detach_topic = f'/devices/{device_id}/detach' if device_id in self.attached_devices: self.attached_devices.remove(device_id) self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY def detach_all_devices(self): self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.') for device in self.attached_devices[:]: self.detach_device(device) while self.attached_devices: # Make sure all devices have been detached time.sleep(0.01) def publish(self, topic, payload): message_info = self.mqtt_client.publish(topic, payload, qos=1) self.pending_messages.append(message_info.mid) self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.') while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive time.sleep(0.01) def send_data(self, msg): device_id = msg.get('device_id') event_type = msg.get('event_type') payload = msg.get('payload') if device_id not in self.attached_devices: self.attach_device(device_id=device_id) if event_type == 'telemetry': topic = f'/devices/{device_id}/events' elif event_type == 'state': topic = f'/devices/{device_id}/state' else: self.log.error(f'Unknown event type {event_type}.') return self.publish(topic, payload) @staticmethod def get_id_from_topic(topic): index_device_id = 2 dir_tree = topic.split('/') if len(dir_tree) != 4 or dir_tree[1] != "devices": return None return dir_tree[index_device_id] def reattach_devices(self): for device in self.attached_devices: self.log.info(f'Re-attaching device {device}.') self.attach_device(device)
class MongoHandler: @dataclass class MongoConfLocal: host: str = 'host_ip' user: str = 'admin' pwd: str = 'mongo_admin_iot' url: str = f'mongodb://{user}:{pwd}@{host}/' def __init__(self, db_name: str) -> None: self.config = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self.mongo_db = self.connect_to_db(db_name=db_name) def connect_to_db(self, db_name: str) -> MongoClient: mongo_host = self.config['mongo_db']['host_ip'] mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host, mongo_host) try: client = self.get_mongo_client(url=mongo_url) client.server_info() db = client[db_name] self.log.success( f'Connected to MongoDB {db_name!r} at {mongo_url}') except errors.ServerSelectionTimeoutError as err: self.log.critical( f'Connection MongoDB error at {mongo_url} with error: {err}') raise RuntimeError from err return db @staticmethod def get_mongo_client(url: str) -> MongoClient: return MongoClient(url) def get(self, collection_name: str, query: dict = None) -> list: collection = self.mongo_db[collection_name] self.log.debug( f'Executing query {query!r} on collection {collection_name!r}') return list(collection.find(query)) def insert(self, collection_name: str, data: dict) -> None: collection = self.mongo_db[collection_name] data_id = collection.insert_one(data) self.log.debug( f'Inserted {data!r} into {collection_name!r} with ID {data_id}') def update(self, collection_name: str, object_id: str, updated_values: dict) -> None: collection = self.mongo_db[collection_name] query = {'_id': object_id} collection.update_one(query, updated_values) self.log.debug( f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully' ) def write(self, collection_name: str, data: Union[list, dict], key: str) -> None: """ Add's data if it does not exist, else update that data based on key """ if isinstance(data, list): for entry in data: self._write(collection=collection_name, data=entry, key=key) else: self._write(collection=collection_name, data=data, key=key) def _write(self, collection: str, data: dict, key: str) -> None: query = {key: data.get(key, None)} object_id = self.get_first_object_id_from_query( collection_name=collection, query=query) print(object_id) if object_id: values = {'$set': data} self.update(collection_name=collection, object_id=object_id, updated_values=values) else: self.insert(collection_name=collection, data=data) def get_first_object_id_from_query(self, collection_name: str, query: dict) -> Union[str, None]: collection = self.mongo_db[collection_name] data = collection.find_one(query) if isinstance(data, dict): return data.get('_id') return None