class MqttClient: """ Basic wrapper around Paho mqtt """ _mqtt_client = None def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None: if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]): self._config = config self._on_connect_callback = connect_callback self._on_message_callback = message_callback self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: pass @staticmethod def valid_arguments(config: dict, callbacks: list) -> bool: callables = [callable(callback) for callback in callbacks] valid = 'broker' in config and any(callables) if valid: return True raise IllegalArgumentError def connect(self) -> bool: self._mqtt_client = paho_mqtt.Client() try: self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None), keepalive=self._config.get("stay_alive", None)) self._mqtt_client.on_connect = self._on_connect self._mqtt_client.on_message = self._on_message self._mqtt_client.loop_start() except (ConnectionRefusedError, TimeoutError) as err: self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}') return False return True def _on_connect(self, _client, _userdata, _flags, _rc) -> None: self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})') self._on_connect_callback() def _on_message(self, _client, _userdata, message) -> None: self.log.debug(f'Received message on topic {message.topic!r}') topic = message.topic payload = message.payload.decode("utf-8") self._on_message_callback(topic=topic, payload=payload) def publish(self, topic: str, msg: Union[dict, str]) -> bool: self.log.debug(f'Publishing message {msg!r} on topic {topic!r}') message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1) if message_info.rc != 0: self.log.warning('Message did not get published successfully') return False return True def subscribe(self, topics: list) -> None: for topic in topics: self._mqtt_client.subscribe(topic=topic, qos=1)
class DeviceManager(Thread): running = True subscribed_event = ['digital_twin'] remote_digital_twin = [] device_status_map = {} poll_timer = None new_devices = False def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self._observer_publish_queue = queue self._thread_ready = thread_event self._observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) self.config = ConfigurationParser().get_config() self.poll_interval = self.config["device_manager"]["poll_interval"] self.device_map_lock = Lock() def __del__(self) -> None: self.running = False def notify(self, msg: ObserverMessage) -> None: self._observer_notify_queue.put(item=msg) def run(self) -> None: self._thread_ready.set() self._fetch_digital_twin() self._start_timer(interval=self.poll_interval, callback=self._timer_callback) while self.running: queue_msg = self._observer_notify_queue.get() if queue_msg.event == "digital_twin": self._handle_digital_twin_event(msg=queue_msg) self.poll_timer.cancel() def _fetch_digital_twin(self): msg = ObserverMessage(event="digital_twin", data={}, subject="fetch_digital_twin") self._observer_publish_queue.put(msg) def _start_timer(self, interval: int, callback: Callable) -> None: self.poll_timer = Timer(interval=interval, function=callback) self.poll_timer.start() def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "retrieved_digital_twin": self._store_remote_digital_twin(data=msg.data) elif msg.subject == "device_status": self._store_device_status(data=msg.data) def _store_remote_digital_twin(self, data: dict): self.remote_digital_twin = data self.log.success("Received remote digital twin") self.log.debug(f"Remote digital twin: {self.remote_digital_twin}") def _store_device_status(self, data: dict): device_id = data.get("device_id", None) status = data.get("active", None) self.log.debug(f"Received device status {status} from {device_id}") if device_id and status: self.device_status_map[device_id] = status def _timer_callback(self): self.log.debug("Starting device status polling stage") self.device_status_map = self._generate_device_map_from_remote_twin() self._publish_device_status_poll() wait_period = self.config["device_manager"]["wait_period"] self._wait_for_status_messages(wait_period=wait_period) digital_twin = self._create_digital_twin_from_device_status() if digital_twin: self._publish_digital_twin(twin=digital_twin) if self.new_devices: self._fetch_digital_twin() self.new_devices = False self._start_timer(interval=self.poll_interval, callback=self._timer_callback) def _generate_device_map_from_remote_twin(self) -> dict: device_map = {} for twin_item in self.remote_digital_twin: device_name = twin_item["device_name"] device_map[device_name] = False return device_map def _publish_device_status_poll(self): msg = ObserverMessage(event="digital_twin", data={}, subject="poll_devices") self._observer_publish_queue.put(msg) def _wait_for_status_messages(self, wait_period: Union[int, float]) -> None: self.log.debug(f"Starting waiting period of {wait_period} seconds") start_time = time() while self.running and (time() - start_time < wait_period): sleep(0.01) self.log.debug("Waiting period over") def _create_digital_twin_from_device_status( self) -> Union[None, List[dict]]: if not self.device_status_map and not self.remote_digital_twin: self.log.debug( "No digital twin created since remote and local twin are empty" ) return None with self.device_map_lock: device_status_map = self.device_status_map.copy() digital_twin = [] for remote_item in self.remote_digital_twin: device_id = remote_item["device_name"] helper = remote_item if device_id in device_status_map: helper["active"] = device_status_map[device_id] del device_status_map[device_id] digital_twin.append(helper) for new_device in device_status_map: new_item = { "device_name": new_device, "active": True, "location": None, "technology": None, "battery_level": None } digital_twin.append(new_item) self.new_devices = True return digital_twin def _publish_digital_twin(self, twin: Union[list, dict]) -> None: msg = ObserverMessage(event="digital_twin", data=twin, subject="save_digital_twin") self._observer_publish_queue.put(msg)
class MqttGateway(Thread): running = False subscribed_event = ['gcp_state_changed', 'digital_twin'] def __init__(self, queue, thread_event: Event): Thread.__init__(self) self.config: dict = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self._observer_notify_queue: Queue = Queue(maxsize=100) self._observer_publish_queue: Queue = queue self._thread_ready: Event = thread_event config = self.get_mqtt_config() self.mqtt_client = MqttClient(config=config, connect_callback=self.on_connect, message_callback=self.on_message) if not self.mqtt_client.connect(): # todo: unscribcribe from subject self.log.critical("TODO: Unsubscribe itself form framework") def __del__(self): self.running = False def run(self): self._thread_ready.set() while self.running: queue_item = self._observer_notify_queue.get() if queue_item.event == "digital_twin": self._handle_digital_twin_event(msg=queue_item) def notify(self, msg: ObserverMessage): self._observer_notify_queue.put(item=msg) def get_mqtt_config(self) -> dict: return { 'broker': self.config['mqtt_gateway']['broker_address'], 'port': 1883, 'stay_alive': 60 } def on_connect(self): topics = ['iot/#'] self.mqtt_client.subscribe(topics=topics) self._thread_ready.set() self.running = True def on_message(self, topic: str, payload: str) -> None: self.log.info(f'Received {payload!r} on topic {topic!r}') self._log_mqtt_traffic(topic=topic, payload=payload) message = IotMessage(mqtt_topic=topic, data=payload) if message.is_valid(): handler = self._select_handler(event=message.event) handler(msg=message) else: self.log.warning('The MQTT message is not valid') def _log_mqtt_traffic(self, topic: str, payload: str) -> None: data = { 'timestamp': datetime.now(), 'source': type(self).__name__, 'topic': topic, 'payload': payload } msg = ObserverMessage(event="iot_traffic", data=data) self._observer_publish_queue.put(msg) def _select_handler(self, event: str) -> Callable: handler_map = { 'state': self._handle_state_change, 'telemetry': self._handle_telemetry, 'system': self._handle_system, 'verification': self._handle_verification } return handler_map.get(event, self._unknown_event) def _unknown_event(self, msg: IotMessage) -> None: self.log.warning(f'Unknown event {msg.event} - No action selected') def _handle_state_change(self, msg: IotMessage) -> None: self.log.debug("Handling state event") message = { 'device_id': msg.device_id, 'event_type': msg.event, 'state': msg.payload.get('state') } item = ObserverMessage(event="device_state_changed", data=message) self._observer_publish_queue.put(item) def _handle_telemetry(self, msg: IotMessage) -> None: self.log.debug("Handling telemetry event") message = {'timestamp': datetime.now(), 'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="device_sensor_data", data=message) self._observer_publish_queue.put(item) def _handle_system(self, msg: IotMessage) -> None: self.log.debug(f"Handling system message from device {msg.device_id}") if msg.device_id != "framework": message = {'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="digital_twin", data=message, subject="device_status") self._observer_publish_queue.put(item) def _handle_verification(self, msg: IotMessage) -> None: self.log.debug(f"Handling verification event {msg.event}") if msg.payload.get("action") == "ping": self.mqtt_client.publish(topic="iot/devices/system/verification", msg={"action": "pong"}) def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "poll_devices": self.mqtt_client.publish(topic="iot/devices/framework/system", msg={"event": "poll"})
class GBridge(threading.Thread): subscribed_event = ['device_state_changed'] mqtt_client = None g_bridge_connected = False attached_devices = [] pending_messages = [] pending_subscribed_topics = [] def __init__(self, queue, thread_event: threading.Event): threading.Thread.__init__(self) self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug) gateway_configuration = MqttGatewayConfiguration() self.observer_notify_queue = Queue(maxsize=100) self.observer_publish_queue = queue self._thread_ready = thread_event keys_dir = get_keys_dir() gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file) gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs) self.gateway_id = gateway_configuration.gateway_id self.connect_to_iot_core_broker(gateway_configuration) def __del__(self): self.detach_all_devices() self.mqtt_client.disconnect() self.mqtt_client.loop_stop() def run(self): self.mqtt_client.loop_start() self.wait_for_connection(5) self._thread_ready.set() while self.g_bridge_connected: queue_item = self.observer_notify_queue.get() self.send_data(msg=queue_item) time.sleep(ONE_MILLISECOND_SECONDS) def notify(self, msg, _) -> None: self.observer_notify_queue.put(item=msg) @staticmethod def poll_events(): return [] def connect_to_iot_core_broker(self, conf): # Create the MQTT client and connect to Cloud IoT. gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \ f'{conf.registry_id}/devices/{conf.gateway_id}' self.mqtt_client = mqtt.Client(gateway_id) jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm) self.mqtt_client.username_pw_set(username='******', password=jwt_pwd) self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2) self.mqtt_client.on_connect = self.on_connect self.mqtt_client.on_publish = self.on_publish self.mqtt_client.on_disconnect = self.on_disconnect self.mqtt_client.on_subscribe = self.on_subscribe self.mqtt_client.on_message = self.on_message self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port) def wait_for_connection(self, timeout): total_time = 0 while not self.g_bridge_connected and total_time < timeout: time.sleep(1) total_time += 1 if not self.g_bridge_connected: self.log.critical('Could not connect to Iot Core MQTT bridge') raise RuntimeError() def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc): self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}') self.g_bridge_connected = True self.subscribe_to_topics(self.gateway_id, True) if self.attached_devices: # Not empty list, Previously already had connected devices self.log.warning('Re-connect occurred! Re-attaching all connected devices.') def subscribe_to_topics(self, dev_id, gateway): config_topic = f'/devices/{dev_id}/config' command_topic = f'/devices/{dev_id}/commands/#' subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}] if gateway: gateway_error_topic = f'/devices/{dev_id}/errors' subscriptions.append({'topic': gateway_error_topic, 'qos': 0}) for subscription in subscriptions: self.subscribe(subscription.get('topic'), subscription.get('qos')) def subscribe(self, topic, qos): _, mid = self.mqtt_client.subscribe(topic, qos) self.pending_subscribed_topics.append(mid) while topic in self.pending_subscribed_topics: time.sleep(0.01) self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.') def on_disconnect(self, _unused_client, _unused_userdata, rc): self.log.warning(f'Disconnected: {error_str(rc)!r}') self.g_bridge_connected = False def on_publish(self, _unused_client, _unused_userdata, mid): self.log.debug(f'ACK received for message {mid!r}') if mid in self.pending_messages: self.pending_messages.remove(mid) def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos): if granted_qos[0] == 128: self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed') else: if mid in self.pending_subscribed_topics: self.pending_subscribed_topics.remove(mid) def on_message(self, _unused_client, _unused_userdata, message): payload = message.payload.decode('utf-8') self.log.info(f'Received message {payload!r} on topic {message.topic!r}.') if not payload: return # todo: fix this so that is better if message.topic.split('/')[3] == "commands": device_id = GBridge.get_id_from_topic(message.topic) queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload} item = {'event': 'gcp_state_changed', 'message': queue_message} self.observer_publish_queue.put(item) def attach_device(self, device_id): self.log.debug(f'Attaching device {device_id!r}.') attach_topic = f'/devices/{device_id}/attach' if device_id not in self.attached_devices: self.attached_devices.append(device_id) self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY self.subscribe_to_topics(device_id, False) def detach_device(self, device_id): self.log.warning(f'Detaching device {device_id!r}.') detach_topic = f'/devices/{device_id}/detach' if device_id in self.attached_devices: self.attached_devices.remove(device_id) self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY def detach_all_devices(self): self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.') for device in self.attached_devices[:]: self.detach_device(device) while self.attached_devices: # Make sure all devices have been detached time.sleep(0.01) def publish(self, topic, payload): message_info = self.mqtt_client.publish(topic, payload, qos=1) self.pending_messages.append(message_info.mid) self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.') while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive time.sleep(0.01) def send_data(self, msg): device_id = msg.get('device_id') event_type = msg.get('event_type') payload = msg.get('payload') if device_id not in self.attached_devices: self.attach_device(device_id=device_id) if event_type == 'telemetry': topic = f'/devices/{device_id}/events' elif event_type == 'state': topic = f'/devices/{device_id}/state' else: self.log.error(f'Unknown event type {event_type}.') return self.publish(topic, payload) @staticmethod def get_id_from_topic(topic): index_device_id = 2 dir_tree = topic.split('/') if len(dir_tree) != 4 or dir_tree[1] != "devices": return None return dir_tree[index_device_id] def reattach_devices(self): for device in self.attached_devices: self.log.info(f'Re-attaching device {device}.') self.attach_device(device)
class MongoHandler: @dataclass class MongoConfLocal: host: str = 'host_ip' user: str = 'admin' pwd: str = 'mongo_admin_iot' url: str = f'mongodb://{user}:{pwd}@{host}/' def __init__(self, db_name: str) -> None: self.config = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self.mongo_db = self.connect_to_db(db_name=db_name) def connect_to_db(self, db_name: str) -> MongoClient: mongo_host = self.config['mongo_db']['host_ip'] mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host, mongo_host) try: client = self.get_mongo_client(url=mongo_url) client.server_info() db = client[db_name] self.log.success( f'Connected to MongoDB {db_name!r} at {mongo_url}') except errors.ServerSelectionTimeoutError as err: self.log.critical( f'Connection MongoDB error at {mongo_url} with error: {err}') raise RuntimeError from err return db @staticmethod def get_mongo_client(url: str) -> MongoClient: return MongoClient(url) def get(self, collection_name: str, query: dict = None) -> list: collection = self.mongo_db[collection_name] self.log.debug( f'Executing query {query!r} on collection {collection_name!r}') return list(collection.find(query)) def insert(self, collection_name: str, data: dict) -> None: collection = self.mongo_db[collection_name] data_id = collection.insert_one(data) self.log.debug( f'Inserted {data!r} into {collection_name!r} with ID {data_id}') def update(self, collection_name: str, object_id: str, updated_values: dict) -> None: collection = self.mongo_db[collection_name] query = {'_id': object_id} collection.update_one(query, updated_values) self.log.debug( f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully' ) def write(self, collection_name: str, data: Union[list, dict], key: str) -> None: """ Add's data if it does not exist, else update that data based on key """ if isinstance(data, list): for entry in data: self._write(collection=collection_name, data=entry, key=key) else: self._write(collection=collection_name, data=data, key=key) def _write(self, collection: str, data: dict, key: str) -> None: query = {key: data.get(key, None)} object_id = self.get_first_object_id_from_query( collection_name=collection, query=query) print(object_id) if object_id: values = {'$set': data} self.update(collection_name=collection, object_id=object_id, updated_values=values) else: self.insert(collection_name=collection, data=data) def get_first_object_id_from_query(self, collection_name: str, query: dict) -> Union[str, None]: collection = self.mongo_db[collection_name] data = collection.find_one(query) if isinstance(data, dict): return data.get('_id') return None