class MqttClient: """ Basic wrapper around Paho mqtt """ _mqtt_client = None def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None: if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]): self._config = config self._on_connect_callback = connect_callback self._on_message_callback = message_callback self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: pass @staticmethod def valid_arguments(config: dict, callbacks: list) -> bool: callables = [callable(callback) for callback in callbacks] valid = 'broker' in config and any(callables) if valid: return True raise IllegalArgumentError def connect(self) -> bool: self._mqtt_client = paho_mqtt.Client() try: self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None), keepalive=self._config.get("stay_alive", None)) self._mqtt_client.on_connect = self._on_connect self._mqtt_client.on_message = self._on_message self._mqtt_client.loop_start() except (ConnectionRefusedError, TimeoutError) as err: self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}') return False return True def _on_connect(self, _client, _userdata, _flags, _rc) -> None: self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})') self._on_connect_callback() def _on_message(self, _client, _userdata, message) -> None: self.log.debug(f'Received message on topic {message.topic!r}') topic = message.topic payload = message.payload.decode("utf-8") self._on_message_callback(topic=topic, payload=payload) def publish(self, topic: str, msg: Union[dict, str]) -> bool: self.log.debug(f'Publishing message {msg!r} on topic {topic!r}') message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1) if message_info.rc != 0: self.log.warning('Message did not get published successfully') return False return True def subscribe(self, topics: list) -> None: for topic in topics: self._mqtt_client.subscribe(topic=topic, qos=1)
class MqttGateway(Thread): running = False subscribed_event = ['gcp_state_changed', 'digital_twin'] def __init__(self, queue, thread_event: Event): Thread.__init__(self) self.config: dict = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self._observer_notify_queue: Queue = Queue(maxsize=100) self._observer_publish_queue: Queue = queue self._thread_ready: Event = thread_event config = self.get_mqtt_config() self.mqtt_client = MqttClient(config=config, connect_callback=self.on_connect, message_callback=self.on_message) if not self.mqtt_client.connect(): # todo: unscribcribe from subject self.log.critical("TODO: Unsubscribe itself form framework") def __del__(self): self.running = False def run(self): self._thread_ready.set() while self.running: queue_item = self._observer_notify_queue.get() if queue_item.event == "digital_twin": self._handle_digital_twin_event(msg=queue_item) def notify(self, msg: ObserverMessage): self._observer_notify_queue.put(item=msg) def get_mqtt_config(self) -> dict: return { 'broker': self.config['mqtt_gateway']['broker_address'], 'port': 1883, 'stay_alive': 60 } def on_connect(self): topics = ['iot/#'] self.mqtt_client.subscribe(topics=topics) self._thread_ready.set() self.running = True def on_message(self, topic: str, payload: str) -> None: self.log.info(f'Received {payload!r} on topic {topic!r}') self._log_mqtt_traffic(topic=topic, payload=payload) message = IotMessage(mqtt_topic=topic, data=payload) if message.is_valid(): handler = self._select_handler(event=message.event) handler(msg=message) else: self.log.warning('The MQTT message is not valid') def _log_mqtt_traffic(self, topic: str, payload: str) -> None: data = { 'timestamp': datetime.now(), 'source': type(self).__name__, 'topic': topic, 'payload': payload } msg = ObserverMessage(event="iot_traffic", data=data) self._observer_publish_queue.put(msg) def _select_handler(self, event: str) -> Callable: handler_map = { 'state': self._handle_state_change, 'telemetry': self._handle_telemetry, 'system': self._handle_system, 'verification': self._handle_verification } return handler_map.get(event, self._unknown_event) def _unknown_event(self, msg: IotMessage) -> None: self.log.warning(f'Unknown event {msg.event} - No action selected') def _handle_state_change(self, msg: IotMessage) -> None: self.log.debug("Handling state event") message = { 'device_id': msg.device_id, 'event_type': msg.event, 'state': msg.payload.get('state') } item = ObserverMessage(event="device_state_changed", data=message) self._observer_publish_queue.put(item) def _handle_telemetry(self, msg: IotMessage) -> None: self.log.debug("Handling telemetry event") message = {'timestamp': datetime.now(), 'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="device_sensor_data", data=message) self._observer_publish_queue.put(item) def _handle_system(self, msg: IotMessage) -> None: self.log.debug(f"Handling system message from device {msg.device_id}") if msg.device_id != "framework": message = {'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="digital_twin", data=message, subject="device_status") self._observer_publish_queue.put(item) def _handle_verification(self, msg: IotMessage) -> None: self.log.debug(f"Handling verification event {msg.event}") if msg.payload.get("action") == "ping": self.mqtt_client.publish(topic="iot/devices/system/verification", msg={"action": "pong"}) def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "poll_devices": self.mqtt_client.publish(topic="iot/devices/framework/system", msg={"event": "poll"})
class GBridge(threading.Thread): subscribed_event = ['device_state_changed'] mqtt_client = None g_bridge_connected = False attached_devices = [] pending_messages = [] pending_subscribed_topics = [] def __init__(self, queue, thread_event: threading.Event): threading.Thread.__init__(self) self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug) gateway_configuration = MqttGatewayConfiguration() self.observer_notify_queue = Queue(maxsize=100) self.observer_publish_queue = queue self._thread_ready = thread_event keys_dir = get_keys_dir() gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file) gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs) self.gateway_id = gateway_configuration.gateway_id self.connect_to_iot_core_broker(gateway_configuration) def __del__(self): self.detach_all_devices() self.mqtt_client.disconnect() self.mqtt_client.loop_stop() def run(self): self.mqtt_client.loop_start() self.wait_for_connection(5) self._thread_ready.set() while self.g_bridge_connected: queue_item = self.observer_notify_queue.get() self.send_data(msg=queue_item) time.sleep(ONE_MILLISECOND_SECONDS) def notify(self, msg, _) -> None: self.observer_notify_queue.put(item=msg) @staticmethod def poll_events(): return [] def connect_to_iot_core_broker(self, conf): # Create the MQTT client and connect to Cloud IoT. gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \ f'{conf.registry_id}/devices/{conf.gateway_id}' self.mqtt_client = mqtt.Client(gateway_id) jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm) self.mqtt_client.username_pw_set(username='******', password=jwt_pwd) self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2) self.mqtt_client.on_connect = self.on_connect self.mqtt_client.on_publish = self.on_publish self.mqtt_client.on_disconnect = self.on_disconnect self.mqtt_client.on_subscribe = self.on_subscribe self.mqtt_client.on_message = self.on_message self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port) def wait_for_connection(self, timeout): total_time = 0 while not self.g_bridge_connected and total_time < timeout: time.sleep(1) total_time += 1 if not self.g_bridge_connected: self.log.critical('Could not connect to Iot Core MQTT bridge') raise RuntimeError() def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc): self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}') self.g_bridge_connected = True self.subscribe_to_topics(self.gateway_id, True) if self.attached_devices: # Not empty list, Previously already had connected devices self.log.warning('Re-connect occurred! Re-attaching all connected devices.') def subscribe_to_topics(self, dev_id, gateway): config_topic = f'/devices/{dev_id}/config' command_topic = f'/devices/{dev_id}/commands/#' subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}] if gateway: gateway_error_topic = f'/devices/{dev_id}/errors' subscriptions.append({'topic': gateway_error_topic, 'qos': 0}) for subscription in subscriptions: self.subscribe(subscription.get('topic'), subscription.get('qos')) def subscribe(self, topic, qos): _, mid = self.mqtt_client.subscribe(topic, qos) self.pending_subscribed_topics.append(mid) while topic in self.pending_subscribed_topics: time.sleep(0.01) self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.') def on_disconnect(self, _unused_client, _unused_userdata, rc): self.log.warning(f'Disconnected: {error_str(rc)!r}') self.g_bridge_connected = False def on_publish(self, _unused_client, _unused_userdata, mid): self.log.debug(f'ACK received for message {mid!r}') if mid in self.pending_messages: self.pending_messages.remove(mid) def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos): if granted_qos[0] == 128: self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed') else: if mid in self.pending_subscribed_topics: self.pending_subscribed_topics.remove(mid) def on_message(self, _unused_client, _unused_userdata, message): payload = message.payload.decode('utf-8') self.log.info(f'Received message {payload!r} on topic {message.topic!r}.') if not payload: return # todo: fix this so that is better if message.topic.split('/')[3] == "commands": device_id = GBridge.get_id_from_topic(message.topic) queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload} item = {'event': 'gcp_state_changed', 'message': queue_message} self.observer_publish_queue.put(item) def attach_device(self, device_id): self.log.debug(f'Attaching device {device_id!r}.') attach_topic = f'/devices/{device_id}/attach' if device_id not in self.attached_devices: self.attached_devices.append(device_id) self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY self.subscribe_to_topics(device_id, False) def detach_device(self, device_id): self.log.warning(f'Detaching device {device_id!r}.') detach_topic = f'/devices/{device_id}/detach' if device_id in self.attached_devices: self.attached_devices.remove(device_id) self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY def detach_all_devices(self): self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.') for device in self.attached_devices[:]: self.detach_device(device) while self.attached_devices: # Make sure all devices have been detached time.sleep(0.01) def publish(self, topic, payload): message_info = self.mqtt_client.publish(topic, payload, qos=1) self.pending_messages.append(message_info.mid) self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.') while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive time.sleep(0.01) def send_data(self, msg): device_id = msg.get('device_id') event_type = msg.get('event_type') payload = msg.get('payload') if device_id not in self.attached_devices: self.attach_device(device_id=device_id) if event_type == 'telemetry': topic = f'/devices/{device_id}/events' elif event_type == 'state': topic = f'/devices/{device_id}/state' else: self.log.error(f'Unknown event type {event_type}.') return self.publish(topic, payload) @staticmethod def get_id_from_topic(topic): index_device_id = 2 dir_tree = topic.split('/') if len(dir_tree) != 4 or dir_tree[1] != "devices": return None return dir_tree[index_device_id] def reattach_devices(self): for device in self.attached_devices: self.log.info(f'Re-attaching device {device}.') self.attach_device(device)