class MqttClient:
""" Basic wrapper around Paho mqtt """
_mqtt_client = None
def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None:
if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]):
self._config = config
self._on_connect_callback = connect_callback
self._on_message_callback = message_callback
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
pass
@staticmethod
def valid_arguments(config: dict, callbacks: list) -> bool:
callables = [callable(callback) for callback in callbacks]
valid = 'broker' in config and any(callables)
if valid:
return True
raise IllegalArgumentError
def connect(self) -> bool:
self._mqtt_client = paho_mqtt.Client()
try:
self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None),
keepalive=self._config.get("stay_alive", None))
self._mqtt_client.on_connect = self._on_connect
self._mqtt_client.on_message = self._on_message
self._mqtt_client.loop_start()
except (ConnectionRefusedError, TimeoutError) as err:
self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}')
return False
return True
def _on_connect(self, _client, _userdata, _flags, _rc) -> None:
self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})')
self._on_connect_callback()
def _on_message(self, _client, _userdata, message) -> None:
self.log.debug(f'Received message on topic {message.topic!r}')
topic = message.topic
payload = message.payload.decode("utf-8")
self._on_message_callback(topic=topic, payload=payload)
def publish(self, topic: str, msg: Union[dict, str]) -> bool:
self.log.debug(f'Publishing message {msg!r} on topic {topic!r}')
message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1)
if message_info.rc != 0:
self.log.warning('Message did not get published successfully')
return False
return True
def subscribe(self, topics: list) -> None:
for topic in topics:
self._mqtt_client.subscribe(topic=topic, qos=1)
class MqttGateway(Thread):
running = False
subscribed_event = ['gcp_state_changed', 'digital_twin']
def __init__(self, queue, thread_event: Event):
Thread.__init__(self)
self.config: dict = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self._observer_notify_queue: Queue = Queue(maxsize=100)
self._observer_publish_queue: Queue = queue
self._thread_ready: Event = thread_event
config = self.get_mqtt_config()
self.mqtt_client = MqttClient(config=config,
connect_callback=self.on_connect,
message_callback=self.on_message)
if not self.mqtt_client.connect():
# todo: unscribcribe from subject
self.log.critical("TODO: Unsubscribe itself form framework")
def __del__(self):
self.running = False
def run(self):
self._thread_ready.set()
while self.running:
queue_item = self._observer_notify_queue.get()
if queue_item.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_item)
def notify(self, msg: ObserverMessage):
self._observer_notify_queue.put(item=msg)
def get_mqtt_config(self) -> dict:
return {
'broker': self.config['mqtt_gateway']['broker_address'],
'port': 1883,
'stay_alive': 60
}
def on_connect(self):
topics = ['iot/#']
self.mqtt_client.subscribe(topics=topics)
self._thread_ready.set()
self.running = True
def on_message(self, topic: str, payload: str) -> None:
self.log.info(f'Received {payload!r} on topic {topic!r}')
self._log_mqtt_traffic(topic=topic, payload=payload)
message = IotMessage(mqtt_topic=topic, data=payload)
if message.is_valid():
handler = self._select_handler(event=message.event)
handler(msg=message)
else:
self.log.warning('The MQTT message is not valid')
def _log_mqtt_traffic(self, topic: str, payload: str) -> None:
data = {
'timestamp': datetime.now(),
'source': type(self).__name__,
'topic': topic,
'payload': payload
}
msg = ObserverMessage(event="iot_traffic", data=data)
self._observer_publish_queue.put(msg)
def _select_handler(self, event: str) -> Callable:
handler_map = {
'state': self._handle_state_change,
'telemetry': self._handle_telemetry,
'system': self._handle_system,
'verification': self._handle_verification
}
return handler_map.get(event, self._unknown_event)
def _unknown_event(self, msg: IotMessage) -> None:
self.log.warning(f'Unknown event {msg.event} - No action selected')
def _handle_state_change(self, msg: IotMessage) -> None:
self.log.debug("Handling state event")
message = {
'device_id': msg.device_id,
'event_type': msg.event,
'state': msg.payload.get('state')
}
item = ObserverMessage(event="device_state_changed", data=message)
self._observer_publish_queue.put(item)
def _handle_telemetry(self, msg: IotMessage) -> None:
self.log.debug("Handling telemetry event")
message = {'timestamp': datetime.now(), 'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="device_sensor_data", data=message)
self._observer_publish_queue.put(item)
def _handle_system(self, msg: IotMessage) -> None:
self.log.debug(f"Handling system message from device {msg.device_id}")
if msg.device_id != "framework":
message = {'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="digital_twin",
data=message,
subject="device_status")
self._observer_publish_queue.put(item)
def _handle_verification(self, msg: IotMessage) -> None:
self.log.debug(f"Handling verification event {msg.event}")
if msg.payload.get("action") == "ping":
self.mqtt_client.publish(topic="iot/devices/system/verification",
msg={"action": "pong"})
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "poll_devices":
self.mqtt_client.publish(topic="iot/devices/framework/system",
msg={"event": "poll"})
class GBridge(threading.Thread):
subscribed_event = ['device_state_changed']
mqtt_client = None
g_bridge_connected = False
attached_devices = []
pending_messages = []
pending_subscribed_topics = []
def __init__(self, queue, thread_event: threading.Event):
threading.Thread.__init__(self)
self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug)
gateway_configuration = MqttGatewayConfiguration()
self.observer_notify_queue = Queue(maxsize=100)
self.observer_publish_queue = queue
self._thread_ready = thread_event
keys_dir = get_keys_dir()
gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file)
gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs)
self.gateway_id = gateway_configuration.gateway_id
self.connect_to_iot_core_broker(gateway_configuration)
def __del__(self):
self.detach_all_devices()
self.mqtt_client.disconnect()
self.mqtt_client.loop_stop()
def run(self):
self.mqtt_client.loop_start()
self.wait_for_connection(5)
self._thread_ready.set()
while self.g_bridge_connected:
queue_item = self.observer_notify_queue.get()
self.send_data(msg=queue_item)
time.sleep(ONE_MILLISECOND_SECONDS)
def notify(self, msg, _) -> None:
self.observer_notify_queue.put(item=msg)
@staticmethod
def poll_events():
return []
def connect_to_iot_core_broker(self, conf):
# Create the MQTT client and connect to Cloud IoT.
gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \
f'{conf.registry_id}/devices/{conf.gateway_id}'
self.mqtt_client = mqtt.Client(gateway_id)
jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm)
self.mqtt_client.username_pw_set(username='******', password=jwt_pwd)
self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2)
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_publish = self.on_publish
self.mqtt_client.on_disconnect = self.on_disconnect
self.mqtt_client.on_subscribe = self.on_subscribe
self.mqtt_client.on_message = self.on_message
self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port)
def wait_for_connection(self, timeout):
total_time = 0
while not self.g_bridge_connected and total_time < timeout:
time.sleep(1)
total_time += 1
if not self.g_bridge_connected:
self.log.critical('Could not connect to Iot Core MQTT bridge')
raise RuntimeError()
def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc):
self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}')
self.g_bridge_connected = True
self.subscribe_to_topics(self.gateway_id, True)
if self.attached_devices: # Not empty list, Previously already had connected devices
self.log.warning('Re-connect occurred! Re-attaching all connected devices.')
def subscribe_to_topics(self, dev_id, gateway):
config_topic = f'/devices/{dev_id}/config'
command_topic = f'/devices/{dev_id}/commands/#'
subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}]
if gateway:
gateway_error_topic = f'/devices/{dev_id}/errors'
subscriptions.append({'topic': gateway_error_topic, 'qos': 0})
for subscription in subscriptions:
self.subscribe(subscription.get('topic'), subscription.get('qos'))
def subscribe(self, topic, qos):
_, mid = self.mqtt_client.subscribe(topic, qos)
self.pending_subscribed_topics.append(mid)
while topic in self.pending_subscribed_topics:
time.sleep(0.01)
self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.')
def on_disconnect(self, _unused_client, _unused_userdata, rc):
self.log.warning(f'Disconnected: {error_str(rc)!r}')
self.g_bridge_connected = False
def on_publish(self, _unused_client, _unused_userdata, mid):
self.log.debug(f'ACK received for message {mid!r}')
if mid in self.pending_messages:
self.pending_messages.remove(mid)
def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos):
if granted_qos[0] == 128:
self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed')
else:
if mid in self.pending_subscribed_topics:
self.pending_subscribed_topics.remove(mid)
def on_message(self, _unused_client, _unused_userdata, message):
payload = message.payload.decode('utf-8')
self.log.info(f'Received message {payload!r} on topic {message.topic!r}.')
if not payload:
return
# todo: fix this so that is better
if message.topic.split('/')[3] == "commands":
device_id = GBridge.get_id_from_topic(message.topic)
queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload}
item = {'event': 'gcp_state_changed', 'message': queue_message}
self.observer_publish_queue.put(item)
def attach_device(self, device_id):
self.log.debug(f'Attaching device {device_id!r}.')
attach_topic = f'/devices/{device_id}/attach'
if device_id not in self.attached_devices:
self.attached_devices.append(device_id)
self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
self.subscribe_to_topics(device_id, False)
def detach_device(self, device_id):
self.log.warning(f'Detaching device {device_id!r}.')
detach_topic = f'/devices/{device_id}/detach'
if device_id in self.attached_devices:
self.attached_devices.remove(device_id)
self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
def detach_all_devices(self):
self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.')
for device in self.attached_devices[:]:
self.detach_device(device)
while self.attached_devices: # Make sure all devices have been detached
time.sleep(0.01)
def publish(self, topic, payload):
message_info = self.mqtt_client.publish(topic, payload, qos=1)
self.pending_messages.append(message_info.mid)
self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.')
while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive
time.sleep(0.01)
def send_data(self, msg):
device_id = msg.get('device_id')
event_type = msg.get('event_type')
payload = msg.get('payload')
if device_id not in self.attached_devices:
self.attach_device(device_id=device_id)
if event_type == 'telemetry':
topic = f'/devices/{device_id}/events'
elif event_type == 'state':
topic = f'/devices/{device_id}/state'
else:
self.log.error(f'Unknown event type {event_type}.')
return
self.publish(topic, payload)
@staticmethod
def get_id_from_topic(topic):
index_device_id = 2
dir_tree = topic.split('/')
if len(dir_tree) != 4 or dir_tree[1] != "devices":
return None
return dir_tree[index_device_id]
def reattach_devices(self):
for device in self.attached_devices:
self.log.info(f'Re-attaching device {device}.')
self.attach_device(device)
