class MqttClient:
""" Basic wrapper around Paho mqtt """
_mqtt_client = None
def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None:
if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]):
self._config = config
self._on_connect_callback = connect_callback
self._on_message_callback = message_callback
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
pass
@staticmethod
def valid_arguments(config: dict, callbacks: list) -> bool:
callables = [callable(callback) for callback in callbacks]
valid = 'broker' in config and any(callables)
if valid:
return True
raise IllegalArgumentError
def connect(self) -> bool:
self._mqtt_client = paho_mqtt.Client()
try:
self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None),
keepalive=self._config.get("stay_alive", None))
self._mqtt_client.on_connect = self._on_connect
self._mqtt_client.on_message = self._on_message
self._mqtt_client.loop_start()
except (ConnectionRefusedError, TimeoutError) as err:
self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}')
return False
return True
def _on_connect(self, _client, _userdata, _flags, _rc) -> None:
self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})')
self._on_connect_callback()
def _on_message(self, _client, _userdata, message) -> None:
self.log.debug(f'Received message on topic {message.topic!r}')
topic = message.topic
payload = message.payload.decode("utf-8")
self._on_message_callback(topic=topic, payload=payload)
def publish(self, topic: str, msg: Union[dict, str]) -> bool:
self.log.debug(f'Publishing message {msg!r} on topic {topic!r}')
message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1)
if message_info.rc != 0:
self.log.warning('Message did not get published successfully')
return False
return True
def subscribe(self, topics: list) -> None:
for topic in topics:
self._mqtt_client.subscribe(topic=topic, qos=1)
class DeviceManager(Thread):
running = True
subscribed_event = ['digital_twin']
remote_digital_twin = []
device_status_map = {}
poll_timer = None
new_devices = False
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self._observer_publish_queue = queue
self._thread_ready = thread_event
self._observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
self.config = ConfigurationParser().get_config()
self.poll_interval = self.config["device_manager"]["poll_interval"]
self.device_map_lock = Lock()
def __del__(self) -> None:
self.running = False
def notify(self, msg: ObserverMessage) -> None:
self._observer_notify_queue.put(item=msg)
def run(self) -> None:
self._thread_ready.set()
self._fetch_digital_twin()
self._start_timer(interval=self.poll_interval,
callback=self._timer_callback)
while self.running:
queue_msg = self._observer_notify_queue.get()
if queue_msg.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_msg)
self.poll_timer.cancel()
def _fetch_digital_twin(self):
msg = ObserverMessage(event="digital_twin",
data={},
subject="fetch_digital_twin")
self._observer_publish_queue.put(msg)
def _start_timer(self, interval: int, callback: Callable) -> None:
self.poll_timer = Timer(interval=interval, function=callback)
self.poll_timer.start()
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "retrieved_digital_twin":
self._store_remote_digital_twin(data=msg.data)
elif msg.subject == "device_status":
self._store_device_status(data=msg.data)
def _store_remote_digital_twin(self, data: dict):
self.remote_digital_twin = data
self.log.success("Received remote digital twin")
self.log.debug(f"Remote digital twin: {self.remote_digital_twin}")
def _store_device_status(self, data: dict):
device_id = data.get("device_id", None)
status = data.get("active", None)
self.log.debug(f"Received device status {status} from {device_id}")
if device_id and status:
self.device_status_map[device_id] = status
def _timer_callback(self):
self.log.debug("Starting device status polling stage")
self.device_status_map = self._generate_device_map_from_remote_twin()
self._publish_device_status_poll()
wait_period = self.config["device_manager"]["wait_period"]
self._wait_for_status_messages(wait_period=wait_period)
digital_twin = self._create_digital_twin_from_device_status()
if digital_twin:
self._publish_digital_twin(twin=digital_twin)
if self.new_devices:
self._fetch_digital_twin()
self.new_devices = False
self._start_timer(interval=self.poll_interval,
callback=self._timer_callback)
def _generate_device_map_from_remote_twin(self) -> dict:
device_map = {}
for twin_item in self.remote_digital_twin:
device_name = twin_item["device_name"]
device_map[device_name] = False
return device_map
def _publish_device_status_poll(self):
msg = ObserverMessage(event="digital_twin",
data={},
subject="poll_devices")
self._observer_publish_queue.put(msg)
def _wait_for_status_messages(self, wait_period: Union[int,
float]) -> None:
self.log.debug(f"Starting waiting period of {wait_period} seconds")
start_time = time()
while self.running and (time() - start_time < wait_period):
sleep(0.01)
self.log.debug("Waiting period over")
def _create_digital_twin_from_device_status(
self) -> Union[None, List[dict]]:
if not self.device_status_map and not self.remote_digital_twin:
self.log.debug(
"No digital twin created since remote and local twin are empty"
)
return None
with self.device_map_lock:
device_status_map = self.device_status_map.copy()
digital_twin = []
for remote_item in self.remote_digital_twin:
device_id = remote_item["device_name"]
helper = remote_item
if device_id in device_status_map:
helper["active"] = device_status_map[device_id]
del device_status_map[device_id]
digital_twin.append(helper)
for new_device in device_status_map:
new_item = {
"device_name": new_device,
"active": True,
"location": None,
"technology": None,
"battery_level": None
}
digital_twin.append(new_item)
self.new_devices = True
return digital_twin
def _publish_digital_twin(self, twin: Union[list, dict]) -> None:
msg = ObserverMessage(event="digital_twin",
data=twin,
subject="save_digital_twin")
self._observer_publish_queue.put(msg)
class MqttGateway(Thread):
running = False
subscribed_event = ['gcp_state_changed', 'digital_twin']
def __init__(self, queue, thread_event: Event):
Thread.__init__(self)
self.config: dict = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self._observer_notify_queue: Queue = Queue(maxsize=100)
self._observer_publish_queue: Queue = queue
self._thread_ready: Event = thread_event
config = self.get_mqtt_config()
self.mqtt_client = MqttClient(config=config,
connect_callback=self.on_connect,
message_callback=self.on_message)
if not self.mqtt_client.connect():
# todo: unscribcribe from subject
self.log.critical("TODO: Unsubscribe itself form framework")
def __del__(self):
self.running = False
def run(self):
self._thread_ready.set()
while self.running:
queue_item = self._observer_notify_queue.get()
if queue_item.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_item)
def notify(self, msg: ObserverMessage):
self._observer_notify_queue.put(item=msg)
def get_mqtt_config(self) -> dict:
return {
'broker': self.config['mqtt_gateway']['broker_address'],
'port': 1883,
'stay_alive': 60
}
def on_connect(self):
topics = ['iot/#']
self.mqtt_client.subscribe(topics=topics)
self._thread_ready.set()
self.running = True
def on_message(self, topic: str, payload: str) -> None:
self.log.info(f'Received {payload!r} on topic {topic!r}')
self._log_mqtt_traffic(topic=topic, payload=payload)
message = IotMessage(mqtt_topic=topic, data=payload)
if message.is_valid():
handler = self._select_handler(event=message.event)
handler(msg=message)
else:
self.log.warning('The MQTT message is not valid')
def _log_mqtt_traffic(self, topic: str, payload: str) -> None:
data = {
'timestamp': datetime.now(),
'source': type(self).__name__,
'topic': topic,
'payload': payload
}
msg = ObserverMessage(event="iot_traffic", data=data)
self._observer_publish_queue.put(msg)
def _select_handler(self, event: str) -> Callable:
handler_map = {
'state': self._handle_state_change,
'telemetry': self._handle_telemetry,
'system': self._handle_system,
'verification': self._handle_verification
}
return handler_map.get(event, self._unknown_event)
def _unknown_event(self, msg: IotMessage) -> None:
self.log.warning(f'Unknown event {msg.event} - No action selected')
def _handle_state_change(self, msg: IotMessage) -> None:
self.log.debug("Handling state event")
message = {
'device_id': msg.device_id,
'event_type': msg.event,
'state': msg.payload.get('state')
}
item = ObserverMessage(event="device_state_changed", data=message)
self._observer_publish_queue.put(item)
def _handle_telemetry(self, msg: IotMessage) -> None:
self.log.debug("Handling telemetry event")
message = {'timestamp': datetime.now(), 'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="device_sensor_data", data=message)
self._observer_publish_queue.put(item)
def _handle_system(self, msg: IotMessage) -> None:
self.log.debug(f"Handling system message from device {msg.device_id}")
if msg.device_id != "framework":
message = {'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="digital_twin",
data=message,
subject="device_status")
self._observer_publish_queue.put(item)
def _handle_verification(self, msg: IotMessage) -> None:
self.log.debug(f"Handling verification event {msg.event}")
if msg.payload.get("action") == "ping":
self.mqtt_client.publish(topic="iot/devices/system/verification",
msg={"action": "pong"})
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "poll_devices":
self.mqtt_client.publish(topic="iot/devices/framework/system",
msg={"event": "poll"})
class GBridge(threading.Thread):
subscribed_event = ['device_state_changed']
mqtt_client = None
g_bridge_connected = False
attached_devices = []
pending_messages = []
pending_subscribed_topics = []
def __init__(self, queue, thread_event: threading.Event):
threading.Thread.__init__(self)
self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug)
gateway_configuration = MqttGatewayConfiguration()
self.observer_notify_queue = Queue(maxsize=100)
self.observer_publish_queue = queue
self._thread_ready = thread_event
keys_dir = get_keys_dir()
gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file)
gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs)
self.gateway_id = gateway_configuration.gateway_id
self.connect_to_iot_core_broker(gateway_configuration)
def __del__(self):
self.detach_all_devices()
self.mqtt_client.disconnect()
self.mqtt_client.loop_stop()
def run(self):
self.mqtt_client.loop_start()
self.wait_for_connection(5)
self._thread_ready.set()
while self.g_bridge_connected:
queue_item = self.observer_notify_queue.get()
self.send_data(msg=queue_item)
time.sleep(ONE_MILLISECOND_SECONDS)
def notify(self, msg, _) -> None:
self.observer_notify_queue.put(item=msg)
@staticmethod
def poll_events():
return []
def connect_to_iot_core_broker(self, conf):
# Create the MQTT client and connect to Cloud IoT.
gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \
f'{conf.registry_id}/devices/{conf.gateway_id}'
self.mqtt_client = mqtt.Client(gateway_id)
jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm)
self.mqtt_client.username_pw_set(username='******', password=jwt_pwd)
self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2)
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_publish = self.on_publish
self.mqtt_client.on_disconnect = self.on_disconnect
self.mqtt_client.on_subscribe = self.on_subscribe
self.mqtt_client.on_message = self.on_message
self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port)
def wait_for_connection(self, timeout):
total_time = 0
while not self.g_bridge_connected and total_time < timeout:
time.sleep(1)
total_time += 1
if not self.g_bridge_connected:
self.log.critical('Could not connect to Iot Core MQTT bridge')
raise RuntimeError()
def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc):
self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}')
self.g_bridge_connected = True
self.subscribe_to_topics(self.gateway_id, True)
if self.attached_devices: # Not empty list, Previously already had connected devices
self.log.warning('Re-connect occurred! Re-attaching all connected devices.')
def subscribe_to_topics(self, dev_id, gateway):
config_topic = f'/devices/{dev_id}/config'
command_topic = f'/devices/{dev_id}/commands/#'
subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}]
if gateway:
gateway_error_topic = f'/devices/{dev_id}/errors'
subscriptions.append({'topic': gateway_error_topic, 'qos': 0})
for subscription in subscriptions:
self.subscribe(subscription.get('topic'), subscription.get('qos'))
def subscribe(self, topic, qos):
_, mid = self.mqtt_client.subscribe(topic, qos)
self.pending_subscribed_topics.append(mid)
while topic in self.pending_subscribed_topics:
time.sleep(0.01)
self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.')
def on_disconnect(self, _unused_client, _unused_userdata, rc):
self.log.warning(f'Disconnected: {error_str(rc)!r}')
self.g_bridge_connected = False
def on_publish(self, _unused_client, _unused_userdata, mid):
self.log.debug(f'ACK received for message {mid!r}')
if mid in self.pending_messages:
self.pending_messages.remove(mid)
def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos):
if granted_qos[0] == 128:
self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed')
else:
if mid in self.pending_subscribed_topics:
self.pending_subscribed_topics.remove(mid)
def on_message(self, _unused_client, _unused_userdata, message):
payload = message.payload.decode('utf-8')
self.log.info(f'Received message {payload!r} on topic {message.topic!r}.')
if not payload:
return
# todo: fix this so that is better
if message.topic.split('/')[3] == "commands":
device_id = GBridge.get_id_from_topic(message.topic)
queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload}
item = {'event': 'gcp_state_changed', 'message': queue_message}
self.observer_publish_queue.put(item)
def attach_device(self, device_id):
self.log.debug(f'Attaching device {device_id!r}.')
attach_topic = f'/devices/{device_id}/attach'
if device_id not in self.attached_devices:
self.attached_devices.append(device_id)
self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
self.subscribe_to_topics(device_id, False)
def detach_device(self, device_id):
self.log.warning(f'Detaching device {device_id!r}.')
detach_topic = f'/devices/{device_id}/detach'
if device_id in self.attached_devices:
self.attached_devices.remove(device_id)
self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
def detach_all_devices(self):
self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.')
for device in self.attached_devices[:]:
self.detach_device(device)
while self.attached_devices: # Make sure all devices have been detached
time.sleep(0.01)
def publish(self, topic, payload):
message_info = self.mqtt_client.publish(topic, payload, qos=1)
self.pending_messages.append(message_info.mid)
self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.')
while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive
time.sleep(0.01)
def send_data(self, msg):
device_id = msg.get('device_id')
event_type = msg.get('event_type')
payload = msg.get('payload')
if device_id not in self.attached_devices:
self.attach_device(device_id=device_id)
if event_type == 'telemetry':
topic = f'/devices/{device_id}/events'
elif event_type == 'state':
topic = f'/devices/{device_id}/state'
else:
self.log.error(f'Unknown event type {event_type}.')
return
self.publish(topic, payload)
@staticmethod
def get_id_from_topic(topic):
index_device_id = 2
dir_tree = topic.split('/')
if len(dir_tree) != 4 or dir_tree[1] != "devices":
return None
return dir_tree[index_device_id]
def reattach_devices(self):
for device in self.attached_devices:
self.log.info(f'Re-attaching device {device}.')
self.attach_device(device)
class MongoHandler:
@dataclass
class MongoConfLocal:
host: str = 'host_ip'
user: str = 'admin'
pwd: str = 'mongo_admin_iot'
url: str = f'mongodb://{user}:{pwd}@{host}/'
def __init__(self, db_name: str) -> None:
self.config = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self.mongo_db = self.connect_to_db(db_name=db_name)
def connect_to_db(self, db_name: str) -> MongoClient:
mongo_host = self.config['mongo_db']['host_ip']
mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host,
mongo_host)
try:
client = self.get_mongo_client(url=mongo_url)
client.server_info()
db = client[db_name]
self.log.success(
f'Connected to MongoDB {db_name!r} at {mongo_url}')
except errors.ServerSelectionTimeoutError as err:
self.log.critical(
f'Connection MongoDB error at {mongo_url} with error: {err}')
raise RuntimeError from err
return db
@staticmethod
def get_mongo_client(url: str) -> MongoClient:
return MongoClient(url)
def get(self, collection_name: str, query: dict = None) -> list:
collection = self.mongo_db[collection_name]
self.log.debug(
f'Executing query {query!r} on collection {collection_name!r}')
return list(collection.find(query))
def insert(self, collection_name: str, data: dict) -> None:
collection = self.mongo_db[collection_name]
data_id = collection.insert_one(data)
self.log.debug(
f'Inserted {data!r} into {collection_name!r} with ID {data_id}')
def update(self, collection_name: str, object_id: str,
updated_values: dict) -> None:
collection = self.mongo_db[collection_name]
query = {'_id': object_id}
collection.update_one(query, updated_values)
self.log.debug(
f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully'
)
def write(self, collection_name: str, data: Union[list, dict],
key: str) -> None:
""" Add's data if it does not exist, else update that data based on key """
if isinstance(data, list):
for entry in data:
self._write(collection=collection_name, data=entry, key=key)
else:
self._write(collection=collection_name, data=data, key=key)
def _write(self, collection: str, data: dict, key: str) -> None:
query = {key: data.get(key, None)}
object_id = self.get_first_object_id_from_query(
collection_name=collection, query=query)
print(object_id)
if object_id:
values = {'$set': data}
self.update(collection_name=collection,
object_id=object_id,
updated_values=values)
else:
self.insert(collection_name=collection, data=data)
def get_first_object_id_from_query(self, collection_name: str,
query: dict) -> Union[str, None]:
collection = self.mongo_db[collection_name]
data = collection.find_one(query)
if isinstance(data, dict):
return data.get('_id')
return None