class MqttGateway(Thread):
running = False
subscribed_event = ['gcp_state_changed', 'digital_twin']
def __init__(self, queue, thread_event: Event):
Thread.__init__(self)
self.config: dict = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self._observer_notify_queue: Queue = Queue(maxsize=100)
self._observer_publish_queue: Queue = queue
self._thread_ready: Event = thread_event
config = self.get_mqtt_config()
self.mqtt_client = MqttClient(config=config,
connect_callback=self.on_connect,
message_callback=self.on_message)
if not self.mqtt_client.connect():
# todo: unscribcribe from subject
self.log.critical("TODO: Unsubscribe itself form framework")
def __del__(self):
self.running = False
def run(self):
self._thread_ready.set()
while self.running:
queue_item = self._observer_notify_queue.get()
if queue_item.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_item)
def notify(self, msg: ObserverMessage):
self._observer_notify_queue.put(item=msg)
def get_mqtt_config(self) -> dict:
return {
'broker': self.config['mqtt_gateway']['broker_address'],
'port': 1883,
'stay_alive': 60
}
def on_connect(self):
topics = ['iot/#']
self.mqtt_client.subscribe(topics=topics)
self._thread_ready.set()
self.running = True
def on_message(self, topic: str, payload: str) -> None:
self.log.info(f'Received {payload!r} on topic {topic!r}')
self._log_mqtt_traffic(topic=topic, payload=payload)
message = IotMessage(mqtt_topic=topic, data=payload)
if message.is_valid():
handler = self._select_handler(event=message.event)
handler(msg=message)
else:
self.log.warning('The MQTT message is not valid')
def _log_mqtt_traffic(self, topic: str, payload: str) -> None:
data = {
'timestamp': datetime.now(),
'source': type(self).__name__,
'topic': topic,
'payload': payload
}
msg = ObserverMessage(event="iot_traffic", data=data)
self._observer_publish_queue.put(msg)
def _select_handler(self, event: str) -> Callable:
handler_map = {
'state': self._handle_state_change,
'telemetry': self._handle_telemetry,
'system': self._handle_system,
'verification': self._handle_verification
}
return handler_map.get(event, self._unknown_event)
def _unknown_event(self, msg: IotMessage) -> None:
self.log.warning(f'Unknown event {msg.event} - No action selected')
def _handle_state_change(self, msg: IotMessage) -> None:
self.log.debug("Handling state event")
message = {
'device_id': msg.device_id,
'event_type': msg.event,
'state': msg.payload.get('state')
}
item = ObserverMessage(event="device_state_changed", data=message)
self._observer_publish_queue.put(item)
def _handle_telemetry(self, msg: IotMessage) -> None:
self.log.debug("Handling telemetry event")
message = {'timestamp': datetime.now(), 'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="device_sensor_data", data=message)
self._observer_publish_queue.put(item)
def _handle_system(self, msg: IotMessage) -> None:
self.log.debug(f"Handling system message from device {msg.device_id}")
if msg.device_id != "framework":
message = {'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="digital_twin",
data=message,
subject="device_status")
self._observer_publish_queue.put(item)
def _handle_verification(self, msg: IotMessage) -> None:
self.log.debug(f"Handling verification event {msg.event}")
if msg.payload.get("action") == "ping":
self.mqtt_client.publish(topic="iot/devices/system/verification",
msg={"action": "pong"})
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "poll_devices":
self.mqtt_client.publish(topic="iot/devices/framework/system",
msg={"event": "poll"})
class HealthMonitor(Thread):
running = True
update_time_sec = 600
subscribed_event = []
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self.observer_publish_queue = queue
self._thread_ready = thread_event
self.observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
self.running = False
def run(self) -> None:
self.log.info(f'Updating system information every {self.update_time_sec} seconds.')
self._thread_ready.set()
while self.running:
start_time = time()
host_data = self._fetch_host_data()
msg = ObserverMessage(event="host_health", data=host_data)
self.observer_publish_queue.put(msg)
sleep_time = self.update_time_sec - ((time() - start_time) % self.update_time_sec)
sleep(sleep_time)
def notify(self, msg: ObserverMessage) -> None:
pass
def _fetch_host_data(self) -> dict:
data = {
'timestamp': self._get_timestamp(),
'temperature': self.poll_system_temp(),
'cpu_load': self.poll_cpu_load()
}
return data
@staticmethod
def _get_timestamp() -> datetime:
return datetime.now()
def poll_system_temp(self) -> float:
temp_file = self._get_temperature_file()
try:
with open(temp_file) as file:
return float(file.readline()) / 1000
except FileNotFoundError:
self.log.critical(f'Temperature file {temp_file!r} does not exist')
return 0
@staticmethod
def _get_temperature_file() -> Path:
return Path('/sys/class/thermal/thermal_zone0/temp')
def poll_cpu_load(self) -> float:
cpu_command = ["cat", "/proc/stat"]
try:
with subprocess.Popen(cpu_command, stdout=subprocess.PIPE) as process_result:
proc_stat, _ = process_result.communicate()
cpu_data = proc_stat.decode('utf-8').split('\n')[0].split()[1:-1]
cpu_data = [int(field) for field in cpu_data]
cpu_usage = ((cpu_data[0] + cpu_data[2]) * 100 / (cpu_data[0] + cpu_data[2] + cpu_data[3]))
return round(cpu_usage, 3)
except FileNotFoundError as error:
self.log.critical(f'Command {" ".join(cpu_command)!r} was not found on the system: {error}')
except ValueError as error:
self.log.error(f'Parsing of the data went wrong: {error}')
return 0
class GBridge(threading.Thread):
subscribed_event = ['device_state_changed']
mqtt_client = None
g_bridge_connected = False
attached_devices = []
pending_messages = []
pending_subscribed_topics = []
def __init__(self, queue, thread_event: threading.Event):
threading.Thread.__init__(self)
self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug)
gateway_configuration = MqttGatewayConfiguration()
self.observer_notify_queue = Queue(maxsize=100)
self.observer_publish_queue = queue
self._thread_ready = thread_event
keys_dir = get_keys_dir()
gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file)
gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs)
self.gateway_id = gateway_configuration.gateway_id
self.connect_to_iot_core_broker(gateway_configuration)
def __del__(self):
self.detach_all_devices()
self.mqtt_client.disconnect()
self.mqtt_client.loop_stop()
def run(self):
self.mqtt_client.loop_start()
self.wait_for_connection(5)
self._thread_ready.set()
while self.g_bridge_connected:
queue_item = self.observer_notify_queue.get()
self.send_data(msg=queue_item)
time.sleep(ONE_MILLISECOND_SECONDS)
def notify(self, msg, _) -> None:
self.observer_notify_queue.put(item=msg)
@staticmethod
def poll_events():
return []
def connect_to_iot_core_broker(self, conf):
# Create the MQTT client and connect to Cloud IoT.
gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \
f'{conf.registry_id}/devices/{conf.gateway_id}'
self.mqtt_client = mqtt.Client(gateway_id)
jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm)
self.mqtt_client.username_pw_set(username='******', password=jwt_pwd)
self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2)
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_publish = self.on_publish
self.mqtt_client.on_disconnect = self.on_disconnect
self.mqtt_client.on_subscribe = self.on_subscribe
self.mqtt_client.on_message = self.on_message
self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port)
def wait_for_connection(self, timeout):
total_time = 0
while not self.g_bridge_connected and total_time < timeout:
time.sleep(1)
total_time += 1
if not self.g_bridge_connected:
self.log.critical('Could not connect to Iot Core MQTT bridge')
raise RuntimeError()
def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc):
self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}')
self.g_bridge_connected = True
self.subscribe_to_topics(self.gateway_id, True)
if self.attached_devices: # Not empty list, Previously already had connected devices
self.log.warning('Re-connect occurred! Re-attaching all connected devices.')
def subscribe_to_topics(self, dev_id, gateway):
config_topic = f'/devices/{dev_id}/config'
command_topic = f'/devices/{dev_id}/commands/#'
subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}]
if gateway:
gateway_error_topic = f'/devices/{dev_id}/errors'
subscriptions.append({'topic': gateway_error_topic, 'qos': 0})
for subscription in subscriptions:
self.subscribe(subscription.get('topic'), subscription.get('qos'))
def subscribe(self, topic, qos):
_, mid = self.mqtt_client.subscribe(topic, qos)
self.pending_subscribed_topics.append(mid)
while topic in self.pending_subscribed_topics:
time.sleep(0.01)
self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.')
def on_disconnect(self, _unused_client, _unused_userdata, rc):
self.log.warning(f'Disconnected: {error_str(rc)!r}')
self.g_bridge_connected = False
def on_publish(self, _unused_client, _unused_userdata, mid):
self.log.debug(f'ACK received for message {mid!r}')
if mid in self.pending_messages:
self.pending_messages.remove(mid)
def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos):
if granted_qos[0] == 128:
self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed')
else:
if mid in self.pending_subscribed_topics:
self.pending_subscribed_topics.remove(mid)
def on_message(self, _unused_client, _unused_userdata, message):
payload = message.payload.decode('utf-8')
self.log.info(f'Received message {payload!r} on topic {message.topic!r}.')
if not payload:
return
# todo: fix this so that is better
if message.topic.split('/')[3] == "commands":
device_id = GBridge.get_id_from_topic(message.topic)
queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload}
item = {'event': 'gcp_state_changed', 'message': queue_message}
self.observer_publish_queue.put(item)
def attach_device(self, device_id):
self.log.debug(f'Attaching device {device_id!r}.')
attach_topic = f'/devices/{device_id}/attach'
if device_id not in self.attached_devices:
self.attached_devices.append(device_id)
self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
self.subscribe_to_topics(device_id, False)
def detach_device(self, device_id):
self.log.warning(f'Detaching device {device_id!r}.')
detach_topic = f'/devices/{device_id}/detach'
if device_id in self.attached_devices:
self.attached_devices.remove(device_id)
self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
def detach_all_devices(self):
self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.')
for device in self.attached_devices[:]:
self.detach_device(device)
while self.attached_devices: # Make sure all devices have been detached
time.sleep(0.01)
def publish(self, topic, payload):
message_info = self.mqtt_client.publish(topic, payload, qos=1)
self.pending_messages.append(message_info.mid)
self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.')
while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive
time.sleep(0.01)
def send_data(self, msg):
device_id = msg.get('device_id')
event_type = msg.get('event_type')
payload = msg.get('payload')
if device_id not in self.attached_devices:
self.attach_device(device_id=device_id)
if event_type == 'telemetry':
topic = f'/devices/{device_id}/events'
elif event_type == 'state':
topic = f'/devices/{device_id}/state'
else:
self.log.error(f'Unknown event type {event_type}.')
return
self.publish(topic, payload)
@staticmethod
def get_id_from_topic(topic):
index_device_id = 2
dir_tree = topic.split('/')
if len(dir_tree) != 4 or dir_tree[1] != "devices":
return None
return dir_tree[index_device_id]
def reattach_devices(self):
for device in self.attached_devices:
self.log.info(f'Re-attaching device {device}.')
self.attach_device(device)
class MongoHandler:
@dataclass
class MongoConfLocal:
host: str = 'host_ip'
user: str = 'admin'
pwd: str = 'mongo_admin_iot'
url: str = f'mongodb://{user}:{pwd}@{host}/'
def __init__(self, db_name: str) -> None:
self.config = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self.mongo_db = self.connect_to_db(db_name=db_name)
def connect_to_db(self, db_name: str) -> MongoClient:
mongo_host = self.config['mongo_db']['host_ip']
mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host,
mongo_host)
try:
client = self.get_mongo_client(url=mongo_url)
client.server_info()
db = client[db_name]
self.log.success(
f'Connected to MongoDB {db_name!r} at {mongo_url}')
except errors.ServerSelectionTimeoutError as err:
self.log.critical(
f'Connection MongoDB error at {mongo_url} with error: {err}')
raise RuntimeError from err
return db
@staticmethod
def get_mongo_client(url: str) -> MongoClient:
return MongoClient(url)
def get(self, collection_name: str, query: dict = None) -> list:
collection = self.mongo_db[collection_name]
self.log.debug(
f'Executing query {query!r} on collection {collection_name!r}')
return list(collection.find(query))
def insert(self, collection_name: str, data: dict) -> None:
collection = self.mongo_db[collection_name]
data_id = collection.insert_one(data)
self.log.debug(
f'Inserted {data!r} into {collection_name!r} with ID {data_id}')
def update(self, collection_name: str, object_id: str,
updated_values: dict) -> None:
collection = self.mongo_db[collection_name]
query = {'_id': object_id}
collection.update_one(query, updated_values)
self.log.debug(
f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully'
)
def write(self, collection_name: str, data: Union[list, dict],
key: str) -> None:
""" Add's data if it does not exist, else update that data based on key """
if isinstance(data, list):
for entry in data:
self._write(collection=collection_name, data=entry, key=key)
else:
self._write(collection=collection_name, data=data, key=key)
def _write(self, collection: str, data: dict, key: str) -> None:
query = {key: data.get(key, None)}
object_id = self.get_first_object_id_from_query(
collection_name=collection, query=query)
print(object_id)
if object_id:
values = {'$set': data}
self.update(collection_name=collection,
object_id=object_id,
updated_values=values)
else:
self.insert(collection_name=collection, data=data)
def get_first_object_id_from_query(self, collection_name: str,
query: dict) -> Union[str, None]:
collection = self.mongo_db[collection_name]
data = collection.find_one(query)
if isinstance(data, dict):
return data.get('_id')
return None
