def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self.mongo = MongoHandler(db_name='iot_db')
self.observer_publish_queue = queue
self._thread_ready = thread_event
self.observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
class MqttClient:
""" Basic wrapper around Paho mqtt """
_mqtt_client = None
def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None:
if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]):
self._config = config
self._on_connect_callback = connect_callback
self._on_message_callback = message_callback
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
pass
@staticmethod
def valid_arguments(config: dict, callbacks: list) -> bool:
callables = [callable(callback) for callback in callbacks]
valid = 'broker' in config and any(callables)
if valid:
return True
raise IllegalArgumentError
def connect(self) -> bool:
self._mqtt_client = paho_mqtt.Client()
try:
self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None),
keepalive=self._config.get("stay_alive", None))
self._mqtt_client.on_connect = self._on_connect
self._mqtt_client.on_message = self._on_message
self._mqtt_client.loop_start()
except (ConnectionRefusedError, TimeoutError) as err:
self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}')
return False
return True
def _on_connect(self, _client, _userdata, _flags, _rc) -> None:
self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})')
self._on_connect_callback()
def _on_message(self, _client, _userdata, message) -> None:
self.log.debug(f'Received message on topic {message.topic!r}')
topic = message.topic
payload = message.payload.decode("utf-8")
self._on_message_callback(topic=topic, payload=payload)
def publish(self, topic: str, msg: Union[dict, str]) -> bool:
self.log.debug(f'Publishing message {msg!r} on topic {topic!r}')
message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1)
if message_info.rc != 0:
self.log.warning('Message did not get published successfully')
return False
return True
def subscribe(self, topics: list) -> None:
for topic in topics:
self._mqtt_client.subscribe(topic=topic, qos=1)
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self._observer_publish_queue = queue
self._thread_ready = thread_event
self._observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
self.config = ConfigurationParser().get_config()
self.poll_interval = self.config["device_manager"]["poll_interval"]
self.device_map_lock = Lock()
def __init__(self) -> None:
self.log = Logging(owner=__file__, config=True)
self.config = ConfigurationParser().get_config()
events = self.config['framework']['events']
self.subject = Subject(events)
self.observer_queue = Queue(maxsize=100)
self._thread_started_event = Event()
self.init_observers()
self.attach_observers()
self.start_observer_threats()
def __init__(self, queue, thread_event: threading.Event):
threading.Thread.__init__(self)
self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug)
gateway_configuration = MqttGatewayConfiguration()
self.observer_notify_queue = Queue(maxsize=100)
self.observer_publish_queue = queue
self._thread_ready = thread_event
keys_dir = get_keys_dir()
gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file)
gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs)
self.gateway_id = gateway_configuration.gateway_id
self.connect_to_iot_core_broker(gateway_configuration)
def __init__(self, queue, thread_event: Event):
Thread.__init__(self)
self.config: dict = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self._observer_notify_queue: Queue = Queue(maxsize=100)
self._observer_publish_queue: Queue = queue
self._thread_ready: Event = thread_event
config = self.get_mqtt_config()
self.mqtt_client = MqttClient(config=config,
connect_callback=self.on_connect,
message_callback=self.on_message)
if not self.mqtt_client.connect():
# todo: unscribcribe from subject
self.log.critical("TODO: Unsubscribe itself form framework")
def test_output_format(self):
log = Logging(owner='t',
log_mode='test',
min_log_lvl=LogLevels.not_set)
truth_list = [
'\033[1;35m', '\033[1;31m', '\033[0;33m', '\033[0;0m', '\033[;1m',
'\033[0;0m', '\033[0;32m'
]
log_levels = [
LogLevels.critical, LogLevels.error, LogLevels.warning,
LogLevels.info, LogLevels.debug, LogLevels.not_set,
LogLevels.success
]
for truth, log_lvl in zip(truth_list, log_levels):
test_result = log._get_output_format(log_lvl)
self.assertEqual(test_result, truth)
def test_leaf_path(self):
paths = [
'a/b/c/', 'a/b/c', '\\a\\b\\c', '\\a\\b\\c\\', 'a\\b\\c',
'a/b/../../a/b/c/', 'a/b/../../a/b/c.py', 'c', '/c.py', '//c.py',
'a.py/b/c'
]
filenames = ['c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c']
for path, filename in zip(paths, filenames):
result = Logging._path_leaf(path)
print(result)
self.assertEqual(result, filename)
def test_different_log_levels(self):
log = Logging(owner='a',
log_mode='test',
min_log_lvl=LogLevels.not_set)
msg = 'a'
time = '1'
source = 'a'
log_levels = [
LogLevels.critical, LogLevels.error, LogLevels.warning,
LogLevels.success, LogLevels.info, LogLevels.debug,
LogLevels.not_set
]
truth_list = [
'1 - a | critical : a', '1 - a | error : a', '1 - a | warning : a',
'1 - a | success : a', '1 - a | info : a', '1 - a | debug : a',
'1 - a | not_set : a'
]
for truth, log_lvl in zip(truth_list, log_levels):
test_result = log._format_log_msg(msg, log_lvl, time, source)
self.assertEqual(test_result, truth)
def test_set_log_lvl(self):
log = Logging(owner='t',
log_mode='test',
min_log_lvl=LogLevels.not_set)
msg = 'a'
log_levels = [
LogLevels.critical, LogLevels.error, LogLevels.warning,
LogLevels.success, LogLevels.info, LogLevels.debug,
LogLevels.not_set
]
min_log_lvls = [
LogLevels.debug, LogLevels.debug, LogLevels.error, LogLevels.debug,
LogLevels.not_set, LogLevels.warning, LogLevels.critical
]
truth_list = [
't | critical : a', 't | error : a', None, 't | success : a',
't | info : a', None, None
]
for truth, log_lvl, min_log_lvl in zip(truth_list, log_levels,
min_log_lvls):
log._set_log_lvl(min_log_lvl)
helper = str(log._log(msg, log_lvl))
test_result = helper.split(' - ')[-1]
self.assertEqual(test_result, str(truth))
class DeviceManager(Thread):
running = True
subscribed_event = ['digital_twin']
remote_digital_twin = []
device_status_map = {}
poll_timer = None
new_devices = False
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self._observer_publish_queue = queue
self._thread_ready = thread_event
self._observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
self.config = ConfigurationParser().get_config()
self.poll_interval = self.config["device_manager"]["poll_interval"]
self.device_map_lock = Lock()
def __del__(self) -> None:
self.running = False
def notify(self, msg: ObserverMessage) -> None:
self._observer_notify_queue.put(item=msg)
def run(self) -> None:
self._thread_ready.set()
self._fetch_digital_twin()
self._start_timer(interval=self.poll_interval,
callback=self._timer_callback)
while self.running:
queue_msg = self._observer_notify_queue.get()
if queue_msg.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_msg)
self.poll_timer.cancel()
def _fetch_digital_twin(self):
msg = ObserverMessage(event="digital_twin",
data={},
subject="fetch_digital_twin")
self._observer_publish_queue.put(msg)
def _start_timer(self, interval: int, callback: Callable) -> None:
self.poll_timer = Timer(interval=interval, function=callback)
self.poll_timer.start()
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "retrieved_digital_twin":
self._store_remote_digital_twin(data=msg.data)
elif msg.subject == "device_status":
self._store_device_status(data=msg.data)
def _store_remote_digital_twin(self, data: dict):
self.remote_digital_twin = data
self.log.success("Received remote digital twin")
self.log.debug(f"Remote digital twin: {self.remote_digital_twin}")
def _store_device_status(self, data: dict):
device_id = data.get("device_id", None)
status = data.get("active", None)
self.log.debug(f"Received device status {status} from {device_id}")
if device_id and status:
self.device_status_map[device_id] = status
def _timer_callback(self):
self.log.debug("Starting device status polling stage")
self.device_status_map = self._generate_device_map_from_remote_twin()
self._publish_device_status_poll()
wait_period = self.config["device_manager"]["wait_period"]
self._wait_for_status_messages(wait_period=wait_period)
digital_twin = self._create_digital_twin_from_device_status()
if digital_twin:
self._publish_digital_twin(twin=digital_twin)
if self.new_devices:
self._fetch_digital_twin()
self.new_devices = False
self._start_timer(interval=self.poll_interval,
callback=self._timer_callback)
def _generate_device_map_from_remote_twin(self) -> dict:
device_map = {}
for twin_item in self.remote_digital_twin:
device_name = twin_item["device_name"]
device_map[device_name] = False
return device_map
def _publish_device_status_poll(self):
msg = ObserverMessage(event="digital_twin",
data={},
subject="poll_devices")
self._observer_publish_queue.put(msg)
def _wait_for_status_messages(self, wait_period: Union[int,
float]) -> None:
self.log.debug(f"Starting waiting period of {wait_period} seconds")
start_time = time()
while self.running and (time() - start_time < wait_period):
sleep(0.01)
self.log.debug("Waiting period over")
def _create_digital_twin_from_device_status(
self) -> Union[None, List[dict]]:
if not self.device_status_map and not self.remote_digital_twin:
self.log.debug(
"No digital twin created since remote and local twin are empty"
)
return None
with self.device_map_lock:
device_status_map = self.device_status_map.copy()
digital_twin = []
for remote_item in self.remote_digital_twin:
device_id = remote_item["device_name"]
helper = remote_item
if device_id in device_status_map:
helper["active"] = device_status_map[device_id]
del device_status_map[device_id]
digital_twin.append(helper)
for new_device in device_status_map:
new_item = {
"device_name": new_device,
"active": True,
"location": None,
"technology": None,
"battery_level": None
}
digital_twin.append(new_item)
self.new_devices = True
return digital_twin
def _publish_digital_twin(self, twin: Union[list, dict]) -> None:
msg = ObserverMessage(event="digital_twin",
data=twin,
subject="save_digital_twin")
self._observer_publish_queue.put(msg)
class HealthMonitor(Thread):
running = True
update_time_sec = 600
subscribed_event = []
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self.observer_publish_queue = queue
self._thread_ready = thread_event
self.observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
self.running = False
def run(self) -> None:
self.log.info(f'Updating system information every {self.update_time_sec} seconds.')
self._thread_ready.set()
while self.running:
start_time = time()
host_data = self._fetch_host_data()
msg = ObserverMessage(event="host_health", data=host_data)
self.observer_publish_queue.put(msg)
sleep_time = self.update_time_sec - ((time() - start_time) % self.update_time_sec)
sleep(sleep_time)
def notify(self, msg: ObserverMessage) -> None:
pass
def _fetch_host_data(self) -> dict:
data = {
'timestamp': self._get_timestamp(),
'temperature': self.poll_system_temp(),
'cpu_load': self.poll_cpu_load()
}
return data
@staticmethod
def _get_timestamp() -> datetime:
return datetime.now()
def poll_system_temp(self) -> float:
temp_file = self._get_temperature_file()
try:
with open(temp_file) as file:
return float(file.readline()) / 1000
except FileNotFoundError:
self.log.critical(f'Temperature file {temp_file!r} does not exist')
return 0
@staticmethod
def _get_temperature_file() -> Path:
return Path('/sys/class/thermal/thermal_zone0/temp')
def poll_cpu_load(self) -> float:
cpu_command = ["cat", "/proc/stat"]
try:
with subprocess.Popen(cpu_command, stdout=subprocess.PIPE) as process_result:
proc_stat, _ = process_result.communicate()
cpu_data = proc_stat.decode('utf-8').split('\n')[0].split()[1:-1]
cpu_data = [int(field) for field in cpu_data]
cpu_usage = ((cpu_data[0] + cpu_data[2]) * 100 / (cpu_data[0] + cpu_data[2] + cpu_data[3]))
return round(cpu_usage, 3)
except FileNotFoundError as error:
self.log.critical(f'Command {" ".join(cpu_command)!r} was not found on the system: {error}')
except ValueError as error:
self.log.error(f'Parsing of the data went wrong: {error}')
return 0
def __init__(self, db_name: str) -> None:
self.config = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self.mongo_db = self.connect_to_db(db_name=db_name)
class MqttGateway(Thread):
running = False
subscribed_event = ['gcp_state_changed', 'digital_twin']
def __init__(self, queue, thread_event: Event):
Thread.__init__(self)
self.config: dict = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self._observer_notify_queue: Queue = Queue(maxsize=100)
self._observer_publish_queue: Queue = queue
self._thread_ready: Event = thread_event
config = self.get_mqtt_config()
self.mqtt_client = MqttClient(config=config,
connect_callback=self.on_connect,
message_callback=self.on_message)
if not self.mqtt_client.connect():
# todo: unscribcribe from subject
self.log.critical("TODO: Unsubscribe itself form framework")
def __del__(self):
self.running = False
def run(self):
self._thread_ready.set()
while self.running:
queue_item = self._observer_notify_queue.get()
if queue_item.event == "digital_twin":
self._handle_digital_twin_event(msg=queue_item)
def notify(self, msg: ObserverMessage):
self._observer_notify_queue.put(item=msg)
def get_mqtt_config(self) -> dict:
return {
'broker': self.config['mqtt_gateway']['broker_address'],
'port': 1883,
'stay_alive': 60
}
def on_connect(self):
topics = ['iot/#']
self.mqtt_client.subscribe(topics=topics)
self._thread_ready.set()
self.running = True
def on_message(self, topic: str, payload: str) -> None:
self.log.info(f'Received {payload!r} on topic {topic!r}')
self._log_mqtt_traffic(topic=topic, payload=payload)
message = IotMessage(mqtt_topic=topic, data=payload)
if message.is_valid():
handler = self._select_handler(event=message.event)
handler(msg=message)
else:
self.log.warning('The MQTT message is not valid')
def _log_mqtt_traffic(self, topic: str, payload: str) -> None:
data = {
'timestamp': datetime.now(),
'source': type(self).__name__,
'topic': topic,
'payload': payload
}
msg = ObserverMessage(event="iot_traffic", data=data)
self._observer_publish_queue.put(msg)
def _select_handler(self, event: str) -> Callable:
handler_map = {
'state': self._handle_state_change,
'telemetry': self._handle_telemetry,
'system': self._handle_system,
'verification': self._handle_verification
}
return handler_map.get(event, self._unknown_event)
def _unknown_event(self, msg: IotMessage) -> None:
self.log.warning(f'Unknown event {msg.event} - No action selected')
def _handle_state_change(self, msg: IotMessage) -> None:
self.log.debug("Handling state event")
message = {
'device_id': msg.device_id,
'event_type': msg.event,
'state': msg.payload.get('state')
}
item = ObserverMessage(event="device_state_changed", data=message)
self._observer_publish_queue.put(item)
def _handle_telemetry(self, msg: IotMessage) -> None:
self.log.debug("Handling telemetry event")
message = {'timestamp': datetime.now(), 'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="device_sensor_data", data=message)
self._observer_publish_queue.put(item)
def _handle_system(self, msg: IotMessage) -> None:
self.log.debug(f"Handling system message from device {msg.device_id}")
if msg.device_id != "framework":
message = {'device_id': msg.device_id}
message.update(msg.payload)
item = ObserverMessage(event="digital_twin",
data=message,
subject="device_status")
self._observer_publish_queue.put(item)
def _handle_verification(self, msg: IotMessage) -> None:
self.log.debug(f"Handling verification event {msg.event}")
if msg.payload.get("action") == "ping":
self.mqtt_client.publish(topic="iot/devices/system/verification",
msg={"action": "pong"})
def _handle_digital_twin_event(self, msg: ObserverMessage):
if msg.subject == "poll_devices":
self.mqtt_client.publish(topic="iot/devices/framework/system",
msg={"event": "poll"})
class MongoHandler:
@dataclass
class MongoConfLocal:
host: str = 'host_ip'
user: str = 'admin'
pwd: str = 'mongo_admin_iot'
url: str = f'mongodb://{user}:{pwd}@{host}/'
def __init__(self, db_name: str) -> None:
self.config = ConfigurationParser().get_config()
self.log = Logging(owner=__file__, config=True)
self.mongo_db = self.connect_to_db(db_name=db_name)
def connect_to_db(self, db_name: str) -> MongoClient:
mongo_host = self.config['mongo_db']['host_ip']
mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host,
mongo_host)
try:
client = self.get_mongo_client(url=mongo_url)
client.server_info()
db = client[db_name]
self.log.success(
f'Connected to MongoDB {db_name!r} at {mongo_url}')
except errors.ServerSelectionTimeoutError as err:
self.log.critical(
f'Connection MongoDB error at {mongo_url} with error: {err}')
raise RuntimeError from err
return db
@staticmethod
def get_mongo_client(url: str) -> MongoClient:
return MongoClient(url)
def get(self, collection_name: str, query: dict = None) -> list:
collection = self.mongo_db[collection_name]
self.log.debug(
f'Executing query {query!r} on collection {collection_name!r}')
return list(collection.find(query))
def insert(self, collection_name: str, data: dict) -> None:
collection = self.mongo_db[collection_name]
data_id = collection.insert_one(data)
self.log.debug(
f'Inserted {data!r} into {collection_name!r} with ID {data_id}')
def update(self, collection_name: str, object_id: str,
updated_values: dict) -> None:
collection = self.mongo_db[collection_name]
query = {'_id': object_id}
collection.update_one(query, updated_values)
self.log.debug(
f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully'
)
def write(self, collection_name: str, data: Union[list, dict],
key: str) -> None:
""" Add's data if it does not exist, else update that data based on key """
if isinstance(data, list):
for entry in data:
self._write(collection=collection_name, data=entry, key=key)
else:
self._write(collection=collection_name, data=data, key=key)
def _write(self, collection: str, data: dict, key: str) -> None:
query = {key: data.get(key, None)}
object_id = self.get_first_object_id_from_query(
collection_name=collection, query=query)
print(object_id)
if object_id:
values = {'$set': data}
self.update(collection_name=collection,
object_id=object_id,
updated_values=values)
else:
self.insert(collection_name=collection, data=data)
def get_first_object_id_from_query(self, collection_name: str,
query: dict) -> Union[str, None]:
collection = self.mongo_db[collection_name]
data = collection.find_one(query)
if isinstance(data, dict):
return data.get('_id')
return None
class IotSubject:
observers = []
running = False
def __init__(self) -> None:
self.log = Logging(owner=__file__, config=True)
self.config = ConfigurationParser().get_config()
events = self.config['framework']['events']
self.subject = Subject(events)
self.observer_queue = Queue(maxsize=100)
self._thread_started_event = Event()
self.init_observers()
self.attach_observers()
self.start_observer_threats()
def init_observers(self) -> None:
active_components = self._get_activated_components()
for component in active_components:
obj = self._get_matching_object(component_name=component)
observer = obj(queue=self.observer_queue,
thread_event=self._thread_started_event)
events = observer.subscribed_event
self.observers.append({'obs_object': observer, 'events': events})
def _get_activated_components(self) -> list:
system_components = self.config['system_components'].keys()
return [
component for component in system_components
if self.config['system_components'][component]
]
@staticmethod
def _get_matching_object(component_name: str) -> Callable:
object_mapper = {
'mqtt_gateway': MqttGateway,
'db': DbHandler,
'host_monitor': HealthMonitor,
'device_manager': DeviceManager
}
return object_mapper.get(component_name)
def attach_observers(self) -> None:
for observer in self.observers:
self.subject.register(**observer)
def start_observer_threats(self) -> None:
for observer in self.observers:
observer.get('obs_object').start()
self._thread_started_event.wait()
observer_names = [key['obs_object'] for key in self.observers]
self.log.success(
f'Started {len(observer_names)} observers. {observer_names}')
self.running = True
def run(self) -> None:
while self.running:
msg = self.get_observer_events()
self.notify_observers(msg=msg)
def notify_observers(self, msg: ObserverMessage) -> None:
self.subject.dispatch(message=msg)
def get_observer_events(self) -> ObserverMessage:
return self.observer_queue.get()
def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None:
if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]):
self._config = config
self._on_connect_callback = connect_callback
self._on_message_callback = message_callback
self.log = Logging(owner=__file__, config=True)
class DbHandler(Thread):
running = True
subscribed_event = [
'gcp_state_changed', 'device_state_changed', 'iot_traffic',
'host_health', 'device_sensor_data', 'digital_twin'
]
def __init__(self, queue: Queue, thread_event: Event) -> None:
Thread.__init__(self)
self.mongo = MongoHandler(db_name='iot_db')
self.observer_publish_queue = queue
self._thread_ready = thread_event
self.observer_notify_queue = Queue(maxsize=100)
self.log = Logging(owner=__file__, config=True)
def __del__(self) -> None:
self.running = False
def run(self) -> None:
self._thread_ready.set()
while self.running:
item = self.observer_notify_queue.get()
action = self.action_selector(event=item.event)
action(msg=item)
def notify(self, msg: ObserverMessage) -> None:
self.observer_notify_queue.put(msg)
def action_selector(self, event: str) -> Callable:
action_map = {
'gcp_state_changed': self.store_state_data,
'device_state_changed': self.store_state_data,
'iot_traffic': self.add_document_row,
'host_health': self.add_document_row,
'device_sensor_data': self.add_document_row,
'digital_twin': self.handle_digital_twin
}
return action_map.get(event, self.action_skip)
@staticmethod
def action_skip():
pass
def get_document_data(self, document: str) -> list:
return self.mongo.get(collection_name=document)
def store_state_data(self, msg: ObserverMessage) -> None:
document_data = {
'device_id': msg.data.get('device_id'),
'event': msg.event,
'change_source': msg.data.get('event'),
'state': msg.data.get('state')
}
self.mongo.write(collection_name="states",
data=document_data,
key="device_id")
def add_document_row(self, msg: ObserverMessage) -> None:
self.mongo.insert(collection_name=msg.event, data=msg.data)
def handle_digital_twin(self, msg: ObserverMessage) -> None:
if msg.subject == "fetch_digital_twin":
self._fetch_digital_twin()
elif msg.subject == "save_digital_twin":
self._save_digital_twin(twin=msg.data)
else:
self.action_skip()
def _fetch_digital_twin(self) -> None:
self.log.info("Fetching digital twin from DB")
twin_data = self.get_document_data(document="digital_twin")
digital_twin = self._outbound_adapter(data=twin_data)
msg = ObserverMessage(event="digital_twin",
data=digital_twin,
subject="retrieved_digital_twin")
self.observer_publish_queue.put(msg)
def _save_digital_twin(self, twin: list) -> None:
self.log.info("Uploading updated digital twin")
self.mongo.write(collection_name='digital_twin',
data=twin,
key='device_name')
@staticmethod
def _outbound_adapter(data: list) -> list:
""" Removes the object_id field from each data entry, preping the data for transportation """
for entry in data:
entry.pop("_id", None)
return data
class GBridge(threading.Thread):
subscribed_event = ['device_state_changed']
mqtt_client = None
g_bridge_connected = False
attached_devices = []
pending_messages = []
pending_subscribed_topics = []
def __init__(self, queue, thread_event: threading.Event):
threading.Thread.__init__(self)
self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug)
gateway_configuration = MqttGatewayConfiguration()
self.observer_notify_queue = Queue(maxsize=100)
self.observer_publish_queue = queue
self._thread_ready = thread_event
keys_dir = get_keys_dir()
gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file)
gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs)
self.gateway_id = gateway_configuration.gateway_id
self.connect_to_iot_core_broker(gateway_configuration)
def __del__(self):
self.detach_all_devices()
self.mqtt_client.disconnect()
self.mqtt_client.loop_stop()
def run(self):
self.mqtt_client.loop_start()
self.wait_for_connection(5)
self._thread_ready.set()
while self.g_bridge_connected:
queue_item = self.observer_notify_queue.get()
self.send_data(msg=queue_item)
time.sleep(ONE_MILLISECOND_SECONDS)
def notify(self, msg, _) -> None:
self.observer_notify_queue.put(item=msg)
@staticmethod
def poll_events():
return []
def connect_to_iot_core_broker(self, conf):
# Create the MQTT client and connect to Cloud IoT.
gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \
f'{conf.registry_id}/devices/{conf.gateway_id}'
self.mqtt_client = mqtt.Client(gateway_id)
jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm)
self.mqtt_client.username_pw_set(username='******', password=jwt_pwd)
self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2)
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_publish = self.on_publish
self.mqtt_client.on_disconnect = self.on_disconnect
self.mqtt_client.on_subscribe = self.on_subscribe
self.mqtt_client.on_message = self.on_message
self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port)
def wait_for_connection(self, timeout):
total_time = 0
while not self.g_bridge_connected and total_time < timeout:
time.sleep(1)
total_time += 1
if not self.g_bridge_connected:
self.log.critical('Could not connect to Iot Core MQTT bridge')
raise RuntimeError()
def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc):
self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}')
self.g_bridge_connected = True
self.subscribe_to_topics(self.gateway_id, True)
if self.attached_devices: # Not empty list, Previously already had connected devices
self.log.warning('Re-connect occurred! Re-attaching all connected devices.')
def subscribe_to_topics(self, dev_id, gateway):
config_topic = f'/devices/{dev_id}/config'
command_topic = f'/devices/{dev_id}/commands/#'
subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}]
if gateway:
gateway_error_topic = f'/devices/{dev_id}/errors'
subscriptions.append({'topic': gateway_error_topic, 'qos': 0})
for subscription in subscriptions:
self.subscribe(subscription.get('topic'), subscription.get('qos'))
def subscribe(self, topic, qos):
_, mid = self.mqtt_client.subscribe(topic, qos)
self.pending_subscribed_topics.append(mid)
while topic in self.pending_subscribed_topics:
time.sleep(0.01)
self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.')
def on_disconnect(self, _unused_client, _unused_userdata, rc):
self.log.warning(f'Disconnected: {error_str(rc)!r}')
self.g_bridge_connected = False
def on_publish(self, _unused_client, _unused_userdata, mid):
self.log.debug(f'ACK received for message {mid!r}')
if mid in self.pending_messages:
self.pending_messages.remove(mid)
def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos):
if granted_qos[0] == 128:
self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed')
else:
if mid in self.pending_subscribed_topics:
self.pending_subscribed_topics.remove(mid)
def on_message(self, _unused_client, _unused_userdata, message):
payload = message.payload.decode('utf-8')
self.log.info(f'Received message {payload!r} on topic {message.topic!r}.')
if not payload:
return
# todo: fix this so that is better
if message.topic.split('/')[3] == "commands":
device_id = GBridge.get_id_from_topic(message.topic)
queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload}
item = {'event': 'gcp_state_changed', 'message': queue_message}
self.observer_publish_queue.put(item)
def attach_device(self, device_id):
self.log.debug(f'Attaching device {device_id!r}.')
attach_topic = f'/devices/{device_id}/attach'
if device_id not in self.attached_devices:
self.attached_devices.append(device_id)
self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
self.subscribe_to_topics(device_id, False)
def detach_device(self, device_id):
self.log.warning(f'Detaching device {device_id!r}.')
detach_topic = f'/devices/{device_id}/detach'
if device_id in self.attached_devices:
self.attached_devices.remove(device_id)
self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY
def detach_all_devices(self):
self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.')
for device in self.attached_devices[:]:
self.detach_device(device)
while self.attached_devices: # Make sure all devices have been detached
time.sleep(0.01)
def publish(self, topic, payload):
message_info = self.mqtt_client.publish(topic, payload, qos=1)
self.pending_messages.append(message_info.mid)
self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.')
while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive
time.sleep(0.01)
def send_data(self, msg):
device_id = msg.get('device_id')
event_type = msg.get('event_type')
payload = msg.get('payload')
if device_id not in self.attached_devices:
self.attach_device(device_id=device_id)
if event_type == 'telemetry':
topic = f'/devices/{device_id}/events'
elif event_type == 'state':
topic = f'/devices/{device_id}/state'
else:
self.log.error(f'Unknown event type {event_type}.')
return
self.publish(topic, payload)
@staticmethod
def get_id_from_topic(topic):
index_device_id = 2
dir_tree = topic.split('/')
if len(dir_tree) != 4 or dir_tree[1] != "devices":
return None
return dir_tree[index_device_id]
def reattach_devices(self):
for device in self.attached_devices:
self.log.info(f'Re-attaching device {device}.')
self.attach_device(device)
