def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self.mongo = MongoHandler(db_name='iot_db') self.observer_publish_queue = queue self._thread_ready = thread_event self.observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True)
class MqttClient: """ Basic wrapper around Paho mqtt """ _mqtt_client = None def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None: if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]): self._config = config self._on_connect_callback = connect_callback self._on_message_callback = message_callback self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: pass @staticmethod def valid_arguments(config: dict, callbacks: list) -> bool: callables = [callable(callback) for callback in callbacks] valid = 'broker' in config and any(callables) if valid: return True raise IllegalArgumentError def connect(self) -> bool: self._mqtt_client = paho_mqtt.Client() try: self._mqtt_client.connect(host=self._config.get("broker"), port=self._config.get("port", None), keepalive=self._config.get("stay_alive", None)) self._mqtt_client.on_connect = self._on_connect self._mqtt_client.on_message = self._on_message self._mqtt_client.loop_start() except (ConnectionRefusedError, TimeoutError) as err: self.log.error(f'Failed to connect to MQTT broker ({self._config.get("broker", None)}). Error: {err}') return False return True def _on_connect(self, _client, _userdata, _flags, _rc) -> None: self.log.success(f'Connected to MQTT broker ({self._config.get("broker")})') self._on_connect_callback() def _on_message(self, _client, _userdata, message) -> None: self.log.debug(f'Received message on topic {message.topic!r}') topic = message.topic payload = message.payload.decode("utf-8") self._on_message_callback(topic=topic, payload=payload) def publish(self, topic: str, msg: Union[dict, str]) -> bool: self.log.debug(f'Publishing message {msg!r} on topic {topic!r}') message_info = self._mqtt_client.publish(topic=topic, payload=dumps(msg), qos=1) if message_info.rc != 0: self.log.warning('Message did not get published successfully') return False return True def subscribe(self, topics: list) -> None: for topic in topics: self._mqtt_client.subscribe(topic=topic, qos=1)
def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self._observer_publish_queue = queue self._thread_ready = thread_event self._observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) self.config = ConfigurationParser().get_config() self.poll_interval = self.config["device_manager"]["poll_interval"] self.device_map_lock = Lock()
def __init__(self) -> None: self.log = Logging(owner=__file__, config=True) self.config = ConfigurationParser().get_config() events = self.config['framework']['events'] self.subject = Subject(events) self.observer_queue = Queue(maxsize=100) self._thread_started_event = Event() self.init_observers() self.attach_observers() self.start_observer_threats()
def __init__(self, queue, thread_event: threading.Event): threading.Thread.__init__(self) self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug) gateway_configuration = MqttGatewayConfiguration() self.observer_notify_queue = Queue(maxsize=100) self.observer_publish_queue = queue self._thread_ready = thread_event keys_dir = get_keys_dir() gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file) gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs) self.gateway_id = gateway_configuration.gateway_id self.connect_to_iot_core_broker(gateway_configuration)
def __init__(self, queue, thread_event: Event): Thread.__init__(self) self.config: dict = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self._observer_notify_queue: Queue = Queue(maxsize=100) self._observer_publish_queue: Queue = queue self._thread_ready: Event = thread_event config = self.get_mqtt_config() self.mqtt_client = MqttClient(config=config, connect_callback=self.on_connect, message_callback=self.on_message) if not self.mqtt_client.connect(): # todo: unscribcribe from subject self.log.critical("TODO: Unsubscribe itself form framework")
def test_output_format(self): log = Logging(owner='t', log_mode='test', min_log_lvl=LogLevels.not_set) truth_list = [ '\033[1;35m', '\033[1;31m', '\033[0;33m', '\033[0;0m', '\033[;1m', '\033[0;0m', '\033[0;32m' ] log_levels = [ LogLevels.critical, LogLevels.error, LogLevels.warning, LogLevels.info, LogLevels.debug, LogLevels.not_set, LogLevels.success ] for truth, log_lvl in zip(truth_list, log_levels): test_result = log._get_output_format(log_lvl) self.assertEqual(test_result, truth)
def test_leaf_path(self): paths = [ 'a/b/c/', 'a/b/c', '\\a\\b\\c', '\\a\\b\\c\\', 'a\\b\\c', 'a/b/../../a/b/c/', 'a/b/../../a/b/c.py', 'c', '/c.py', '//c.py', 'a.py/b/c' ] filenames = ['c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c'] for path, filename in zip(paths, filenames): result = Logging._path_leaf(path) print(result) self.assertEqual(result, filename)
def test_different_log_levels(self): log = Logging(owner='a', log_mode='test', min_log_lvl=LogLevels.not_set) msg = 'a' time = '1' source = 'a' log_levels = [ LogLevels.critical, LogLevels.error, LogLevels.warning, LogLevels.success, LogLevels.info, LogLevels.debug, LogLevels.not_set ] truth_list = [ '1 - a | critical : a', '1 - a | error : a', '1 - a | warning : a', '1 - a | success : a', '1 - a | info : a', '1 - a | debug : a', '1 - a | not_set : a' ] for truth, log_lvl in zip(truth_list, log_levels): test_result = log._format_log_msg(msg, log_lvl, time, source) self.assertEqual(test_result, truth)
def test_set_log_lvl(self): log = Logging(owner='t', log_mode='test', min_log_lvl=LogLevels.not_set) msg = 'a' log_levels = [ LogLevels.critical, LogLevels.error, LogLevels.warning, LogLevels.success, LogLevels.info, LogLevels.debug, LogLevels.not_set ] min_log_lvls = [ LogLevels.debug, LogLevels.debug, LogLevels.error, LogLevels.debug, LogLevels.not_set, LogLevels.warning, LogLevels.critical ] truth_list = [ 't | critical : a', 't | error : a', None, 't | success : a', 't | info : a', None, None ] for truth, log_lvl, min_log_lvl in zip(truth_list, log_levels, min_log_lvls): log._set_log_lvl(min_log_lvl) helper = str(log._log(msg, log_lvl)) test_result = helper.split(' - ')[-1] self.assertEqual(test_result, str(truth))
class DeviceManager(Thread): running = True subscribed_event = ['digital_twin'] remote_digital_twin = [] device_status_map = {} poll_timer = None new_devices = False def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self._observer_publish_queue = queue self._thread_ready = thread_event self._observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) self.config = ConfigurationParser().get_config() self.poll_interval = self.config["device_manager"]["poll_interval"] self.device_map_lock = Lock() def __del__(self) -> None: self.running = False def notify(self, msg: ObserverMessage) -> None: self._observer_notify_queue.put(item=msg) def run(self) -> None: self._thread_ready.set() self._fetch_digital_twin() self._start_timer(interval=self.poll_interval, callback=self._timer_callback) while self.running: queue_msg = self._observer_notify_queue.get() if queue_msg.event == "digital_twin": self._handle_digital_twin_event(msg=queue_msg) self.poll_timer.cancel() def _fetch_digital_twin(self): msg = ObserverMessage(event="digital_twin", data={}, subject="fetch_digital_twin") self._observer_publish_queue.put(msg) def _start_timer(self, interval: int, callback: Callable) -> None: self.poll_timer = Timer(interval=interval, function=callback) self.poll_timer.start() def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "retrieved_digital_twin": self._store_remote_digital_twin(data=msg.data) elif msg.subject == "device_status": self._store_device_status(data=msg.data) def _store_remote_digital_twin(self, data: dict): self.remote_digital_twin = data self.log.success("Received remote digital twin") self.log.debug(f"Remote digital twin: {self.remote_digital_twin}") def _store_device_status(self, data: dict): device_id = data.get("device_id", None) status = data.get("active", None) self.log.debug(f"Received device status {status} from {device_id}") if device_id and status: self.device_status_map[device_id] = status def _timer_callback(self): self.log.debug("Starting device status polling stage") self.device_status_map = self._generate_device_map_from_remote_twin() self._publish_device_status_poll() wait_period = self.config["device_manager"]["wait_period"] self._wait_for_status_messages(wait_period=wait_period) digital_twin = self._create_digital_twin_from_device_status() if digital_twin: self._publish_digital_twin(twin=digital_twin) if self.new_devices: self._fetch_digital_twin() self.new_devices = False self._start_timer(interval=self.poll_interval, callback=self._timer_callback) def _generate_device_map_from_remote_twin(self) -> dict: device_map = {} for twin_item in self.remote_digital_twin: device_name = twin_item["device_name"] device_map[device_name] = False return device_map def _publish_device_status_poll(self): msg = ObserverMessage(event="digital_twin", data={}, subject="poll_devices") self._observer_publish_queue.put(msg) def _wait_for_status_messages(self, wait_period: Union[int, float]) -> None: self.log.debug(f"Starting waiting period of {wait_period} seconds") start_time = time() while self.running and (time() - start_time < wait_period): sleep(0.01) self.log.debug("Waiting period over") def _create_digital_twin_from_device_status( self) -> Union[None, List[dict]]: if not self.device_status_map and not self.remote_digital_twin: self.log.debug( "No digital twin created since remote and local twin are empty" ) return None with self.device_map_lock: device_status_map = self.device_status_map.copy() digital_twin = [] for remote_item in self.remote_digital_twin: device_id = remote_item["device_name"] helper = remote_item if device_id in device_status_map: helper["active"] = device_status_map[device_id] del device_status_map[device_id] digital_twin.append(helper) for new_device in device_status_map: new_item = { "device_name": new_device, "active": True, "location": None, "technology": None, "battery_level": None } digital_twin.append(new_item) self.new_devices = True return digital_twin def _publish_digital_twin(self, twin: Union[list, dict]) -> None: msg = ObserverMessage(event="digital_twin", data=twin, subject="save_digital_twin") self._observer_publish_queue.put(msg)
class HealthMonitor(Thread): running = True update_time_sec = 600 subscribed_event = [] def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self.observer_publish_queue = queue self._thread_ready = thread_event self.observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: self.running = False def run(self) -> None: self.log.info(f'Updating system information every {self.update_time_sec} seconds.') self._thread_ready.set() while self.running: start_time = time() host_data = self._fetch_host_data() msg = ObserverMessage(event="host_health", data=host_data) self.observer_publish_queue.put(msg) sleep_time = self.update_time_sec - ((time() - start_time) % self.update_time_sec) sleep(sleep_time) def notify(self, msg: ObserverMessage) -> None: pass def _fetch_host_data(self) -> dict: data = { 'timestamp': self._get_timestamp(), 'temperature': self.poll_system_temp(), 'cpu_load': self.poll_cpu_load() } return data @staticmethod def _get_timestamp() -> datetime: return datetime.now() def poll_system_temp(self) -> float: temp_file = self._get_temperature_file() try: with open(temp_file) as file: return float(file.readline()) / 1000 except FileNotFoundError: self.log.critical(f'Temperature file {temp_file!r} does not exist') return 0 @staticmethod def _get_temperature_file() -> Path: return Path('/sys/class/thermal/thermal_zone0/temp') def poll_cpu_load(self) -> float: cpu_command = ["cat", "/proc/stat"] try: with subprocess.Popen(cpu_command, stdout=subprocess.PIPE) as process_result: proc_stat, _ = process_result.communicate() cpu_data = proc_stat.decode('utf-8').split('\n')[0].split()[1:-1] cpu_data = [int(field) for field in cpu_data] cpu_usage = ((cpu_data[0] + cpu_data[2]) * 100 / (cpu_data[0] + cpu_data[2] + cpu_data[3])) return round(cpu_usage, 3) except FileNotFoundError as error: self.log.critical(f'Command {" ".join(cpu_command)!r} was not found on the system: {error}') except ValueError as error: self.log.error(f'Parsing of the data went wrong: {error}') return 0
def __init__(self, db_name: str) -> None: self.config = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self.mongo_db = self.connect_to_db(db_name=db_name)
class MqttGateway(Thread): running = False subscribed_event = ['gcp_state_changed', 'digital_twin'] def __init__(self, queue, thread_event: Event): Thread.__init__(self) self.config: dict = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self._observer_notify_queue: Queue = Queue(maxsize=100) self._observer_publish_queue: Queue = queue self._thread_ready: Event = thread_event config = self.get_mqtt_config() self.mqtt_client = MqttClient(config=config, connect_callback=self.on_connect, message_callback=self.on_message) if not self.mqtt_client.connect(): # todo: unscribcribe from subject self.log.critical("TODO: Unsubscribe itself form framework") def __del__(self): self.running = False def run(self): self._thread_ready.set() while self.running: queue_item = self._observer_notify_queue.get() if queue_item.event == "digital_twin": self._handle_digital_twin_event(msg=queue_item) def notify(self, msg: ObserverMessage): self._observer_notify_queue.put(item=msg) def get_mqtt_config(self) -> dict: return { 'broker': self.config['mqtt_gateway']['broker_address'], 'port': 1883, 'stay_alive': 60 } def on_connect(self): topics = ['iot/#'] self.mqtt_client.subscribe(topics=topics) self._thread_ready.set() self.running = True def on_message(self, topic: str, payload: str) -> None: self.log.info(f'Received {payload!r} on topic {topic!r}') self._log_mqtt_traffic(topic=topic, payload=payload) message = IotMessage(mqtt_topic=topic, data=payload) if message.is_valid(): handler = self._select_handler(event=message.event) handler(msg=message) else: self.log.warning('The MQTT message is not valid') def _log_mqtt_traffic(self, topic: str, payload: str) -> None: data = { 'timestamp': datetime.now(), 'source': type(self).__name__, 'topic': topic, 'payload': payload } msg = ObserverMessage(event="iot_traffic", data=data) self._observer_publish_queue.put(msg) def _select_handler(self, event: str) -> Callable: handler_map = { 'state': self._handle_state_change, 'telemetry': self._handle_telemetry, 'system': self._handle_system, 'verification': self._handle_verification } return handler_map.get(event, self._unknown_event) def _unknown_event(self, msg: IotMessage) -> None: self.log.warning(f'Unknown event {msg.event} - No action selected') def _handle_state_change(self, msg: IotMessage) -> None: self.log.debug("Handling state event") message = { 'device_id': msg.device_id, 'event_type': msg.event, 'state': msg.payload.get('state') } item = ObserverMessage(event="device_state_changed", data=message) self._observer_publish_queue.put(item) def _handle_telemetry(self, msg: IotMessage) -> None: self.log.debug("Handling telemetry event") message = {'timestamp': datetime.now(), 'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="device_sensor_data", data=message) self._observer_publish_queue.put(item) def _handle_system(self, msg: IotMessage) -> None: self.log.debug(f"Handling system message from device {msg.device_id}") if msg.device_id != "framework": message = {'device_id': msg.device_id} message.update(msg.payload) item = ObserverMessage(event="digital_twin", data=message, subject="device_status") self._observer_publish_queue.put(item) def _handle_verification(self, msg: IotMessage) -> None: self.log.debug(f"Handling verification event {msg.event}") if msg.payload.get("action") == "ping": self.mqtt_client.publish(topic="iot/devices/system/verification", msg={"action": "pong"}) def _handle_digital_twin_event(self, msg: ObserverMessage): if msg.subject == "poll_devices": self.mqtt_client.publish(topic="iot/devices/framework/system", msg={"event": "poll"})
class MongoHandler: @dataclass class MongoConfLocal: host: str = 'host_ip' user: str = 'admin' pwd: str = 'mongo_admin_iot' url: str = f'mongodb://{user}:{pwd}@{host}/' def __init__(self, db_name: str) -> None: self.config = ConfigurationParser().get_config() self.log = Logging(owner=__file__, config=True) self.mongo_db = self.connect_to_db(db_name=db_name) def connect_to_db(self, db_name: str) -> MongoClient: mongo_host = self.config['mongo_db']['host_ip'] mongo_url = self.MongoConfLocal.url.replace(self.MongoConfLocal.host, mongo_host) try: client = self.get_mongo_client(url=mongo_url) client.server_info() db = client[db_name] self.log.success( f'Connected to MongoDB {db_name!r} at {mongo_url}') except errors.ServerSelectionTimeoutError as err: self.log.critical( f'Connection MongoDB error at {mongo_url} with error: {err}') raise RuntimeError from err return db @staticmethod def get_mongo_client(url: str) -> MongoClient: return MongoClient(url) def get(self, collection_name: str, query: dict = None) -> list: collection = self.mongo_db[collection_name] self.log.debug( f'Executing query {query!r} on collection {collection_name!r}') return list(collection.find(query)) def insert(self, collection_name: str, data: dict) -> None: collection = self.mongo_db[collection_name] data_id = collection.insert_one(data) self.log.debug( f'Inserted {data!r} into {collection_name!r} with ID {data_id}') def update(self, collection_name: str, object_id: str, updated_values: dict) -> None: collection = self.mongo_db[collection_name] query = {'_id': object_id} collection.update_one(query, updated_values) self.log.debug( f'Data with ID {object_id!r} in collection {collection_name!r} updated successfully' ) def write(self, collection_name: str, data: Union[list, dict], key: str) -> None: """ Add's data if it does not exist, else update that data based on key """ if isinstance(data, list): for entry in data: self._write(collection=collection_name, data=entry, key=key) else: self._write(collection=collection_name, data=data, key=key) def _write(self, collection: str, data: dict, key: str) -> None: query = {key: data.get(key, None)} object_id = self.get_first_object_id_from_query( collection_name=collection, query=query) print(object_id) if object_id: values = {'$set': data} self.update(collection_name=collection, object_id=object_id, updated_values=values) else: self.insert(collection_name=collection, data=data) def get_first_object_id_from_query(self, collection_name: str, query: dict) -> Union[str, None]: collection = self.mongo_db[collection_name] data = collection.find_one(query) if isinstance(data, dict): return data.get('_id') return None
class IotSubject: observers = [] running = False def __init__(self) -> None: self.log = Logging(owner=__file__, config=True) self.config = ConfigurationParser().get_config() events = self.config['framework']['events'] self.subject = Subject(events) self.observer_queue = Queue(maxsize=100) self._thread_started_event = Event() self.init_observers() self.attach_observers() self.start_observer_threats() def init_observers(self) -> None: active_components = self._get_activated_components() for component in active_components: obj = self._get_matching_object(component_name=component) observer = obj(queue=self.observer_queue, thread_event=self._thread_started_event) events = observer.subscribed_event self.observers.append({'obs_object': observer, 'events': events}) def _get_activated_components(self) -> list: system_components = self.config['system_components'].keys() return [ component for component in system_components if self.config['system_components'][component] ] @staticmethod def _get_matching_object(component_name: str) -> Callable: object_mapper = { 'mqtt_gateway': MqttGateway, 'db': DbHandler, 'host_monitor': HealthMonitor, 'device_manager': DeviceManager } return object_mapper.get(component_name) def attach_observers(self) -> None: for observer in self.observers: self.subject.register(**observer) def start_observer_threats(self) -> None: for observer in self.observers: observer.get('obs_object').start() self._thread_started_event.wait() observer_names = [key['obs_object'] for key in self.observers] self.log.success( f'Started {len(observer_names)} observers. {observer_names}') self.running = True def run(self) -> None: while self.running: msg = self.get_observer_events() self.notify_observers(msg=msg) def notify_observers(self, msg: ObserverMessage) -> None: self.subject.dispatch(message=msg) def get_observer_events(self) -> ObserverMessage: return self.observer_queue.get()
def __init__(self, config: dict, connect_callback: Callable, message_callback: Callable) -> None: if self.valid_arguments(config=config, callbacks=[connect_callback, message_callback]): self._config = config self._on_connect_callback = connect_callback self._on_message_callback = message_callback self.log = Logging(owner=__file__, config=True)
class DbHandler(Thread): running = True subscribed_event = [ 'gcp_state_changed', 'device_state_changed', 'iot_traffic', 'host_health', 'device_sensor_data', 'digital_twin' ] def __init__(self, queue: Queue, thread_event: Event) -> None: Thread.__init__(self) self.mongo = MongoHandler(db_name='iot_db') self.observer_publish_queue = queue self._thread_ready = thread_event self.observer_notify_queue = Queue(maxsize=100) self.log = Logging(owner=__file__, config=True) def __del__(self) -> None: self.running = False def run(self) -> None: self._thread_ready.set() while self.running: item = self.observer_notify_queue.get() action = self.action_selector(event=item.event) action(msg=item) def notify(self, msg: ObserverMessage) -> None: self.observer_notify_queue.put(msg) def action_selector(self, event: str) -> Callable: action_map = { 'gcp_state_changed': self.store_state_data, 'device_state_changed': self.store_state_data, 'iot_traffic': self.add_document_row, 'host_health': self.add_document_row, 'device_sensor_data': self.add_document_row, 'digital_twin': self.handle_digital_twin } return action_map.get(event, self.action_skip) @staticmethod def action_skip(): pass def get_document_data(self, document: str) -> list: return self.mongo.get(collection_name=document) def store_state_data(self, msg: ObserverMessage) -> None: document_data = { 'device_id': msg.data.get('device_id'), 'event': msg.event, 'change_source': msg.data.get('event'), 'state': msg.data.get('state') } self.mongo.write(collection_name="states", data=document_data, key="device_id") def add_document_row(self, msg: ObserverMessage) -> None: self.mongo.insert(collection_name=msg.event, data=msg.data) def handle_digital_twin(self, msg: ObserverMessage) -> None: if msg.subject == "fetch_digital_twin": self._fetch_digital_twin() elif msg.subject == "save_digital_twin": self._save_digital_twin(twin=msg.data) else: self.action_skip() def _fetch_digital_twin(self) -> None: self.log.info("Fetching digital twin from DB") twin_data = self.get_document_data(document="digital_twin") digital_twin = self._outbound_adapter(data=twin_data) msg = ObserverMessage(event="digital_twin", data=digital_twin, subject="retrieved_digital_twin") self.observer_publish_queue.put(msg) def _save_digital_twin(self, twin: list) -> None: self.log.info("Uploading updated digital twin") self.mongo.write(collection_name='digital_twin', data=twin, key='device_name') @staticmethod def _outbound_adapter(data: list) -> list: """ Removes the object_id field from each data entry, preping the data for transportation """ for entry in data: entry.pop("_id", None) return data
class GBridge(threading.Thread): subscribed_event = ['device_state_changed'] mqtt_client = None g_bridge_connected = False attached_devices = [] pending_messages = [] pending_subscribed_topics = [] def __init__(self, queue, thread_event: threading.Event): threading.Thread.__init__(self) self.log = Logging(owner=__file__, log_mode='terminal', min_log_lvl=LogLevels.debug) gateway_configuration = MqttGatewayConfiguration() self.observer_notify_queue = Queue(maxsize=100) self.observer_publish_queue = queue self._thread_ready = thread_event keys_dir = get_keys_dir() gateway_configuration.private_key_file = Path(keys_dir, gateway_configuration.private_key_file) gateway_configuration.ca_certs = Path(keys_dir, gateway_configuration.ca_certs) self.gateway_id = gateway_configuration.gateway_id self.connect_to_iot_core_broker(gateway_configuration) def __del__(self): self.detach_all_devices() self.mqtt_client.disconnect() self.mqtt_client.loop_stop() def run(self): self.mqtt_client.loop_start() self.wait_for_connection(5) self._thread_ready.set() while self.g_bridge_connected: queue_item = self.observer_notify_queue.get() self.send_data(msg=queue_item) time.sleep(ONE_MILLISECOND_SECONDS) def notify(self, msg, _) -> None: self.observer_notify_queue.put(item=msg) @staticmethod def poll_events(): return [] def connect_to_iot_core_broker(self, conf): # Create the MQTT client and connect to Cloud IoT. gateway_id = f'projects/{conf.project_id}/locations/{conf.cloud_region}/registries/' \ f'{conf.registry_id}/devices/{conf.gateway_id}' self.mqtt_client = mqtt.Client(gateway_id) jwt_pwd = create_jwt(conf.project_id, conf.private_key_file, conf.algorithm) self.mqtt_client.username_pw_set(username='******', password=jwt_pwd) self.mqtt_client.tls_set(ca_certs=conf.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2) self.mqtt_client.on_connect = self.on_connect self.mqtt_client.on_publish = self.on_publish self.mqtt_client.on_disconnect = self.on_disconnect self.mqtt_client.on_subscribe = self.on_subscribe self.mqtt_client.on_message = self.on_message self.mqtt_client.connect(conf.mqtt_bridge_hostname, conf.mqtt_bridge_port) def wait_for_connection(self, timeout): total_time = 0 while not self.g_bridge_connected and total_time < timeout: time.sleep(1) total_time += 1 if not self.g_bridge_connected: self.log.critical('Could not connect to Iot Core MQTT bridge') raise RuntimeError() def on_connect(self, _unused_client, _unused_userdata, _unused_flags, rc): self.log.success(f'Connected to GCP IoT core MQTT Broker with connection Result: {error_str(rc)}') self.g_bridge_connected = True self.subscribe_to_topics(self.gateway_id, True) if self.attached_devices: # Not empty list, Previously already had connected devices self.log.warning('Re-connect occurred! Re-attaching all connected devices.') def subscribe_to_topics(self, dev_id, gateway): config_topic = f'/devices/{dev_id}/config' command_topic = f'/devices/{dev_id}/commands/#' subscriptions = [{'topic': config_topic, 'qos': 1}, {'topic': command_topic, 'qos': 1}] if gateway: gateway_error_topic = f'/devices/{dev_id}/errors' subscriptions.append({'topic': gateway_error_topic, 'qos': 0}) for subscription in subscriptions: self.subscribe(subscription.get('topic'), subscription.get('qos')) def subscribe(self, topic, qos): _, mid = self.mqtt_client.subscribe(topic, qos) self.pending_subscribed_topics.append(mid) while topic in self.pending_subscribed_topics: time.sleep(0.01) self.log.debug(f'Successfully subscribed to topic {topic!r} with Qos {qos!r}.') def on_disconnect(self, _unused_client, _unused_userdata, rc): self.log.warning(f'Disconnected: {error_str(rc)!r}') self.g_bridge_connected = False def on_publish(self, _unused_client, _unused_userdata, mid): self.log.debug(f'ACK received for message {mid!r}') if mid in self.pending_messages: self.pending_messages.remove(mid) def on_subscribe(self, _unused_client, _unused_userdata, mid, granted_qos): if granted_qos[0] == 128: self.log.error(f'Subscription result: {granted_qos[0]!r} - Subscription failed') else: if mid in self.pending_subscribed_topics: self.pending_subscribed_topics.remove(mid) def on_message(self, _unused_client, _unused_userdata, message): payload = message.payload.decode('utf-8') self.log.info(f'Received message {payload!r} on topic {message.topic!r}.') if not payload: return # todo: fix this so that is better if message.topic.split('/')[3] == "commands": device_id = GBridge.get_id_from_topic(message.topic) queue_message = {'device_id': device_id, 'event_type': 'command', 'payload': payload} item = {'event': 'gcp_state_changed', 'message': queue_message} self.observer_publish_queue.put(item) def attach_device(self, device_id): self.log.debug(f'Attaching device {device_id!r}.') attach_topic = f'/devices/{device_id}/attach' if device_id not in self.attached_devices: self.attached_devices.append(device_id) self.publish(attach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY self.subscribe_to_topics(device_id, False) def detach_device(self, device_id): self.log.warning(f'Detaching device {device_id!r}.') detach_topic = f'/devices/{device_id}/detach' if device_id in self.attached_devices: self.attached_devices.remove(device_id) self.publish(detach_topic, "") # Message content is empty because gateway auth-method=ASSOCIATION_ONLY def detach_all_devices(self): self.log.info(f'Detaching all devices. Currently all connected devices: {self.attached_devices}.') for device in self.attached_devices[:]: self.detach_device(device) while self.attached_devices: # Make sure all devices have been detached time.sleep(0.01) def publish(self, topic, payload): message_info = self.mqtt_client.publish(topic, payload, qos=1) self.pending_messages.append(message_info.mid) self.log.info(f'Publishing payload: {payload!r} on Topic {topic!r} with mid {message_info.mid!r}.') while message_info.mid in self.pending_messages: # Waiting for message ACK to arrive time.sleep(0.01) def send_data(self, msg): device_id = msg.get('device_id') event_type = msg.get('event_type') payload = msg.get('payload') if device_id not in self.attached_devices: self.attach_device(device_id=device_id) if event_type == 'telemetry': topic = f'/devices/{device_id}/events' elif event_type == 'state': topic = f'/devices/{device_id}/state' else: self.log.error(f'Unknown event type {event_type}.') return self.publish(topic, payload) @staticmethod def get_id_from_topic(topic): index_device_id = 2 dir_tree = topic.split('/') if len(dir_tree) != 4 or dir_tree[1] != "devices": return None return dir_tree[index_device_id] def reattach_devices(self): for device in self.attached_devices: self.log.info(f'Re-attaching device {device}.') self.attach_device(device)