def unsubscribe(self, mqttclient: MqttClient) -> None:
super().unsubscribe(mqttclient)
mqttclient.unsubscribe(self.stateTopic)
mqttclient.unsubscribe(self.heatingTopic)
mqttclient.unsubscribe(self.setpointTopic)
mqttclient.unsubscribe(self.ackAlarmTopic)
for topic in self.tempReferences:
mqttclient.unsubscribe(topic)
class MqttPlugin(plugins.SimplePlugin):
"""
Plugin that listens to MQTT topics and publishes the payload
'unmodified' to a channel on the CherryPy bus. The cherrypy channel name
is the same as the MQTT topic
Requires PAHO
"""
def __init__(
self, bus: wspbus, broker: str, port: int, topic_list: Union[str, List[str]]
) -> None:
"""
Setup the plugin
:param bus: Cherrypy internal Bus
:param broker: Mqtt broker
:param port: Port of the Mqtt broker
:param topic_list: topic to subscribe
"""
# Cherrypy plugins.SimplePlugin doesn't accept the super().__init__()
# Inside cherrypy's docs it's like this
# https://docs.cherrypy.org/en/latest/extend.html#create-a-plugin
plugins.SimplePlugin.__init__(self, bus)
self.broker = broker
self.port = port
self.topic_list = topic_list
self.client = Client(client_id=f"Catalog{randrange(1, 100000)}")
self.client.on_message = Bus(bus).my_on_message
def start(self):
self.bus.log("Setup mqttcherrypy")
self.client.connect(self.broker, self.port)
self.bus.log(f"Connected to broker: {self.broker} port: {self.port})")
self.client.loop_start()
self.client.subscribe(self.topic_list)
self.bus.log(f"Subscribed to {self.topic_list}")
def stop(self):
self.bus.log("Shut down mqttcherrypy")
self.client.unsubscribe(self.topic_list)
self.bus.log(f"Unsubscribed from {self.topic_list}")
self.client.loop_stop(force=True)
self.client.disconnect()
self.bus.log(f"Disconnected from: {self.broker} port: {self.port}")
def unsubscribe(self, topics):
"""
Unsubscribe of a given topic
"""
if not isinstance(topics, list):
topics = [topics]
for topic in topics:
rc, mid = Mosquitto.unsubscribe(self, topic)
self._subscriptions[mid] = topic
self.log(logging.INFO,
"Sent unsubscription request of topic %s" % topic)
class Cep2Zigbee2mqttClient:
""" This class implements a simple zigbee2mqtt client.
By default it subscribes to all events of the default topic (zigbee2mqtt/#). No methods for
explicitly publishing to zigbee2mqtt are provided, since the class can provide higher level
abstraction methods for this. An example implemented example is this class' check_health().
Since all events from zigbee2mqtt are subscribed, the events filtering and management are up to
the class user. For that, a callback can be set in the initializer (on_message_clbk) for
processing the received messages. This callback is blocking, i.e. once the subscriber receives
an event and invokes the callback, no new events will be processed. Careful should be taken with
methods that might take too much time to process the events or that might eventually block (for
example, sending an event to another service).
"""
ROOT_TOPIC = "zigbee2mqtt/#"
def __init__(self,
host: str,
on_message_clbk: Callable[[Optional[Cep2Zigbee2mqttMessage]], None],
port: int = 1883,
topics: List[str] = [ROOT_TOPIC]):
""" Class initializer where the MQTT broker's host and port can be set, the list of topics
to subscribe and a callback to handle events from zigbee2mqtt.
Args:
host (str): string with the hostname, or IP address, of the MQTT broker.
on_message_clbk (Callable[[Zigbee2mqttMessage], None]): a function that is called when
a message is received from zigbee2mqtt. This returns None if the
port (int): network port of the MQTT broker. Defaults to 1883.
topics (List[str], optional): a list of topics that the client will subscribe to.
Defaults to ["zigbee2mqtt/#"].
"""
self.__client = MqttClient()
self.__client.on_connect = self.__on_connect
self.__client.on_disconnect = self.__on_disconnect
self.__client.on_message = self.__on_message
self.__connected = False
self.__events_queue = Queue()
self.__host = host
self.__on_message_clbk = on_message_clbk
self.__port = port
self.__stop_worker = Event()
self.__subscriber_thread = Thread(target=self.__worker,
daemon=True)
self.__topics = topics
def connect(self) -> None:
""" Connects to the MQTT broker specified in the initializer. This is a blocking function.
"""
# In the client is already connected then stop here.
if self.__connected:
return
# Connect to the host given in initializer.
self.__client.connect(self.__host,
self.__port)
self.__client.loop_start()
# Subscribe to all topics given in the initializer.
for t in self.__topics:
self.__client.subscribe(t)
# Start the subscriber thread.
self.__subscriber_thread.start()
def change_state(self, device_id: str, state: str) -> None:
if not self.__connected:
raise RuntimeError("The client is not connected. Connect first.")
self.__client.publish(topic=f"zigbee2mqtt/{device_id}/set",
payload=json.dumps({"state": f"{state}"}))
def check_health(self) -> str:
""" Allows to check whether zigbee2mqtt is healthy, i.e. the service is running properly.
Refer to zigbee2mqtt for more information. This is a blocking function that waits for a
response to the health request.
Returns:
A string with a description of zigbee2mqtt's health. This can be 'ok' or 'fail'.
"""
health_status = "fail"
health_response_received = Event()
# This function will run the subscriber on a thread. The subscriber must be started first,
# so that the health_check message is received, even if the broker does not have message
# persistence active. This should also avoid that messages with QoS 0 are not received.
# Also, if the subscriber is started after, it could happen the message to be received by
# another subscriber and never by this one.
# More information can be found in
# https://pagefault.blog/2020/02/05/how-to-set-up-persistent-storage-for-mosquitto-mqtt-broker
def health_check_subscriber():
message = subscribe.simple(hostname=self.__host,
port=self.__port,
topics="zigbee2mqtt/bridge/response/health_check")
if message:
# Decode and parse JSON payload.
payload = message.payload.decode("utf-8")
health = json.loads(payload)
# The nonlocal keyword is used to set the variable health_status that is not defined
# in this function's scope. For more information got to
# https://www.programiz.com/python-programming/global-local-nonlocal-variables
nonlocal health_status
health_status = health.get("status", "fail")
# Set the flag so that the function exits.
health_response_received.set()
# Start a thread with the subscriber that will wait for the response of the health_check.
Thread(target=health_check_subscriber, daemon=True).start()
# Wait for the subscriber to establish a connection with the broker.
sleep(.5)
# Publish the health_check request.
publish.single(hostname=self.__host,
port=self.__port,
topic="zigbee2mqtt/bridge/request/health_check")
# Wait until the response is received. If it is not received within 5 seconds, then return
# the default state: "fail".
health_response_received.wait(timeout=5)
return health_status
def disconnect(self) -> None:
""" Disconnects from the MQTT broker.
"""
self.__stop_worker.set()
self.__client.loop_stop()
# Unsubscribe from all topics given in the initializer.
for t in self.__topics:
self.__client.unsubscribe(t)
self.__client.disconnect()
def __on_connect(self, client, userdata, flags, rc) -> None:
""" Callback invoked when a connection with the MQTT broker is established.
Refer to paho-mqtt documentation for more information on this callback:
https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks
"""
# Set connected flag to true. This is later used if multiple calls to connect are made. This
# way the user does not need to very if the client is connected.
self.__connected = True
print("MQTT client connected")
def __on_disconnect(self, client, userdata, rc) -> None:
""" Callback invoked when the client disconnects from the MQTT broker occurs.
Refer to paho-mqtt documentation for more information on this callback:
https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks
"""
# Set connected flag to false. This is later used if multiple calls to connect are made.
# This way the user does not need to very if the client is connected.
self.__connected = False
print("MQTT client disconnected")
def __on_message(self, client, userdata, message: MQTTMessage) -> None:
""" Callback invoked when a message has been received on a topic that the client subscribed.
Refer to paho-mqtt documentation for more information on this callback:
https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks
"""
# Push a message to the queue. This will later be processed by the worker.
self.__events_queue.put(message)
def __worker(self) -> None:
""" This method pulls zigbee2mqtt messages from the queue of received messages, pushed when
a message is received, i.e. by the __on_message() callback. This method will be stopped when
the instance of zigbee2mqttClient disconnects, i.e. disconnect() is called and sets the
__stop_worker event.
"""
while not self.__stop_worker.is_set():
try:
# Pull a message from the queue.
message = self.__events_queue.get(timeout=1)
except Empty:
# This exception is raised when the queue pull times out. Ignore it and retry a new
# pull.
pass
else:
# If a message was successfully pulled from the queue, then process it.
# NOTE: this else condition is part of the try and it is executed when the action
# inside the try does not throws and exception.
# The decode() transforms a byte array into a string, following the utf-8 encoding.
if message:
self.__on_message_clbk(Cep2Zigbee2mqttMessage.parse(message.topic,
message.payload.decode("utf-8")))
class Mqtt():
def __init__(self, app=None):
self.app = app
self.client = Client()
self.refresh_time = 1.0
self.topics = []
self.mqtt_thread = Thread(target=self._loop_forever)
self.mqtt_thread.daemon = True
if app is not None:
self.init_app(app)
def init_app(self, app):
self.refresh_time = app.config.get('MQTT_REFRESH_TIME', 1.0)
self._connect(
username=app.config.get('MQTT_USERNAME'),
password=app.config.get('MQTT_PASSWORD'),
broker_url=app.config.get('MQTT_BROKER_URL', 'localhost'),
broker_port=app.config.get('MQTT_BROKER_PORT', 1883)
)
def _connect(self, username, password, broker_url, broker_port):
if not self.mqtt_thread.is_alive():
if username is not None:
self.client.username_pw_set(username, password)
res = self.client.connect(broker_url, broker_port)
if res == 0:
self.mqtt_thread.start()
def _disconnect(self):
self.client.disconnect()
def _loop_forever(self):
while True:
time.sleep(self.refresh_time)
self.client.loop(timeout=1.0, max_packets=1)
def on_topic(self, topic: str):
"""
Decorator to add a callback function that is called when a certain
topic has been published. The callback function is expected to have the
following form: `handle_topic(client, userdata, message)`
:parameter topic: a string specifying the subscription topic to subscribe to
**Example usage:**
::
@on_topic('home/mytopic')
def handle_mytopic(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
self.client.message_callback_add(topic, handler)
return handler
return decorator
def subscribe(self, topic: str, qos: int=0):
"""
Subscribe to a certain topic.
:param topic: a string specifying the subscription topic to subscribe to.
:param qos: the desired quality of service level for the subscription.
Defaults to 0.
A topic is a UTF-8 string, which is used by the broker to filter
messages for each connected client. A topic consists of one or more
topic levels. Each topic level is separated by a forward slash
(topic level separator).
**Topic example:** `myhome/groundfloor/livingroom/temperature`
"""
# TODO: add support for list of topics
# don't subscribe if already subscribed
if topic in self.topics:
return
# try to subscribe
result, mid = self.client.subscribe(topic, qos)
# if successful add to topics
if result == MQTT_ERR_SUCCESS:
self.topics.append(topic)
return result
def unsubscribe(self, topic: str):
"""
Unsubscribe from a single topic.
:param topic: a single string that is the subscription topic to
unsubscribe from
"""
# don't unsubscribe if not in topics
if topic not in self.topics:
return
result, mid = self.client.unsubscribe(topic)
# if successful remove from topics
if result == MQTT_ERR_SUCCESS:
self.topics.remove(topic)
return result
def unsubscribe_all(self):
"""
Unsubscribe from all topics.
"""
topics = self.topics[:]
for topic in topics:
self.unsubscribe(topic)
def publish(self, topic: str, payload: bytes=None, qos: int=0,
retain: bool=False) -> Tuple[int, int]:
"""
Send a message to the broker.
:param topic: the topic that the message should be published on
:param payload: the actual message to send. If not given, or set to
None a zero length message will be used. Passing an
int or float will result in the payload being
converted to a string representing that number.
If you wish to send a true int/float, use struct.pack()
to create the payload you require.
:param qos: the quality of service level to use
:param retain: if set to True, the message will be set as the
"last known good"/retained message for the topic
:returns: Returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN
if the client is not currently connected. mid is the message
ID for the publish request.
"""
return self.client.publish(topic, payload, qos, retain)
def on_message(self):
"""
Decorator to handle all messages that have been subscribed.
**Example Usage:**
::
@mqtt.on_message()
def handle_messages(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
self.client.on_message = handler
return handler
return decorator
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__gateway = gateway # Reference to TB Gateway
self._connector_type = connector_type # Should be "mqtt"
self.config = config # mqtt.json contents
self.__log = log
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__subscribes_sent = {}
# Extract main sections from configuration ---------------------------------------------------------------------
self.__broker = config.get('broker')
self.__mapping = []
self.__server_side_rpc = []
self.__connect_requests = []
self.__disconnect_requests = []
self.__attribute_requests = []
self.__attribute_updates = []
mandatory_keys = {
"mapping": ['topicFilter', 'converter'],
"serverSideRpc": [
'deviceNameFilter', 'methodFilter', 'requestTopicExpression',
'valueExpression'
],
"connectRequests": ['topicFilter'],
"disconnectRequests": ['topicFilter'],
"attributeRequests":
['topicFilter', 'topicExpression', 'valueExpression'],
"attributeUpdates": [
'deviceNameFilter', 'attributeFilter', 'topicExpression',
'valueExpression'
]
}
# Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file
self.load_handlers('mapping', mandatory_keys['mapping'],
self.__mapping)
# RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers
self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'],
self.__server_side_rpc)
# Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry
self.load_handlers('connectRequests',
mandatory_keys['connectRequests'],
self.__connect_requests)
# Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet
self.load_handlers('disconnectRequests',
mandatory_keys['disconnectRequests'],
self.__disconnect_requests)
# Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value
self.load_handlers('attributeRequests',
mandatory_keys['attributeRequests'],
self.__attribute_requests)
# Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute
self.load_handlers('attributeUpdates',
mandatory_keys['attributeUpdates'],
self.__attribute_updates)
# Setup topic substitution lists for each class of handlers ----------------------------------------------------
self.__mapping_sub_topics = {}
self.__connect_requests_sub_topics = {}
self.__disconnect_requests_sub_topics = {}
self.__attribute_requests_sub_topics = {}
# Set up external MQTT broker connection -----------------------------------------------------------------------
client_id = self.__broker.get(
"clientId",
''.join(random.choice(string.ascii_lowercase) for _ in range(23)))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] \
or self.__broker["security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. "
"Please check your configuration.\nError: ",
self.get_name())
self.__log.exception(e)
if self.__broker["security"].get("insecure", False):
self._client.tls_insecure_set(True)
else:
self._client.tls_insecure_set(False)
# Set up external MQTT broker callbacks ------------------------------------------------------------------------
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self._client.on_disconnect = self._on_disconnect
# self._client.on_log = self._on_log
# Set up lifecycle flags ---------------------------------------------------------------------------------------
self._connected = False
self.__stopped = False
self.daemon = True
self.__msg_queue = Queue()
self.__workers_thread_pool = []
self.__max_msg_number_for_worker = config.get(
'maxMessageNumberPerWorker', 10)
self.__max_number_of_workers = config.get('maxNumberOfWorkers', 100)
def load_handlers(self, handler_flavor, mandatory_keys,
accepted_handlers_list):
if handler_flavor not in self.config:
self.__log.error("'%s' section missing from configuration",
handler_flavor)
else:
for handler in self.config.get(handler_flavor):
discard = False
for key in mandatory_keys:
if key not in handler:
# Will report all missing fields to user before discarding the entry => no break here
discard = True
self.__log.error(
"Mandatory key '%s' missing from %s handler: %s",
key, handler_flavor, simplejson.dumps(handler))
else:
self.__log.debug(
"Mandatory key '%s' found in %s handler: %s", key,
handler_flavor, simplejson.dumps(handler))
if discard:
self.__log.error(
"%s handler is missing some mandatory keys => rejected: %s",
handler_flavor, simplejson.dumps(handler))
else:
accepted_handlers_list.append(handler)
self.__log.debug(
"%s handler has all mandatory keys => accepted: %s",
handler_flavor, simplejson.dumps(handler))
self.__log.info("Number of accepted %s handlers: %d",
handler_flavor, len(accepted_handlers_list))
self.__log.info(
"Number of rejected %s handlers: %d", handler_flavor,
len(self.config.get(handler_flavor)) -
len(accepted_handlers_list))
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
self.__connect()
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
elif not self._connected:
self.__connect()
self.__threads_manager()
sleep(.2)
def __connect(self):
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
sleep(1)
except ConnectionRefusedError as e:
self.__log.error(e)
sleep(10)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic, qos):
message = self._client.subscribe(topic, qos)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug(
"Client %s, userdata %s, flags %s, extra_params %s",
str(client), str(userdata), str(flags), extra_params)
self.__mapping_sub_topics = {}
# Setup data upload requests handling ----------------------------------------------------------------------
for mapping in self.__mapping:
try:
# Load converter for this mapping entry ------------------------------------------------------------
# mappings are guaranteed to have topicFilter and converter fields. See __init__
default_converter_class_name = "JsonMqttUplinkConverter"
# Get converter class from "extension" parameter or default converter
converter_class_name = mapping["converter"].get(
"extension", default_converter_class_name)
# Find and load required class
module = TBModuleLoader.import_module(
self._connector_type, converter_class_name)
if module:
self.__log.debug('Converter %s for topic %s - found!',
converter_class_name,
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
continue
# Setup regexp topic acceptance list ---------------------------------------------------------------
regex_topic = TBUtility.topic_to_regex(
mapping["topicFilter"])
# There may be more than one converter per topic, so I'm using vectors
if not self.__mapping_sub_topics.get(regex_topic):
self.__mapping_sub_topics[regex_topic] = []
self.__mapping_sub_topics[regex_topic].append(converter)
# Subscribe to appropriate topic -------------------------------------------------------------------
self.__subscribe(mapping["topicFilter"],
mapping.get("subscriptionQos", 1))
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
except Exception as e:
self.__log.exception(e)
# Setup connection requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__connect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__connect_requests_sub_topics[topic_filter] = request
# Setup disconnection requests handling --------------------------------------------------------------------
for request in [
entry for entry in self.__disconnect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__disconnect_requests_sub_topics[topic_filter] = request
# Setup attributes requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__attribute_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__attribute_requests_sub_topics[topic_filter] = request
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), result_code,
result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self._connected = False
self.__log.debug('"%s" was disconnected. %s', self.get_name(),
str(args))
def _on_log(self, *args):
self.__log.debug(args)
def _on_subscribe(self, _, __, mid, granted_qos, *args):
log.info(args)
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
except Exception as e:
self.__log.exception(e)
# Success or not, remove this topic from the list of pending subscription requests
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
def put_data_to_convert(self, converter, message, content) -> bool:
if not self.__msg_queue.full():
self.__msg_queue.put((converter.convert, message.topic, content),
True, 100)
return True
return False
def _save_converted_msg(self, topic, data):
self.__gateway.send_to_storage(self.name, data)
self.statistics['MessagesSent'] += 1
self.__log.debug("Successfully converted message from topic %s", topic)
def __threads_manager(self):
if len(self.__workers_thread_pool) == 0:
worker = MqttConnector.ConverterWorker("Main", self.__msg_queue,
self._save_converted_msg)
self.__workers_thread_pool.append(worker)
worker.start()
number_of_needed_threads = round(
self.__msg_queue.qsize() / self.__max_msg_number_for_worker, 0)
threads_count = len(self.__workers_thread_pool)
if number_of_needed_threads > threads_count < self.__max_number_of_workers:
thread = MqttConnector.ConverterWorker(
"Worker " + ''.join(
random.choice(string.ascii_lowercase) for _ in range(5)),
self.__msg_queue, self._save_converted_msg)
self.__workers_thread_pool.append(thread)
thread.start()
elif number_of_needed_threads < threads_count and threads_count > 1:
worker: MqttConnector.ConverterWorker = self.__workers_thread_pool[
-1]
if not worker.in_progress:
worker.stopped = True
self.__workers_thread_pool.remove(worker)
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
# Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" ---------------------------
topic_handlers = [
regex for regex in self.__mapping_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
# Note: every topic may be associated to one or more converter. This means that a single MQTT message
# may produce more than one message towards ThingsBoard. This also means that I cannot return after
# the first successful conversion: I got to use all the available ones.
# I will use a flag to understand whether at least one converter succeeded
request_handled = False
for topic in topic_handlers:
available_converters = self.__mapping_sub_topics[topic]
for converter in available_converters:
try:
if isinstance(content, list):
for item in content:
request_handled = self.put_data_to_convert(
converter, message, item)
if not request_handled:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client),
str(userdata))
else:
request_handled = self.put_data_to_convert(
converter, message, content)
except Exception as e:
log.exception(e)
if not request_handled:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client), str(userdata))
# Note: if I'm in this branch, this was for sure a telemetry/attribute push message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in connection handlers "i.e., I'm connecting a device" -------------------------
topic_handlers = [
regex for regex in self.__connect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__connect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
found_device_type = device_type_match.group(
0) if device_type_match is not None else handler[
"deviceTypeTopicExpression"]
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from connection request")
continue
# Note: device must be added even if it is already known locally: else ThingsBoard
# will not send RPCs and attribute updates
self.__log.info("Connecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.add_device(found_device_name,
{"connector": self},
device_type=found_device_type)
# Note: if I'm in this branch, this was for sure a connection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" -------------------
topic_handlers = [
regex for regex in self.__disconnect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__disconnect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
if device_type_match is not None:
found_device_type = device_type_match.group(0)
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from disconnection request")
continue
if found_device_name in self.__gateway.get_devices():
self.__log.info("Disconnecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.del_device(found_device_name)
else:
self.__log.info("Device %s was not connected",
found_device_name)
break
# Note: if I'm in this branch, this was for sure a disconnection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" --------
topic_handlers = [
regex for regex in self.__attribute_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
try:
for topic in topic_handlers:
handler = self.__attribute_requests_sub_topics[topic]
found_device_name = None
found_attribute_name = None
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"],
message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get attribute name, either from topic or from content
if handler.get("attributeNameTopicExpression"):
attribute_name_match = search(
handler["attributeNameTopicExpression"],
message.topic)
if attribute_name_match is not None:
found_attribute_name = attribute_name_match.group(
0)
elif handler.get("attributeNameJsonExpression"):
found_attribute_name = TBUtility.get_value(
handler["attributeNameJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from attribute request")
continue
if found_attribute_name is None:
self.__log.error(
"Attribute name missing from attribute request")
continue
self.__log.info("Will retrieve attribute %s of %s",
found_attribute_name, found_device_name)
self.__gateway.tb_client.client.gw_request_shared_attributes(
found_device_name, [found_attribute_name],
lambda data, *args: self.notify_attribute(
data, found_attribute_name,
handler.get("topicExpression"),
handler.get("valueExpression"),
handler.get('retain', False)))
break
except Exception as e:
log.exception(e)
# Note: if I'm in this branch, this was for sure an attribute request message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in RPC handlers ----------------------------------------------------------------
# The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is
if self.__gateway.is_rpc_in_progress(message.topic):
log.info("RPC response arrived. Forwarding it to thingsboard.")
self.__gateway.rpc_with_reply_processing(message.topic, content)
return None
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def notify_attribute(self, incoming_data, attribute_name, topic_expression,
value_expression, retain):
if incoming_data.get("device") is None or incoming_data.get(
"value") is None:
return
device_name = incoming_data.get("device")
attribute_value = incoming_data.get("value")
topic = topic_expression \
.replace("${deviceName}", str(device_name)) \
.replace("${attributeKey}", str(attribute_name))
data = value_expression.replace("${attributeKey}", str(attribute_name)) \
.replace("${attributeValue}", str(attribute_value))
self._client.publish(topic, data, retain=retain).wait_for_publish()
def on_attributes_update(self, content):
if self.__attribute_updates:
for attribute_update in self.__attribute_updates:
if match(attribute_update["deviceNameFilter"],
content["device"]):
for attribute_key in content["data"]:
if match(attribute_update["attributeFilter"],
attribute_key):
try:
topic = attribute_update["topicExpression"] \
.replace("${deviceName}", str(content["device"])) \
.replace("${attributeKey}", str(attribute_key)) \
.replace("${attributeValue}", str(content["data"][attribute_key]))
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
try:
data = attribute_update["valueExpression"] \
.replace("${attributeKey}", str(attribute_key)) \
.replace("${attributeValue}", str(content["data"][attribute_key]))
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
self._client.publish(
topic,
data,
retain=attribute_update.get(
'retain', False)).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Cannot find attributeName by filter in message with data: %s",
content)
else:
self.__log.error(
"Cannot find deviceName by filter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
self.__log.info("Incoming server-side RPC: %s", content)
# Check whether one of my RPC handlers can handle this request
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# This handler seems able to handle the request
self.__log.info("Candidate RPC handler found")
expects_response = rpc_config.get("responseTopicExpression")
defines_timeout = rpc_config.get("responseTimeout")
# 2-way RPC setup
if expects_response and defines_timeout:
expected_response_topic = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", str(content["device"])) \
.replace("${methodName}", str(content["data"]["method"])) \
.replace("${requestId}", str(content["data"]["id"]))
expected_response_topic = TBUtility.replace_params_tags(
expected_response_topic, content)
timeout = time() * 1000 + rpc_config.get("responseTimeout")
# Start listenting on the response topic
self.__log.info("Subscribing to: %s",
expected_response_topic)
self.__subscribe(expected_response_topic,
rpc_config.get("responseTopicQoS", 1))
# Wait for subscription to be carried out
sub_response_timeout = 10
while expected_response_topic in self.__subscribes_sent.values(
):
sub_response_timeout -= 1
sleep(0.1)
if sub_response_timeout == 0:
break
# Ask the gateway to enqueue this as an RPC response
self.__gateway.register_rpc_request_timeout(
content, timeout, expected_response_topic,
self.rpc_cancel_processing)
# Wait for RPC to be successfully enqueued, which never fails.
while self.__gateway.is_rpc_in_progress(
expected_response_topic):
sleep(0.1)
elif expects_response and not defines_timeout:
self.__log.info(
"2-way RPC without timeout: treating as 1-way")
# Actually reach out for the device
request_topic: str = rpc_config.get("requestTopicExpression") \
.replace("${deviceName}", str(content["device"])) \
.replace("${methodName}", str(content["data"]["method"])) \
.replace("${requestId}", str(content["data"]["id"]))
request_topic = TBUtility.replace_params_tags(
request_topic, content)
data_to_send_tags = TBUtility.get_values(
rpc_config.get('valueExpression'),
content['data'],
'params',
get_tag=True)
data_to_send_values = TBUtility.get_values(
rpc_config.get('valueExpression'),
content['data'],
'params',
expression_instead_none=True)
data_to_send = rpc_config.get('valueExpression')
for (tag, value) in zip(data_to_send_tags,
data_to_send_values):
data_to_send = data_to_send.replace(
'${' + tag + '}', str(value))
try:
self.__log.info("Publishing to: %s with data %s",
request_topic, data_to_send)
self._client.publish(request_topic,
data_to_send,
retain=rpc_config.get(
'retain', False))
if not expects_response or not defines_timeout:
self.__log.info(
"One-way RPC: sending ack to ThingsBoard immediately"
)
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
# Everything went out smoothly: RPC is served
return
except Exception as e:
self.__log.exception(e)
self.__log.error("RPC not handled: %s", content)
def rpc_cancel_processing(self, topic):
log.info("RPC canceled or terminated. Unsubscribing from %s", topic)
self._client.unsubscribe(topic)
class ConverterWorker(Thread):
def __init__(self, name, incoming_queue, send_result):
super().__init__()
self.stopped = False
self.setName(name)
self.setDaemon(True)
self.__msg_queue = incoming_queue
self.in_progress = False
self.__send_result = send_result
def run(self):
while not self.stopped:
if not self.__msg_queue.empty():
self.in_progress = True
convert_function, config, incoming_data = self.__msg_queue.get(
True, 100)
converted_data = convert_function(config, incoming_data)
log.debug(converted_data)
self.__send_result(config, converted_data)
self.in_progress = False
else:
sleep(.2)
class Mqtt():
"""Main Mqtt class.
:param app: flask application object
:param connect_async: if True then connect_aync will be used to connect to MQTT broker
:param mqtt_logging: if True then messages from MQTT client will be logged
"""
def __init__(self, app=None, connect_async=False, mqtt_logging=False):
# type: (Flask, bool, bool) -> None
self.app = app
self._connect_async = connect_async # type: bool
self._connect_handler = None # type: Optional[Callable]
self._disconnect_handler = None # type: Optional[Callable]
self.topics = {} # type: Dict[str, TopicQos]
self.connected = False
self.client = Client()
if mqtt_logging:
self.client.enable_logger(logger)
if app is not None:
self.init_app(app)
def init_app(self, app):
# type: (Flask) -> None
"""Init the Flask-MQTT addon."""
self.client_id = app.config.get("MQTT_CLIENT_ID", "")
self.clean_session = app.config.get("MQTT_CLEAN_SESSION", True)
if isinstance(self.client_id, unicode):
self.client._client_id = self.client_id.encode('utf-8')
else:
self.client._client_id = self.client_id
self.client._clean_session = self.clean_session
self.client._transport = app.config.get("MQTT_TRANSPORT",
"tcp").lower()
self.client._protocol = app.config.get("MQTT_PROTOCOL_VERSION",
MQTTv311)
self.client.on_connect = self._handle_connect
self.client.on_disconnect = self._handle_disconnect
self.username = app.config.get("MQTT_USERNAME")
self.password = app.config.get("MQTT_PASSWORD")
self.broker_url = app.config.get("MQTT_BROKER_URL", "localhost")
self.broker_port = app.config.get("MQTT_BROKER_PORT", 1883)
self.tls_enabled = app.config.get("MQTT_TLS_ENABLED", False)
self.keepalive = app.config.get("MQTT_KEEPALIVE", 60)
self.last_will_topic = app.config.get("MQTT_LAST_WILL_TOPIC")
self.last_will_message = app.config.get("MQTT_LAST_WILL_MESSAGE")
self.last_will_qos = app.config.get("MQTT_LAST_WILL_QOS", 0)
self.last_will_retain = app.config.get("MQTT_LAST_WILL_RETAIN", False)
if self.tls_enabled:
self.tls_ca_certs = app.config["MQTT_TLS_CA_CERTS"]
self.tls_certfile = app.config.get("MQTT_TLS_CERTFILE")
self.tls_keyfile = app.config.get("MQTT_TLS_KEYFILE")
self.tls_cert_reqs = app.config.get("MQTT_TLS_CERT_REQS",
ssl.CERT_REQUIRED)
self.tls_version = app.config.get("MQTT_TLS_VERSION",
ssl.PROTOCOL_TLSv1)
self.tls_ciphers = app.config.get("MQTT_TLS_CIPHERS")
self.tls_insecure = app.config.get("MQTT_TLS_INSECURE", False)
# set last will message
if self.last_will_topic is not None:
self.client.will_set(
self.last_will_topic,
self.last_will_message,
self.last_will_qos,
self.last_will_retain,
)
self._connect()
def _connect(self):
# type: () -> None
if self.username is not None:
self.client.username_pw_set(self.username, self.password)
# security
if self.tls_enabled:
self.client.tls_set(
ca_certs=self.tls_ca_certs,
certfile=self.tls_certfile,
keyfile=self.tls_keyfile,
cert_reqs=self.tls_cert_reqs,
tls_version=self.tls_version,
ciphers=self.tls_ciphers,
)
if self.tls_insecure:
self.client.tls_insecure_set(self.tls_insecure)
if self._connect_async:
# if connect_async is used
self.client.connect_async(self.broker_url,
self.broker_port,
keepalive=self.keepalive)
else:
res = self.client.connect(self.broker_url,
self.broker_port,
keepalive=self.keepalive)
if res == 0:
logger.debug("Connected client '{0}' to broker {1}:{2}".format(
self.client_id, self.broker_url, self.broker_port))
else:
logger.error(
"Could not connect to MQTT Broker, Error Code: {0}".format(
res))
self.client.loop_start()
def _disconnect(self):
# type: () -> None
self.client.loop_stop()
self.client.disconnect()
logger.debug('Disconnected from Broker')
def _handle_connect(self, client, userdata, flags, rc):
# type: (Client, Any, Dict, int) -> None
if rc == MQTT_ERR_SUCCESS:
self.connected = True
for key, item in self.topics.items():
self.client.subscribe(topic=item.topic, qos=item.qos)
if self._connect_handler is not None:
self._connect_handler(client, userdata, flags, rc)
def _handle_disconnect(self, client, userdata, rc):
# type: (str, Any, int) -> None
self.connected = False
if self._disconnect_handler is not None:
self._disconnect_handler()
def on_topic(self, topic):
# type: (str) -> Callable
"""Decorator.
Decorator to add a callback function that is called when a certain
topic has been published. The callback function is expected to have the
following form: `handle_topic(client, userdata, message)`
:parameter topic: a string specifying the subscription topic to
subscribe to
The topic still needs to be subscribed via mqtt.subscribe() before the
callback function can be used to handle a certain topic. This way it is
possible to subscribe and unsubscribe during runtime.
**Example usage:**::
app = Flask(__name__)
mqtt = Mqtt(app)
mqtt.subscribe('home/mytopic')
@mqtt.on_topic('home/mytopic')
def handle_mytopic(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable[[str], None]) -> Callable[[str], None]
self.client.message_callback_add(topic, handler)
return handler
return decorator
def subscribe(self, topic, qos=0):
# type: (str, int) -> Tuple[int, int]
"""
Subscribe to a certain topic.
:param topic: a string specifying the subscription topic to
subscribe to.
:param qos: the desired quality of service level for the subscription.
Defaults to 0.
:rtype: (int, int)
:result: (result, mid)
A topic is a UTF-8 string, which is used by the broker to filter
messages for each connected client. A topic consists of one or more
topic levels. Each topic level is separated by a forward slash
(topic level separator).
The function returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the
client is not currently connected. mid is the message ID for the
subscribe request. The mid value can be used to track the subscribe
request by checking against the mid argument in the on_subscribe()
callback if it is defined.
**Topic example:** `myhome/groundfloor/livingroom/temperature`
"""
# TODO: add support for list of topics
# don't subscribe if already subscribed
# try to subscribe
result, mid = self.client.subscribe(topic=topic, qos=qos)
# if successful add to topics
if result == MQTT_ERR_SUCCESS:
self.topics[topic] = TopicQos(topic=topic, qos=qos)
logger.debug('Subscribed to topic: {0}, qos: {1}'.format(
topic, qos))
else:
logger.error('Error {0} subscribing to topic: {1}'.format(
result, topic))
return (result, mid)
def unsubscribe(self, topic):
# type: (str) -> Optional[Tuple[int, int]]
"""
Unsubscribe from a single topic.
:param topic: a single string that is the subscription topic to
unsubscribe from
:rtype: (int, int)
:result: (result, mid)
Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS
to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not
currently connected.
mid is the message ID for the unsubscribe request. The mid value can be
used to track the unsubscribe request by checking against the mid
argument in the on_unsubscribe() callback if it is defined.
"""
# don't unsubscribe if not in topics
if topic in self.topics:
result, mid = self.client.unsubscribe(topic)
if result == MQTT_ERR_SUCCESS:
self.topics.pop(topic)
logger.debug('Unsubscribed from topic: {0}'.format(topic))
else:
logger.debug('Error {0} unsubscribing from topic: {1}'.format(
result, topic))
# if successful remove from topics
return result, mid
return None
def unsubscribe_all(self):
# type: () -> None
"""Unsubscribe from all topics."""
topics = list(self.topics.keys())
for topic in topics:
self.unsubscribe(topic)
def publish(self, topic, payload=None, qos=0, retain=False):
# type: (str, bytes, int, bool) -> Tuple[int, int]
"""
Send a message to the broker.
:param topic: the topic that the message should be published on
:param payload: the actual message to send. If not given, or set to
None a zero length message will be used. Passing an
int or float will result in the payload being
converted to a string representing that number.
If you wish to send a true int/float, use struct.pack()
to create the payload you require.
:param qos: the quality of service level to use
:param retain: if set to True, the message will be set as the
"last known good"/retained message for the topic
:returns: Returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN
if the client is not currently connected. mid is the message
ID for the publish request.
"""
if not self.connected:
self.client.reconnect()
result, mid = self.client.publish(topic, payload, qos, retain)
if result == MQTT_ERR_SUCCESS:
logger.debug('Published topic {0}: {1}'.format(topic, payload))
else:
logger.error('Error {0} publishing topic {1}'.format(
result, topic))
return (result, mid)
def on_connect(self):
# type: () -> Callable
"""Decorator.
Decorator to handle the event when the broker responds to a connection
request. Only the last decorated function will be called.
"""
def decorator(handler):
# type: (Callable) -> Callable
self._connect_handler = handler
return handler
return decorator
def on_disconnect(self):
# type: () -> Callable
"""Decorator.
Decorator to handle the event when client disconnects from broker. Only
the last decorated function will be called.
"""
def decorator(handler):
# type: (Callable) -> Callable
self._disconnect_handler = handler
return handler
return decorator
def on_message(self):
# type: () -> Callable
"""Decorator.
Decorator to handle all messages that have been subscribed and that
are not handled via the `on_message` decorator.
**Note:** Unlike as written in the paho mqtt documentation this
callback will not be called if there exists an topic-specific callback
added by the `on_topic` decorator.
**Example Usage:**::
@mqtt.on_message()
def handle_messages(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_message = handler
return handler
return decorator
def on_publish(self):
# type: () -> Callable
"""Decorator.
Decorator to handle all messages that have been published by the
client.
**Example Usage:**::
@mqtt.on_publish()
def handle_publish(client, userdata, mid):
print('Published message with mid {}.'
.format(mid))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_publish = handler
return handler
return decorator
def on_subscribe(self):
# type: () -> Callable
"""Decorate a callback function to handle subscritions.
**Usage:**::
@mqtt.on_subscribe()
def handle_subscribe(client, userdata, mid, granted_qos):
print('Subscription id {} granted with qos {}.'
.format(mid, granted_qos))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_subscribe = handler
return handler
return decorator
def on_unsubscribe(self):
# type: () -> Callable
"""Decorate a callback funtion to handle unsubscribtions.
**Usage:**::
@mqtt.unsubscribe()
def handle_unsubscribe(client, userdata, mid)
print('Unsubscribed from topic (id: {})'
.format(mid)')
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_unsubscribe = handler
return handler
return decorator
def on_log(self):
# type: () -> Callable
"""Decorate a callback function to handle MQTT logging.
**Example Usage:**
::
@mqtt.on_log()
def handle_logging(client, userdata, level, buf):
print(client, userdata, level, buf)
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_log = handler
return handler
return decorator
class MqttConnection:
def __init__(self, ip, port, username, password, connection_callback):
self.logger = logging.getLogger("mqtt")
self.mqtt = Client()
if username is not None:
self.mqtt.username_pw_set(username, password)
self.mqtt.on_connect = self._on_connect
self.mqtt.on_message = self._on_message
self.ip = ip
self.port = port
self.connection_callback = connection_callback
self.queue = []
def _on_connect(self, client, userdata, flags, rc):
"""
The callback for when the client receives a CONNACK response from the server.
"""
self.logger.debug("connected to mqtt with result code %d" % rc)
# subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
self.connection_callback.on_mqtt_connected(self)
if len(self.queue) > 0:
self.logger.debug("found %d queued messages" % len(self.queue))
for msg in self.queue:
self._internal_send_message(msg[0], msg[1], False)
self.queue.clear()
self.logger.debug("handled all queued messages")
def _on_message(self, client, userdata, msg):
"""
The callback for when a PUBLISH message is received from the server.
"""
self.logger.debug("received mqtt publish of %s with data \"%s\"" % (msg.topic, msg.payload))
self.connection_callback.on_mqtt_message_received(msg.topic, msg.payload)
def send_message(self, topic, payload):
return self._internal_send_message(topic, payload, True)
def subscribe(self, topic):
self.logger.debug("subscribing to topic %s" % topic)
result = self.mqtt.subscribe(topic)
if result[0] == MQTT_ERR_NO_CONN:
self.logger.warning("no connection while trying to subscribe to topic %s" % topic)
return False
return result[0] == MQTT_ERR_SUCCESS
def unsubscribe(self, topic):
self.logger.debug("unsubscribing from topic %s" % topic)
result = self.mqtt.unsubscribe(topic)
if result[0] == MQTT_ERR_NO_CONN:
self.logger.warning("no connection while trying to unsubscribe from topic %s" % topic)
return False
return result[0] == MQTT_ERR_SUCCESS
def _internal_send_message(self, topic, payload, queue):
self.logger.debug("sending topic %s with value \"%s\"" % (topic, payload))
result = self.mqtt.publish(topic, payload, retain=True)
if result == MQTT_ERR_NO_CONN and queue:
self.logger.debug("no connection, saving message with topic %s to queue" % topic)
self.queue.append([topic, payload])
elif result[0] != MQTT_ERR_SUCCESS:
self.logger.warn("failed sending message %s, mqtt error %s" % (topic, result))
return False
return True
def start_connection(self):
try:
self.mqtt.connect(self.ip, self.port)
except ConnectionError:
self.logger.exception("failed connecting to mqtt")
return False
self.mqtt.loop_start()
return True
def stop_connection(self):
self.mqtt.disconnect()
self.mqtt.loop_stop()
class Service:
"""Service that sends emails"""
_mqtt_client: Dict[str, Client] = {}
_service_list: Dict[str, dict] = {}
_broker: DefaultDict[str, set] = defaultdict(set)
_broker_port: Dict[str, int] = {}
_topic: DefaultDict[str, set] = defaultdict(set)
_update_thread: Timer = None
alarm_user_topic = "labsw4/arduino/contacted/user"
_user_list: Dict[str, dict] = {}
# Email
from_email = "******" #E-mail (Gmail) associated to the service
password = "******" #Password of this e-mail
subject = "Temperature alarm"
signature = "\n\nBest regards,\n\nSmart Home - IoT distributed platform developers" # signature block
def __init__(self):
"""
Instantiate the service
"""
self.service = Client(client_id=f"EMailService{randrange(1, 100000)}")
self.service.on_message = partial(self.my_on_message,
email_service=self.service)
self.user_lock = Lock()
self.service_lock = Lock()
def _update(self, service_list: List[dict]):
"""
Update reserved dicts
:param service_list: service list to add
"""
for alarm in service_list:
service = alarm["serviceID"]
broker = alarm["end_points"]["MQTT"]["broker"]["ip"]
port = alarm["end_points"]["MQTT"]["broker"]["port"]
topics = {
topic
for topic in alarm["end_points"]["MQTT"]["subscribe"]
if "alarm_temperature" in topic
}
self._service_list[service] = {
"ip": broker,
"topics": topics,
}
self._broker_port[broker] = port
self._broker[broker].update({service})
self._topic[broker].update(topics)
print(f"[{time.ctime()}] SERVICE {service} ONLINE")
def setup(self, first_time: bool = True):
"""
Setup the service
"""
try:
print(
f"[{time.ctime()}] EXTRACT info about all the services registered"
)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all"
)
while result.status_code != 200:
print(
f"[{time.ctime()}] WARNING no service registered found, retrying after 30 seconds"
)
time.sleep(30)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}"
f"/catalog/services/all")
print(f"[{time.ctime()}] SERVICES found")
data = json.loads(result.content.decode())
print(
f"[{time.ctime()}] EXTRACT from the services list info about Alarm_Temperature"
)
service_list = [alarm for alarm in data if find_service(alarm)]
if len(service_list) == 0:
print(
f"[{time.ctime()}] No ServiceTemperatureAlarm found... retrying in 10 seconds"
)
time.sleep(10)
self.setup(first_time)
return
print(
f"[{time.ctime()}] EXTRACT info about all the users registered"
)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/users/all"
)
user_list = {}
if result.status_code == 200:
data = json.loads(result.content.decode())
user_list = {
user["userID"]: {
user["email_addresses"][email]
for email in user["email_addresses"]
}
for user in data
}
except KeyboardInterrupt:
print(f"[{time.ctime()}] EXIT")
if first_time:
sys.exit()
return
if first_time:
# Connect the service
self.service.connect(host=SERVICE_BROKER_PORT["ip"],
port=SERVICE_BROKER_PORT["port"])
print(f"[{time.ctime()}] SERVICE CONNECTED to "
f"broker: {SERVICE_BROKER_PORT['ip']} "
f"on port: port={SERVICE_BROKER_PORT['port']}")
# Update registered services and users and subscribe to all the topics
with self.service_lock:
self._update(service_list)
self.subscribe(service_list)
with self.user_lock:
self._user_list.update(user_list)
# Register inside the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}")
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
def start(self):
"""
Start the service.
"""
# Setup the service
self.setup()
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
# Schedule the update of the registration
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
try:
# Run the service forever
self.service.loop_forever()
except KeyboardInterrupt:
self.stop()
def my_on_message(self,
client: Client,
userdata: Any,
msg: MQTTMessage,
email_service: Client = None):
"""
Check if alarm is true. If so, send emails.
:param client: MQTT client
:param userdata: They could be any type
:param msg: MQTT message
:param email_service: Service Client
"""
data = json.loads(msg.payload.decode())
if data["alarm"]:
message_body = f"{data['device']} is out of range of good functioning"
else:
return
with self.user_lock:
for user_id in self._user_list:
for email in self._user_list[user_id]:
msg = MIMEMultipart()
msg["From"] = f"Smart Home - IoT distributed platform <{self.from_email}>"
msg["To"] = email
msg["Subject"] = self.subject
msg.attach(MIMEText(message_body + self.signature,
'plain'))
server = smtplib.SMTP("smtp.gmail.com", 587)
server.starttls()
server.login(self.from_email, self.password)
text = msg.as_string()
server.sendmail(self.from_email, email, text)
server.quit()
print(f"[{time.ctime()}] EMAIL SENT TO {email}")
email_service.publish(self.alarm_user_topic,
payload=json.dumps({
"userID":
[user_id for user_id in self._user_list]
}))
print(
f"[{time.ctime()}] EMAILS SENT TO USERS: {[user_id for user_id in self._user_list]}"
)
def update_registration(self):
"""
Update service registration to the catalog
"""
print(f"[{time.ctime()}] EXTRACT info about all the users registered")
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/users/all"
)
user_list = {}
if result.status_code == 200:
data = json.loads(result.content.decode())
user_list = {
user["userID"]: {
user["email_addresses"][email]
for email in user["email_addresses"]
}
for user in data
}
with self.user_lock:
self._user_list.update(user_list)
print(
f"[{time.ctime()}] EXTRACT info about all the services registered")
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all"
)
# No device found
if result.status_code != 200:
self.reset()
return
data = json.loads(result.content.decode())
service_list = [alarm for alarm in data if find_service(alarm)]
if len(service_list) == 0:
self.reset()
return
# New Devices Found
new_services = {alarm["serviceID"] for alarm in service_list}
# Old devices list
old_services = set(self._service_list.keys())
# Check if there are update to do
if old_services == new_services:
# No update, ping the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}"
)
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
return
with self.service_lock:
# Delete inactive devices
delete_list = {
alarm: self._service_list[alarm]
for alarm in self._service_list if alarm not in new_services
}
# Unsubscribe from devices
self.unsubscribe(delete_list)
# Update registered devices and subscribe
new_services = [
alarm for alarm in service_list
if alarm["serviceID"] not in old_services
]
self._update(new_services)
self.subscribe(new_services)
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def reset(self):
"""
Reset the service
"""
with self.service_lock:
self._clear()
self.setup(first_time=False)
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def unsubscribe(self, service_list: dict):
"""
Unsubscribe from services
:param service_list: service list
"""
for alarm in service_list:
print(f"[{time.ctime()}] SERVICE {alarm} OFFLINE")
broker = service_list[alarm]["ip"]
topics = service_list[alarm]["topics"]
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
else:
for topic in topics:
self._mqtt_client[broker].unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
# Update topics
self._topic[broker] = self._topic[broker].difference(topics)
# Disconnect from external broker
if len(self._topic[broker]) == 0:
if broker != SERVICE_BROKER_PORT["ip"]:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(
f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
del self._mqtt_client[broker]
# Clean
del self._topic[broker]
del self._broker_port[broker]
# Delete device
del self._service_list[alarm]
self._broker[broker].discard(alarm)
def subscribe(self, service_list: List[dict]):
"""
Subscribe to services
:param service_list: list of services
"""
for alarm in service_list:
broker = alarm["end_points"]["MQTT"]["broker"]["ip"]
topics = {
topic
for topic in alarm["end_points"]["MQTT"]["subscribe"]
}
# Subscribe to topics
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
# Connect to external brokers
else:
if broker in self._mqtt_client:
# Connection already made
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
else:
# Connect to the broker
self._mqtt_client[broker] = Client(
f"EmailAlarmService{randrange(1, 1000000)}")
self._mqtt_client[broker].on_message = partial(
self.my_on_message, email_service=self.service)
self._mqtt_client[broker].connect(
host=broker, port=self._broker_port[broker])
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
def _clear(self):
"""
Clear all the data stored
"""
# Unsubscribe from all topics
self.unsubscribe(self._service_list.copy())
# Disconnect from all the external brokers
for broker in self._mqtt_client:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
# Clear
self._mqtt_client.clear()
self._service_list.clear()
self._broker.clear()
self._broker_port.clear()
self._topic.clear()
def stop(self):
"""
Stop the service
"""
# Stop the schedule
self._update_thread.cancel()
# Wait for the threads to finish
if self._update_thread.is_alive():
self._update_thread.join()
# Disconnect the clients
print(f"[{time.ctime()}] SHUTTING DOWN")
for client in self._mqtt_client:
self._mqtt_client[client].disconnect()
self._mqtt_client[client].loop_stop()
# Disconnect the service
self.service.disconnect()
print(f"[{time.ctime()}] EXIT")
class Bettor:
def __init__(self,
name,
money,
broker='localhost',
auth=None,
interact=False):
self.name = name
self.lock = Lock()
self.money = money
self.client = Client()
self.interest_objects = []
self.interact = interact
self.lst = []
self.broker = broker
self.auth = auth
if auth != None:
(usr, pwd) = self.auth
self.client.username_pw_set(usr, pwd)
self.client.connect(broker)
def on_message(self, client, userdata, msg):
if msg.topic == 'Available-items':
self.lst = msg.payload
self.lst = self.lst.decode("utf-8").split()
if self.interact == False:
self.interesting_items()
else:
'''
Manage bid in terms of available money.
Mutual exclusion to ensure money limitations between all
bid processes.
'''
(current_bet, bid_owner) = msg.payload.decode("utf-8").split()
current_bet = int(current_bet)
if self.name != bid_owner:
with self.lock:
if current_bet > self.money:
print('Limit reached for', self.name, ' for item ',
msg.topic.replace('results/', ''))
self.client.unsubscribe(msg.topic)
self.interest_objects.remove(
msg.topic.replace('results/', ''))
else:
bet = current_bet + random.randint(1, 50)
topic = msg.topic.replace('results', 'items')
print(self.name, 'bets', bet, 'for', topic)
self.client.publish(topic,
str(bet) + ' ' + str(self.name))
self.money -= bet
def bet_process(self):
self.client.on_message = self.on_message
for it in self.interest_objects:
self.client.subscribe('results/' + str(it))
self.client.loop_forever()
def interesting_items(self):
n = random.randint(1, len(self.lst))
self.interest_objects = random.sample(self.lst, n)
print(self.name, 'will play for', self.interest_objects)
Process(target=self.bet_process, args=()).start()
def get_items(self):
self.client.subscribe('Available-items')
self.client.on_message = self.on_message
self.client.loop_forever()
def parse_input_int(self, a):
return a.isdigit() and int(a) > 0 and int(a) <= self.money
def get_input(self):
print('Insert cuantity to bid (you have ', self.money, '): ')
value = input()
while not self.parse_input_int(value):
print(
'Please, insert a valid number, or check if you have enough money'
)
value = input('Insert cuantity to bid: ')
return value
def bet_process_interact(self):
(usr, pwd) = self.auth
alive = True
while alive:
cuantity = self.get_input()
publish.single('items/' + str(self.interest_objects),
str(cuantity) + ' ' + str(self.name),
hostname=self.broker,
auth={
'username': usr,
'password': pwd
})
self.money -= int(cuantity)
alive = self.money != 0
print('Out of money, you are out!')
def parse_list(self, elem):
return str(elem) in self.lst
def start(self):
if self.interact:
print('Interact mode: make sure you have more than one bidder ')
print('(if you do not want to compete against yourself)... ')
print('maybe it does not even work :)')
self.client.loop_start()
(usr, pwd) = self.auth
available = subscribe.simple('Available-items',
hostname=self.broker,
auth={
'username': usr,
'password': pwd
})
objs = available.payload.decode("utf-8")
print('This are the available objects: ', objs)
self.lst = objs.split()
in_obj = input('Wich object do you want to bid for? ')
while not self.parse_list(in_obj):
in_obj = input('Wich object do you want to bid for? ')
self.interest_objects = str(in_obj)
self.bet_process_interact()
else:
Process(target=self.get_items, args=()).start()
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__log = log
self.config = config
self._connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__gateway = gateway
self.__broker = config.get('broker')
self.__mapping = config.get('mapping')
self.__server_side_rpc = config.get('serverSideRpc', [])
self.__service_config = {
"connectRequests": [],
"disconnectRequests": []
}
self.__attribute_updates = config.get("attributeUpdates")
self.__get_service_config(config)
self.__sub_topics = {}
client_id = ''.join(
random.choice(string.ascii_lowercase) for _ in range(23))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] or self.__broker[
"security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: ",
self.get_name())
self.__log.exception(e)
self._client.tls_insecure_set(False)
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self.__subscribes_sent = {} # For logging the subscriptions
self._client.on_disconnect = self._on_disconnect
# self._client.on_log = self._on_log
self._connected = False
self.__stopped = False
self.daemon = True
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
time.sleep(1)
except Exception as e:
self.__log.exception(e)
time.sleep(10)
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
time.sleep(.01)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic):
message = self._client.subscribe(topic)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug(
"Client %s, userdata %s, flags %s, extra_params %s",
str(client), str(userdata), str(flags), extra_params)
for mapping in self.__mapping:
try:
converter = None
if mapping["converter"]["type"] == "custom":
module = TBUtility.check_and_import(
self._connector_type,
mapping["converter"]["extension"])
if module is not None:
self.__log.debug(
'Custom converter for topic %s - found!',
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error(
"\n\nCannot find extension module for %s topic.\nPlease check your configuration.\n",
mapping["topicFilter"])
else:
converter = JsonMqttUplinkConverter(mapping)
if converter is not None:
regex_topic = TBUtility.topic_to_regex(
mapping.get("topicFilter"))
if not self.__sub_topics.get(regex_topic):
self.__sub_topics[regex_topic] = []
self.__sub_topics[regex_topic].append(
{converter: None})
# self._client.subscribe(TBUtility.regex_to_topic(regex_topic))
self.__subscribe(mapping["topicFilter"])
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
except Exception as e:
self.__log.exception(e)
try:
for request in self.__service_config:
if self.__service_config.get(request) is not None:
for request_config in self.__service_config.get(
request):
self.__subscribe(request_config["topicFilter"])
except Exception as e:
self.__log.error(e)
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), result_code,
result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self.__log.debug('"%s" was disconnected. %s', self.get_name(),
str(args))
def _on_log(self, *args):
self.__log.debug(args)
def _on_subscribe(self, _, __, mid, granted_qos):
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
except Exception as e:
self.__log.exception(e)
def __get_service_config(self, config):
for service_config in self.__service_config:
if config.get(service_config):
self.__service_config[service_config] = config[service_config]
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
# Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes"
regex_topic = [
regex for regex in self.__sub_topics
if fullmatch(regex, message.topic)
]
if regex_topic:
try:
for regex in regex_topic:
if self.__sub_topics.get(regex):
for converter_value in range(
len(self.__sub_topics.get(regex))):
if self.__sub_topics[regex][converter_value]:
for converter in self.__sub_topics.get(
regex)[converter_value]:
converted_content = converter.convert(
message.topic, content)
if converted_content:
try:
self.__sub_topics[regex][
converter_value][
converter] = converted_content
except Exception as e:
self.__log.exception(e)
self.__gateway.send_to_storage(
self.name, converted_content)
self.statistics['MessagesSent'] += 1
else:
continue
else:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client), str(userdata))
return None
except Exception as e:
log.exception(e)
return None
# Check if message topic is matched by an existing connection request handler
if self.__service_config.get("connectRequests"):
for request in self.__service_config["connectRequests"]:
# Check that the current connection request handler defines a topic filter (mandatory)
if request.get("topicFilter") is None:
continue
found_device_name = None
found_device_type = 'default'
# Extract device name and type from regexps, if any.
# This cannot be postponed because message topic may contain wildcards
if request.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"], content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
device_name_match = search(device_name_expression,
message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
if request.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
request["deviceTypeJsonExpression"], content)
if request.get("deviceTypeTopicExpression"):
device_type_expression = request[
"deviceTypeTopicExpression"]
found_device_type = search(device_type_expression,
message.topic)
# Check if request topic matches with message topic before of after regexp substitution
if message.topic not in request.get("topicFilter"):
sub_topic = message.topic
# Substitute device name (if defined) in topic
if found_device_name is not None:
sub_topic = sub(found_device_name, "+", sub_topic)
# Substitute device type in topic
sub_topic = sub(found_device_type, "+", sub_topic)
# If topic still not matches, this is not the correct handler
if sub_topic not in request.get("topicFilter"):
continue
# I'm now sure that this message must be handled by this connection request handler
if found_device_name is None:
self.__log.error(
"Device name missing from connection request")
return None
# Note: device must be added even if it is already known locally: else ThingsBoard
# will not send RPCs and attribute updates
self.__gateway.add_device(found_device_name,
{"connector": self},
device_type=found_device_type)
return None
# Check if message topic is matched by an existing disconnection request handler
if self.__service_config.get("disconnectRequests"):
for request in self.__service_config["disconnectRequests"]:
# Check that the current disconnection request handler defines a topic filter (mandatory)
if request.get("topicFilter") is None:
continue
found_device_name = None
found_device_type = 'default'
# Extract device name and type from regexps, if any.
# This cannot be postponed because message topic may contain wildcards
if request.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"], content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
device_name_match = search(device_name_expression,
message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
if request.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
request["deviceTypeJsonExpression"], content)
if request.get("deviceTypeTopicExpression"):
device_type_expression = request[
"deviceTypeTopicExpression"]
found_device_type = search(device_type_expression,
message.topic)
# Check if request topic matches with message topic before of after regexp substitution
if message.topic not in request.get("topicFilter"):
sub_topic = message.topic
# Substitute device name (if defined) in topic
if found_device_name is not None:
sub_topic = sub(found_device_name, "+", sub_topic)
# Substitute device type in topic
sub_topic = sub(found_device_type, "+", sub_topic)
# If topic still not matches, this is not the correct handler
if sub_topic not in request.get("topicFilter"):
continue
# I'm now sure that this message must be handled by this connection request handler
if found_device_name is None:
self.__log.error(
"Device name missing from disconnection request")
return None
if found_device_name in self.__gateway.get_devices():
self.__log.info("Device %s of type %s disconnected",
found_device_name, found_device_type)
self.__gateway.del_device(found_device_name)
else:
self.__log.info("Device %s is already disconnected",
found_device_name)
return None
if message.topic in self.__gateway.rpc_requests_in_progress:
self.__gateway.rpc_with_reply_processing(message.topic, content)
else:
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def on_attributes_update(self, content):
if self.__attribute_updates:
for attribute_update in self.__attribute_updates:
if match(attribute_update["deviceNameFilter"],
content["device"]):
for attribute_key in content["data"]:
if match(attribute_update["attributeFilter"],
attribute_key):
try:
topic = attribute_update["topicExpression"]\
.replace("${deviceName}", content["device"])\
.replace("${attributeKey}", attribute_key)\
.replace("${attributeValue}", content["data"][attribute_key])
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
try:
data = attribute_update["valueExpression"]\
.replace("${attributeKey}", attribute_key)\
.replace("${attributeValue}", content["data"][attribute_key])
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
self._client.publish(topic,
data).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Cannot find attributeName by filter in message with data: %s",
content)
else:
self.__log.error(
"Cannot find deviceName by filter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# Subscribe to RPC response topic
if rpc_config.get("responseTopicExpression"):
topic_for_subscribe = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", content["device"]) \
.replace("${methodName}", content["data"]["method"]) \
.replace("${requestId}", str(content["data"]["id"])) \
.replace("${params}", content["data"]["params"])
if rpc_config.get("responseTimeout"):
timeout = time.time() * 1000 + rpc_config.get(
"responseTimeout")
self.__gateway.register_rpc_request_timeout(
content, timeout, topic_for_subscribe,
self.rpc_cancel_processing)
# Maybe we need to wait for the command to execute successfully before publishing the request.
self._client.subscribe(topic_for_subscribe)
else:
self.__log.error(
"Not found RPC response timeout in config, sending without waiting for response"
)
# Publish RPC request
if rpc_config.get("requestTopicExpression") is not None\
and rpc_config.get("valueExpression"):
topic = rpc_config.get("requestTopicExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
data_to_send = rpc_config.get("valueExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
try:
self._client.publish(topic, data_to_send)
self.__log.debug(
"Send RPC with no response request to topic: %s with data %s",
topic, data_to_send)
if rpc_config.get("responseTopicExpression") is None:
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
except Exception as e:
self.__log.exception(e)
def rpc_cancel_processing(self, topic):
self._client.unsubscribe(topic)
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__gateway = gateway # Reference to TB Gateway
self._connector_type = connector_type # Should be "mqtt"
self.config = config # mqtt.json contents
self.__log = log
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__subscribes_sent = {}
# Extract main sections from configuration ---------------------------------------------------------------------
self.__broker = config.get('broker')
self.__mapping = []
self.__server_side_rpc = []
self.__connect_requests = []
self.__disconnect_requests = []
self.__attribute_requests = []
self.__attribute_updates = []
mandatory_keys = {
"mapping": ['topicFilter', 'converter'],
"serverSideRpc": ['deviceNameFilter', 'methodFilter'],
"connectRequests": ['topicFilter'],
"disconnectRequests": ['topicFilter'],
"attributeRequests":
['topicFilter', 'topicExpression', 'valueExpression'],
"attributeUpdates": [
'deviceNameFilter', 'attributeFilter', 'topicExpression',
'valueExpression'
]
}
# Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file
self.load_handlers('mapping', mandatory_keys['mapping'],
self.__mapping)
# RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers
self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'],
self.__server_side_rpc)
# Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry
self.load_handlers('connectRequests',
mandatory_keys['connectRequests'],
self.__connect_requests)
# Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet
self.load_handlers('disconnectRequests',
mandatory_keys['disconnectRequests'],
self.__disconnect_requests)
# Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value
self.load_handlers('attributeRequests',
mandatory_keys['attributeRequests'],
self.__attribute_requests)
# Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute
self.load_handlers('attributeUpdates',
mandatory_keys['attributeUpdates'],
self.__attribute_updates)
# Setup topic substitution lists for each class of handlers ----------------------------------------------------
self.__mapping_sub_topics = {}
self.__connect_requests_sub_topics = {}
self.__disconnect_requests_sub_topics = {}
self.__attribute_requests_sub_topics = {}
# Set up external MQTT broker connection -----------------------------------------------------------------------
client_id = ''.join(
random.choice(string.ascii_lowercase) for _ in range(23))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] \
or self.__broker["security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. "
"Please check your configuration.\nError: ",
self.get_name())
self.__log.exception(e)
self._client.tls_insecure_set(False)
# Set up external MQTT broker callbacks ------------------------------------------------------------------------
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self._client.on_disconnect = self._on_disconnect
# self._client.on_log = self._on_log
# Set up lifecycle flags ---------------------------------------------------------------------------------------
self._connected = False
self.__stopped = False
self.daemon = True
def load_handlers(self, handler_flavor, mandatory_keys,
accepted_handlers_list):
if handler_flavor not in self.config:
self.__log.error("'%s' section missing from configuration",
handler_flavor)
else:
for handler in self.config.get(handler_flavor):
discard = False
for key in mandatory_keys:
if key not in handler:
# Will report all missing fields to user before discarding the entry => no break here
discard = True
self.__log.error(
"Mandatory key '%s' missing from %s handler: %s",
key, handler_flavor, json.dumps(handler))
else:
self.__log.debug(
"Mandatory key '%s' found in %s handler: %s", key,
handler_flavor, json.dumps(handler))
if discard:
self.__log.error(
"%s handler is missing some mandatory keys => rejected: %s",
handler_flavor, json.dumps(handler))
else:
accepted_handlers_list.append(handler)
self.__log.debug(
"%s handler has all mandatory keys => accepted: %s",
handler_flavor, json.dumps(handler))
self.__log.info("Number of accepted %s handlers: %d",
handler_flavor, len(accepted_handlers_list))
self.__log.info(
"Number of rejected %s handlers: %d", handler_flavor,
len(self.config.get(handler_flavor)) -
len(accepted_handlers_list))
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
time.sleep(1)
except Exception as e:
self.__log.exception(e)
time.sleep(10)
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
time.sleep(.01)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic, qos):
message = self._client.subscribe(topic, qos)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug(
"Client %s, userdata %s, flags %s, extra_params %s",
str(client), str(userdata), str(flags), extra_params)
# Setup data upload requests handling ----------------------------------------------------------------------
for mapping in self.__mapping:
try:
converter = None
# Load converter for this mapping entry ------------------------------------------------------------
# mappings are guaranteed to have topicFilter and converter fields. See __init__
converter_type = mapping["converter"]["type"]
converter_extension = mapping["converter"]["extension"]
if converter_type:
if converter_extension:
module = TBUtility.check_and_import(
self._connector_type, converter_extension)
if module:
self.__log.debug(
'Custom converter for topic %s - found!',
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error(
"\n\nCannot find extension module for %s topic."
"\nPlease check your configuration.\n",
mapping["topicFilter"])
else:
converter = JsonMqttUplinkConverter(mapping)
if converter is None:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
continue
# Setup regexp topic acceptance list ---------------------------------------------------------------
regex_topic = TBUtility.topic_to_regex(
mapping["topicFilter"])
# There may be more than one converter per topic, so I'm using vectors
if not self.__mapping_sub_topics.get(regex_topic):
self.__mapping_sub_topics[regex_topic] = []
self.__mapping_sub_topics[regex_topic].append(converter)
# Subscribe to appropriate topic -------------------------------------------------------------------
self.__subscribe(mapping["topicFilter"],
mapping.get("subscriptionQos", 0))
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
except Exception as e:
self.__log.exception(e)
# Setup connection requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__connect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 0))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__connect_requests_sub_topics[topic_filter] = request
# Setup disconnection requests handling --------------------------------------------------------------------
for request in [
entry for entry in self.__disconnect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 0))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__disconnect_requests_sub_topics[topic_filter] = request
# Setup attributes requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__attribute_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 0))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__attribute_requests_sub_topics[topic_filter] = request
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), result_code,
result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self.__log.debug('"%s" was disconnected. %s', self.get_name(),
str(args))
def _on_log(self, *args):
self.__log.debug(args)
def _on_subscribe(self, _, __, mid, granted_qos):
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
except Exception as e:
self.__log.exception(e)
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
# Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" ---------------------------
topic_handlers = [
regex for regex in self.__mapping_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
# Note: every topic may be associated to one or more converter. This means that a single MQTT message
# may produce more than one message towards ThingsBoard. This also means that I cannot return after
# the first successful conversion: I got to use all the available ones.
# I will use a flag to understand whether at least one converter succeeded
request_handled = False
for topic in topic_handlers:
available_converters = self.__mapping_sub_topics[topic]
for converter in available_converters:
converted_content = converter.convert(
message.topic, content)
if converted_content:
request_handled = True
self.__gateway.send_to_storage(self.name,
converted_content)
self.statistics['MessagesSent'] += 1
self.__log.info(
"Successfully converted message from topic %s",
message.topic)
else:
continue
if not request_handled:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client), str(userdata))
# Note: if I'm in this branch, this was for sure a telemetry/attribute push message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in connection handlers "i.e., I'm connecting a device" -------------------------
topic_handlers = [
regex for regex in self.__connect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__connect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
if device_type_match is not None:
found_device_type = device_type_match.group(0)
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from connection request")
continue
# Note: device must be added even if it is already known locally: else ThingsBoard
# will not send RPCs and attribute updates
self.__log.info("Connecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.add_device(found_device_name,
{"connector": self},
device_type=found_device_type)
# Note: if I'm in this branch, this was for sure a connection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" -------------------
topic_handlers = [
regex for regex in self.__disconnect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__disconnect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
if device_type_match is not None:
found_device_type = device_type_match.group(0)
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from disconnection request")
continue
if found_device_name in self.__gateway.get_devices():
self.__log.info("Disconnecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.del_device(found_device_name)
else:
self.__log.info("Device %s was not connected",
found_device_name)
break
# Note: if I'm in this branch, this was for sure a disconnection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" --------
topic_handlers = [
regex for regex in self.__attribute_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
try:
for topic in topic_handlers:
handler = self.__attribute_requests_sub_topics[topic]
found_device_name = None
found_attribute_name = None
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"],
message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get attribute name, either from topic or from content
if handler.get("attributeNameTopicExpression"):
attribute_name_match = search(
handler["attributeNameTopicExpression"],
message.topic)
if attribute_name_match is not None:
found_attribute_name = attribute_name_match.group(
0)
elif handler.get("attributeNameJsonExpression"):
found_attribute_name = TBUtility.get_value(
handler["attributeNameJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from attribute request")
continue
if found_attribute_name is None:
self.__log.error(
"Attribute name missing from attribute request")
continue
self.__log.info("Will retrieve attribute %s of %s",
found_attribute_name, found_device_name)
self.__gateway.tb_client.client.gw_request_shared_attributes(
found_device_name, [found_attribute_name],
lambda data, *args: self.notify_attribute(
data, found_attribute_name,
handler.get("topicExpression"),
handler.get("valueExpression")))
break
except Exception as e:
log.exception(e)
# Note: if I'm in this branch, this was for sure an attribute request message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in RPC handlers ----------------------------------------------------------------
# The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is
if message.topic in self.__gateway.rpc_requests_in_progress:
self.__gateway.rpc_with_reply_processing(message.topic, content)
return None
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def notify_attribute(self, incoming_data, attribute_name, topic_expression,
value_expression):
if incoming_data.get("device") is None or incoming_data.get(
"value") is None:
return
device_name = incoming_data.get("device")
attribute_value = incoming_data.get("value")
topic = topic_expression \
.replace("${deviceName}", device_name) \
.replace("${attributeKey}", attribute_name)
data = value_expression.replace("${attributeKey}", attribute_name) \
.replace("${attributeValue}", attribute_value)
self._client.publish(topic, data).wait_for_publish()
def on_attributes_update(self, content):
if self.__attribute_updates:
for attribute_update in self.__attribute_updates:
if match(attribute_update["deviceNameFilter"],
content["device"]):
for attribute_key in content["data"]:
if match(attribute_update["attributeFilter"],
attribute_key):
try:
topic = attribute_update["topicExpression"]\
.replace("${deviceName}", content["device"])\
.replace("${attributeKey}", attribute_key)\
.replace("${attributeValue}", content["data"][attribute_key])
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
try:
data = attribute_update["valueExpression"]\
.replace("${attributeKey}", attribute_key)\
.replace("${attributeValue}", content["data"][attribute_key])
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
self._client.publish(topic,
data).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Cannot find attributeName by filter in message with data: %s",
content)
else:
self.__log.error(
"Cannot find deviceName by filter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# Subscribe to RPC response topic
if rpc_config.get("responseTopicExpression"):
topic_for_subscribe = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", content["device"]) \
.replace("${methodName}", content["data"]["method"]) \
.replace("${requestId}", str(content["data"]["id"])) \
.replace("${params}", content["data"]["params"])
if rpc_config.get("responseTimeout"):
timeout = time.time() * 1000 + rpc_config.get(
"responseTimeout")
self.__gateway.register_rpc_request_timeout(
content, timeout, topic_for_subscribe,
self.rpc_cancel_processing)
# Maybe we need to wait for the command to execute successfully before publishing the request.
self._client.subscribe(topic_for_subscribe)
else:
self.__log.error(
"Not found RPC response timeout in config, sending without waiting for response"
)
# Publish RPC request
if rpc_config.get("requestTopicExpression") is not None\
and rpc_config.get("valueExpression"):
topic = rpc_config.get("requestTopicExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
data_to_send = rpc_config.get("valueExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
try:
self._client.publish(topic, data_to_send)
self.__log.debug(
"Send RPC with no response request to topic: %s with data %s",
topic, data_to_send)
if rpc_config.get("responseTopicExpression") is None:
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
except Exception as e:
self.__log.exception(e)
def rpc_cancel_processing(self, topic):
self._client.unsubscribe(topic)
class Judge:
max_time = 0
elapsed_time = 0
initial_time = 0
lock = Lock()
lst = [] # List of available objects
lst_results = Array('i', 0) # Max value for each object
lst_names = [] # Name of winner for each object
def __init__(self, max_timeout, broker='localhost', auth=None):
self.max_time = max_timeout
self.auth = auth
self.client = Client()
if auth != None:
(usr, pwd) = auth
self.client.username_pw_set(usr, pwd)
self.client.connect(broker)
def on_message(self, client, userdata, msg):
if msg.topic == 'Available-items':
'''
This msg contains all available objects. Once received,
update the self list to contain them, then create the
results list for keeping maximum value for each object.
'''
self.lst = msg.payload
self.lst = self.lst.decode("utf-8")
self.lst = list(map(int, self.lst.split()))
self.lst_results = Array('i', len(msg.payload))
print('Available objects', self.lst)
for i in range(len(self.lst)):
self.lst_names.append('')
Process(target=self.collect, args=()).start()
time.sleep(2 * random.random())
for i in range(1, len(self.lst) + 1):
self.client.publish('results/' + str(i),
str(0) + ' ' + str('None'))
self.initial_time = time.time()
self.elapsed_time = time.time() - self.initial_time
else:
with self.lock:
time.sleep(random.random())
self.elapsed_time = time.time() - self.initial_time
if self.elapsed_time < self.max_time:
self.update_value(msg)
else:
if len(self.lst_names) != 0:
print('Timeout reached, no more bids are accepted')
for i in self.lst:
top = 'results/' + str(i)
own = str(self.lst_names[i - 1])
client.publish(top, 'Winner is ' + own)
self.client.unsubscribe('items/#')
self.lst_names = []
def update_value(self, msg):
'''
Manage every msg received:
1- update the current value for each object (mutex with self.lock)
2- publish the value in the corresponding topic
'''
item_topic = msg.topic
(rest, bid_owner) = msg.payload.decode("utf-8").split()
result_topic = item_topic.replace('items', 'results')
item_pos = int(item_topic.replace('items/', '')) - 1
item_value = int(rest)
last_value = self.lst_results[item_pos]
if last_value < item_value:
self.lst_results[item_pos] = item_value
self.lst_names[item_pos] = bid_owner
current_value = self.lst_results[item_pos]
self.client.publish(
result_topic,
str(current_value) + ' ' + str(self.lst_names[item_pos]))
print('New winner for',
str(msg.topic).replace('items/', 'item '), ':',
self.lst_names[item_pos])
def self_process(self, topic):
self.client.subscribe('items/' + str(topic))
self.client.on_message = self.on_message
self.client.loop_forever()
def collect(self):
print('Waitting users to bid ...')
for it in self.lst:
self.client.subscribe('items/' + str(it))
self.client.on_message = self.on_message
self.client.loop_forever()
def get_items(self):
self.client.subscribe('Available-items')
self.client.on_message = self.on_message
self.client.loop_forever()
def start(self):
self.get_items()
class MQTTBus(BaseBus):
class MQTTPubSub(BasePubSub):
def __init__(self, channel, bus, handler):
try:
from Queue import Queue
except:
from queue import Queue
self.bus = bus
self.channel = channel
self.handler = handler
self.queue = Queue()
listened = False
subscribed = True
def listen(self):
self.listened = True
while self.listened:
data = self.queue.get()
yield data
def close(self):
self.listened = False
self.subscribed = False
def _handle_data(self, data):
if self.handler:
self.handler(data)
elif self.listened:
self.queue.put(data)
connected = False
subscriber_map = {}
def __init__(self, host='127.0.0.1', port=1883, username=None, password=None, channel_prefix=None):
import uuid
from paho.mqtt.client import Client
self.host = host
self.port = port
self.username = username
self.password = password
self.channel_prefix = channel_prefix
clientid = str(uuid.uuid4())
self.mqttclient = Client(client_id=clientid)
if self.username:
self.mqttclient.username_pw_set(self.username, self.password or None)
self.mqttclient.on_connect = self._on_connect
self.mqttclient.on_message = self._on_message
self._connect()
def _connect(self):
import threading
if self.username:
self.mqttclient.username_pw_set(self.username, self.password)
self.mqttclient.connect(self.host, self.port)
th = threading.Thread(target=self._forloop)
th.setDaemon(True)
th.start()
def _disconnect(self):
self.mqttclient.disconnect()
def subscriber_with_handler(self, channel, handler):
ful_channel = self._get_ful_channel(channel)
with self.lock:
if ful_channel not in self.subscriber_map:
self.mqttclient.subscribe(topic=ful_channel)
self.subscriber_map[ful_channel] = {
'mpss': [],
}
mps = MQTTBus.MQTTPubSub(channel, self, handler)
self.subscriber_map[ful_channel]['mpss'].append(mps)
return mps
def subscriber(self, channel):
return self.subscriber_with_handler(channel, None)
def publish(self, channel, data):
return self.mqttclient.publish(self._get_ful_channel(channel), payload=encode_data(data))
def _forloop(self):
self.mqttclient.loop_forever()
def _on_connect(self, client, userdata, flags, rc):
# print("_on_connect", client, userdata, flags, rc)
self.connected = True
def _on_message(self, client, userdata, msg):
# print("_on_message", client, userdata, msg.topic, msg.payload)
try:
ful_channel = msg.topic
data = decode_data(msg.payload.decode('utf-8'))
if ful_channel in self.subscriber_map:
changed = True
for mps in self.subscriber_map[ful_channel]['mpss']:
if mps.subscribed:
mps._handle_data(data)
else:
changed = True
if changed:
with self.lock:
mpss = filter(lambda mps: mps.subscribed, self.subscriber_map[ful_channel]['mpss'])
if not mpss:
self.mqttclient.unsubscribe(topic=ful_channel)
del self.subscriber_map[ful_channel]
else:
self.subscriber_map[ful_channel]['mpss'] = mpss
except:
import traceback
traceback.print_exc()
class Bridge(object):
WEBSOCKETS = "websocket"
SSE = "sse"
def __init__(self, args, ioloop, dynamic_subscriptions):
""" parse config values and setup Routes.
"""
self.mqtt_topics = []
try:
self.mqtt_host = args["mqtt-to-server"]["broker"]["host"]
self.mqtt_port = args["mqtt-to-server"]["broker"]["port"]
self.bridge_port = args["server-to-client"]["port"]
self.stream_protocol = args["server-to-client"]["protocol"]
logger.info("Using protocol %s" % self.stream_protocol.lower())
if self.stream_protocol.lower(
) != "websocket" and self.stream_protocol.lower() != "sse":
raise ConfigException("Invalid protocol")
self.dynamic_subscriptions = dynamic_subscriptions
if not self.dynamic_subscriptions:
self.mqtt_topics = args["mqtt-to-server"]["topics"]
except KeyError as e:
raise ConfigException("Error when accessing field", e) from e
logger.info("connecting to mqtt")
self.topic_dict = {}
self.ioloop = ioloop
self.mqtt_client = Client()
self.mqtt_client.on_message = self.on_mqtt_message
self.mqtt_client.on_connect = self.on_mqtt_connect
self.mqtt_client.connect_async(host=self.mqtt_host,
port=self.mqtt_port)
self.mqtt_client.loop_start()
self._app = web.Application([
(r'.*', WebsocketHandler if self.stream_protocol == "websocket"
else ServeSideEventsHandler, dict(parent=self)),
],
debug=True)
def get_app(self):
return self._app
def get_port(self):
return self.bridge_port
async def parse_req_path(self, req_path):
candidate_path = req_path
if len(candidate_path) is 1 and candidate_path[0] is "/":
return "#"
if candidate_path[len(req_path) - 1] is "/":
candidate_path = candidate_path + "#"
if candidate_path[0] == "/":
candidate_path = candidate_path[1:]
return candidate_path
def on_mqtt_message(self, client, userdata, message):
logger.debug("received message on topic %s" % message.topic)
async def socket_write_message(self, socket, message):
try:
await socket.write_message(json.dumps(message))
except Exception as e:
logger.error(e)
try:
await socket.write_message(message)
except Exception as e:
logger.error(e)
def append_dynamic(self, topic):
logger.info("adding dynamic subscription for %s " % topic)
self.message_callback_add_with_sub_topic(topic, dynamic=True)
def remove_dynamic(self, topic):
logger.info("removing dynamic subscription for %s " % topic)
self.topic_dict.pop(topic, None)
self.mqtt_client.unsubscribe(topic)
def message_callback_add_with_sub_topic(self, sub_topic, dynamic):
logger.info("adding callback for mqtt topic: %s" % sub_topic)
def message_callback(client, userdata, message):
logger.debug(
"Recieved Mqtt Message on %s as result of subscription on %s" %
(message.topic, sub_topic))
if sub_topic is not message.topic:
if message.topic not in self.topic_dict[sub_topic]["matches"]:
self.topic_dict[sub_topic]["matches"].append(message.topic)
for topic in self.topic_dict:
if topic == message.topic:
for socket in self.topic_dict[topic]["sockets"]:
logger.debug("sending to socket:")
logger.debug(socket)
self.ioloop.add_callback(
self.socket_write_message,
socket=socket,
message={
"topic": message.topic,
"payload": message.payload.decode('utf-8')
})
elif message.topic in self.topic_dict[topic]["matches"]:
for socket in self.topic_dict[topic]["sockets"]:
logger.debug("sending to socket:")
logger.debug(socket)
self.ioloop.add_callback(
self.socket_write_message,
socket=socket,
message={
"topic": message.topic,
"payload": message.payload.decode('utf-8')
})
if sub_topic not in self.topic_dict:
self.mqtt_client.message_callback_add(sub_topic, message_callback)
self.topic_dict[sub_topic] = {
"matches": [],
"sockets": [],
"dynamic": dynamic
}
self.mqtt_client.subscribe(sub_topic)
def on_mqtt_connect(self, client, userdata, flags, rc):
logger.info("mqtt connected to broker %s:%s" %
(self.mqtt_host, str(self.mqtt_port)))
for topic in self.mqtt_topics:
self.message_callback_add_with_sub_topic(topic, dynamic=False)
async def mqtt_disconnect(self):
t = Thread(target=self.mqtt_client.disconnect, daemon=True)
t.start()
self.mqtt_client.loop_stop()
class Service:
"""
Service that publish informations about whether or not the devices are in expected range
of good functioning
"""
_mqtt_client: Dict[str, Client] = {}
_device_list: Dict[str, dict] = {}
_broker: DefaultDict[str, set] = defaultdict(set)
_broker_port: Dict[str, int] = {}
_topic: DefaultDict[str, set] = defaultdict(set)
_update_thread: Timer = None
alarm_topic = "labsw3/arduino/alarm"
def __init__(self):
"""
Instantiate the service
"""
self.service = Client(client_id=f"AlarmService{randrange(1, 100000)}")
self.service.on_message = partial(self.my_on_message,
led=self.service,
service=self.service)
self.lock = Lock()
self.device_lock = Lock()
def _update(self, device_list: List[dict]):
"""
Update reserved dicts
:param device_list: device list to add
"""
for arduino in device_list:
device = arduino["deviceID"]
broker = arduino["end_points"]["MQTT"]["ip"]
port = arduino["end_points"]["MQTT"]["port"]
topics = {
topic
for topic in arduino["end_points"]["MQTT"]["end_points"]
["subscribe"] if "temp" in topic
}
led_topics = {
topic
for topic in arduino["end_points"]["MQTT"]["end_points"]
["publish"] if "led" in topic
}
self._device_list[device] = {
"ip": broker,
"temperature_topics": topics,
"led_topics": led_topics
}
self._broker_port[broker] = port
self._broker[broker].update({device})
self._topic[broker].update(topics)
print(f"[{time.ctime()}] DEVICE {device} CONNECTED")
def setup(self, first_time: bool = True):
"""
Setup the service
"""
try:
print(
f"[{time.ctime()}] EXTRACT info about all the devices registered"
)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/devices/all"
)
while result.status_code != 200:
print(
f"[{time.ctime()}] WARNING no device registered found, retrying after 30 seconds"
)
time.sleep(30)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}"
f"/catalog/devices/all")
print(f"[{time.ctime()}] DEVICES found")
data = json.loads(result.content.decode())
print(
f"[{time.ctime()}] EXTRACT from the devices list info about ArduinoYUN"
)
device_list = [device for device in data if find_arduino(device)]
if len(device_list) == 0:
print(
f"[{time.ctime()}] No ArduinoYUN found... retrying in 10 seconds"
)
time.sleep(10)
self.setup(first_time)
return
except KeyboardInterrupt:
print(f"[{time.ctime()}] EXIT")
if first_time:
sys.exit()
return
if first_time:
# Connect the service
self.service.connect(host=SERVICE_BROKER_PORT["ip"],
port=SERVICE_BROKER_PORT["port"])
print(f"[{time.ctime()}] SERVICE CONNECTED to "
f"broker: {SERVICE_BROKER_PORT['ip']} "
f"on port: port={SERVICE_BROKER_PORT['port']}")
# Update registered devices and subscribe to all the topics
with self.device_lock:
self._update(device_list)
self.subscribe(device_list)
# Register inside the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}")
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
def start(self):
"""
Start the service.
"""
# Setup the service
self.setup()
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
# Schedule the update of the registration
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
try:
# Run the service forever
self.service.loop_forever()
except KeyboardInterrupt:
self.stop()
def my_on_message(self,
client: Client,
userdata: Any,
msg: MQTTMessage,
led: Client = None,
service: Client = None):
"""
Check if the temperature is good. In case of bad values,
turn on a led and send the alarm
:param client: MQTT client
:param userdata: They could be any type
:param msg: MQTT message
:param led: Led Client
:param service: Service Client
"""
def _device_led():
"""
Find the arduino that generates the temperature telemetry
and the topic to control the led
"""
with self.device_lock:
for device in self._device_list:
if msg.topic in self._device_list[device][
"temperature_topics"]:
return device, self._device_list[device]["led_topics"]
data = json.loads(msg.payload.decode())
arduino, led_topics = _device_led()
with self.lock:
if RANGE["min"] < data["v"] < RANGE["max"]:
for topic in led_topics:
led.publish(topic,
payload=json.dumps({
"n": "led",
"v": 0,
"u": None
}))
print(
f"[{time.ctime()}] PUBLISHING Alarm status on topic: {self.alarm_topic}"
)
service.publish(self.alarm_topic,
payload=json.dumps({
"device": arduino,
"alarm": False
}))
else:
for topic in led_topics:
led.publish(topic,
payload=json.dumps({
"n": "led",
"v": 1,
"u": None
}))
print(
f"[{time.ctime()}] PUBLISHING Alarm status on topic: {self.alarm_topic}"
)
service.publish(self.alarm_topic,
payload=json.dumps({
"device": arduino,
"alarm": True
}))
def update_registration(self):
"""
Update service registration to the catalog
"""
print(
f"[{time.ctime()}] EXTRACT info about all the devices registered")
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/devices/all"
)
# No device found
if result.status_code != 200:
self.reset()
return
data = json.loads(result.content.decode())
device_list = [device for device in data if find_arduino(device)]
if len(device_list) == 0:
self.reset()
return
# New Devices Found
new_devices = {arduino["deviceID"] for arduino in device_list}
# Old devices list
old_devices = set(self._device_list.keys())
# Check if there are updates to do
if old_devices == new_devices:
# No update, ping the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}"
)
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
return
with self.device_lock:
# Delete inactive devices
delete_list = {
arduino: self._device_list[arduino]
for arduino in self._device_list if arduino not in new_devices
}
# Unsubscribe from devices
self.unsubscribe(delete_list)
# Update registered devices and subscribe
new_devices = [
arduino for arduino in device_list
if arduino["deviceID"] not in old_devices
]
self._update(new_devices)
self.subscribe(new_devices)
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def reset(self):
"""
Reset the service
"""
with self.device_lock:
self._clear()
self.setup(first_time=False)
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def unsubscribe(self, device_list: dict):
"""
Unsubscribe from devices
:param device_list: device list
"""
for arduino in device_list:
print(f"[{time.ctime()}] DEVICE {arduino} DISCONNECTED")
broker = device_list[arduino]["ip"]
topics = device_list[arduino]["temperature_topics"]
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
else:
for topic in topics:
self._mqtt_client[broker].unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
# Update topics
self._topic[broker] = self._topic[broker].difference(topics)
# Disconnect from external broker
if len(self._topic[broker]) == 0:
if broker != SERVICE_BROKER_PORT["ip"]:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(
f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
del self._mqtt_client[broker]
# Clean
del self._topic[broker]
del self._broker_port[broker]
# Delete device
del self._device_list[arduino]
self._broker[broker].discard(arduino)
def subscribe(self, device_list: List[dict]):
"""
Subscribe to all the devices registered
:param device_list: list of device
"""
for arduino in device_list:
broker = arduino["end_points"]["MQTT"]["ip"]
topics = {
topic
for topic in arduino["end_points"]["MQTT"]["end_points"]
["subscribe"] if "temp" in topic
}
# Subscribe to topics
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
# Connect to external brokers
else:
if broker in self._mqtt_client:
# Connection already made
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
else:
# Connect to the broker
self._mqtt_client[broker] = Client(
f"TemperatureMeanService{randrange(1, 1000000)}")
self._mqtt_client[broker].on_message = partial(
self.my_on_message,
led=self._mqtt_client[broker],
service=self.service)
self._mqtt_client[broker].connect(
host=broker, port=self._broker_port[broker])
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
def _clear(self):
"""
Clear all the data stored
"""
# Unsubscribe from all topics
self.unsubscribe(self._device_list.copy())
# Disconnect from all the external brokers
for broker in self._mqtt_client:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
# Clear
self._mqtt_client.clear()
self._device_list.clear()
self._broker.clear()
self._broker_port.clear()
self._topic.clear()
def stop(self):
"""
Stop the service
"""
# Stop the schedule
self._update_thread.cancel()
# Wait for the threads to finish
if self._update_thread.is_alive():
self._update_thread.join()
# Disconnect the clients
print(f"[{time.ctime()}] SHUTTING DOWN")
for client in self._mqtt_client:
self._mqtt_client[client].disconnect()
self._mqtt_client[client].loop_stop()
# Disconnect the service
self.service.disconnect()
print(f"[{time.ctime()}] EXIT")
class Mqtt(Abstract):
def __init__(self, service_info=None):
self.connected = False
self.client = None
self.queue = Queue()
super().__init__(service_info)
def start(self):
self.connect()
self.receive_msgs()
def stop(self):
self.stop_receiving_msgs()
self.disconnect()
def add_stream(self, stream):
self.streams[stream.name] = stream
def remove_stream(self, name):
if name in self.streams.keys():
self.streams.pop(name)
def remove_all_streams(self):
self.streams = {}
def is_connected(self):
if self.client and self.connected:
return True
return False
def connect(self):
if not self.client:
host = self.service_info['host']
port = self.service_info['port']
self.client = Client()
self.client.on_message = self.on_mqtt_msg
self.client.on_connect = self.on_connect
self.client.on_disconnect = self.on_disconnect
self.client.connect(host, port, 60)
def on_connect(self, client, userdata, flags, rc):
self.connected = True
def disconnect(self):
if self.client:
self.client.disconnect()
def on_disconnect(self, client, userdata, rc):
self.client = None
self.connected = False
def send_msg(self, msg, tx_info=None):
if self.client:
topic = None
if tx_info:
topic = tx_info['topic']
elif self.service_info:
topic = self.service_info['publish']['default_topic']
if self.client and topic:
self.client.publish(topic, msg)
else:
raise MqttError('Cannot send message without a client')
def receive_msgs(self):
topics = self.service_info['subscribe']['topics']
if self.client:
for t in topics:
self.client.subscribe(t, 0)
sub_type = self.service_info['subscribe']['type']
if sub_type == 'blocking':
self.client.loop_forever()
elif sub_type == 'unblocking':
self.client.loop_start()
elif sub_type == 'polled':
self.client.loop(self.service_info['subscribe']['loop_time'])
else:
self.client.loop(.1)
else:
raise MqttError('Cannot receive message without a client')
def stop_receiving_msgs(self):
topics = self.service_info['subscribe']['topics']
if self.client:
for t in topics:
self.client.unsubscribe(t)
self.client.loop_stop()
def handle_msg(self):
msg = self.queue.get()
print(f'Received: {msg["topic"]}::{msg["msg"]}')
data = super().handle_msg(msg)
for k, s in self.streams.items():
events = []
for d in data:
timestamp = d.timestamp
sample = getattr(d, s.handle)
events.append(Event(timestamp, sample))
s.handle_msg(events)
def on_mqtt_msg(self, client, userdata, msg):
payload = {}
payload['topic'] = msg.topic
payload['msg'] = msg.payload
self.queue.put(payload)
self.handle_msg()
def unsubscribe(self, mqttclient: MqttClient):
mqttclient.unsubscribe(self.updatedDeviceTopic)
with self.devicesLock:
for device in self.devices.values():
device.unsubscribe(mqttclient)
class MqttClient():
default_config = {
"host": "api.raptorbox.eu",
"port": 1883,
"reconnect_min_delay": 1,
"reconnect_max_delay": 127,
}
def __init__(self, config):
'''
Initialize information for the client
provide configurations as parameter, if None provided the configurations would be defaults
see MqttClient.default_config for configurations format
'''
self.config = MqttClient.default_config
if config:
for k, v in config.iteritems():
self.config[k] = v
self.mqtt_client = Client()
self.mqtt_client.username_pw_set(config["username"],
config["password"])
if "reconnect_min_delay" in config and "reconnect_max_delay" in config:
self.mqtt_client.reconnect_delay_set(
config["reconnect_min_delay"],
config["reconnect_max_delay"],
)
# self.connection_thread = None
# self.mqtt_client.tls_set()
def connect(self): #TODO consider srv field in dns
'''
connect the client to mqtt server with configurations provided by constructor
and starts listening for topics
'''
self.mqtt_client.connect(self.config["host"], self.config["port"])
# self.mqtt_client.loop_forever()
self.mqtt_client.loop_start()
def subscribe(self, topic, callback):
'''
register 'callback' to "topic". Every message will be passed to callback in order to be executed
'''
logging.debug("subscribing to topic: {}".format(topic))
self.mqtt_client.subscribe(topic)
self.mqtt_client.message_callback_add(topic, callback)
def unsubscribe(self, topic):
'''
unsubscribe to topic
topic could be either a string or al list of strings containing to topics to unsubscribe to.
Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS
to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not
currently connected.
mid is the message ID for the unsubscribe request. The mid value can be
used to track the unsubscribe request by checking against the mid
argument in the on_unsubscribe() callback if it is defined.
'''
return self.mqtt_client.unsubscribe(topic)
def disconnect(self):
'''
Disconnects from the server
'''
self.mqtt_client.disconnect()
def reconnect(self):
'''
Reconnects after a disconnection, this could be called after connection only
'''
self.mqtt_client.reconnect()
class Mqtt():
def __init__(self, app=None):
# type: (Flask) -> None
self.app = app
self.client = Client()
self.client.on_connect = self._handle_connect
self.client.on_disconnect = self._handle_disconnect
self.topics = [] # type: List[str]
self.connected = False
if app is not None:
self.init_app(app)
def init_app(self, app):
# type: (Flask) -> None
self.username = app.config.get('MQTT_USERNAME')
self.password = app.config.get('MQTT_PASSWORD')
self.broker_url = app.config.get('MQTT_BROKER_URL', 'localhost')
self.broker_port = app.config.get('MQTT_BROKER_PORT', 1883)
self.tls_enabled = app.config.get('MQTT_TLS_ENABLED', False)
self.keepalive = app.config.get('MQTT_KEEPALIVE', 60)
self.last_will_topic = app.config.get('MQTT_LAST_WILL_TOPIC')
self.last_will_message = app.config.get('MQTT_LAST_WILL_MESSAGE')
self.last_will_qos = app.config.get('MQTT_LAST_WILL_QOS', 0)
self.last_will_retain = app.config.get('MQTT_LAST_WILL_RETAIN', False)
if self.tls_enabled:
self.tls_ca_certs = app.config['MQTT_TLS_CA_CERTS']
self.tls_certfile = app.config.get('MQTT_TLS_CERTFILE')
self.tls_keyfile = app.config.get('MQTT_TLS_KEYFILE')
self.tls_cert_reqs = app.config.get('MQTT_TLS_CERT_REQS',
ssl.CERT_REQUIRED)
self.tls_version = app.config.get('MQTT_TLS_VERSION',
ssl.PROTOCOL_TLSv1)
self.tls_ciphers = app.config.get('MQTT_TLS_CIPHERS')
self.tls_insecure = app.config.get('MQTT_TLS_INSECURE', False)
# set last will message
if self.last_will_topic is not None:
self.client.will_set(self.last_will_topic, self.last_will_message,
self.last_will_qos, self.last_will_retain)
self.app = app
self._connect()
def _connect(self):
# type: () -> None
if self.username is not None:
self.client.username_pw_set(self.username, self.password)
# security
if self.tls_enabled:
if self.tls_insecure:
self.client.tls_insecure_set(self.tls_insecure)
self.client.tls_set(
ca_certs=self.tls_ca_certs,
certfile=self.tls_certfile,
keyfile=self.tls_keyfile,
cert_reqs=self.tls_cert_reqs,
tls_version=self.tls_version,
ciphers=self.tls_ciphers,
)
self.client.loop_start()
res = self.client.connect(self.broker_url,
self.broker_port,
keepalive=self.keepalive)
def _disconnect(self):
# type: () -> None
self.client.loop_stop()
self.client.disconnect()
def _handle_connect(self, client, userdata, flags, rc):
# type: (Client, Any, Dict, int) -> None
if rc == MQTT_ERR_SUCCESS:
self.connected = True
for topic in self.topics:
self.client.subscribe(topic)
def _handle_disconnect(self, client, userdata, rc):
# type: (str, Any, int) -> None
self.connected = False
def on_topic(self, topic):
# type: (str) -> Callable
"""
Decorator to add a callback function that is called when a certain
topic has been published. The callback function is expected to have the
following form: `handle_topic(client, userdata, message)`
:parameter topic: a string specifying the subscription topic to subscribe to
The topic still needs to be subscribed via mqtt.subscribe() before the
callback function can be used to handle a certain topic. This way it is
possible to subscribe and unsubscribe during runtime.
**Example usage:**::
app = Flask(__name__)
mqtt = Mqtt(app)
mqtt.subscribe('home/mytopic')
@mqtt.on_topic('home/mytopic')
def handle_mytopic(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable[[str], None]) -> Callable[[str], None]
self.client.message_callback_add(topic, handler)
return handler
return decorator
def subscribe(self, topic, qos=0):
# type: (str, int) -> tuple(int, int)
"""
Subscribe to a certain topic.
:param topic: a string specifying the subscription topic to subscribe to.
:param qos: the desired quality of service level for the subscription.
Defaults to 0.
:rtype: (int, int)
:result: (result, mid)
A topic is a UTF-8 string, which is used by the broker to filter
messages for each connected client. A topic consists of one or more
topic levels. Each topic level is separated by a forward slash
(topic level separator).
The function returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the
client is not currently connected. mid is the message ID for the
subscribe request. The mid value can be used to track the subscribe
request by checking against the mid argument in the on_subscribe()
callback if it is defined.
**Topic example:** `myhome/groundfloor/livingroom/temperature`
"""
# TODO: add support for list of topics
# don't subscribe if already subscribed
# try to subscribe
result, mid = self.client.subscribe(topic, qos)
# if successful add to topics
if result == MQTT_ERR_SUCCESS:
if topic not in self.topics:
self.topics.append(topic)
return (result, mid)
def unsubscribe(self, topic):
# type: (str) -> tuple(int, int)
"""
Unsubscribe from a single topic.
:param topic: a single string that is the subscription topic to
unsubscribe from
:rtype: (int, int)
:result: (result, mid)
Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS
to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not
currently connected.
mid is the message ID for the unsubscribe request. The mid value can be
used to track the unsubscribe request by checking against the mid
argument in the on_unsubscribe() callback if it is defined.
"""
# don't unsubscribe if not in topics
if topic not in self.topics:
return
result, mid = self.client.unsubscribe(topic)
# if successful remove from topics
if result == MQTT_ERR_SUCCESS:
self.topics.remove(topic)
return result, mid
def unsubscribe_all(self):
# type: () -> None
"""
Unsubscribe from all topics.
"""
topics = self.topics[:]
for topic in topics:
self.unsubscribe(topic)
def publish(self, topic, payload=None, qos=0, retain=False):
# type: (str, bytes, int, bool) -> Tuple[int, int]
"""
Send a message to the broker.
:param topic: the topic that the message should be published on
:param payload: the actual message to send. If not given, or set to
None a zero length message will be used. Passing an
int or float will result in the payload being
converted to a string representing that number.
If you wish to send a true int/float, use struct.pack()
to create the payload you require.
:param qos: the quality of service level to use
:param retain: if set to True, the message will be set as the
"last known good"/retained message for the topic
:returns: Returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN
if the client is not currently connected. mid is the message
ID for the publish request.
"""
if not self.connected:
self.client.reconnect()
return self.client.publish(topic, payload, qos, retain)
def on_message(self):
# type: () -> Callable
"""
Decorator to handle all messages that have been subscribed and that
are not handled via the `on_message` decorator.
**Note:** Unlike as written in the paho mqtt documentation this
callback will not be called if there exists an topic-specific callback
added by the `on_topic` decorator.
**Example Usage:**::
@mqtt.on_message()
def handle_messages(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_message = handler
return handler
return decorator
def on_publish(self):
"""
Decorator to handle all messages that have been published by the
client.
**Example Usage:**::
@mqtt.on_publish()
def handle_publish(client, userdata, mid):
print('Published message with mid {}.'
.format(mid))
"""
def decorator(handler):
self.client.on_publish = handler
return handler
return decorator
def on_subscribe(self):
"""
Decorator to handle subscribe callbacks.
**Usage:**::
@mqtt.on_subscribe()
def handle_subscribe(client, userdata, mid, granted_qos):
print('Subscription id {} granted with qos {}.'
.format(mid, granted_qos))
"""
def decorator(handler):
self.client.on_subscribe = handler
return handler
return decorator
def on_unsubscribe(self):
"""
Decorator to handle unsubscribe callbacks.
**Usage:**::
@mqtt.unsubscribe()
def handle_unsubscribe(client, userdata, mid)
print('Unsubscribed from topic (id: {})'
.format(mid)')
"""
def decorator(handler):
self.client.on_unsubscribe = handler
return handler
return decorator
def on_log(self):
# type: () -> Callable
"""
Decorator to handle MQTT logging.
**Example Usage:**
::
@mqtt.on_log()
def handle_logging(client, userdata, level, buf):
print(client, userdata, level, buf)
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_log = handler
return handler
return decorator
def unsubscribe(self, mqttclient: MqttClient) -> None:
super().unsubscribe(mqttclient)
mqttclient.unsubscribe(self.stateTopic)
class MqttClient:
"""A wrapper around the paho mqtt client specifically for device automation bus
This wrapper is mean to handle connect/disconnect as well as sending and
receiving messages.
Clients can register handlers to be called when certain messages are received.
Clients also use this class to publish messages onto the bus.
"""
class MqttClientEvents:
"""Container for MqttClient thread events"""
def __init__(self):
self.connected_event = Event()
self.disconnected_event = Event()
self.apps_discovered_event = Event()
self.capabilities_discovered_event = Event()
def __init__(self, host, port):
self.host = host
self.port = port
self.mqtt_client_events = MqttClient.MqttClientEvents()
self.thread = None
self.client = Client(userdata=self.mqtt_client_events)
self.app_registry = {}
self.app_registry_timer = None
self.app_registry_timer_lock = Lock()
self.capabilities_registry = {}
self.capabilities_registry_timer = None
self.capabilities_registry_timer_lock = Lock()
self.topic_handlers = [{
'topic': 'apps/+',
'regex': 'apps/[^/]*',
'handler': self.on_app_message
}, {
'topic': 'platform/+',
'regex': 'platform/[^/]*',
'handler': self.on_app_message
}, {
'topic': 'platform/telemetry/monitor/#',
'regex': 'platform/telemetry/monitor/.*',
'handler': self.on_monitor_message
}]
self.logger = logging.getLogger(__name__)
def on_connect(self, client, mqtt_client_events, flags, rc):
"""Set the connected state and subscribe to existing known topics."""
del mqtt_client_events, flags, rc # Delete unused parameters to prevent warnings.
for topic_handler in self.topic_handlers:
client.subscribe(topic_handler["topic"])
self.mqtt_client_events.connected_event.set()
self.mqtt_client_events.disconnected_event.clear()
def on_disconnect(self, client, user_data, rc):
"""Set the disconnected state."""
del client, user_data, rc # Delete unused parameters to prevent warnings.
self.mqtt_client_events.connected_event.clear()
self.mqtt_client_events.disconnected_event.set()
def on_message(self, client, user_data, packet):
"""The main message dispatcher
This function is called for all messages on all registered topics.
It handles dispatching messages to the registered handlers.
"""
del client, user_data # Delete unused parameters to prevent warnings.
try:
if packet:
self.logger.info("topic: " + packet.topic)
# Message payload can come in as a char array instead of string.
# Normalize it to a string for easier access by handlers.
if type(packet.payload) is str:
message = packet.payload
else:
message = packet.payload.decode("utf-8")
self.logger.info("message: " + message)
# Since this function receives messages for all topics, use the specified regex
# to call the correct handler(s).
for topic_handler in self.topic_handlers:
if re.fullmatch(topic_handler["regex"], packet.topic):
topic_handler["handler"](packet.topic, message)
# Since paho eats all the exceptions, catch them here beforehand
# and make sure a stack trace is printed.
except Exception:
traceback.print_exc()
raise
def on_app_message(self, topic, message):
"""Local handler for handling registration of media apps.
This function receives a message for each registered media application.
Since these messages are published as retained, normally this will be
called with all the messages in on batch.
To know that we have received all of the initial batch, use a simple timeout
to measure how long it has been since the last message. When enough time goes
by, allow callers to get the list of apps from the registry.
"""
app = json.loads(message)
app["app_id"] = os.path.basename(topic)
self.app_registry.update({app["name"]: app})
self.app_registry_timer_lock.acquire(True)
if self.app_registry_timer:
self.app_registry_timer.cancel()
while self.app_registry_timer.is_alive():
pass
self.app_registry_timer = Timer(
1.0, self.mqtt_client_events.apps_discovered_event.set).start()
self.app_registry_timer_lock.release()
def on_platform_message(self, topic, message):
"""Local handler for handling platform messages
This function receives a message for each registered platform capability.
"""
capability = json.loads(message)
capability_id = os.path.basename(topic)
self.capabilities_registry.update({capability_id: capability})
self.capabilities_registry_timer_lock.acquire(True)
if self.capabilities_registry_timer:
self.capabilities_registry_timer.cancel()
while self.capabilities_registry_timer.is_alive():
pass
self.capabilities_registry_timer = Timer(
1.0,
self.mqtt_client_events.capabilities_discovered_event.set).start()
self.capabilities_registry_timer_lock.release()
def on_monitor_message(self, topic, message):
"""Local handler for handling process monitor messages
"""
message = json.loads(message)
def get_discovered_apps(self):
"""Method to get the discovered apps.
TODO: Get the app registry out of here into it's own class.
"""
self.mqtt_client_events.apps_discovered_event.wait()
self.app_registry_timer_lock.acquire(True)
discovered_apps = self.app_registry.copy().values()
self.app_registry_timer_lock.release()
return discovered_apps
def get_discovered_capabilities(self):
"""Method to get the discovered platform capabilities.
TODO: Get the registry out of here into it's own class.
"""
self.mqtt_client_events.capabilities_discovered_event.wait()
self.capabilities_registry_timer_lock.acquire(True)
discovered_capabilities = self.capabilities_registry.copy().values()
self.capabilities_registry_timer_lock.release()
return discovered_capabilities
def _start(self):
"""Private start method that does th actual work of starting the client in a new thread."""
self.client.enable_logger(self.logger)
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.connect(self.host, self.port)
self.client.loop_start()
self.mqtt_client_events.connected_event.wait(15.0)
if not self.mqtt_client_events.connected_event.is_set():
raise Exception("Connect timed out after 15 seconds.")
self.mqtt_client_events.disconnected_event.wait() # yes, forever
def start(self):
"""Public start method used by callers to start the client."""
self.thread = Thread(target=self._start)
self.thread.start()
def stop(self):
"""Stop method used to shutdown the client."""
for topic_handler in self.topic_handlers:
self.client.unsubscribe(topic_handler["topic"])
self.client.disconnect()
self.mqtt_client_events.apps_discovered_event.set()
def publish(self, topic, message):
"""Publish a message to the specified topic."""
self.logger.info("publish: Sending '{}' to topic '{}'".format(
message, topic))
self.client.publish(topic, message)
def subscribe(self, topic, handler, regex=None):
"""Add a handler for a particular topic
The handler is a function that will be called when a message is received on the specified topic.
An optional regex can be provided to filter the topic even more.
"""
if not regex:
regex = topic
self.topic_handlers.append({
'topic': topic,
'handler': handler,
'regex': regex
})
self.client.subscribe(topic)
class MqttBus(IBus):
"""
Args transport from pusher to subscribers via MQTT
"""
_subscribers = dict()
def __init__(self,
host=None,
*args,
publish_time_out=0.05,
publish_waiting=0.1,
mid_max_len=64,
qos=2,
retain=True,
**kwargs):
super().__init__()
if host is None:
host = 'localhost'
self.client = Client(*args, **kwargs)
self.client.connect(host)
self.client.loop_start()
self.client.on_publish = self._on_publish
self._received_messages = []
self.mid_max_len = mid_max_len
self.publish_time_out = publish_time_out
self.publish_waiting = publish_waiting
self.qos = qos
self.retain = retain
@staticmethod
def _loads(payload):
try:
return pickle.loads(payload,
fix_imports=True,
encoding="utf-8",
errors="strict")
except Exception as e:
log.error('mqtt loads error:' + str(e))
return None
@staticmethod
def _dumps(data):
try:
return pickle.dumps(data, protocol=3, fix_imports=True)
except Exception as e:
log.error('mqtt dumps error:' + str(e))
return None
def _on_publish(self, client, userdata, mid):
if len(self._received_messages) >= self.mid_max_len:
self._received_messages.pop(0)
self._received_messages.append(mid)
else:
self._received_messages.append(mid)
def _on_message(self, client, userdata, msg):
args, kwargs = self._loads(msg.payload)
if self._subscribers.get(msg.topic):
for fn in self._subscribers[msg.topic]:
fn(*args, **kwargs)
else:
log.warning(
'mqtt Callback _subscribers is not set for topic={}'.format(
msg.topic))
def subscribe(self, topic, fn=None):
"""
Add function fn to subscribers list
"""
if fn:
assert hasattr(fn, '__call__')
self.client.subscribe(topic, qos=self.qos)
self.client.message_callback_add(
topic, lambda client, userdata, msg: self._on_message(
client, userdata, msg))
if self._subscribers.get(topic) is None:
self._subscribers[topic] = [fn]
log.info('mqtt Function {} subscribed to topic "{}"'.format(
fn.__name__, topic))
else:
self._subscribers[topic].append(fn)
log.info('mqtt Function {} subscribed to topic "{}"'.format(
fn.__name__, topic))
else:
self.client.message_callback_remove(topic)
self.client.unsubscribe(topic)
def push(self, topic, *args, **kwargs):
"""
Call all the subscribers
"""
_args = [arg for arg in args if isinstance(arg, (float, int, str))]
_kwargs = {
key: arg
for key, arg in kwargs.items()
if isinstance(arg, (float, int, str))
}
if _args or _kwargs:
log.debug(
'pushed topic "{}" with args {} and kwargs {} and {}'.format(
topic, str(_args), str(_kwargs),
str(set(kwargs.keys()) - set(_kwargs.keys()))))
send_data = self._dumps((args, kwargs))
log.info('mqtt Pushing data with size={} to topic={}...'.format(
len(send_data), topic))
rc, mid = self.client.publish(topic,
send_data,
qos=self.qos,
retain=self.retain)
publish_time = time.time()
while mid not in self._received_messages:
time.sleep(self.publish_time_out)
if (time.time() - publish_time) >= self.publish_waiting:
break
if rc == 0:
log.info('mqtt Pushed data to topic={} with rc {}'.format(
topic, rc))
else:
log.warning('mqtt Not pushed data to topic={} with rc {}'.format(
topic, rc))
class JMSClient(object):
"""Class JMSClient
"""
_mh = None
_client = None
_host = None
_port = None
_user = None
_passw = None
_verbose = None
_is_connected = None
_messages = []
def __init__(self, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
verbose (bool): verbose mode
"""
try:
self._mh = MasterHead.get_head()
self._client = Client()
self._client.on_message = self._on_message
self._verbose = verbose
if (self._verbose):
self._client.on_log = self._on_log
except MQTTException as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
@property
def client(self):
""" MQTT client property getter """
return self._client
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def verbose(self):
""" verbose property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def _on_log(self, client, obj, level, string):
""" Callback for on_log event """
print(string)
def _on_message(self, client, obj, msg):
""" Callback for on_message event """
self._messages.append(msg.payload.decode())
def connect(self, host, port=1883, user=None, passw=None, timeout=10):
"""Method connects to server
Args:
host (str): hostname
port (str): port
user (str): username
passw (str): password
timeout (int): timeout
Returns:
bool: result
Raises:
event: jms_before_connect
event: jms_after_connected
"""
try:
msg = 'host:{0}, port:{1}, user:{2}, passw:{3}, timeout:{4}'.format(
host, port, user, passw, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connecting', msg), self._mh.fromhere())
ev = event.Event(
'jms_before_connect', host, port, user, passw, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
timeout = ev.argv(4)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
if (self._user != None):
self._client.username_pw_set(self._user, self._passw)
setdefaulttimeout(timeout)
self._client.connect(self._host, self._port)
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_connected'), self._mh.fromhere())
ev = event.Event('jms_after_connect')
self._mh.fire_event(ev)
return True
except (MQTTException, error, ValueError) as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_disconnecting'), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return False
else:
self._client.disconnect()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_disconnected'), self._mh.fromhere())
return True
except (MQTTException, error, ValueError) as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def send(self, destination_name, message):
"""Method sends message
Args:
destination_name (str): topic name
message (str): message
Returns:
bool: result
Raises:
event: jms_before_send
event: jms_after_send
"""
try:
msg = 'destination_name:{0}, message:{1}'.format(
destination_name, message)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_sending_msg', msg), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return False
ev = event.Event('jms_before_send', destination_name, message)
if (self._mh.fire_event(ev) > 0):
destination_name = ev.argv(0)
message = ev.argv(1)
if (ev.will_run_default()):
res, id = self._client.publish(destination_name, message)
if (res != 0):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_jms_sending_error'), self._mh.fromhere())
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_msg_sent'), self._mh.fromhere())
ev = event.Event('jms_after_send')
self._mh.fire_event(ev)
return True
except (MQTTException, error, ValueError) as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return False
def receive(self, destination_name, cnt=1, timeout=10):
"""Method receives messages
Args:
destination_name (str): queue name
cnt (int): count of messages
timeout (int): timeout to receive message
Returns:
list: messages
Raises:
event: jms_before_receive
event: jms_after_receive
"""
try:
msg = 'destination_name:{0}, count:{1}'.format(
destination_name, cnt)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_receiving_msg', msg), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_jms_not_connected'), self._mh.fromhere())
return None
ev = event.Event('jms_before_receive', destination_name, cnt)
if (self._mh.fire_event(ev) > 0):
destination_name = ev.argv(0)
cnt = ev.argv(1)
if (ev.will_run_default()):
res, id = self._client.subscribe(destination_name)
if (res != 0):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_jms_sending_error'), self._mh.fromhere())
return None
res = 0
cnt_before = 0
start = time()
while (res == 0):
res = self._client.loop()
cnt_after = len(self._messages)
if (cnt_after > cnt_before and cnt_after < cnt):
cnt_before = cnt_after
elif (cnt_after == cnt or time() > start + timeout):
res = -1
messages = self._messages
self._client.unsubscribe(destination_name)
self._messages = []
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_jms_msg_received', len(messages)), self._mh.fromhere())
ev = event.Event('jms_after_receive')
self._mh.fire_event(ev)
return messages
except (MQTTException, error, ValueError) as ex:
self._mh.demsg('htk_on_error', ex, self._mh.fromhere())
return None
class BroadcastBridge:
"""
Main Class of the Broadcast Bridge
"""
def __init__(self, broker_host: str="localhost", broker_port: int=1883, http_host="localhost", http_port=8080,
mqtt_prefix='broadcast_bridge/', persistence_file=None):
self.mqtt_client = Client()
self.mqtt_client.connect(broker_host, broker_port)
self.mqtt_client.loop_start()
self.mqtt_client.on_message = self.__on_mqtt_message
self.__mqtt_prefix = mqtt_prefix
self.http_port = http_port
self.http_host = http_host
self.available_modules = {}
self.instances = {}
self.log = logging.getLogger('mqttbb')
self.log.setLevel(logging.DEBUG)
self.__persistence_file = persistence_file
self.__read_persistence_file()
self.__init_http_server()
def __read_persistence_file(self):
if self.__persistence_file is None:
return
try:
persistence = json.load(open(self.__persistence_file, 'r'))
for instance in persistence:
if instance['module'] not in self.available_modules:
success = self.add_module(instance['module'])
if not success:
raise PersistenceException('Module "{}" is needed for load persistence but '
'seems not to be installed.'
.format(instance['module']))
self.add_module_instance(instance['module'],
instance['config'],
instance['short_description'],
instance['uuid'])
except FileNotFoundError:
return
def __update_persistence_file(self):
if self.__persistence_file is None:
return
instances = []
for i, instance in self.instances.items():
instances.append({
'module': instance['module'],
'config': instance['instance'].config_values,
'uuid': i,
'short_description': instance['short_description'],
})
json.dump(instances, open(self.__persistence_file, 'w+'))
def __on_mqtt_message(self, client, user_data, message: MQTTMessage):
"""
receives MQTT message and forward it to the right module
:param client:
:param user_data:
:param message:
:return:
"""
if message.topic.startswith(self.__mqtt_prefix):
last = message.topic[len(self.__mqtt_prefix):]
match = re.match(r'^(.*)/(.*)$', last)
if match.group(1) in self.instances:
t = threading.Thread(target=self.instances[match.group(1)]['instance'].on_message,
args=(match.group(2), message.payload))
t.start()
def add_module(self, name):
"""
Add module to bridge and check if the given module exists and is compatible.
:param name: name of the module
:return: Success state
"""
try:
module = __import__(name)
module_class = module.Module
if module_class == BaseModule:
self.log.error('The module class seems to be a copy of base Module in "{}"'.format(name))
if not issubclass(module.Module, BaseModule):
self.log.error('Given module "{}" is not valid: Class Module is not inherit from mqttbb.modules.Module'
.format(name))
for config in module_class.config:
if 'name' not in config:
self.log.error('Config definition error, missing name in module "{}"'.format(name))
return False
if 'title' not in config:
config['title'] = config['name']
if 'type' not in config:
self.log.error('Config definition error, missing type in module "{}"'.format(name))
return False
if name in self.available_modules:
self.log.warning('Module already loaded.')
return False
self.available_modules[name] = module_class
return True
except ModuleNotFoundError:
self.log.error('Module not found "{}"'.format(name))
except AttributeError:
self.log.error('Given module "{}" is not valid: No class "Module" found'.format(name))
return False
def add_module_instance(self, name, config, short_description="", instance_id=None):
"""
Add module instance to broadcast bridge
:param name: name of the module
:param config: config object for the module
:param short_description: short description
:param instance_id: id of the instance if already exists
:return: void
"""
if not isinstance(config, dict):
raise TypeError("config must be dict")
if name not in self.available_modules:
raise ModuleNotFoundError()
if instance_id is None:
instance_id = str(uuid.uuid4())
if instance_id in self.instances:
raise InstanceAlreadyExists('Instance id is already registered.')
def on_path_register(path):
full_path = "{}{}/{}".format(self.__mqtt_prefix, instance_id, path)
self.mqtt_client.subscribe(full_path)
self.log.debug('Module "{}" registered topic "{}"'.format(name, full_path))
def on_publish(path, payload, retain=False):
full_path = "{}{}/{}".format(self.__mqtt_prefix, instance_id, path)
self.mqtt_client.publish(full_path, payload, retain=retain)
self.instances[instance_id] = {
'instance': self.available_modules[name](config,
on_path_register=on_path_register,
on_publish=on_publish),
'short_description': short_description,
'module': name
}
self.__update_mqtt_service_meta()
def remove_module_instance(self, uid):
if uid not in self.instances:
return
instance = self.instances[uid]['instance']
for path in instance.paths:
full_path = "{}{}/{}".format(self.__mqtt_prefix, uid, path)
self.mqtt_client.unsubscribe(full_path)
instance.shutdown()
del self.instances[uid]
self.__update_mqtt_service_meta()
def __update_mqtt_service_meta(self):
instances = []
for i, instance in self.instances.items():
instances.append({
'id': i,
'name': instance['instance'].name,
'short_description':instance['short_description']
})
self.mqtt_client.publish('{}meta'.format(self.__mqtt_prefix), json.dumps(instances), retain=True)
self.__update_persistence_file()
def __init_http_server(self):
"""
initialize http server (creating routes and config)
:return: void
"""
bootstrap_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'static/bootstrap/dist/css/bootstrap.min.css')
template_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static/templates/')
view = functools.partial(bottle.jinja2_view, template_lookup=[template_dir])
@bottle.get('/css/bootstrap')
def bootstrap():
return bottle.static_file(os.path.basename(bootstrap_path), os.path.dirname(bootstrap_path))
@bottle.get('/css/bootstrap.min.css.map')
def bootstrap_map():
return bottle.static_file(os.path.basename(bootstrap_path)+'.map', os.path.dirname(bootstrap_path))
@bottle.get('/')
@view('overview.j2', )
def index():
available_modules = []
for i, module in self.available_modules.items():
available_modules.append({
'name': module.name,
'id': i,
})
instances = []
for i, instance in self.instances.items():
instances.append({
'short_description': instance['short_description'],
'uid': i,
'module': instance['module'],
'module_name': instance['instance'].name,
})
return {'available_modules': available_modules, 'instances': instances}
@bottle.get('/info/<uid>')
@view('info.j2')
def info(uid):
if uid not in self.instances:
bottle.redirect('/')
instance = self.instances[uid]
return {
'short_description': instance['short_description'],
'uid': uid,
'module': instance['module'],
'module_name': instance['instance'].name,
'config': instance['instance'].config,
'config_values': instance['instance'].config_values,
'paths': instance['instance'].paths.values(),
'path_prefix': '{}{}/'.format(self.__mqtt_prefix, uid),
}
@bottle.route('/delete/<uid>', ['GET', 'POST'])
@view('delete.j2')
def delete(uid):
if uid not in self.instances:
bottle.redirect('/')
return
if bottle.request.method == 'POST':
self.remove_module_instance(uid)
bottle.redirect('/')
return
instance = self.instances[uid]
return {
'short_description': instance['short_description'],
'uid': uid,
'module': instance['module'],
'module_name': instance['instance'].name,
}
@bottle.get('/add', ['GET', 'POST'])
@view('form.j2')
def add():
errors = {}
module = bottle.request.GET.get('module_id')
if module is None or module not in self.available_modules:
return bottle.redirect('/')
config = self.available_modules[module].config
config_values = {}
short_description = ""
if bottle.request.method == 'POST':
form_dict = dict(bottle.request.forms)
short_description = form_dict['short_description']
if short_description == '':
errors['short_description'] = 'Angabe erforderlich'
for element in config:
if element['name'] in form_dict:
config_values[element['name']] = form_dict[element['name']]
try:
if len(errors) == 0:
self.add_module_instance(module, form_dict, short_description=short_description)
bottle.redirect('/')
except ConfigError as e:
errors[e.config_name] = e.message
return {'errors': errors, 'config': config, 'module': module, 'values': config_values,
'short_description': short_description}
def http_loop(self):
"""
start http server as blocking loop
:return:
"""
bottle.run(host=self.http_host, port=self.http_port)
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__log = log
self.config = config
self.__connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__gateway = gateway
self.__broker = config.get('broker')
self.__mapping = config.get('mapping')
self.__server_side_rpc = config.get('serverSideRpc')
self.__service_config = {
"connectRequests": None,
"disconnectRequests": None
}
self.__attribute_updates = []
self.__get_service_config(config)
self.__sub_topics = {}
client_id = ''.join(
random.choice(string.ascii_lowercase) for _ in range(23))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] or self.__broker[
"security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: %s",
self.get_name(), e)
self._client.tls_insecure_set(False)
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self.__subscribes_sent = {} # For logging the subscriptions
self._client.on_disconnect = self._on_disconnect
self._client.on_log = self._on_log
self._connected = False
self.__stopped = False
self.daemon = True
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
time.sleep(1)
except Exception as e:
self.__log.exception(e)
time.sleep(10)
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
else:
time.sleep(1)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic):
message = self._client.subscribe(topic)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, rc, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if rc == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
for mapping in self.__mapping:
try:
converter = None
if mapping["converter"]["type"] == "custom":
try:
module = TBUtility.check_and_import(
self.__connector_type,
mapping["converter"]["extension"])
if module is not None:
self.__log.debug(
'Custom converter for topic %s - found!',
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error(
"\n\nCannot find extension module for %s topic.\n\Please check your configuration.\n",
mapping["topicFilter"])
except Exception as e:
self.__log.exception(e)
else:
converter = JsonMqttUplinkConverter(mapping)
if converter is not None:
regex_topic = TBUtility.topic_to_regex(
mapping.get("topicFilter"))
if not self.__sub_topics.get(regex_topic):
self.__sub_topics[regex_topic] = []
self.__sub_topics[regex_topic].append(
{converter: None})
# self._client.subscribe(TBUtility.regex_to_topic(regex_topic))
self.__subscribe(mapping["topicFilter"])
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
except Exception as e:
self.__log.exception(e)
try:
for request in self.__service_config:
if self.__service_config.get(request) is not None:
for request_config in self.__service_config.get(
request):
self.__subscribe(request_config["topicFilter"])
except Exception as e:
self.__log.error(e)
else:
if rc in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), rc, result_codes[rc])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self.__log.debug('"%s" was disconnected.', self.get_name())
def _on_log(self, *args):
self.__log.debug(args)
# pass
def _on_subscribe(self, client, userdata, mid, granted_qos):
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
except Exception as e:
self.__log.exception(e)
def __get_service_config(self, config):
for service_config in self.__service_config:
if service_config != "attributeUpdates" and config.get(
service_config):
self.__service_config[service_config] = config[service_config]
else:
self.__attribute_updates = config[service_config]
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
regex_topic = [
regex for regex in self.__sub_topics
if fullmatch(regex, message.topic)
]
if regex_topic:
try:
for regex in regex_topic:
if self.__sub_topics.get(regex):
for converter_value in range(
len(self.__sub_topics.get(regex))):
if self.__sub_topics[regex][converter_value]:
for converter in self.__sub_topics.get(
regex)[converter_value]:
converted_content = converter.convert(
message.topic, content)
if converted_content:
try:
self.__sub_topics[regex][
converter_value][
converter] = converted_content
except Exception as e:
self.__log.exception(e)
self.__gateway.send_to_storage(
self.name, converted_content)
self.statistics['MessagesSent'] += 1
else:
continue
else:
self.__log.error(
'Cannot find converter for topic:"%s"!',
message.topic)
return
except Exception as e:
log.exception(e)
return
elif self.__service_config.get("connectRequests"):
connect_requests = [
connect_request for connect_request in
self.__service_config.get("connectRequests")
]
if connect_requests:
for request in connect_requests:
if request.get("topicFilter"):
if message.topic in request.get("topicFilter") or\
(request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)):
founded_device_name = None
if request.get("deviceNameJsonExpression"):
founded_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"],
content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
founded_device_name = search(
device_name_expression, message.topic)
if founded_device_name is not None and founded_device_name not in self.__gateway.get_devices(
):
self.__gateway.add_device(
founded_device_name, {"connector": self})
else:
self.__log.error(
"Cannot find connect request for device from message from topic: %s and with data: %s",
message.topic, content)
else:
self.__log.error(
"\"topicFilter\" in connect requests config not found."
)
else:
self.__log.error("Connection requests in config not found.")
elif self.__service_config.get("disconnectRequests") is not None:
disconnect_requests = [
disconnect_request for disconnect_request in
self.__service_config.get("disconnectRequests")
]
if disconnect_requests:
for request in disconnect_requests:
if request.get("topicFilter") is not None:
if message.topic in request.get("topicFilter") or\
(request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)):
founded_device_name = None
if request.get("deviceNameJsonExpression"):
founded_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"],
content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
founded_device_name = search(
device_name_expression, message.topic)
if founded_device_name is not None and founded_device_name in self.__gateway.get_devices(
):
self.__gateway.del_device(founded_device_name)
else:
self.__log.error(
"Cannot find connect request for device from message from topic: %s and with data: %s",
message.topic, content)
else:
self.__log.error(
"\"topicFilter\" in connect requests config not found."
)
else:
self.__log.error("Disconnection requests in config not found.")
elif message.topic in self.__gateway.rpc_requests_in_progress:
self.__gateway.rpc_with_reply_processing(message.topic, content)
else:
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def on_attributes_update(self, content):
attribute_updates_config = [
update for update in self.__attribute_updates
]
if attribute_updates_config:
for attribute_update in attribute_updates_config:
if match(attribute_update["deviceNameFilter"], content["device"]) and \
content["data"].get(attribute_update["attributeFilter"]):
topic = attribute_update["topicExpression"]\
.replace("${deviceName}", content["device"])\
.replace("${attributeKey}", attribute_update["attributeFilter"])\
.replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]])
data = ''
try:
data = attribute_update["valueExpression"]\
.replace("${attributeKey}", attribute_update["attributeFilter"])\
.replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]])
except Exception as e:
self.__log.error(e)
self._client.publish(topic, data).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Not found deviceName by filter in message or attributeFilter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# Subscribe to RPC response topic
if rpc_config.get("responseTopicExpression"):
topic_for_subscribe = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", content["device"]) \
.replace("${methodName}", content["data"]["method"]) \
.replace("${requestId}", str(content["data"]["id"])) \
.replace("${params}", content["data"]["params"])
if rpc_config.get("responseTimeout"):
timeout = time.time() * 1000 + rpc_config.get(
"responseTimeout")
self.__gateway.register_rpc_request_timeout(
content, timeout, topic_for_subscribe,
self.rpc_cancel_processing)
# Maybe we need to wait for the command to execute successfully before publishing the request.
self._client.subscribe(topic_for_subscribe)
else:
self.__log.error(
"Not found RPC response timeout in config, sending without waiting for response"
)
# Publish RPC request
if rpc_config.get("requestTopicExpression") is not None\
and rpc_config.get("valueExpression"):
topic = rpc_config.get("requestTopicExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
data_to_send = rpc_config.get("valueExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
try:
self._client.publish(topic, data_to_send)
self.__log.debug(
"Send RPC with no response request to topic: %s with data %s",
topic, data_to_send)
if rpc_config.get("responseTopicExpression") is None:
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
except Exception as e:
self.__log.exception(e)
def rpc_cancel_processing(self, topic):
self._client.unsubscribe(topic)
class Service:
"""Service that sends messages via Telegram bot"""
_mqtt_client: Dict[str, Client] = {}
_service_list: Dict[str, dict] = {}
_broker: DefaultDict[str, set] = defaultdict(set)
_broker_port: Dict[str, int] = {}
_topic: DefaultDict[str, set] = defaultdict(set)
_update_thread: Timer = None
chat_id_topic = "labsw4/telegram/user/chat_id"
_user_list: Dict[str, dict] = {}
# Telegram
bot: Bot = Bot(token=TELEGRAM_TOKEN)
chat_id_list = set()
def __init__(self):
"""
Instantiate the service
"""
self.service = Client(
client_id=f"TelegramService{randrange(1, 100000)}")
self.service.on_message = self.my_on_message
self.char_id_lock = Lock()
self.service_lock = Lock()
def _update(self, service_list: List[dict]):
"""
Update reserved dicts
:param service_list: service list to add
"""
for alarm in service_list:
service = alarm["serviceID"]
broker = alarm["end_points"]["MQTT"]["broker"]["ip"]
port = alarm["end_points"]["MQTT"]["broker"]["port"]
topics = {
topic
for topic in alarm["end_points"]["MQTT"]["subscribe"]
if "alarm_temperature" in topic
}
self._service_list[service] = {"ip": broker, "topics": topics}
self._broker_port[broker] = port
self._broker[broker].update({service})
self._topic[broker].update(topics)
print(f"[{time.ctime()}] SERVICE {service} ONLINE")
def setup(self, first_time: bool = True):
"""
Setup the service
"""
try:
print(
f"[{time.ctime()}] EXTRACT info about all the services registered"
)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all"
)
while result.status_code != 200:
print(
f"[{time.ctime()}] WARNING no service registered found, retrying after 30 seconds"
)
time.sleep(30)
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}"
f"/catalog/services/all")
print(f"[{time.ctime()}] SERVICES found")
data = json.loads(result.content.decode())
print(
f"[{time.ctime()}] EXTRACT from the services list info about Alarm_Temperature"
)
service_list = [alarm for alarm in data if find_service(alarm)]
if len(service_list) == 0:
print(
f"[{time.ctime()}] No ServiceTemperatureAlarm found... retrying in 10 seconds"
)
time.sleep(10)
self.setup(first_time)
return
except KeyboardInterrupt:
print(f"[{time.ctime()}] EXIT")
if first_time:
sys.exit()
return
if first_time:
# Connect the service
self.service.connect(host=SERVICE_BROKER_PORT["ip"],
port=SERVICE_BROKER_PORT["port"])
print(f"[{time.ctime()}] SERVICE CONNECTED to "
f"broker: {SERVICE_BROKER_PORT['ip']} "
f"on port: port={SERVICE_BROKER_PORT['port']}")
self.service.subscribe(self.chat_id_topic)
# Update registered services and subscribe to all the topics
with self.service_lock:
self._update(service_list)
self.subscribe(service_list)
# Register inside the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}")
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
def start(self):
"""
Start the service.
"""
# Setup the service
self.setup()
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
# Schedule the update of the registration
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
try:
# Run the service forever
self.service.loop_forever()
except KeyboardInterrupt:
self.stop()
def my_on_message(self, client: Client, userdata: Any, msg: MQTTMessage):
"""
Receive new chat ids. Receive alarm status and check if it is true. If so, send messages
via Telegram Bot.
:param client: MQTT client
:param userdata: They could be any type
:param msg: MQTT message
"""
data = json.loads(msg.payload.decode())
if msg.topic == self.chat_id_topic:
with self.char_id_lock:
self.chat_id_list.add(data["chat_id"])
return
if data["alarm"]:
message_body = f"{data['device']} is out of range of good functioning{SIGNATURE}"
with self.char_id_lock:
for chat_id in self.chat_id_list:
self.bot.sendMessage(chat_id=chat_id, text=message_body)
print(
f"[{time.ctime()}] TELEGRAM MESSAGE SENT TO CHAT_ID: {chat_id}"
)
else:
return
def update_registration(self):
"""
Update service registration to the catalog
"""
print(
f"[{time.ctime()}] EXTRACT info about all the services registered")
result: requests.Response = requests.get(
url=
f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all"
)
# No device found
if result.status_code != 200:
self.reset()
return
data = json.loads(result.content.decode())
service_list = [alarm for alarm in data if find_service(alarm)]
if len(service_list) == 0:
self.reset()
return
# New Devices Found
new_services = {alarm["serviceID"] for alarm in service_list}
# Old devices list
old_services = set(self._service_list.keys())
# Check if there are update to do
if old_services == new_services:
# No update, ping the catalog
print(
f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}"
)
requests.post(
f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services',
data=SERVICE_INFO,
headers={"Content-Type": "application/json"})
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
return
with self.service_lock:
# Delete inactive devices
delete_list = {
alarm: self._service_list[alarm]
for alarm in self._service_list if alarm not in new_services
}
# Unsubscribe from devices
self.unsubscribe(delete_list)
# Update registered devices and subscribe
new_services = [
alarm for alarm in service_list
if alarm["serviceID"] not in old_services
]
self._update(new_services)
self.subscribe(new_services)
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def reset(self):
"""
Reset the service
"""
with self.service_lock:
self._clear()
self.setup(first_time=False)
# Start all the external mqtt_clients
for broker in self._mqtt_client:
self._mqtt_client[broker].loop_start()
self._update_thread = Timer(60, self.update_registration)
self._update_thread.start()
def unsubscribe(self, service_list: dict):
"""
Unsubscribe from services
:param service_list: service list
"""
for alarm in service_list:
print(f"[{time.ctime()}] SERVICE {alarm} OFFLINE")
broker = service_list[alarm]["ip"]
topics = service_list[alarm]["topics"]
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
else:
for topic in topics:
self._mqtt_client[broker].unsubscribe(topic)
print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}")
# Update topics
self._topic[broker] = self._topic[broker].difference(topics)
# Disconnect from external broker
if len(self._topic[broker]) == 0:
if broker != SERVICE_BROKER_PORT["ip"]:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(
f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
del self._mqtt_client[broker]
# Clean
del self._topic[broker]
del self._broker_port[broker]
# Delete device
del self._service_list[alarm]
self._broker[broker].discard(alarm)
def subscribe(self, service_list: List[dict]):
"""
Subscribe to services
:param service_list: list of services
"""
for alarm in service_list:
broker = alarm["end_points"]["MQTT"]["broker"]["ip"]
topics = {
topic
for topic in alarm["end_points"]["MQTT"]["subscribe"]
}
# Subscribe to topics
if broker == SERVICE_BROKER_PORT["ip"]:
for topic in topics:
self.service.subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
# Connect to external brokers
else:
if broker in self._mqtt_client:
# Connection already made
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
else:
# Connect to the broker
self._mqtt_client[broker] = Client(
f"EmailAlarmService{randrange(1, 1000000)}")
self._mqtt_client[broker].on_message = self.my_on_message
self._mqtt_client[broker].connect(
host=broker, port=self._broker_port[broker])
for topic in topics:
self._mqtt_client[broker].subscribe(topic)
print(f"[{time.ctime()}] SUBSCRIBED to : {topic}")
def _clear(self):
"""
Clear all the data stored
"""
# Unsubscribe from all topics
self.unsubscribe(self._service_list.copy())
# Disconnect from all the external brokers
for broker in self._mqtt_client:
self._mqtt_client[broker].disconnect()
self._mqtt_client[broker].loop_stop()
print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}")
# Clear
self._mqtt_client.clear()
self._service_list.clear()
self._broker.clear()
self._broker_port.clear()
self._topic.clear()
def stop(self):
"""
Stop the service
"""
# Stop the schedule
self._update_thread.cancel()
# Wait for the threads to finish
if self._update_thread.is_alive():
self._update_thread.join()
# Disconnect the clients
print(f"[{time.ctime()}] SHUTTING DOWN")
for client in self._mqtt_client:
self._mqtt_client[client].disconnect()
self._mqtt_client[client].loop_stop()
# Disconnect the service
self.service.disconnect()
print(f"[{time.ctime()}] EXIT")
