class MQTTValuePub(object):
'''
Use MQTT to send values on network
pip install paho-mqtt
'''
def __init__(self, name, broker="iot.eclipse.org"):
from paho.mqtt.client import Client
self.name = name
self.message = None
self.client = Client()
print("connecting to broker", broker)
self.client.connect(broker)
self.client.loop_start()
print("connected.")
def run(self, values):
packet = {"name": self.name, "val": values}
p = pickle.dumps(packet)
z = zlib.compress(p)
self.client.publish(self.name, z)
def shutdown(self):
self.client.disconnect()
self.client.loop_stop()
class MqttController:
def __init__(self):
self.client = Client()
self.client.on_connect = self.__on_connect
self.client.on_message = self.__on_message
self.evtCallback: Optional[EventCallback] = None
def connect(self, connectCallback, host, port=1883, keepalive=60):
log.info("Trying to connect MQTT client.")
self.connectCallback = connectCallback
self.client.connect(host, port, keepalive)
self.client.loop_start()
def disconnect(self):
self.client.loop_stop()
self.client.disconnect()
def setCallback(self, callback: EventCallback):
self.evtCallback = callback
def delCallback(self):
self.evtCallback = None
def __on_connect(self, client, userdata, flags, rc):
log.info("MQTT client connected, registering subscriptions.")
self.client.subscribe("/baresip/event")
self.connectCallback()
def __on_message(self, client, userdata, msg: MQTTMessage):
log.info("MQTT message received for path=%s.", msg.topic)
log.debug("payload=%s", msg.payload)
# parse message
try:
msgObj = json.loads(msg.payload)
evtType = EventType[msgObj["type"]]
# notify respective callback
cb = self.evtCallback
if cb is not None:
log.debug("Calling event callback in a thread.")
t = threading.Thread(target=lambda: cb(evtType, msgObj))
t.start()
else:
log.debug("No callback registered.")
except JSONDecodeError:
log.error("Received invalid JSON message.")
except KeyError:
log.warn("Received unhandled type code or type code is missing.")
def send_command(self, msg):
log.debug("Trying to send message %s to phone.", msg)
info = self.client.publish("/baresip/command/", msg)
log.debug("Waiting for publish")
log.debug(info.rc)
info.wait_for_publish()
if info.rc != MQTT_ERR_SUCCESS:
log.error("Failed to publish message")
else:
log.debug("Message sent successfully.")
def main(broker, port, interface):
"""Scan for all MiPlant devicesself.
Send via MQTT if broker is provided. Print to console otherwise.
"""
sensors = discover_and_scan(interface)
if broker:
client = Client()
client.connect(broker, port)
client.loop_start()
for sensor in sensors:
payload = json.dumps({
'battery': sensor.battery,
'temperature': sensor.temperature,
'light': sensor.light,
'moisture': sensor.moisture,
'conductivity': sensor.conductivity
})
channel = "sensors/plantsensors/{addr}".format(addr=sensor.address)
client.publish(channel, json.dumps(payload))
client.loop_stop()
else:
for sensor in sensors:
print('--------------------------')
print('Address: {addr}'.format(addr=sensor.address))
print('Battery level: {bat}'.format(bat=sensor.battery))
print('Temperature: {temp} °C'.format(temp=sensor.temperature))
print('Light: {light} lx'.format(light=sensor.light))
print('Moisture: {moist}'.format(moist=sensor.moisture))
print(
'Conductivity: {cond} µS/cm'.format(cond=sensor.conductivity))
class MqttBridge(BaseBridge):
def __init__(self):
super(MqttBridge, self).__init__()
self.client = MqttClient()
self.client.on_message = self.onMqttMessage
self.client.on_connect = self.onConnect
def publishApiMessage(self, heat_pump_id, base_topic, topic, value):
self.client.publish(base_topic + topic, value)
# noinspection PyUnusedLocal
def onConnect(self, client, userdata, flags, rc):
# type: (MqttBridge, MqttClient, object, dict, object) -> None
topics = []
for topic in self.binding.topics:
topics.append((topic, 0))
if len(topics) == 0:
return
print "mqtt subscribing to topics: ", topics
client.subscribe(topics)
# noinspection PyUnusedLocal
def onMqttMessage(self, client, userdata, msg):
# type: (MqttBridge, MqttClient, object, MQTTMessage) -> None
topic = str(msg.topic)
self.binding.onApiMessage(topic, msg.payload)
def start(self):
print "mqtt connect to:", config.MQTT['host']
self.client.connect(config.MQTT['host'])
self.client.loop_start()
def stop(self):
self.client.disconnect()
self.client.loop_stop()
def run(self):
LOG.critical("Starting")
mqtt_channels = {
e.name: e
for e in self.get_channels_by_application("MQTTSubscriber")
}
LOG.critical("mqtt_channels: {}".format(dir(mqtt_channels)))
for channel_name, channel in mqtt_channels.items():
LOG.critical("CHANNEL NAME: {}".format(channel_name))
LOG.critical("CHANNEL: {}".format(dir(channel)))
ip_address = channel.protocol_config.app_specific_config[
'ip_address']
port = channel.protocol_config.app_specific_config['port']
if ip_address not in self.mqtt_clients:
client = MQTTClient()
self.mqtt_clients[ip_address] = client
if not hasattr(client, 'channels'):
setattr(client, 'channels', {})
self.mqtt_clients[ip_address].channels[
channel.protocol_config.app_specific_config['topic']] = channel
def on_message(client, userdata, msg):
""" Default on_message function for tunable logging. """
LOG.debug(
"userdata: {} dup: {} info: {} mid: {} payload: {} qos: {} retain: {} state: {} timestamp: {} topic: {}"
.format(userdata, msg.dup, msg.info, msg.mid, msg.payload,
msg.qos, msg.retain, msg.state, msg.timestamp,
msg.topic))
for ch in client.channels:
if topic_matches_sub(ch, msg.topic):
LOG.critical("{}->{} got {}".format(
client, client.channels[ch].name, msg.payload))
client.channels[ch].put_sample(msg.payload)
self.mqtt_clients[ip_address].on_message = on_message
self.mqtt_clients[ip_address].connect(ip_address, port)
for client in self.mqtt_clients.values():
client.loop_start()
for channel in client.channels:
client.subscribe(channel)
while not self.is_stopped():
time.sleep(0.25)
for client in self.mqtt_clients.values():
client.loop_stop()
LOG.critical("{} HAS BEEN STOPPED.".format(self.name))
def on_disconnect(client: mq.Client, userdata, rc):
"""
Disconnection callback.
"""
if rc == 0:
logger.info("Peacefully disconnected.")
else:
logger.warning("Unexpected disconnection")
client.loop_stop()
logger.info("Stopped client loop")
class MqttClientConnector():
_host = None
_port = None
_brokerAddr = None
_mqttClient = None
#Allow user to use this constructor to pass custom callBack methods
def __init__(self, host, port, on_connect, on_message, on_publish,
on_subscribe):
self._brokerAddr = host + ":" + str(port)
self._host = host
self._port = port
self._mqttClient = Client()
self._mqttClient.on_connect = on_connect
self._mqttClient.on_message = on_message
self._mqttClient.on_publish = on_publish
self._mqttClient.on_subscribe = on_subscribe
def connect(self):
try:
print("Connecting to broker:" + self._brokerAddr + ".....")
self._mqttClient.connect(self._host, self._port, 60)
self._mqttClient.loop_start()
except Exception as e:
print("Cloud not connect to broker " + self._brokerAddr + " " +
str(e))
def disconnect(self):
self._mqttClient.disconnect()
self._mqttClient.loop_stop()
def publishMessage(self, topic, message, qos):
print("Publishing message:" + message + " to broker: " +
self._brokerAddr + " Topic:" + topic)
self._mqttClient.publish(topic, message, qos)
def subscribeTopic(self, topic, qos):
print("Subscribing to topic:" + topic + ".....")
self._mqttClient.subscribe(topic, qos)
class MoodyBLEWrapper(Thread):
def __init__(self, mac, host, ca_cert):
Thread.__init__(self)
self._mac = mac
self._host = host
self._ca_cert = ca_cert
self._running = False
self._mutex = Lock()
self._client = Client(f"Moody{randint(100, 999)}")
self._client.tls_set(ca_certs=ca_cert)
# When receiving data with a delegate, you also receive a characteristic handle
# This is a mapping of those characteristic for later usage
self._handle_mappings = {}
def run(self):
self._running = True
self._connect(host=self._host)
with Peripheral(self._mac) as peripheral:
for service in list(peripheral.getServices())[2:]:
print(service, service.uuid.getCommonName())
char_uuids = [str(c.uuid) for c in service.getCharacteristics()]
name_char = service.getCharacteristics(char_uuids[0])[0]
value_char = service.getCharacteristics(char_uuids[1])[0]
service_name = name_char.read().decode()
self._handle_mappings[value_char.valHandle] = service_name
mqtt_delegate = _MQTTDelegate(client=self._client, client_mutex=self._mutex,
handle_map=self._handle_mappings)
peripheral.withDelegate(mqtt_delegate)
peripheral.writeCharacteristic(value_char.valHandle + 1, b"\x01\x00")
while self._running:
peripheral.waitForNotifications(1)
peripheral.disconnect()
def _connect(self, host, port=None):
if not port:
port = 8883
self._client.connect(host=host, port=port)
self._client.loop_start()
def stop(self):
self._running = False
self._client.loop_stop()
self._client.disconnect()
def main():
client = Client(client_id="SHELL/UTILITY")
client.connect(host=SERVICE_BROKER_PORT["ip"], port=SERVICE_BROKER_PORT["port"])
client.loop_start()
chat_id = int(
input_dialog(
title='Chat ID',
text='Please activate the bot on your phone:'
' https://t.me/SmartHome_IoTbot and type here your'
' telegram chat id .. to obtain it go to '
'https://telegram.me/get_id_bot:').run()
)
client.publish(topic=SERVICE_TOPIC, payload=json.dumps({"chat_id": chat_id}))
client.loop_stop()
client.disconnect()
def on_connect(_client: mqtt.Client, _userdata: dict, _flags: dict,
rc: int) -> None:
"""
Callback function when a MQTT client connects to the server
Check if the connection is succeeded.
Stop the loop regardless of whether the connection is succeeded or not.
The rc argument is a connection result.
"""
_client.disconnect()
_client.loop_stop()
if rc != mqtt.MQTT_ERR_SUCCESS:
demisto.info(mqtt.connack_string(rc))
raise paho.mqtt.MQTTException(mqtt.connack_string(rc))
else:
demisto.info("connection was succeeded for test")
class MqttPlugin(plugins.SimplePlugin):
"""
Plugin that listens to MQTT topics and publishes the payload
'unmodified' to a channel on the CherryPy bus. The cherrypy channel name
is the same as the MQTT topic
Requires PAHO
"""
def __init__(
self, bus: wspbus, broker: str, port: int, topic_list: Union[str, List[str]]
) -> None:
"""
Setup the plugin
:param bus: Cherrypy internal Bus
:param broker: Mqtt broker
:param port: Port of the Mqtt broker
:param topic_list: topic to subscribe
"""
# Cherrypy plugins.SimplePlugin doesn't accept the super().__init__()
# Inside cherrypy's docs it's like this
# https://docs.cherrypy.org/en/latest/extend.html#create-a-plugin
plugins.SimplePlugin.__init__(self, bus)
self.broker = broker
self.port = port
self.topic_list = topic_list
self.client = Client(client_id=f"Catalog{randrange(1, 100000)}")
self.client.on_message = Bus(bus).my_on_message
def start(self):
self.bus.log("Setup mqttcherrypy")
self.client.connect(self.broker, self.port)
self.bus.log(f"Connected to broker: {self.broker} port: {self.port})")
self.client.loop_start()
self.client.subscribe(self.topic_list)
self.bus.log(f"Subscribed to {self.topic_list}")
def stop(self):
self.bus.log("Shut down mqttcherrypy")
self.client.unsubscribe(self.topic_list)
self.bus.log(f"Unsubscribed from {self.topic_list}")
self.client.loop_stop(force=True)
self.client.disconnect()
self.bus.log(f"Disconnected from: {self.broker} port: {self.port}")
class MqttClientConnector():
_host = None
_port = None
_brokerAddr = None
_mqttClient = None
def __init__(self, host, port, on_connect, on_message, on_publish,
on_subscribe):
self._brokerAddr = host + ":" + str(port)
self._host = host
self._port = port
self._mqttClient = Client()
self._mqttClient.on_connect = on_connect
self._mqttClient.on_message = on_message
self._mqttClient.on_publish = on_publish
self._mqttClient.on_subscribe = on_subscribe
def connect(self):
try:
# make a connection with broker
print("Connect to broker:" + self._brokerAddr + ".....")
self._mqttClient.connect(self._host, self._port, 60)
self._mqttClient.loop_start()
except Exception as e:
print("Broker error, Connected failed" + str(e))
def disconnect(self):
# disconnect with broker
self._mqttClient.disconnect()
self._mqttClient.loop_stop()
def publishMessage(self, topic, message, qos):
# publish msg to remote boroker
print("Start publishing message : " + message)
self._mqttClient.publish(topic, message, qos)
def subscribeTopic(self, topic, qos):
# subscribe msg to remote boroker
print("Start subscribing to topic: " + topic)
self._mqttClient.subscribe(topic, qos)
def mqtt_client(server):
# ESP8266: import ubinascii, utime
import time
machine_id = b'ESP8266-999999'
# setup Mosquitto
# on Python3:
# pip install paho-mqtt
from paho.mqtt.client import Client as MQTTClient
mqttc = MQTTClient(machine_id, server)
mqttc.connect(server)
mqttc.on_message = callback_function_PahoMQTT
mqttc.subscribe('test/#')
mqttc.loop_start()
while True:
time.sleep(1)
mqttc.loop_stop()
def main():
p = argparse.ArgumentParser()
p.add_argument('-a',
'--account-path',
help='Path to JSON file containing Google account details')
args = p.parse_args()
google_session = get_google_session(args.account_path)
brokers = {}
clients = []
reporter = Reporter(google_session)
for device_id, device, desc in generate_devices_from_config():
if not device.url.startswith('mqtt://'):
continue
device_name, host, port, username, password = parse_mqtt_url(
device.url)
broker = (host, port, username, password)
brokers.setdefault(broker, {}).setdefault(device_name,
[]).append(device)
for (host, port, username, password), devices in brokers.items():
client = Client(clean_session=True, userdata=devices)
client.on_message = reporter.on_msg
if username is not None:
client.username_pw_set(username, password)
client.connect(host, port)
for device_name in devices:
# log(f'{device_name} --> STATE')
client.subscribe(f'stat/{device_name}/RESULT')
client.publish(f'cmnd/{device_name}/state')
client.loop_start()
clients.append(client)
try:
while 1:
time.sleep(5)
except KeyboardInterrupt:
print('Ctrl-C')
for client in clients:
client.loop_stop()
def simple_publish(self, payload, routing_key='', **kwargs):
sp_client_id = 'spc-' + str(uuid.uuid4())
sp_client = Client(client_id=sp_client_id, transport="websockets")
sp_client.ws_set_options(self.endpoint)
sp_client.username_pw_set(self.credentials[0], self.credentials[1])
if self.tls_context:
sp_client.tls_set_context(self.tls_context)
sp_client.connect(self.host,
self.port,
keepalive=self.heartbeat_interval)
sp_client.loop_start()
res = sp_client.publish(routing_key, payload=payload)
res.wait_for_publish()
sp_client.loop_stop()
self.log.debug("rc={!r}".format(res.rc))
return res.rc == 0
class MqttClient(object):
"""Mqtt通讯封装"""
def __init__(self, address):
if not isinstance(address, tuple) or len(address) != 2:
raise ValueError("Invalid address.")
def on_connect(client, userdata, flags, rc):
self.handleConnected()
def on_message(client, userdata, msg):
self.handleMessage(msg.topic, msg.payload)
self.client = Mqtt()
self.address = address
self.client.on_connect = on_connect
self.client.on_message = on_message
def handleConnected(self):
pass
def handleMessage(self, topic, payload):
pass
def publish(self, topic, payload=None, qos=0, retain=False):
self.client.publish(topic, payload, qos, retain)
def subscribe(self, topic, qos=0):
self.client.subscribe(topic, qos)
def start(self):
self.client.connect_async(self.address[0], self.address[1])
self.client.loop_start()
def stop(self):
self.client.loop_stop()
def username_pw_set(self, username, password=None):
self.client.username_pw_set(username, password)
def will_set(self, topic, payload=None, qos=0, retain=False):
self.client.will_set(topic, payload, qos, retain)
def __init__(self, client: mqtt.Client, password: str):
self.client = client
self.client.username_pw_set("management/devicebootstrap", password)
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.loop_start()
self.client.tls_set()
self.client.connect("ubirch.cumulocity.com", 8883)
self.connected = False
while not self.connected:
time.sleep(10)
self.authorized = False
while not self.authorized:
self.client.publish("s/ucr")
time.sleep(10)
client.disconnect()
client.loop_stop()
class MqttClient:
def __init__(self, name):
self.client = None
self.name = name
self.connected = False
self.client = Client(client_id=self.name)
self.client.on_connect = self.on_connect
self.client.connect(host='localhost', port=1883, keepalive=60)
print("New MQTT client: " + str(self.name))
self.client.loop_start()
def on_connect(self, client, userdata, flags, rc):
self.connected = True
print("MQTT Client(" + str(self.name) + ") Connected")
def stop(self):
self.client.loop_stop()
self.client.disconnect()
print("MQTT Client Stoped")
class MQTTValueSub(object):
'''
Use MQTT to recv values on network
pip install paho-mqtt
'''
def __init__(self, name, broker="iot.eclipse.org", def_value=None):
from paho.mqtt.client import Client
self.name = name
self.data = None
self.client = Client(clean_session=True)
self.client.on_message = self.on_message
print("(clean_session) connecting to broker", broker)
self.client.connect(broker)
self.client.loop_start()
self.client.subscribe(self.name)
self.def_value = def_value
print("connected.")
def on_message(self, client, userdata, message):
self.data = message.payload
def run(self):
if self.data is None:
return self.def_value
p = zlib.decompress(self.data)
obj = pickle.loads(p)
if self.name == obj['name']:
self.last = obj['val']
#print("steering, throttle", obj['val'])
return obj['val']
return self.def_value
def shutdown(self):
self.client.disconnect()
self.client.loop_stop()
def get_mqtt_payload(topic):
mqttBroker = "localhost"
client = Client("Temperature_Inside")
if topic == "PYWATERING":
client.on_connect = on_connect_st
elif topic == "PLANT1":
client.on_connect = on_connect_plant1
elif topic == "PLANT2":
client.on_connect = on_connect_plant2
elif topic == "PLANT3":
client.on_connect = on_connect_plant3
elif topic == "PLANT4":
client.on_connect = on_connect_plant4
client.on_message = on_message
client.username_pw_set(username=uname, password=passwd)
client.connect("localhost")
client.loop_start()
time.sleep(2)
client.loop_stop()
return global_message
def mqtt_cmnd(topic,
payload=None,
resp_topic=None,
host=None,
port=1883,
username=None,
password=None):
resp_q = queue.Queue(1)
c = MQTTClient(clean_session=True)
if username is not None:
c.username_pw_set(username, password)
c.connect(host, port)
c.subscribe(resp_topic or topic)
c.on_message = lambda client, userdata, msg: resp_q.put(msg.payload)
c.publish(topic, payload)
c.loop_start()
try:
return resp_q.get(timeout=1)
except queue.Empty:
raise TimeoutError from None
finally:
c.loop_stop()
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__log = log
self.config = config
self.__connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__gateway = gateway
self.__broker = config.get('broker')
self.__mapping = config.get('mapping')
self.__server_side_rpc = config.get('serverSideRpc')
self.__service_config = {
"connectRequests": None,
"disconnectRequests": None
}
self.__attribute_updates = []
self.__get_service_config(config)
self.__sub_topics = {}
client_id = ''.join(
random.choice(string.ascii_lowercase) for _ in range(23))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] or self.__broker[
"security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: %s",
self.get_name(), e)
self._client.tls_insecure_set(False)
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self.__subscribes_sent = {} # For logging the subscriptions
self._client.on_disconnect = self._on_disconnect
self._client.on_log = self._on_log
self._connected = False
self.__stopped = False
self.daemon = True
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
time.sleep(1)
except Exception as e:
self.__log.exception(e)
time.sleep(10)
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
else:
time.sleep(1)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic):
message = self._client.subscribe(topic)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, rc, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if rc == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
for mapping in self.__mapping:
try:
converter = None
if mapping["converter"]["type"] == "custom":
try:
module = TBUtility.check_and_import(
self.__connector_type,
mapping["converter"]["extension"])
if module is not None:
self.__log.debug(
'Custom converter for topic %s - found!',
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error(
"\n\nCannot find extension module for %s topic.\n\Please check your configuration.\n",
mapping["topicFilter"])
except Exception as e:
self.__log.exception(e)
else:
converter = JsonMqttUplinkConverter(mapping)
if converter is not None:
regex_topic = TBUtility.topic_to_regex(
mapping.get("topicFilter"))
if not self.__sub_topics.get(regex_topic):
self.__sub_topics[regex_topic] = []
self.__sub_topics[regex_topic].append(
{converter: None})
# self._client.subscribe(TBUtility.regex_to_topic(regex_topic))
self.__subscribe(mapping["topicFilter"])
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
except Exception as e:
self.__log.exception(e)
try:
for request in self.__service_config:
if self.__service_config.get(request) is not None:
for request_config in self.__service_config.get(
request):
self.__subscribe(request_config["topicFilter"])
except Exception as e:
self.__log.error(e)
else:
if rc in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), rc, result_codes[rc])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self.__log.debug('"%s" was disconnected.', self.get_name())
def _on_log(self, *args):
self.__log.debug(args)
# pass
def _on_subscribe(self, client, userdata, mid, granted_qos):
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
except Exception as e:
self.__log.exception(e)
def __get_service_config(self, config):
for service_config in self.__service_config:
if service_config != "attributeUpdates" and config.get(
service_config):
self.__service_config[service_config] = config[service_config]
else:
self.__attribute_updates = config[service_config]
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
regex_topic = [
regex for regex in self.__sub_topics
if fullmatch(regex, message.topic)
]
if regex_topic:
try:
for regex in regex_topic:
if self.__sub_topics.get(regex):
for converter_value in range(
len(self.__sub_topics.get(regex))):
if self.__sub_topics[regex][converter_value]:
for converter in self.__sub_topics.get(
regex)[converter_value]:
converted_content = converter.convert(
message.topic, content)
if converted_content:
try:
self.__sub_topics[regex][
converter_value][
converter] = converted_content
except Exception as e:
self.__log.exception(e)
self.__gateway.send_to_storage(
self.name, converted_content)
self.statistics['MessagesSent'] += 1
else:
continue
else:
self.__log.error(
'Cannot find converter for topic:"%s"!',
message.topic)
return
except Exception as e:
log.exception(e)
return
elif self.__service_config.get("connectRequests"):
connect_requests = [
connect_request for connect_request in
self.__service_config.get("connectRequests")
]
if connect_requests:
for request in connect_requests:
if request.get("topicFilter"):
if message.topic in request.get("topicFilter") or\
(request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)):
founded_device_name = None
if request.get("deviceNameJsonExpression"):
founded_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"],
content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
founded_device_name = search(
device_name_expression, message.topic)
if founded_device_name is not None and founded_device_name not in self.__gateway.get_devices(
):
self.__gateway.add_device(
founded_device_name, {"connector": self})
else:
self.__log.error(
"Cannot find connect request for device from message from topic: %s and with data: %s",
message.topic, content)
else:
self.__log.error(
"\"topicFilter\" in connect requests config not found."
)
else:
self.__log.error("Connection requests in config not found.")
elif self.__service_config.get("disconnectRequests") is not None:
disconnect_requests = [
disconnect_request for disconnect_request in
self.__service_config.get("disconnectRequests")
]
if disconnect_requests:
for request in disconnect_requests:
if request.get("topicFilter") is not None:
if message.topic in request.get("topicFilter") or\
(request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)):
founded_device_name = None
if request.get("deviceNameJsonExpression"):
founded_device_name = TBUtility.get_value(
request["deviceNameJsonExpression"],
content)
if request.get("deviceNameTopicExpression"):
device_name_expression = request[
"deviceNameTopicExpression"]
founded_device_name = search(
device_name_expression, message.topic)
if founded_device_name is not None and founded_device_name in self.__gateway.get_devices(
):
self.__gateway.del_device(founded_device_name)
else:
self.__log.error(
"Cannot find connect request for device from message from topic: %s and with data: %s",
message.topic, content)
else:
self.__log.error(
"\"topicFilter\" in connect requests config not found."
)
else:
self.__log.error("Disconnection requests in config not found.")
elif message.topic in self.__gateway.rpc_requests_in_progress:
self.__gateway.rpc_with_reply_processing(message.topic, content)
else:
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def on_attributes_update(self, content):
attribute_updates_config = [
update for update in self.__attribute_updates
]
if attribute_updates_config:
for attribute_update in attribute_updates_config:
if match(attribute_update["deviceNameFilter"], content["device"]) and \
content["data"].get(attribute_update["attributeFilter"]):
topic = attribute_update["topicExpression"]\
.replace("${deviceName}", content["device"])\
.replace("${attributeKey}", attribute_update["attributeFilter"])\
.replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]])
data = ''
try:
data = attribute_update["valueExpression"]\
.replace("${attributeKey}", attribute_update["attributeFilter"])\
.replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]])
except Exception as e:
self.__log.error(e)
self._client.publish(topic, data).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Not found deviceName by filter in message or attributeFilter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# Subscribe to RPC response topic
if rpc_config.get("responseTopicExpression"):
topic_for_subscribe = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", content["device"]) \
.replace("${methodName}", content["data"]["method"]) \
.replace("${requestId}", str(content["data"]["id"])) \
.replace("${params}", content["data"]["params"])
if rpc_config.get("responseTimeout"):
timeout = time.time() * 1000 + rpc_config.get(
"responseTimeout")
self.__gateway.register_rpc_request_timeout(
content, timeout, topic_for_subscribe,
self.rpc_cancel_processing)
# Maybe we need to wait for the command to execute successfully before publishing the request.
self._client.subscribe(topic_for_subscribe)
else:
self.__log.error(
"Not found RPC response timeout in config, sending without waiting for response"
)
# Publish RPC request
if rpc_config.get("requestTopicExpression") is not None\
and rpc_config.get("valueExpression"):
topic = rpc_config.get("requestTopicExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
data_to_send = rpc_config.get("valueExpression")\
.replace("${deviceName}", content["device"])\
.replace("${methodName}", content["data"]["method"])\
.replace("${requestId}", str(content["data"]["id"]))\
.replace("${params}", content["data"]["params"])
try:
self._client.publish(topic, data_to_send)
self.__log.debug(
"Send RPC with no response request to topic: %s with data %s",
topic, data_to_send)
if rpc_config.get("responseTopicExpression") is None:
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
except Exception as e:
self.__log.exception(e)
def rpc_cancel_processing(self, topic):
self._client.unsubscribe(topic)
class HotWord(object):
def __init__(self):
self.received_lamp_state = None
self.color_database = json.load(open('color.json'))
self.client = Client(client_id='google_home')
self.client.on_connect = self.on_connect
self.client.connect('localhost', port=1883, keepalive=60)
self._wait_for_lamp_state()
self.client.loop_start()
def _receive_lamp_state(self, client, userdata, message):
print(message.payload)
self.received_lamp_state = json.loads(message.payload.decode("utf-8"))
def on_connect(self, client, userdata, flags, rc):
client.message_callback_add('/lamp/changed', self._receive_lamp_state)
client.subscribe('/lamp/changed', qos=1)
def _wait_for_lamp_state(self):
for i in range(10):
if self.received_lamp_state:
return
self.client.loop(0.05)
raise Exception("Timeout waiting for lamp state")
def process_device_actions(self, event, device_id):
if 'inputs' in event.args:
for i in event.args['inputs']:
if i['intent'] == 'action.devices.EXECUTE':
for c in i['payload']['commands']:
for device in c['devices']:
if device['id'] == device_id:
if 'execution' in c:
for e in c['execution']:
if 'params' in e:
yield e['command'], e['params']
else:
yield e['command'], None
def process_event(self, event, device_id):
"""Pretty prints events.
Prints all events that occur with two spaces between each new
conversation and a single space between turns of a conversation.
Args:
event(event.Event): The current event to process.
device_id(str): The device ID of the new instance.
"""
if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
print()
print(event)
if (event.type == EventType.ON_CONVERSATION_TURN_FINISHED
and event.args and not event.args['with_follow_on_turn']):
print()
if event.type == EventType.ON_DEVICE_ACTION:
for command, params in self.process_device_actions(
event, device_id):
print('Do command', command, 'with params', str(params))
if command == "action.devices.commands.OnOff":
if params['on']:
self.received_lamp_state['client'] = 'google_home'
self.received_lamp_state['on'] = True
print('Turning the LED on.')
else:
self.received_lamp_state['client'] = 'google_home'
self.received_lamp_state['on'] = False
print('Turning the LED off.')
self.client.publish('/lamp/set_config',
json.dumps(self.received_lamp_state),
qos=1)
if command == "action.devices.commands.ColorAbsolute":
if params['color']:
print("hello it is me color")
color = params['color'].get('name')
hue = self.color_database[color]['hue']
saturation = self.color_database[color]['saturation']
self.received_lamp_state['color']['h'] = round(hue, 2)
self.received_lamp_state['color']['s'] = round(
saturation, 2)
self.received_lamp_state['client'] = 'google_home'
self.client.publish('/lamp/set_config',
json.dumps(
self.received_lamp_state),
qos=1)
if command == "action.devices.commands.BrightnessAbsolute":
if params['brightness']:
print("hello")
brightness = (params['brightness']) / 100
print(brightness)
self.received_lamp_state['brightness'] = brightness
self.received_lamp_state['client'] = 'google_home'
self.client.publish('/lamp/set_config',
json.dumps(
self.received_lamp_state),
qos=1)
sleep(0.1)
self.client.loop_stop()
def register_device(self, project_id, credentials, device_model_id,
device_id):
"""Register the device if needed.
Registers a new assistant device if an instance with the given id
does not already exists for this model.
Args:
project_id(str): The project ID used to register device instance.
credentials(google.oauth2.credentials.Credentials): The Google
OAuth2 credentials of the user to associate the device
instance with.
device_model_id(str): The registered device model ID.
device_id(str): The device ID of the new instance.
"""
base_url = '/'.join(
[DEVICE_API_URL, 'projects', project_id, 'devices'])
device_url = '/'.join([base_url, device_id])
session = google.auth.transport.requests.AuthorizedSession(credentials)
r = session.get(device_url)
print(device_url, r.status_code)
if r.status_code == 404:
print('Registering....')
r = session.post(base_url,
data=json.dumps({
'id': device_id,
'model_id': device_model_id,
'client_type': 'SDK_LIBRARY'
}))
if r.status_code != 200:
raise Exception('failed to register device: ' + r.text)
print('\rDevice registered.')
def main(self):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--credentials',
type=existing_file,
metavar='OAUTH2_CREDENTIALS_FILE',
default=os.path.join(
os.path.expanduser('~/.config'),
'google-oauthlib-tool', 'credentials.json'),
help='Path to store and read OAuth2 credentials')
parser.add_argument('--device_model_id',
type=str,
metavar='DEVICE_MODEL_ID',
required=True,
help='The device model ID registered with Google')
parser.add_argument(
'--project_id',
type=str,
metavar='PROJECT_ID',
required=False,
help='The project ID used to register device instances.')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + Assistant.__version_str__())
args = parser.parse_args()
with open(args.credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
with Assistant(credentials, args.device_model_id) as assistant:
events = assistant.start()
print('device_model_id:',
args.device_model_id + '\n' + 'device_id:',
assistant.device_id + '\n')
if args.project_id:
register_device(args.project_id, credentials,
args.device_model_id, assistant.device_id)
for event in events:
self.process_event(event, assistant.device_id)
class MqttConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.__gateway = gateway # Reference to TB Gateway
self._connector_type = connector_type # Should be "mqtt"
self.config = config # mqtt.json contents
self.__log = log
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__subscribes_sent = {}
# Extract main sections from configuration ---------------------------------------------------------------------
self.__broker = config.get('broker')
self.__mapping = []
self.__server_side_rpc = []
self.__connect_requests = []
self.__disconnect_requests = []
self.__attribute_requests = []
self.__attribute_updates = []
mandatory_keys = {
"mapping": ['topicFilter', 'converter'],
"serverSideRpc": [
'deviceNameFilter', 'methodFilter', 'requestTopicExpression',
'valueExpression'
],
"connectRequests": ['topicFilter'],
"disconnectRequests": ['topicFilter'],
"attributeRequests":
['topicFilter', 'topicExpression', 'valueExpression'],
"attributeUpdates": [
'deviceNameFilter', 'attributeFilter', 'topicExpression',
'valueExpression'
]
}
# Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file
self.load_handlers('mapping', mandatory_keys['mapping'],
self.__mapping)
# RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers
self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'],
self.__server_side_rpc)
# Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry
self.load_handlers('connectRequests',
mandatory_keys['connectRequests'],
self.__connect_requests)
# Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet
self.load_handlers('disconnectRequests',
mandatory_keys['disconnectRequests'],
self.__disconnect_requests)
# Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value
self.load_handlers('attributeRequests',
mandatory_keys['attributeRequests'],
self.__attribute_requests)
# Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute
self.load_handlers('attributeUpdates',
mandatory_keys['attributeUpdates'],
self.__attribute_updates)
# Setup topic substitution lists for each class of handlers ----------------------------------------------------
self.__mapping_sub_topics = {}
self.__connect_requests_sub_topics = {}
self.__disconnect_requests_sub_topics = {}
self.__attribute_requests_sub_topics = {}
# Set up external MQTT broker connection -----------------------------------------------------------------------
client_id = self.__broker.get(
"clientId",
''.join(random.choice(string.ascii_lowercase) for _ in range(23)))
self._client = Client(client_id)
self.setName(
config.get(
"name",
self.__broker.get(
"name", 'Mqtt Broker ' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(5)))))
if "username" in self.__broker["security"]:
self._client.username_pw_set(self.__broker["security"]["username"],
self.__broker["security"]["password"])
if "caCert" in self.__broker["security"] \
or self.__broker["security"].get("type", "none").lower() == "tls":
ca_cert = self.__broker["security"].get("caCert")
private_key = self.__broker["security"].get("privateKey")
cert = self.__broker["security"].get("cert")
if ca_cert is None:
self._client.tls_set_context(
ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
else:
try:
self._client.tls_set(ca_certs=ca_cert,
certfile=cert,
keyfile=private_key,
cert_reqs=ssl.CERT_REQUIRED,
tls_version=ssl.PROTOCOL_TLSv1_2,
ciphers=None)
except Exception as e:
self.__log.error(
"Cannot setup connection to broker %s using SSL. "
"Please check your configuration.\nError: ",
self.get_name())
self.__log.exception(e)
if self.__broker["security"].get("insecure", False):
self._client.tls_insecure_set(True)
else:
self._client.tls_insecure_set(False)
# Set up external MQTT broker callbacks ------------------------------------------------------------------------
self._client.on_connect = self._on_connect
self._client.on_message = self._on_message
self._client.on_subscribe = self._on_subscribe
self._client.on_disconnect = self._on_disconnect
# self._client.on_log = self._on_log
# Set up lifecycle flags ---------------------------------------------------------------------------------------
self._connected = False
self.__stopped = False
self.daemon = True
self.__msg_queue = Queue()
self.__workers_thread_pool = []
self.__max_msg_number_for_worker = config.get(
'maxMessageNumberPerWorker', 10)
self.__max_number_of_workers = config.get('maxNumberOfWorkers', 100)
def load_handlers(self, handler_flavor, mandatory_keys,
accepted_handlers_list):
if handler_flavor not in self.config:
self.__log.error("'%s' section missing from configuration",
handler_flavor)
else:
for handler in self.config.get(handler_flavor):
discard = False
for key in mandatory_keys:
if key not in handler:
# Will report all missing fields to user before discarding the entry => no break here
discard = True
self.__log.error(
"Mandatory key '%s' missing from %s handler: %s",
key, handler_flavor, simplejson.dumps(handler))
else:
self.__log.debug(
"Mandatory key '%s' found in %s handler: %s", key,
handler_flavor, simplejson.dumps(handler))
if discard:
self.__log.error(
"%s handler is missing some mandatory keys => rejected: %s",
handler_flavor, simplejson.dumps(handler))
else:
accepted_handlers_list.append(handler)
self.__log.debug(
"%s handler has all mandatory keys => accepted: %s",
handler_flavor, simplejson.dumps(handler))
self.__log.info("Number of accepted %s handlers: %d",
handler_flavor, len(accepted_handlers_list))
self.__log.info(
"Number of rejected %s handlers: %d", handler_flavor,
len(self.config.get(handler_flavor)) -
len(accepted_handlers_list))
def is_connected(self):
return self._connected
def open(self):
self.__stopped = False
self.start()
def run(self):
try:
self.__connect()
except Exception as e:
self.__log.exception(e)
try:
self.close()
except Exception as e:
self.__log.exception(e)
while True:
if self.__stopped:
break
elif not self._connected:
self.__connect()
self.__threads_manager()
sleep(.2)
def __connect(self):
while not self._connected and not self.__stopped:
try:
self._client.connect(self.__broker['host'],
self.__broker.get('port', 1883))
self._client.loop_start()
if not self._connected:
sleep(1)
except ConnectionRefusedError as e:
self.__log.error(e)
sleep(10)
def close(self):
self.__stopped = True
try:
self._client.disconnect()
except Exception as e:
log.exception(e)
self._client.loop_stop()
self.__log.info('%s has been stopped.', self.get_name())
def get_name(self):
return self.name
def __subscribe(self, topic, qos):
message = self._client.subscribe(topic, qos)
try:
self.__subscribes_sent[message[1]] = topic
except Exception as e:
self.__log.exception(e)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug(
"Client %s, userdata %s, flags %s, extra_params %s",
str(client), str(userdata), str(flags), extra_params)
self.__mapping_sub_topics = {}
# Setup data upload requests handling ----------------------------------------------------------------------
for mapping in self.__mapping:
try:
# Load converter for this mapping entry ------------------------------------------------------------
# mappings are guaranteed to have topicFilter and converter fields. See __init__
default_converter_class_name = "JsonMqttUplinkConverter"
# Get converter class from "extension" parameter or default converter
converter_class_name = mapping["converter"].get(
"extension", default_converter_class_name)
# Find and load required class
module = TBModuleLoader.import_module(
self._connector_type, converter_class_name)
if module:
self.__log.debug('Converter %s for topic %s - found!',
converter_class_name,
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
continue
# Setup regexp topic acceptance list ---------------------------------------------------------------
regex_topic = TBUtility.topic_to_regex(
mapping["topicFilter"])
# There may be more than one converter per topic, so I'm using vectors
if not self.__mapping_sub_topics.get(regex_topic):
self.__mapping_sub_topics[regex_topic] = []
self.__mapping_sub_topics[regex_topic].append(converter)
# Subscribe to appropriate topic -------------------------------------------------------------------
self.__subscribe(mapping["topicFilter"],
mapping.get("subscriptionQos", 1))
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
except Exception as e:
self.__log.exception(e)
# Setup connection requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__connect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__connect_requests_sub_topics[topic_filter] = request
# Setup disconnection requests handling --------------------------------------------------------------------
for request in [
entry for entry in self.__disconnect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__disconnect_requests_sub_topics[topic_filter] = request
# Setup attributes requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__attribute_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__attribute_requests_sub_topics[topic_filter] = request
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), result_code,
result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def _on_disconnect(self, *args):
self._connected = False
self.__log.debug('"%s" was disconnected. %s', self.get_name(),
str(args))
def _on_log(self, *args):
self.__log.debug(args)
def _on_subscribe(self, _, __, mid, granted_qos, *args):
log.info(args)
try:
if granted_qos[0] == 128:
self.__log.error(
'"%s" subscription failed to topic %s subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
else:
self.__log.info(
'"%s" subscription success to topic %s, subscription message id = %i',
self.get_name(), self.__subscribes_sent.get(mid), mid)
except Exception as e:
self.__log.exception(e)
# Success or not, remove this topic from the list of pending subscription requests
if self.__subscribes_sent.get(mid) is not None:
del self.__subscribes_sent[mid]
def put_data_to_convert(self, converter, message, content) -> bool:
if not self.__msg_queue.full():
self.__msg_queue.put((converter.convert, message.topic, content),
True, 100)
return True
return False
def _save_converted_msg(self, topic, data):
self.__gateway.send_to_storage(self.name, data)
self.statistics['MessagesSent'] += 1
self.__log.debug("Successfully converted message from topic %s", topic)
def __threads_manager(self):
if len(self.__workers_thread_pool) == 0:
worker = MqttConnector.ConverterWorker("Main", self.__msg_queue,
self._save_converted_msg)
self.__workers_thread_pool.append(worker)
worker.start()
number_of_needed_threads = round(
self.__msg_queue.qsize() / self.__max_msg_number_for_worker, 0)
threads_count = len(self.__workers_thread_pool)
if number_of_needed_threads > threads_count < self.__max_number_of_workers:
thread = MqttConnector.ConverterWorker(
"Worker " + ''.join(
random.choice(string.ascii_lowercase) for _ in range(5)),
self.__msg_queue, self._save_converted_msg)
self.__workers_thread_pool.append(thread)
thread.start()
elif number_of_needed_threads < threads_count and threads_count > 1:
worker: MqttConnector.ConverterWorker = self.__workers_thread_pool[
-1]
if not worker.in_progress:
worker.stopped = True
self.__workers_thread_pool.remove(worker)
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
# Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" ---------------------------
topic_handlers = [
regex for regex in self.__mapping_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
# Note: every topic may be associated to one or more converter. This means that a single MQTT message
# may produce more than one message towards ThingsBoard. This also means that I cannot return after
# the first successful conversion: I got to use all the available ones.
# I will use a flag to understand whether at least one converter succeeded
request_handled = False
for topic in topic_handlers:
available_converters = self.__mapping_sub_topics[topic]
for converter in available_converters:
try:
if isinstance(content, list):
for item in content:
request_handled = self.put_data_to_convert(
converter, message, item)
if not request_handled:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client),
str(userdata))
else:
request_handled = self.put_data_to_convert(
converter, message, content)
except Exception as e:
log.exception(e)
if not request_handled:
self.__log.error(
'Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic, str(client), str(userdata))
# Note: if I'm in this branch, this was for sure a telemetry/attribute push message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in connection handlers "i.e., I'm connecting a device" -------------------------
topic_handlers = [
regex for regex in self.__connect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__connect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
found_device_type = device_type_match.group(
0) if device_type_match is not None else handler[
"deviceTypeTopicExpression"]
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from connection request")
continue
# Note: device must be added even if it is already known locally: else ThingsBoard
# will not send RPCs and attribute updates
self.__log.info("Connecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.add_device(found_device_name,
{"connector": self},
device_type=found_device_type)
# Note: if I'm in this branch, this was for sure a connection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" -------------------
topic_handlers = [
regex for regex in self.__disconnect_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
for topic in topic_handlers:
handler = self.__disconnect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(
handler["deviceTypeTopicExpression"], message.topic)
if device_type_match is not None:
found_device_type = device_type_match.group(0)
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(
handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from disconnection request")
continue
if found_device_name in self.__gateway.get_devices():
self.__log.info("Disconnecting device %s of type %s",
found_device_name, found_device_type)
self.__gateway.del_device(found_device_name)
else:
self.__log.info("Device %s was not connected",
found_device_name)
break
# Note: if I'm in this branch, this was for sure a disconnection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" --------
topic_handlers = [
regex for regex in self.__attribute_requests_sub_topics
if fullmatch(regex, message.topic)
]
if topic_handlers:
try:
for topic in topic_handlers:
handler = self.__attribute_requests_sub_topics[topic]
found_device_name = None
found_attribute_name = None
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(
handler["deviceNameTopicExpression"],
message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(
handler["deviceNameJsonExpression"], content)
# Get attribute name, either from topic or from content
if handler.get("attributeNameTopicExpression"):
attribute_name_match = search(
handler["attributeNameTopicExpression"],
message.topic)
if attribute_name_match is not None:
found_attribute_name = attribute_name_match.group(
0)
elif handler.get("attributeNameJsonExpression"):
found_attribute_name = TBUtility.get_value(
handler["attributeNameJsonExpression"], content)
if found_device_name is None:
self.__log.error(
"Device name missing from attribute request")
continue
if found_attribute_name is None:
self.__log.error(
"Attribute name missing from attribute request")
continue
self.__log.info("Will retrieve attribute %s of %s",
found_attribute_name, found_device_name)
self.__gateway.tb_client.client.gw_request_shared_attributes(
found_device_name, [found_attribute_name],
lambda data, *args: self.notify_attribute(
data, found_attribute_name,
handler.get("topicExpression"),
handler.get("valueExpression"),
handler.get('retain', False)))
break
except Exception as e:
log.exception(e)
# Note: if I'm in this branch, this was for sure an attribute request message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in RPC handlers ----------------------------------------------------------------
# The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is
if self.__gateway.is_rpc_in_progress(message.topic):
log.info("RPC response arrived. Forwarding it to thingsboard.")
self.__gateway.rpc_with_reply_processing(message.topic, content)
return None
self.__log.debug(
"Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic, content)
def notify_attribute(self, incoming_data, attribute_name, topic_expression,
value_expression, retain):
if incoming_data.get("device") is None or incoming_data.get(
"value") is None:
return
device_name = incoming_data.get("device")
attribute_value = incoming_data.get("value")
topic = topic_expression \
.replace("${deviceName}", str(device_name)) \
.replace("${attributeKey}", str(attribute_name))
data = value_expression.replace("${attributeKey}", str(attribute_name)) \
.replace("${attributeValue}", str(attribute_value))
self._client.publish(topic, data, retain=retain).wait_for_publish()
def on_attributes_update(self, content):
if self.__attribute_updates:
for attribute_update in self.__attribute_updates:
if match(attribute_update["deviceNameFilter"],
content["device"]):
for attribute_key in content["data"]:
if match(attribute_update["attributeFilter"],
attribute_key):
try:
topic = attribute_update["topicExpression"] \
.replace("${deviceName}", str(content["device"])) \
.replace("${attributeKey}", str(attribute_key)) \
.replace("${attributeValue}", str(content["data"][attribute_key]))
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
try:
data = attribute_update["valueExpression"] \
.replace("${attributeKey}", str(attribute_key)) \
.replace("${attributeValue}", str(content["data"][attribute_key]))
except KeyError as e:
log.exception(
"Cannot form topic, key %s - not found", e)
raise e
self._client.publish(
topic,
data,
retain=attribute_update.get(
'retain', False)).wait_for_publish()
self.__log.debug(
"Attribute Update data: %s for device %s to topic: %s",
data, content["device"], topic)
else:
self.__log.error(
"Cannot find attributeName by filter in message with data: %s",
content)
else:
self.__log.error(
"Cannot find deviceName by filter in message with data: %s",
content)
else:
self.__log.error("Attribute updates config not found.")
def server_side_rpc_handler(self, content):
self.__log.info("Incoming server-side RPC: %s", content)
# Check whether one of my RPC handlers can handle this request
for rpc_config in self.__server_side_rpc:
if search(rpc_config["deviceNameFilter"], content["device"]) \
and search(rpc_config["methodFilter"], content["data"]["method"]) is not None:
# This handler seems able to handle the request
self.__log.info("Candidate RPC handler found")
expects_response = rpc_config.get("responseTopicExpression")
defines_timeout = rpc_config.get("responseTimeout")
# 2-way RPC setup
if expects_response and defines_timeout:
expected_response_topic = rpc_config["responseTopicExpression"] \
.replace("${deviceName}", str(content["device"])) \
.replace("${methodName}", str(content["data"]["method"])) \
.replace("${requestId}", str(content["data"]["id"]))
expected_response_topic = TBUtility.replace_params_tags(
expected_response_topic, content)
timeout = time() * 1000 + rpc_config.get("responseTimeout")
# Start listenting on the response topic
self.__log.info("Subscribing to: %s",
expected_response_topic)
self.__subscribe(expected_response_topic,
rpc_config.get("responseTopicQoS", 1))
# Wait for subscription to be carried out
sub_response_timeout = 10
while expected_response_topic in self.__subscribes_sent.values(
):
sub_response_timeout -= 1
sleep(0.1)
if sub_response_timeout == 0:
break
# Ask the gateway to enqueue this as an RPC response
self.__gateway.register_rpc_request_timeout(
content, timeout, expected_response_topic,
self.rpc_cancel_processing)
# Wait for RPC to be successfully enqueued, which never fails.
while self.__gateway.is_rpc_in_progress(
expected_response_topic):
sleep(0.1)
elif expects_response and not defines_timeout:
self.__log.info(
"2-way RPC without timeout: treating as 1-way")
# Actually reach out for the device
request_topic: str = rpc_config.get("requestTopicExpression") \
.replace("${deviceName}", str(content["device"])) \
.replace("${methodName}", str(content["data"]["method"])) \
.replace("${requestId}", str(content["data"]["id"]))
request_topic = TBUtility.replace_params_tags(
request_topic, content)
data_to_send_tags = TBUtility.get_values(
rpc_config.get('valueExpression'),
content['data'],
'params',
get_tag=True)
data_to_send_values = TBUtility.get_values(
rpc_config.get('valueExpression'),
content['data'],
'params',
expression_instead_none=True)
data_to_send = rpc_config.get('valueExpression')
for (tag, value) in zip(data_to_send_tags,
data_to_send_values):
data_to_send = data_to_send.replace(
'${' + tag + '}', str(value))
try:
self.__log.info("Publishing to: %s with data %s",
request_topic, data_to_send)
self._client.publish(request_topic,
data_to_send,
retain=rpc_config.get(
'retain', False))
if not expects_response or not defines_timeout:
self.__log.info(
"One-way RPC: sending ack to ThingsBoard immediately"
)
self.__gateway.send_rpc_reply(
device=content["device"],
req_id=content["data"]["id"],
success_sent=True)
# Everything went out smoothly: RPC is served
return
except Exception as e:
self.__log.exception(e)
self.__log.error("RPC not handled: %s", content)
def rpc_cancel_processing(self, topic):
log.info("RPC canceled or terminated. Unsubscribing from %s", topic)
self._client.unsubscribe(topic)
class ConverterWorker(Thread):
def __init__(self, name, incoming_queue, send_result):
super().__init__()
self.stopped = False
self.setName(name)
self.setDaemon(True)
self.__msg_queue = incoming_queue
self.in_progress = False
self.__send_result = send_result
def run(self):
while not self.stopped:
if not self.__msg_queue.empty():
self.in_progress = True
convert_function, config, incoming_data = self.__msg_queue.get(
True, 100)
converted_data = convert_function(config, incoming_data)
log.debug(converted_data)
self.__send_result(config, converted_data)
self.in_progress = False
else:
sleep(.2)
class Mqtt():
"""Main Mqtt class.
:param app: flask application object
:param connect_async: if True then connect_aync will be used to connect to MQTT broker
:param mqtt_logging: if True then messages from MQTT client will be logged
"""
def __init__(self, app=None, connect_async=False, mqtt_logging=False):
# type: (Flask, bool, bool) -> None
self.app = app
self._connect_async = connect_async # type: bool
self._connect_handler = None # type: Optional[Callable]
self._disconnect_handler = None # type: Optional[Callable]
self.topics = {} # type: Dict[str, TopicQos]
self.connected = False
self.client = Client()
if mqtt_logging:
self.client.enable_logger(logger)
if app is not None:
self.init_app(app)
def init_app(self, app):
# type: (Flask) -> None
"""Init the Flask-MQTT addon."""
self.client_id = app.config.get("MQTT_CLIENT_ID", "")
self.clean_session = app.config.get("MQTT_CLEAN_SESSION", True)
if isinstance(self.client_id, unicode):
self.client._client_id = self.client_id.encode('utf-8')
else:
self.client._client_id = self.client_id
self.client._clean_session = self.clean_session
self.client._transport = app.config.get("MQTT_TRANSPORT",
"tcp").lower()
self.client._protocol = app.config.get("MQTT_PROTOCOL_VERSION",
MQTTv311)
self.client.on_connect = self._handle_connect
self.client.on_disconnect = self._handle_disconnect
self.username = app.config.get("MQTT_USERNAME")
self.password = app.config.get("MQTT_PASSWORD")
self.broker_url = app.config.get("MQTT_BROKER_URL", "localhost")
self.broker_port = app.config.get("MQTT_BROKER_PORT", 1883)
self.tls_enabled = app.config.get("MQTT_TLS_ENABLED", False)
self.keepalive = app.config.get("MQTT_KEEPALIVE", 60)
self.last_will_topic = app.config.get("MQTT_LAST_WILL_TOPIC")
self.last_will_message = app.config.get("MQTT_LAST_WILL_MESSAGE")
self.last_will_qos = app.config.get("MQTT_LAST_WILL_QOS", 0)
self.last_will_retain = app.config.get("MQTT_LAST_WILL_RETAIN", False)
if self.tls_enabled:
self.tls_ca_certs = app.config["MQTT_TLS_CA_CERTS"]
self.tls_certfile = app.config.get("MQTT_TLS_CERTFILE")
self.tls_keyfile = app.config.get("MQTT_TLS_KEYFILE")
self.tls_cert_reqs = app.config.get("MQTT_TLS_CERT_REQS",
ssl.CERT_REQUIRED)
self.tls_version = app.config.get("MQTT_TLS_VERSION",
ssl.PROTOCOL_TLSv1)
self.tls_ciphers = app.config.get("MQTT_TLS_CIPHERS")
self.tls_insecure = app.config.get("MQTT_TLS_INSECURE", False)
# set last will message
if self.last_will_topic is not None:
self.client.will_set(
self.last_will_topic,
self.last_will_message,
self.last_will_qos,
self.last_will_retain,
)
self._connect()
def _connect(self):
# type: () -> None
if self.username is not None:
self.client.username_pw_set(self.username, self.password)
# security
if self.tls_enabled:
self.client.tls_set(
ca_certs=self.tls_ca_certs,
certfile=self.tls_certfile,
keyfile=self.tls_keyfile,
cert_reqs=self.tls_cert_reqs,
tls_version=self.tls_version,
ciphers=self.tls_ciphers,
)
if self.tls_insecure:
self.client.tls_insecure_set(self.tls_insecure)
if self._connect_async:
# if connect_async is used
self.client.connect_async(self.broker_url,
self.broker_port,
keepalive=self.keepalive)
else:
res = self.client.connect(self.broker_url,
self.broker_port,
keepalive=self.keepalive)
if res == 0:
logger.debug("Connected client '{0}' to broker {1}:{2}".format(
self.client_id, self.broker_url, self.broker_port))
else:
logger.error(
"Could not connect to MQTT Broker, Error Code: {0}".format(
res))
self.client.loop_start()
def _disconnect(self):
# type: () -> None
self.client.loop_stop()
self.client.disconnect()
logger.debug('Disconnected from Broker')
def _handle_connect(self, client, userdata, flags, rc):
# type: (Client, Any, Dict, int) -> None
if rc == MQTT_ERR_SUCCESS:
self.connected = True
for key, item in self.topics.items():
self.client.subscribe(topic=item.topic, qos=item.qos)
if self._connect_handler is not None:
self._connect_handler(client, userdata, flags, rc)
def _handle_disconnect(self, client, userdata, rc):
# type: (str, Any, int) -> None
self.connected = False
if self._disconnect_handler is not None:
self._disconnect_handler()
def on_topic(self, topic):
# type: (str) -> Callable
"""Decorator.
Decorator to add a callback function that is called when a certain
topic has been published. The callback function is expected to have the
following form: `handle_topic(client, userdata, message)`
:parameter topic: a string specifying the subscription topic to
subscribe to
The topic still needs to be subscribed via mqtt.subscribe() before the
callback function can be used to handle a certain topic. This way it is
possible to subscribe and unsubscribe during runtime.
**Example usage:**::
app = Flask(__name__)
mqtt = Mqtt(app)
mqtt.subscribe('home/mytopic')
@mqtt.on_topic('home/mytopic')
def handle_mytopic(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable[[str], None]) -> Callable[[str], None]
self.client.message_callback_add(topic, handler)
return handler
return decorator
def subscribe(self, topic, qos=0):
# type: (str, int) -> Tuple[int, int]
"""
Subscribe to a certain topic.
:param topic: a string specifying the subscription topic to
subscribe to.
:param qos: the desired quality of service level for the subscription.
Defaults to 0.
:rtype: (int, int)
:result: (result, mid)
A topic is a UTF-8 string, which is used by the broker to filter
messages for each connected client. A topic consists of one or more
topic levels. Each topic level is separated by a forward slash
(topic level separator).
The function returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the
client is not currently connected. mid is the message ID for the
subscribe request. The mid value can be used to track the subscribe
request by checking against the mid argument in the on_subscribe()
callback if it is defined.
**Topic example:** `myhome/groundfloor/livingroom/temperature`
"""
# TODO: add support for list of topics
# don't subscribe if already subscribed
# try to subscribe
result, mid = self.client.subscribe(topic=topic, qos=qos)
# if successful add to topics
if result == MQTT_ERR_SUCCESS:
self.topics[topic] = TopicQos(topic=topic, qos=qos)
logger.debug('Subscribed to topic: {0}, qos: {1}'.format(
topic, qos))
else:
logger.error('Error {0} subscribing to topic: {1}'.format(
result, topic))
return (result, mid)
def unsubscribe(self, topic):
# type: (str) -> Optional[Tuple[int, int]]
"""
Unsubscribe from a single topic.
:param topic: a single string that is the subscription topic to
unsubscribe from
:rtype: (int, int)
:result: (result, mid)
Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS
to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not
currently connected.
mid is the message ID for the unsubscribe request. The mid value can be
used to track the unsubscribe request by checking against the mid
argument in the on_unsubscribe() callback if it is defined.
"""
# don't unsubscribe if not in topics
if topic in self.topics:
result, mid = self.client.unsubscribe(topic)
if result == MQTT_ERR_SUCCESS:
self.topics.pop(topic)
logger.debug('Unsubscribed from topic: {0}'.format(topic))
else:
logger.debug('Error {0} unsubscribing from topic: {1}'.format(
result, topic))
# if successful remove from topics
return result, mid
return None
def unsubscribe_all(self):
# type: () -> None
"""Unsubscribe from all topics."""
topics = list(self.topics.keys())
for topic in topics:
self.unsubscribe(topic)
def publish(self, topic, payload=None, qos=0, retain=False):
# type: (str, bytes, int, bool) -> Tuple[int, int]
"""
Send a message to the broker.
:param topic: the topic that the message should be published on
:param payload: the actual message to send. If not given, or set to
None a zero length message will be used. Passing an
int or float will result in the payload being
converted to a string representing that number.
If you wish to send a true int/float, use struct.pack()
to create the payload you require.
:param qos: the quality of service level to use
:param retain: if set to True, the message will be set as the
"last known good"/retained message for the topic
:returns: Returns a tuple (result, mid), where result is
MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN
if the client is not currently connected. mid is the message
ID for the publish request.
"""
if not self.connected:
self.client.reconnect()
result, mid = self.client.publish(topic, payload, qos, retain)
if result == MQTT_ERR_SUCCESS:
logger.debug('Published topic {0}: {1}'.format(topic, payload))
else:
logger.error('Error {0} publishing topic {1}'.format(
result, topic))
return (result, mid)
def on_connect(self):
# type: () -> Callable
"""Decorator.
Decorator to handle the event when the broker responds to a connection
request. Only the last decorated function will be called.
"""
def decorator(handler):
# type: (Callable) -> Callable
self._connect_handler = handler
return handler
return decorator
def on_disconnect(self):
# type: () -> Callable
"""Decorator.
Decorator to handle the event when client disconnects from broker. Only
the last decorated function will be called.
"""
def decorator(handler):
# type: (Callable) -> Callable
self._disconnect_handler = handler
return handler
return decorator
def on_message(self):
# type: () -> Callable
"""Decorator.
Decorator to handle all messages that have been subscribed and that
are not handled via the `on_message` decorator.
**Note:** Unlike as written in the paho mqtt documentation this
callback will not be called if there exists an topic-specific callback
added by the `on_topic` decorator.
**Example Usage:**::
@mqtt.on_message()
def handle_messages(client, userdata, message):
print('Received message on topic {}: {}'
.format(message.topic, message.payload.decode()))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_message = handler
return handler
return decorator
def on_publish(self):
# type: () -> Callable
"""Decorator.
Decorator to handle all messages that have been published by the
client.
**Example Usage:**::
@mqtt.on_publish()
def handle_publish(client, userdata, mid):
print('Published message with mid {}.'
.format(mid))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_publish = handler
return handler
return decorator
def on_subscribe(self):
# type: () -> Callable
"""Decorate a callback function to handle subscritions.
**Usage:**::
@mqtt.on_subscribe()
def handle_subscribe(client, userdata, mid, granted_qos):
print('Subscription id {} granted with qos {}.'
.format(mid, granted_qos))
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_subscribe = handler
return handler
return decorator
def on_unsubscribe(self):
# type: () -> Callable
"""Decorate a callback funtion to handle unsubscribtions.
**Usage:**::
@mqtt.unsubscribe()
def handle_unsubscribe(client, userdata, mid)
print('Unsubscribed from topic (id: {})'
.format(mid)')
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_unsubscribe = handler
return handler
return decorator
def on_log(self):
# type: () -> Callable
"""Decorate a callback function to handle MQTT logging.
**Example Usage:**
::
@mqtt.on_log()
def handle_logging(client, userdata, level, buf):
print(client, userdata, level, buf)
"""
def decorator(handler):
# type: (Callable) -> Callable
self.client.on_log = handler
return handler
return decorator
class Messenger(object):
"""
MQTT client for Herald transport.
"""
def __init__(self, peer):
"""
Initialize client
:param peer: The peer behind the MQTT client.
:return:
"""
self.__peer = peer
self.__mqtt = MqttClient()
self.__mqtt.on_connect = self._on_connect
self.__mqtt.on_disconnect = self._on_disconnect
self.__mqtt.on_message = self._on_message
self.__callback_handler = None
self.__WILL_TOPIC = "/".join(
(TOPIC_PREFIX, peer.app_id, RIP_TOPIC))
def __make_uid_topic(self, subtopic):
"""
Constructs a complete UID topic.
:param subtopic: The UID
:return: Fully qualified topic
:rtype : str
"""
return "/".join(
(TOPIC_PREFIX, self.__peer.app_id, UID_TOPIC, subtopic))
def __make_group_topic(self, subtopic):
"""
Constructs a complete group topic.
:param subtopic: The group name
:return: Fully qualified topic
:rtype : str
"""
return "/".join(
(TOPIC_PREFIX, self.__peer.app_id, GROUP_TOPIC, subtopic))
def __handle_will(self, message):
if self.__callback_handler and self.__callback_handler.on_peer_down:
self.__callback_handler.on_peer_down(
message.payload.decode('utf-8'))
else:
_log.debug("Missing callback for on_peer_down.")
def _on_connect(self, *args, **kwargs):
"""
Handles a connection-established event.
:param args: unnamed arguments
:param kwargs: named arguments
:return:
"""
_log.info("Connection established.")
_log.debug("Subscribing for topic %s.",
self.__make_uid_topic(self.__peer.uid))
self.__mqtt.subscribe(self.__make_uid_topic(self.__peer.uid))
self.__mqtt.subscribe(self.__make_group_topic("all"))
self.__mqtt.subscribe(self.__WILL_TOPIC)
for group in self.__peer.groups:
_log.debug("Subscribing for topic %s.",
self.__make_group_topic(group))
self.__mqtt.subscribe(self.__make_group_topic(group))
if self.__callback_handler and self.__callback_handler.on_connected:
self.__callback_handler.on_connected()
else:
_log.warning("Missing callback for on_connect.")
def _on_disconnect(self, *args, **kwargs):
"""
Handles a connection-lost event.
:param args: unnamed arguments
:param kwargs: named arguments
:return:
"""
_log.info("Connection lost.")
if self.__callback_handler and self.__callback_handler.on_disconnected:
self.__callback_handler.on_disconnected()
def _on_message(self, client, data, message):
"""
Handles an incoming message.
:param client: the client instance for this callback
:param data: the private user data
:param message: an instance of MQTTMessage
:type message: paho.mqtt.client.MQTTMessage
:return:
"""
_log.info("Message received.")
if message.topic == self.__WILL_TOPIC:
self.__handle_will(message)
return
if self.__callback_handler and self.__callback_handler.on_message:
self.__callback_handler.on_message(message.payload.decode('utf-8'))
else:
_log.warning("Missing callback for on_message.")
def fire(self, peer_uid, message):
"""
Sends a message to another peer.
:param peer_uid: Peer UID
:param message: Message content
:return:
"""
self.__mqtt.publish(
self.__make_uid_topic(peer_uid),
message,
1
)
def fire_group(self, group, message):
"""
Sends a message to a group of peers.
:param group: Group's name
:param message: Message content
:return:
"""
self.__mqtt.publish(
self.__make_group_topic(group),
message,
1
)
def set_callback_listener(self, listener):
"""
Sets callback listener.
:param listener: the listener
:return:
"""
self.__callback_handler = listener
def login(self, username, password):
"""
Set credentials for an MQTT broker.
:param username: Username
:param password: Password
:return:
"""
self.__mqtt.username_pw_set(username, password)
def connect(self, host, port):
"""
Connects to an MQTT broker.
:param host: broker's host name
:param port: broker's port number
:return:
"""
_log.info("Connecting to MQTT broker at %s:%s ...", host, port)
self.__mqtt.will_set(self.__WILL_TOPIC, self.__peer.uid, 1)
self.__mqtt.connect(host, port)
self.__mqtt.loop_start()
def disconnect(self):
"""
Diconnects from an MQTT broker.
:return:
"""
_log.info("Disconnecting from MQTT broker...")
self.__mqtt.publish(self.__WILL_TOPIC, self.__peer.uid, 1)
self.__mqtt.loop_stop()
self.__mqtt.disconnect()
class GatewayEntry(Thread, GatewayStats):
"""Main class for working with the gateway via Telnet (23), MQTT (1883) and
miIO (54321) protocols.
"""
time_offset = 0
pair_model = None
pair_payload = None
pair_payload2 = None
telnet_cmd = None
def __init__(self, host: str, token: str, config: dict, **options):
super().__init__(daemon=True, name=f"{host}_main")
self.host = host
self.options = options
self.miio = SyncmiIO(host, token)
self.mqtt = Client()
self.mqtt.on_connect = self.on_connect
self.mqtt.on_disconnect = self.on_disconnect
self.mqtt.on_message = self.on_message
self._ble = options.get('ble', True) # for fast access
self._debug = options.get('debug', '') # for fast access
self.parent_scan_interval = options.get('parent', -1)
self.default_devices = config['devices'] if config else None
self.telnet_cmd = options.get('telnet_cmd') or TELNET_CMD
if 'true' in self._debug:
self.miio.debug = True
self.setups = {}
self.stats = {}
@property
def device(self):
return self.devices[self.did]
def debug(self, message: str):
# basic logs
if 'true' in self._debug:
_LOGGER.debug(f"{self.host} | {message}")
def stop(self, *args):
self.enabled = False
self.mqtt.loop_stop()
for device in self.devices.values():
if self in device['gateways']:
device['gateways'].remove(self)
def run(self):
"""Main thread loop."""
self.debug("Start main thread")
self.mqtt.connect_async(self.host)
self.enabled = True
while self.enabled:
# if not telnet - enable it
if not self._check_port(23) and not self._enable_telnet():
time.sleep(30)
continue
if not self.did:
devices = self._get_devices()
if not devices:
time.sleep(60)
continue
self.setup_devices(devices)
self.update_time_offset()
self.mesh_start()
# if not mqtt - enable it (handle Mi Home and ZHA mode)
if not self._prepare_gateway() or not self._mqtt_connect():
time.sleep(60)
continue
self.mqtt.loop_forever()
self.debug("Stop main thread")
def update_time_offset(self):
gw_time = ntp_time(self.host)
if gw_time:
self.time_offset = gw_time - time.time()
self.debug(f"Gateway time offset: {self.time_offset}")
def _check_port(self, port: int):
"""Check if gateway port open."""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
return s.connect_ex((self.host, port)) == 0
finally:
s.close()
def _enable_telnet(self):
"""Enable telnet with miio protocol."""
raw = json.loads(self.telnet_cmd)
if self.miio.send(raw['method'], raw.get('params')) != ['ok']:
self.debug(f"Can't enable telnet")
return False
return True
def _prepare_gateway(self):
"""Launching the required utilities on the hub, if they are not already
running.
"""
self.debug("Prepare Gateway")
try:
shell = TelnetShell(self.host)
self.debug(f"Version: {shell.ver}")
ps = shell.get_running_ps()
if "mosquitto -d" not in ps:
self.debug("Run public mosquitto")
shell.run_public_mosquitto()
if "ntpd" not in ps:
# run NTPd for sync time
shell.run_ntpd()
bt_fix = shell.check_bt()
if bt_fix is None:
self.debug("Fixed BT don't supported")
elif bt_fix is False:
self.debug("Download fixed BT")
shell.download_bt()
# check after download
if shell.check_bt():
self.debug("Run fixed BT")
shell.run_bt()
elif "-t log/ble" not in ps:
self.debug("Run fixed BT")
shell.run_bt()
if "-t log/miio" not in ps:
# all data or only necessary events
pattern = ('\\{"' if 'miio' in self._debug else
"ot_agent_recv_handler_one.+"
"ble_event|properties_changed|heartbeat")
self.debug(f"Redirect miio to MQTT")
shell.redirect_miio2mqtt(pattern)
if self.options.get('buzzer'):
if "dummy:basic_gw" not in ps:
self.debug("Disable buzzer")
shell.stop_buzzer()
else:
if "dummy:basic_gw" in ps:
self.debug("Enable buzzer")
shell.run_buzzer()
if self.options.get('zha'):
if "Lumi_Z3GatewayHost_MQTT" in ps:
self.debug("Stop Lumi Zigbee")
shell.stop_lumi_zigbee()
if "tcp-l:8888" not in ps:
if "Received" in shell.check_or_download_socat():
self.debug("Download socat")
self.debug("Run Zigbee TCP")
shell.run_zigbee_tcp()
else:
if "tcp-l:8888" in ps:
self.debug("Stop Zigbee TCP")
shell.stop_zigbee_tcp()
if (self.parent_scan_interval >= 0 and
"Lumi_Z3GatewayHost_MQTT -n 1 -b 115200 -l" not in ps):
self.debug("Run public Zigbee console")
shell.run_public_zb_console()
elif "Lumi_Z3GatewayHost_MQTT" not in ps:
self.debug("Run Lumi Zigbee")
shell.run_lumi_zigbee()
shell.close()
return True
except (ConnectionRefusedError, socket.timeout):
return False
except Exception as e:
self.debug(f"Can't prepare gateway: {e}")
return False
def _mqtt_connect(self) -> bool:
try:
self.mqtt.reconnect()
return True
except:
return False
def _get_devices(self):
"""Load devices info for Coordinator, Zigbee and Mesh."""
try:
shell = TelnetShell(self.host)
# 1. Read coordinator info
raw = shell.read_file('/data/zigbee/coordinator.info')
device = json.loads(raw)
devices = [{
'did': shell.get_did(),
'model': 'lumi.gateway.mgl03',
'mac': device['mac'],
'wlan_mac': shell.get_wlan_mac(),
'type': 'gateway',
'fw_ver': shell.ver,
'online': True,
'init': {
'firmware lock': shell.check_firmware_lock(),
}
}]
# 2. Read zigbee devices
if not self.options.get('zha'):
# read Silicon devices DB
nwks = {}
try:
raw = shell.read_file(
'/data/silicon_zigbee_host/devices.txt')
raw = raw.decode().split(' ')
for i in range(0, len(raw) - 1, 32):
ieee = reversed(raw[i + 3:i + 11])
ieee = ''.join(f"{i:>02s}" for i in ieee)
nwks[ieee] = f"{raw[i]:>04s}"
except:
_LOGGER.exception("Can't read Silicon devices DB")
# read Xiaomi devices DB
raw = shell.read_file(shell.zigbee_db, as_base64=True)
# self.debug(f"Devices RAW: {raw}")
if raw.startswith(b'unqlite'):
db = Unqlite(raw)
data = db.read_all()
else:
raw = re.sub(br'}\s*{', b',', raw)
data = json.loads(raw)
# data = {} or data = {'dev_list': 'null'}
dev_list = json.loads(data.get('dev_list', 'null')) or []
for did in dev_list:
model = data.get(did + '.model')
if not model:
self.debug(f"{did} has not in devices DB")
continue
desc = zigbee.get_device(model)
# skip unknown model
if desc is None:
self.debug(f"{did} has an unsupported modell: {model}")
continue
retain = json.loads(data[did + '.prop'])['props']
self.debug(f"{did} {model} retain: {retain}")
params = {
p[2]: retain.get(p[1])
for p in (desc['params'] or desc['mi_spec'])
if p[1] is not None
}
ieee = f"{data[did + '.mac']:>016s}"
device = {
'did': did,
'mac': '0x' + data[did + '.mac'],
'ieee': ieee,
'nwk': nwks.get(ieee),
'model': model,
'type': 'zigbee',
'fw_ver': retain.get('fw_ver'),
'init': zigbee.fix_xiaomi_props(model, params),
'online': retain.get('alive', 1) == 1
}
devices.append(device)
# 3. Read bluetooth devices
if self._ble:
raw = shell.read_file('/data/miio/mible_local.db',
as_base64=True)
db = SQLite(raw)
# load BLE devices
rows = db.read_table('gateway_authed_table')
for row in rows:
device = {
'did': row[4],
'mac': RE_REVERSE.sub(r'\6\5\4\3\2\1', row[1]),
'model': row[2],
'type': 'ble',
'online': True,
'init': {}
}
devices.append(device)
# load Mesh groups
try:
mesh_groups = {}
rows = db.read_table(shell.mesh_group_table)
for row in rows:
# don't know if 8 bytes enougth
mac = int(row[0]).to_bytes(8, 'big').hex()
device = {
'did': 'group.' + row[0],
'mac': mac,
'model': 0,
'childs': [],
'type': 'mesh',
'online': True
}
devices.append(device)
group_addr = row[1]
mesh_groups[group_addr] = device
# load Mesh bulbs
rows = db.read_table(shell.mesh_device_table)
for row in rows:
device = {
'did': row[0],
'mac': row[1].replace(':', ''),
'model': row[2],
'type': 'mesh',
'online': False
}
devices.append(device)
group_addr = row[5]
if group_addr in mesh_groups:
# add bulb to group if exist
mesh_groups[group_addr]['childs'].append(row[0])
except:
_LOGGER.exception("Can't read mesh devices")
# for testing purposes
for k, v in self.default_devices.items():
if k[0] == '_':
devices.append(v)
return devices
except (ConnectionRefusedError, socket.timeout):
return None
except Exception as e:
_LOGGER.exception(f"{self.host} | Can't read devices: {e}")
return None
def lock_firmware(self, enable: bool):
self.debug(f"Set firmware lock to {enable}")
try:
shell = TelnetShell(self.host)
if "Received" in shell.check_or_download_busybox():
self.debug("Download busybox")
shell.lock_firmware(enable)
locked = shell.check_firmware_lock()
shell.close()
return enable == locked
except Exception as e:
self.debug(f"Can't set firmware lock: {e}")
return False
def update_entities_states(self):
for device in list(self.devices.values()):
if self in device['gateways']:
for entity in device['entities'].values():
if entity:
entity.schedule_update_ha_state()
def on_connect(self, client, userdata, flags, rc):
self.debug("MQTT connected")
self.mqtt.subscribe('#')
self.available = True
self.process_gw_stats()
self.update_entities_states()
def on_disconnect(self, client, userdata, rc):
self.debug("MQTT disconnected")
# force end mqtt.loop_forever()
self.mqtt.disconnect()
self.available = False
self.process_gw_stats()
self.update_entities_states()
def on_message(self, client: Client, userdata, msg: MQTTMessage):
# for debug purpose
enabled = self.enabled
try:
topic = msg.topic
if 'mqtt' in self._debug:
_LOGGER.debug(f"{self.host} | MQTT | {topic} {msg.payload}")
if topic == 'zigbee/send':
payload = json.loads(msg.payload)
self.process_message(payload)
elif topic == 'log/miio':
# don't need to process another data
if b'ot_agent_recv_handler_one' not in msg.payload:
return
for raw in utils.extract_jsons(msg.payload):
if self._ble and b'_async.ble_event' in raw:
data = json.loads(raw)['params']
self.process_ble_event(data)
self.process_ble_stats(data)
elif self._ble and b'properties_changed' in raw:
data = json.loads(raw)['params']
self.debug(f"Process props {data}")
self.process_mesh_data(data)
elif b'event.gw.heartbeat' in raw:
payload = json.loads(raw)['params'][0]
self.process_gw_stats(payload)
# time offset may changed right after gw.heartbeat
self.update_time_offset()
elif topic == 'log/ble':
payload = json.loads(msg.payload)
self.process_ble_event_fix(payload)
self.process_ble_stats(payload)
elif topic == 'log/z3':
self.process_z3(msg.payload.decode())
elif topic.endswith('/heartbeat'):
payload = json.loads(msg.payload)
self.process_gw_stats(payload)
elif topic.endswith(('/MessageReceived', '/devicestatechange')):
payload = json.loads(msg.payload)
self.process_zb_stats(payload)
# read only retained ble
elif topic.startswith('ble') and msg.retain:
payload = json.loads(msg.payload)
self.process_ble_retain(topic[4:], payload)
elif self.pair_model and topic.endswith('/commands'):
self.process_pair(msg.payload)
except:
_LOGGER.exception(f"Processing MQTT: {msg.topic} {msg.payload}")
def setup_devices(self, devices: list):
"""Add devices to hass."""
for device in devices:
did = device['did']
type_ = device['type']
if type_ == 'gateway':
self.did = device['did']
self.gw_topic = f"gw/{device['mac'][2:].upper()}/"
# if device already exists - take it from registry
if did not in self.devices:
if type_ in ('gateway', 'zigbee'):
desc = zigbee.get_device(device['model'])
elif type_ == 'mesh':
desc = bluetooth.get_device(device['model'], 'Mesh')
elif type_ == 'ble':
desc = bluetooth.get_device(device['model'], 'BLE')
else:
raise NotImplemented
device.update(desc)
# update params from config
default_config = (self.default_devices.get(device['mac'])
or self.default_devices.get(device['did']))
if default_config:
device.update(default_config)
self.debug(f"Setup {type_} device {device}")
device['entities'] = {}
device['gateways'] = []
self.devices[did] = device
else:
device = self.devices[did]
if type_ in ('gateway', 'zigbee', 'mesh'):
for param in (device['params'] or device['mi_spec']):
self.add_entity(param[3], device, param[2])
if self.options.get('stats') and type_ != 'mesh':
self.add_entity('sensor', device, device['type'])
def process_message(self, data: dict):
if data['cmd'] == 'heartbeat':
# don't know if only one item
assert len(data['params']) == 1, data
data = data['params'][0]
pkey = 'res_list'
elif data['cmd'] == 'report':
pkey = 'params' if 'params' in data else 'mi_spec'
elif data['cmd'] in ('write_rsp', 'read_rsp'):
pkey = 'results' if 'results' in data else 'mi_spec'
elif data['cmd'] == 'write_ack':
return
else:
_LOGGER.warning(f"Unsupported cmd: {data}")
return
did = data['did'] if data['did'] != 'lumi.0' else self.did
# skip without callback and without data
if did not in self.devices or pkey not in data:
return
ts = time.time()
device = self.devices[did]
payload = {}
# convert codes to names
for param in data[pkey]:
if param.get('error_code', 0) != 0:
continue
if 'res_name' in param:
prop = param['res_name']
elif 'piid' in param:
prop = f"{param['siid']}.{param['piid']}"
elif 'eiid' in param:
prop = f"{param['siid']}.{param['eiid']}"
else:
_LOGGER.warning(f"Unsupported param: {data}")
return
if prop in zigbee.GLOBAL_PROP:
prop = zigbee.GLOBAL_PROP[prop]
else:
prop = next((p[2]
for p in (device['params'] or device['mi_spec'])
if p[0] == prop), prop)
# https://github.com/Koenkk/zigbee2mqtt/issues/798
# https://www.maero.dk/aqara-temperature-humidity-pressure-sensor-teardown/
if (prop == 'temperature'
and device['model'] != 'lumi.airmonitor.acn01'):
if -4000 < param['value'] < 12500:
payload[prop] = param['value'] / 100.0
elif (prop == 'humidity'
and device['model'] != 'lumi.airmonitor.acn01'):
if 0 <= param['value'] <= 10000:
payload[prop] = param['value'] / 100.0
elif prop == 'pressure':
payload[prop] = param['value'] / 100.0
elif prop in ('battery', 'voltage'):
# sometimes voltage and battery came in one payload
if prop == 'voltage' and 'battery' in payload:
continue
# I do not know if the formula is correct, so battery is more
# important than voltage
payload['battery'] = (param['value']
if param['value'] < 1000 else int(
(min(param['value'], 3200) - 2600) /
6))
elif prop == 'alive' and param['value']['status'] == 'offline':
device['online'] = False
elif prop == 'angle':
# xiaomi cube 100 points = 360 degrees
payload[prop] = param['value'] * 4
elif prop == 'duration':
# xiaomi cube
payload[prop] = param['value'] / 1000.0
elif prop in ('consumption', 'power'):
payload[prop] = round(param['value'], 2)
elif 'value' in param:
payload[prop] = param['value']
elif 'arguments' in param:
if prop == 'motion':
payload[prop] = 1
else:
payload[prop] = param['arguments']
# no time in device add command
ts = round(ts - data['time'] * 0.001 + self.time_offset, 2) \
if 'time' in data else '?'
self.debug(f"{device['did']} {device['model']} <= {payload} [{ts}]")
if payload:
device['online'] = True
for entity in device['entities'].values():
if entity:
entity.update(payload)
# TODO: move code earlier!!!
if 'added_device' in payload:
# {'did': 'lumi.fff', 'mac': 'fff', 'model': 'lumi.sen_ill.mgl01',
# 'version': '21', 'zb_ver': '3.0'}
device = payload['added_device']
device['mac'] = '0x' + device['mac']
device['type'] = 'zigbee'
device['init'] = payload
self.setup_devices([device])
# return for tests purpose
return payload
def process_ble_event(self, data: dict):
self.debug(f"Process BLE {data}")
pdid = data['dev'].get('pdid')
did = data['dev']['did']
if did not in self.devices:
mac = data['dev']['mac'].replace(':', '').lower() \
if 'mac' in data['dev'] else \
'ble_' + did.replace('blt.3.', '')
self.devices[did] = device = {
'did': did,
'mac': mac,
'init': {},
'type': 'bluetooth'
}
desc = bluetooth.get_device(pdid, 'BLE')
device.update(desc)
# update params from config
default_config = self.default_devices.get(did)
if default_config:
device.update(default_config)
else:
device = self.devices[did]
if device.get('seq') == data['frmCnt']:
return
device['seq'] = data['frmCnt']
if isinstance(data['evt'], list):
# check if only one
assert len(data['evt']) == 1, data
payload = bluetooth.parse_xiaomi_ble(data['evt'][0], pdid)
elif isinstance(data['evt'], dict):
payload = bluetooth.parse_xiaomi_ble(data['evt'], pdid)
else:
payload = None
if payload:
self._process_ble_event(device, payload)
def process_ble_event_fix(self, data: dict):
self.debug(f"Process BLE Fix {data}")
did = data['did']
if did not in self.devices:
self.debug(f"Unregistered BLE device {did}")
return
device = self.devices[did]
if device.get('seq') == data['seq']:
return
device['seq'] = data['seq']
payload = bluetooth.parse_xiaomi_ble(data, data['pdid'])
if payload:
self._process_ble_event(device, payload)
def _process_ble_event(self, device: dict, payload: dict):
did = device['did']
# init entities if needed
init = device['init']
for k in payload.keys():
if k in init:
# update for retain
init[k] = payload[k]
continue
init[k] = payload[k]
domain = bluetooth.get_ble_domain(k)
self.add_entity(domain, device, k)
for entity in device['entities'].values():
if entity:
entity.update(payload)
raw = json.dumps(init, separators=(',', ':'))
self.mqtt.publish(f"ble/{did}", raw, retain=True)
def process_ble_retain(self, did: str, payload: dict):
if did not in self.devices:
self.debug(f"BLE device {did} is no longer on the gateway")
return
self.debug(f"{did} retain: {payload}")
device = self.devices[did]
# init entities if needed
for k in payload.keys():
# don't retain action and motion
if k in device['entities']:
continue
if k in ('action', 'motion'):
device['init'][k] = ''
else:
device['init'][k] = payload[k]
domain = bluetooth.get_ble_domain(k)
self.add_entity(domain, device, k)
for entity in device['entities'].values():
if entity:
entity.update(payload)
def process_pair(self, raw: bytes):
_LOGGER.debug(f"!!! {raw}")
# get shortID and eui64 of paired device
if b'lumi send-nwk-key' in raw:
# create model response
payload = f"0x08020105000042{len(self.pair_model):02x}" \
f"{self.pair_model.encode().hex()}"
m = RE_NWK_KEY.search(raw.decode())
self.pair_payload = json.dumps(
{
'sourceAddress': m[1],
'eui64': '0x' + m[2],
'profileId': '0x0104',
'clusterId': '0x0000',
'sourceEndpoint': '0x01',
'destinationEndpoint': '0x01',
'APSCounter': '0x01',
'APSPlayload': payload
},
separators=(',', ':'))
self.pair_payload2 = json.dumps(
{
'sourceAddress': m[1],
'eui64': '0x' + m[2],
'profileId': '0x0104',
'clusterId': '0x0000',
'sourceEndpoint': '0x01',
'destinationEndpoint': '0x01',
'APSCounter': '0x01',
'APSPlayload': '0x0801010100002001'
},
separators=(',', ':'))
# send model response "from device"
elif b'zdo active ' in raw:
self.mqtt.publish(self.gw_topic + 'MessageReceived',
self.pair_payload2)
self.mqtt.publish(self.gw_topic + 'MessageReceived',
self.pair_payload)
def send(self, device: dict, data: dict):
did = device['did'] if device['did'] != self.did else 'lumi.0'
payload = {'cmd': 'write', 'did': did}
# convert hass prop to lumi prop
if device['mi_spec']:
params = []
for k, v in data.items():
if k == 'switch':
v = bool(v)
k = next(p[0] for p in device['mi_spec'] if p[2] == k)
params.append({
'siid': int(k[0]),
'piid': int(k[2]),
'value': v
})
payload['mi_spec'] = params
else:
params = [{
'res_name':
next(p[0] for p in device['params'] if p[2] == k),
'value':
v
} for k, v in data.items()]
payload = {
'cmd': 'write',
'did': device['did'],
'params': params,
}
self.debug(f"{device['did']} {device['model']} => {payload}")
payload = json.dumps(payload, separators=(',', ':')).encode()
self.mqtt.publish('zigbee/recv', payload)
def send_telnet(self, *args: str):
try:
shell = TelnetShell(self.host)
for command in args:
if command == 'ftp':
shell.check_or_download_busybox()
shell.run_ftp()
elif command == 'dump':
raw = shell.tar_data()
filename = Path().absolute() / f"{self.host}.tar.gz"
with open(filename, 'wb') as f:
f.write(raw)
else:
shell.exec(command)
shell.close()
except Exception as e:
_LOGGER.exception(f"Telnet command error: {e}")
def send_mqtt(self, cmd: str):
if cmd == 'publishstate':
self.mqtt.publish(self.gw_topic + 'publishstate')
def get_device(self, mac: str) -> Optional[dict]:
for device in self.devices.values():
if device.get('mac') == mac:
return device
return None
class Gateway3(Thread, GatewayV, GatewayMesh, GatewayStats):
pair_model = None
pair_payload = None
def __init__(self, host: str, token: str, config: dict, **options):
super().__init__(daemon=True)
self.host = host
self.options = options
self.miio = SyncmiIO(host, token)
self.mqtt = Client()
self.mqtt.on_connect = self.on_connect
self.mqtt.on_disconnect = self.on_disconnect
self.mqtt.on_message = self.on_message
self.mqtt.connect_async(host)
self._ble = options.get('ble') # for fast access
self._debug = options.get('debug', '') # for fast access
self.parent_scan_interval = options.get('parent') # for fast access
self.default_devices = config['devices']
self.devices = {}
self.updates = {}
self.setups = {}
self.stats = {}
@property
def device(self):
return self.devices['lumi.0']
def add_update(self, did: str, handler):
"""Add handler to device update event."""
self.updates.setdefault(did, []).append(handler)
def remove_update(self, did: str, handler):
self.updates.setdefault(did, []).remove(handler)
def add_setup(self, domain: str, handler):
"""Add hass device setup funcion."""
self.setups[domain] = handler
def debug(self, message: str):
_LOGGER.debug(f"{self.host} | {message}")
def stop(self):
self.enabled = False
self.mqtt.loop_stop()
def run(self):
"""Main thread loop."""
self.enabled = True
while self.enabled:
# if not telnet - enable it
if not self._check_port(23) and not self._enable_telnet():
time.sleep(30)
continue
devices = self._prepeare_gateway(with_devices=True)
if devices:
self.setup_devices(devices)
break
self.mesh_start()
while self.enabled:
# if not telnet - enable it
if not self._check_port(23) and not self._enable_telnet():
time.sleep(30)
continue
# if not mqtt - enable it (handle Mi Home and ZHA mode)
if not self._mqtt_connect() and not self._prepeare_gateway():
time.sleep(60)
continue
self.mqtt.loop_forever()
def _check_port(self, port: int):
"""Check if gateway port open."""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
return s.connect_ex((self.host, port)) == 0
finally:
s.close()
def _enable_telnet(self):
"""Enable telnet with miio protocol."""
if self.miio.send("enable_telnet_service") != 'ok':
self.debug(f"Can't enable telnet")
return False
return True
def _prepeare_gateway(self, with_devices: bool = False):
"""Launching the required utilities on the hub, if they are not already
running.
"""
self.debug("Prepare Gateway")
try:
shell = TelnetShell(self.host)
self.ver = shell.get_version()
self.debug(f"Version: {self.ver}")
ps = shell.get_running_ps()
if "mosquitto -d" not in ps:
self.debug("Run public mosquitto")
shell.run_public_mosquitto()
# all data or only necessary events
pattern = '\\{"' if 'miio' in self._debug \
else "ble_event|properties_changed|heartbeat"
if f"awk /{pattern} {{" not in ps:
self.debug(f"Redirect miio to MQTT")
shell.redirect_miio2mqtt(pattern, self.ver_miio)
if self.options.get('buzzer'):
if "basic_gw -b" in ps:
self.debug("Disable buzzer")
shell.stop_buzzer()
else:
if "dummy:basic_gw" in ps:
self.debug("Enable buzzer")
shell.run_buzzer()
if self.options.get('zha'):
if "socat" not in ps:
if "Received" in shell.check_or_download_socat():
self.debug("Download socat")
self.debug("Run socat")
shell.run_socat()
if "Lumi_Z3GatewayHost_MQTT" in ps:
self.debug("Stop Lumi Zigbee")
shell.stop_lumi_zigbee()
else:
if "socat" in ps:
self.debug("Stop socat")
shell.stop_socat()
if (self.parent_scan_interval is not None and
"Lumi_Z3GatewayHost_MQTT -n 1 -b 115200 -v" not in ps):
self.debug("Run public Zigbee console")
shell.run_public_zb_console()
elif "Lumi_Z3GatewayHost_MQTT" not in ps:
self.debug("Run Lumi Zigbee")
shell.run_lumi_zigbee()
if with_devices:
self.debug("Get devices")
return self._get_devices(shell)
return True
except (ConnectionRefusedError, socket.timeout):
return False
except Exception as e:
self.debug(f"Can't read devices: {e}")
return False
def _mqtt_connect(self) -> bool:
try:
self.mqtt.reconnect()
return True
except:
return False
def _miio_connect(self) -> bool:
if not self.miio.ping():
self.debug("Can't send handshake")
return False
return True
def _get_devices(self, shell: TelnetShell):
"""Load devices info for Coordinator, Zigbee and Mesh."""
# 1. Read coordinator info
raw = shell.read_file('/data/zigbee/coordinator.info')
device = json.loads(raw)
devices = [{
'did': 'lumi.0',
'model': 'lumi.gateway.mgl03',
'mac': device['mac'],
'type': 'gateway',
'init': {
'firmware lock': shell.check_firmware_lock()
}
}]
# 2. Read zigbee devices
if not self.options.get('zha'):
raw = shell.read_file('/data/zigbee_gw/' + self.ver_zigbee_db,
as_base64=True)
if raw.startswith(b'unqlite'):
db = Unqlite(raw)
data = db.read_all()
else:
raw = re.sub(br'}\s+{', b',', raw)
data = json.loads(raw)
# data = {} or data = {'dev_list': 'null'}
dev_list = json.loads(data.get('dev_list', 'null')) or []
for did in dev_list:
model = data[did + '.model']
desc = utils.get_device(model)
# skip unknown model
if desc is None:
self.debug(f"{did} has an unsupported modell: {model}")
continue
retain = json.loads(data[did + '.prop'])['props']
self.debug(f"{did} {model} retain: {retain}")
params = {
p[2]: retain.get(p[1])
for p in (desc['params'] or desc['mi_spec'])
if p[1] is not None
}
device = {
'did': did,
'mac': '0x' + data[did + '.mac'],
'model': data[did + '.model'],
'type': 'zigbee',
'zb_ver': data[did + '.version'],
'init': utils.fix_xiaomi_props(params),
'online': retain.get('alive', 1) == 1
}
devices.append(device)
# 3. Read bluetooth devices
if self._ble:
raw = shell.read_file('/data/miio/mible_local.db', as_base64=True)
db = SQLite(raw)
# load BLE devices
rows = db.read_table('gateway_authed_table')
for row in rows:
device = {
'did': row[4],
'mac': RE_REVERSE.sub(r'\6\5\4\3\2\1', row[1]),
'model': row[2],
'type': 'ble'
}
devices.append(device)
# load Mesh groups
try:
mesh_groups = {}
rows = db.read_table(self.ver_mesh_group)
for row in rows:
# don't know if 8 bytes enougth
mac = int(row[0]).to_bytes(8, 'big').hex()
device = {
'did': 'group.' + row[0],
'mac': mac,
'model': 0,
'childs': [],
'type': 'mesh'
}
group_addr = row[1]
mesh_groups[group_addr] = device
# load Mesh bulbs
rows = db.read_table('mesh_device')
for row in rows:
device = {
'did': row[0],
'mac': row[1].replace(':', ''),
'model': row[2],
'type': 'mesh'
}
devices.append(device)
group_addr = row[5]
if group_addr in mesh_groups:
# add bulb to group if exist
mesh_groups[group_addr]['childs'].append(row[0])
for device in mesh_groups.values():
if device['childs']:
devices.append(device)
except:
_LOGGER.exception("Can't read mesh devices")
# for testing purposes
for k, v in self.default_devices.items():
if k[0] == '_':
devices.append(v)
return devices
def lock_firmware(self, enable: bool):
self.debug(f"Set firmware lock to {enable}")
try:
shell = TelnetShell(self.host)
if "Received" in shell.check_or_download_busybox():
self.debug("Download busybox")
shell.lock_firmware(enable)
locked = shell.check_firmware_lock()
shell.close()
return enable == locked
except Exception as e:
self.debug(f"Can't set firmware lock: {e}")
return False
def on_connect(self, client, userdata, flags, rc):
self.debug("MQTT connected")
self.mqtt.subscribe('#')
self.available = True
self.process_gw_stats()
def on_disconnect(self, client, userdata, rc):
self.debug("MQTT disconnected")
# force end mqtt.loop_forever()
self.mqtt.disconnect()
self.available = False
self.process_gw_stats()
def on_message(self, client: Client, userdata, msg: MQTTMessage):
if 'mqtt' in self._debug:
self.debug(f"[MQ] {msg.topic} {msg.payload.decode()}")
if msg.topic == 'zigbee/send':
payload = json.loads(msg.payload)
self.process_message(payload)
elif msg.topic == 'log/miio':
if 'miio' in self._debug:
self.debug(f"[MI] {msg.payload}")
if self._ble and (b'_async.ble_event' in msg.payload
or b'properties_changed' in msg.payload
or b'event.gw.heartbeat' in msg.payload):
try:
for raw in utils.extract_jsons(msg.payload):
if b'_async.ble_event' in raw:
data = json.loads(raw)['params']
self.process_ble_event(data)
self.process_ble_stats(data)
elif b'properties_changed' in raw:
data = json.loads(raw)['params']
self.debug(f"Process props {data}")
self.process_mesh_data(data)
elif b'event.gw.heartbeat' in raw:
payload = json.loads(raw)['params'][0]
self.process_gw_stats(payload)
except:
_LOGGER.warning(f"Can't read BT: {msg.payload}")
elif msg.topic.endswith('/heartbeat'):
payload = json.loads(msg.payload)
self.process_gw_stats(payload)
elif msg.topic.endswith(('/MessageReceived', '/devicestatechange')):
payload = json.loads(msg.payload)
self.process_zb_stats(payload)
# read only retained ble
elif msg.topic.startswith('ble') and msg.retain:
payload = json.loads(msg.payload)
self.process_ble_retain(msg.topic[4:], payload)
elif self.pair_model and msg.topic.endswith('/commands'):
self.process_pair(msg.payload)
def setup_devices(self, devices: list):
"""Add devices to hass."""
for device in devices:
if device['type'] in ('gateway', 'zigbee'):
desc = utils.get_device(device['model'])
if not desc:
self.debug(f"Unsupported model: {device}")
continue
self.debug(f"Setup Zigbee device {device}")
device.update(desc)
# update params from config
default_config = (self.default_devices.get(device['mac'])
or self.default_devices.get(device['did']))
if default_config:
device.update(default_config)
self.devices[device['did']] = device
for param in (device['params'] or device['mi_spec']):
domain = param[3]
if not domain:
continue
# wait domain init
while domain not in self.setups:
time.sleep(1)
attr = param[2]
self.setups[domain](self, device, attr)
elif device['type'] == 'mesh':
desc = bluetooth.get_device(device['model'], 'Mesh')
device.update(desc)
self.debug(f"Setup Mesh device {device}")
# update params from config
default_config = self.default_devices.get(device['did'])
if default_config:
device.update(default_config)
device['online'] = False
self.devices[device['did']] = device
# wait domain init
while 'light' not in self.setups:
time.sleep(1)
self.setups['light'](self, device, 'light')
elif device['type'] == 'ble':
# only save info for future
desc = bluetooth.get_device(device['model'], 'BLE')
device.update(desc)
# update params from config
default_config = self.default_devices.get(device['did'])
if default_config:
device.update(default_config)
self.devices[device['did']] = device
device['init'] = {}
if self.options.get('stats') and device['type'] != 'mesh':
while 'sensor' not in self.setups:
time.sleep(1)
self.setups['sensor'](self, device, device['type'])
def process_message(self, data: dict):
if data['cmd'] == 'heartbeat':
# don't know if only one item
assert len(data['params']) == 1, data
data = data['params'][0]
pkey = 'res_list'
elif data['cmd'] == 'report':
pkey = 'params' if 'params' in data else 'mi_spec'
elif data['cmd'] in ('write_rsp', 'read_rsp'):
pkey = 'results'
elif data['cmd'] == 'write_ack':
return
else:
_LOGGER.warning(f"Unsupported cmd: {data}")
return
did = data['did']
# skip without callback
if did not in self.updates:
return
device = self.devices[did]
payload = {}
# convert codes to names
for param in data[pkey]:
if param.get('error_code', 0) != 0:
continue
prop = param['res_name'] if 'res_name' in param else \
f"{param['siid']}.{param['piid']}"
if prop in GLOBAL_PROP:
prop = GLOBAL_PROP[prop]
else:
prop = next((p[2]
for p in (device['params'] or device['mi_spec'])
if p[0] == prop), prop)
if prop in ('temperature', 'humidity', 'pressure'):
payload[prop] = param['value'] / 100.0
elif prop == 'battery' and param['value'] > 1000:
# xiaomi light sensor
payload[prop] = round((min(param['value'], 3200) - 2500) / 7)
elif prop == 'alive':
# {'res_name':'8.0.2102','value':{'status':'online','time':0}}
device['online'] = (param['value']['status'] == 'online')
elif prop == 'angle':
# xiaomi cube 100 points = 360 degrees
payload[prop] = param['value'] * 4
elif prop == 'duration':
# xiaomi cube
payload[prop] = param['value'] / 1000.0
elif prop in ('consumption', 'power'):
payload[prop] = round(param['value'], 2)
else:
payload[prop] = param['value']
self.debug(f"{device['did']} {device['model']} <= {payload}")
for handler in self.updates[did]:
handler(payload)
if 'added_device' in payload:
# {'did': 'lumi.fff', 'mac': 'fff', 'model': 'lumi.sen_ill.mgl01',
# 'version': '21', 'zb_ver': '3.0'}
device = payload['added_device']
device['mac'] = '0x' + device['mac']
device['type'] = 'zigbee'
device['init'] = payload
self.setup_devices([device])
def process_ble_event(self, data: dict):
self.debug(f"Process BLE {data}")
pdid = data['dev'].get('pdid')
did = data['dev']['did']
if did not in self.devices:
mac = data['dev']['mac'].replace(':', '').lower() \
if 'mac' in data['dev'] else \
'ble_' + did.replace('blt.3.', '')
self.devices[did] = device = {
'did': did,
'mac': mac,
'init': {},
'type': 'bluetooth'
}
desc = bluetooth.get_device(pdid, 'BLE')
device.update(desc)
# update params from config
default_config = self.default_devices.get(did)
if default_config:
device.update(default_config)
else:
device = self.devices[did]
if isinstance(data['evt'], list):
# check if only one
assert len(data['evt']) == 1, data
payload = bluetooth.parse_xiaomi_ble(data['evt'][0], pdid)
elif isinstance(data['evt'], dict):
payload = bluetooth.parse_xiaomi_ble(data['evt'], pdid)
else:
payload = None
if payload is None:
self.debug(f"Unsupported BLE {data}")
return
# init entities if needed
init = device['init']
for k in payload.keys():
if k in init:
# update for retain
init[k] = payload[k]
continue
init[k] = payload[k]
domain = bluetooth.get_ble_domain(k)
if not domain:
continue
# wait domain init
while domain not in self.setups:
time.sleep(1)
self.setups[domain](self, device, k)
if did in self.updates:
for handler in self.updates[did]:
handler(payload)
raw = json.dumps(init, separators=(',', ':'))
self.mqtt.publish(f"ble/{did}", raw, retain=True)
def process_ble_retain(self, did: str, payload: dict):
if did not in self.devices:
self.debug(f"BLE device {did} is no longer on the gateway")
return
self.debug(f"{did} retain: {payload}")
device = self.devices[did]
# init entities if needed
for k in payload.keys():
# don't retain action
if k in device['init'] or k == 'action':
continue
device['init'][k] = payload[k]
domain = bluetooth.get_ble_domain(k)
if not domain:
continue
# wait domain init
while domain not in self.setups:
time.sleep(1)
self.setups[domain](self, device, k)
if did in self.updates:
for handler in self.updates[did]:
handler(payload)
def process_pair(self, raw: bytes):
# get shortID and eui64 of paired device
if b'lumi send-nwk-key' in raw:
# create model response
payload = f"0x18010105000042{len(self.pair_model):02x}" \
f"{self.pair_model.encode().hex()}"
m = RE_NWK_KEY.search(raw.decode())
self.pair_payload = json.dumps(
{
'sourceAddress': m[1],
'eui64': '0x' + m[2],
'profileId': '0x0104',
'clusterId': '0x0000',
'sourceEndpoint': '0x01',
'destinationEndpoint': '0x01',
'APSCounter': '0x01',
'APSPlayload': payload
},
separators=(',', ':'))
# send model response "from device"
elif b'zdo active ' in raw:
mac = self.device['mac'][2:].upper()
self.mqtt.publish(f"gw/{mac}/MessageReceived", self.pair_payload)
def send(self, device: dict, data: dict):
payload = {'cmd': 'write', 'did': device['did']}
# convert hass prop to lumi prop
if device['mi_spec']:
params = []
for k, v in data.items():
if k == 'switch':
v = bool(v)
k = next(p[0] for p in device['mi_spec'] if p[2] == k)
params.append({'siid': k[0], 'piid': k[1], 'value': v})
payload['mi_spec'] = params
else:
params = [{
'res_name':
next(p[0] for p in device['params'] if p[2] == k),
'value':
v
} for k, v in data.items()]
payload = {
'cmd': 'write',
'did': device['did'],
'params': params,
}
self.debug(f"{device['did']} {device['model']} => {payload}")
payload = json.dumps(payload, separators=(',', ':')).encode()
self.mqtt.publish('zigbee/recv', payload)
def send_telnet(self, *args: str):
try:
shell = TelnetShell(self.host)
for command in args:
if command == 'ftp':
shell.check_or_download_busybox()
shell.run_ftp()
else:
shell.exec(command)
shell.close()
except Exception as e:
_LOGGER.exception(f"Telnet command error: {e}")
def send_mqtt(self, cmd: str):
if cmd == 'publishstate':
mac = self.device['mac'][2:].upper()
self.mqtt.publish(f"gw/{mac}/publishstate")
def get_device(self, mac: str) -> Optional[dict]:
for device in self.devices.values():
if device.get('mac') == mac:
return device
return None
from paho.mqtt.client import Client
DEVICE = "172.25.11.1"
TOPIC = "/junos/events/kernel/firewall/filter/#"
def callback(client, data, message):
print "Message with topic %s received: %s" % (message.topic, message.payload)
if __name__ == "__main__":
print "Connecting to MQTT broker"
client = Client()
client.on_message = callback
client.connect(DEVICE)
client.loop_start()
client.subscribe(TOPIC)
raw_input("Press Enter to stop")
client.loop_stop()
client.disconnect()
def subscribe(
topics: str | list[str],
hostname=leader_hostname,
retries: int = 10,
timeout: Optional[float] = None,
allow_retained: bool = True,
**mqtt_kwargs,
) -> Optional[MQTTMessage]:
"""
Modeled closely after the paho version, this also includes some try/excepts and
a timeout. Note that this _does_ disconnect after receiving a single message.
A failure case occurs if this is called in a thread (eg: a callback) and is waiting
indefinitely for a message. The parent job may not exit properly.
"""
retry_count = 1
for retry_count in range(retries):
try:
lock: Optional[threading.Lock]
def on_connect(client, userdata, flags, rc):
client.subscribe(userdata["topics"])
return
def on_message(client, userdata, message: MQTTMessage):
if not allow_retained and message.retain:
return
userdata["messages"] = message
client.disconnect()
if userdata["lock"]:
userdata["lock"].release()
return
if timeout:
lock = threading.Lock()
else:
lock = None
topics = [topics] if isinstance(topics, str) else topics
userdata: dict[str, Any] = {
"topics": [(topic, mqtt_kwargs.pop("qos", 0)) for topic in topics],
"messages": None,
"lock": lock,
}
client = Client(userdata=userdata)
client.on_connect = on_connect
client.on_message = on_message
client.connect(leader_hostname)
if timeout is None:
client.loop_forever()
else:
assert lock is not None
lock.acquire()
client.loop_start()
lock.acquire(timeout=timeout)
client.loop_stop()
client.disconnect()
return userdata["messages"]
except (ConnectionRefusedError, socket.gaierror, OSError, socket.timeout):
from pioreactor.logging import create_logger
logger = create_logger("pubsub.subscribe", to_mqtt=False)
logger.debug(
f"Attempt {retry_count}: Unable to connect to host: {hostname}",
exc_info=True,
)
time.sleep(5 * retry_count) # linear backoff
else:
logger = create_logger("pubsub.subscribe", to_mqtt=False)
logger.error(f"Unable to connect to host: {hostname}. Exiting.")
raise ConnectionRefusedError(f"Unable to connect to host: {hostname}.")
class MqttHandler:
def __init__(self, client_id='DEFAULT_CLIENT_ID', topic='DEFAULT_TOPIC', broker_host='localhost',
broker_port=MQTT_BROKER_PORT):
self.subscribed = False
self.client_id = client_id
self.client = Client(client_id=self.client_id, protocol=MQTT_PROTOCOL_VERSION)
self.client.on_message = self.on_message_callback
self.client.on_publish = self.on_publish_callback
self.client.on_connect = self.connect_callback
self.client.on_disconnect = self.disconnect_callback
self.topic = topic
self.broker_host = broker_host
self.broker_port = broker_port
self.message_received = 0
userdata = {
USER_DATA_MESSAGE_RECEIVED: 0,
}
self.client.user_data_set(userdata)
def connect(self):
self.client.connect(host=self.broker_host, port=self.broker_port)
def connect_async(self):
self.client.connect_async(host=self.broker_host, port=self.broker_port)
def connect_callback(self, client, userdata, flags, rc):
print('connect_callback: result code[' + str(rc) + ']')
(result, _) = client.subscribe(topic=self.topic)
self.subscribed = result
def disconnect(self):
self.client.disconnect()
def disconnect_callback(self, client, userdata, rc):
print('disconnect_callback')
def is_valid(self, my_json: json):
if app.config.get('DEBUG', False):
print("json_validation")
# try:
# if my_json['id'] is None or my_json['byte_stream'] is None:
# return False
# except KeyError:
# return False
return True
def on_message_callback(self, client, userdata, message):
from core.socketio_runner import emit_command
userdata[USER_DATA_MESSAGE_RECEIVED] += 1
topic = message.topic
payload = json.loads(message.payload)
if app.config.get('DEBUG', False):
print('on_message_callback: topic[' + topic + ']')
if self.is_valid(payload):
emit_command(topic, payload)
else:
raise Exception('Message payload not valid')
@staticmethod
def publish_single_message(topic, payload=None, qos=0, retain=False, hostname="localhost",
port=MQTT_BROKER_PORT, client_id="", keepalive=60, will=None, auth=None, tls=None):
if app.config.get('DEBUG', False):
print("publish_single_message")
single(topic=topic, payload=payload, qos=qos, retain=retain, hostname=hostname, port=port, client_id=client_id,
keepalive=keepalive, will=will, auth=auth, tls=tls)
def on_publish_callback(self, client, userdata, mid):
print('on_publish_callback')
def loop_for_ever(self):
self.client.loop_forever()
def loop_start(self):
self.client.loop_start()
def loop_stop(self, force=False):
self.client.loop_stop(force=force)
