class Mqtt(Connector):
def __init__(self, config, asm=None):
self.name = "mqtt"
self.config = config
self.asm = asm
self.default_room = "MyDefaultRoom"
self.client = None
async def connect(self):
self.client = MQTTClient(self.asm.name)
self.client.on_message = on_message
self.client.set_auth_credentials(
os.getenv('MQTT_USER', "arcus"),
os.getenv('MQTT_PASSWORD', "arcusarcus"))
await self.client.connect(os.getenv('MQTT_HOST', "mqtt"),
1883,
keepalive=60,
version=MQTTv311)
_LOGGER.info("Connected to MQTT")
async def listen(self):
self.client.subscribe(self.asm.name + "/#", qos=1)
stop = asyncio.Event()
await stop.wait()
@register_event(Message)
async def respond(self, message):
self.client.publish(self.asm.name,
'Message payload',
response_topic='RESPONSE/TOPIC')
async def disconnect(self):
# Disconnect from the service
await self.client.disconnect()
async def main(broker_host, username, password):
client = MQTTClient("client-id-mitch", user_property=('hello', 'there'))
# client = MQTTClient("client-id-mitch")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(username, password.encode())
await client.connect(host=broker_host, port=1883)
# await client.connect()
print("connected, now ready to send...")
data = f"This is a test! {str(time.time())}"
hash = hashlib.sha256(data.encode()).hexdigest()
client.publish('test/time1',
"hello test/time1..",
qos=1,
message_expiry_interval=5,
content_type="json",
response_topic='RESPONSE/TOPIC2',
user_property=[('hash', hash), ('time', str(time.time()))])
client.publish('test/time2',
"hello test/time2..",
qos=1,
message_expiry_interval=5,
content_type="json",
response_topic='RESPONSE/TOPIC',
user_property=[('hash', hash), ('time', str(time.time()))])
await STOP.wait()
await client.disconnect()
async def connect(self):
try:
print(
'""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""'
)
print('Attempting MQTT connection...')
print('MQTT host : ', self.broker_host)
print('MQTT user : '******'MQTT error, restarting in 8s...')
await asyncio.sleep(8)
await self.connect()
async def main(broker_host, token):
client = MQTTClient(__name__)
client.on_connect = on_connect
client.on_disconnect = on_disconnect
client.on_message = on_message
client.on_subscribe = on_subscribe
if token:
client.set_auth_credentials(token)
log.debug(f'Connecting to {broker_host}')
await client.connect(broker_host)
def on_app_changed(status, payload):
client.publish(BASETOPIC + "/app", payload)
def on_volume_changed(status, payload):
client.publish(BASETOPIC + "/volume", payload)
lg = LGClient()
lg.ac.subscribe_get_current(on_app_changed)
lg.mc.subscribe_get_volume(on_volume_changed)
# Periodic heartbeat from the bridge
def heartbeat():
while True:
client.publish(f"{BASETOPIC}/heartbeat/{client._client_id}",
str(datetime.datetime.now()))
time.sleep(1)
threading.Thread(target=heartbeat, daemon=True).start()
await stopEvent.wait()
await client.disconnect()
class MQTTCli:
def __init__(self, loop: asyncio.AbstractEventLoop, device_registry):
self.loop = loop
self.device_registry = device_registry
self.client = MQTTClient("client-id")
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
async def connect():
user = "******"
pw = "oochi8eengehuYohgeBu1foobooceeZ7to5ieng7pis8saephaetah0hoaphiK3F"
broker_host = "192.168.50.95"
self.client.set_auth_credentials(user, pw)
await self.client.connect(broker_host)
def on_connect(client, flags, rc, properties):
print('Connected')
self.client.subscribe('home-assistant/command', qos=0)
def on_message(client, topic, payload, qos, properties):
print('RECV MSG:', payload)
device_registry.bluetooth_devices.send_message(payload, True, False)
publish('home-assistant/response', payload, qos=1)
def on_disconnect(client, packet, exc=None):
print('Disconnected')
def on_subscribe(client, mid, qos, properties):
print('SUBSCRIBED')
def ask_exit(*args):
STOP.set()
async def main(broker_host, username, password):
client = MQTTClient(client_id="pub-client-id", receive_maximum=24000)
client.on_disconnect = on_disconnect
client.set_auth_credentials(username=username, password=password)
await client.connect(broker_host)
for i in range(10000):
client.publish(message_or_topic='TEST/T1', payload=str(i), qos=1, retain=True, message_expiry_interval=60)
def prepare_client(token):
CLIENT_ID = create_uid()
client = MQTTClient(CLIENT_ID)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
return client
async def main():
client = MQTTClient(mqtt_client_id,
session_expiry_interval=86400 * 10,
clean_session=False)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(username, None)
await client.connect(broker_host)
await STOP.wait()
await client.disconnect()
async def main(broker_host, username, password):
client = MQTTClient(client_id="sub-client-id",
receive_maximum=24000,
clean_session=False,
session_expiry_interval=60)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(username=username, password=password)
await client.connect(broker_host)
await STOP.wait()
await client.disconnect()
async def main(broker_host, token):
client = MQTTClient("vscode-client")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
await client.connect(broker_host)
# client.publish('TEST/TIME', str(time.time()), qos=0)
await STOP.wait()
await client.disconnect()
async def main(broker_host):
client = MQTTClient("client-id")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials('admin', 'admin')
await client.connect(broker_host, 8080, keepalive=60, version=MQTTv50)
#client.publish('TEST/TIME', str(time.time()), qos=1)
time.sleep(30)
await STOP.wait()
await client.disconnect()
async def main():
client = MQTTClient('flespi-examples-mqtt-client-python',
clean_session=True)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
# see https://flespi.com/kb/tokens-access-keys-to-flespi-platform to read about flespi tokens
client.set_auth_credentials(
'FlespiToken {}'.format(os.environ.get("FlespiToken")), None)
print('mqtt client created, connecting...')
await client.connect('mqtt.flespi.io', port=8883, ssl=True)
await STOP.wait()
await client.disconnect()
print('disconnected')
async def main(broker_host, token):
client_id = create_uid()
client = MQTTClient(client_id)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
await client.connect(broker_host)
# Watchdog print out online_clients every 5s
t = threading.Thread(target=watchdog_clients)
t.setDaemon(True)
t.start()
await STOP.wait()
await client.disconnect()
async def main(broker_host, token):
global con
con = pymysql.connect(host=mysql_host,
user=mysql_user,
passwd=mysql_passwd,
db=mysql_db,
autocommit=True)
client = MQTTClient('message_listener')
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
await client.connect(broker_host)
await STOP.wait()
await client.disconnect()
async def main(loop, broker_host, token):
engine = create_engine(SQLALCHEMY_DATABASE_URI)
conn = engine.connect()
Session = sessionmaker(bind=engine)
session = Session()
client = MQTTClient("client-id")
client.on_connect = on_connect
client.on_message = partial(on_message, conn, session)
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
await client.connect(broker_host)
await STOP.wait()
await client.disconnect()
async def main(broker_host, token, loop):
global MQTTClient, QueuePersister, CustomMicroUser, logging, after_get, STOP, asyncio
logging.info("Client ID %s", os.environ.get('MQTT_CLIENT_ID'))
client = MQTTClient(os.environ.get('MQTT_CLIENT_ID'))
client.set_auth_credentials(token, None)
await client.connect(broker_host)
queuePersist = QueuePersister(client, loop)
users_to_get = []
for i in range(1, 10000):
users_to_get.append(asyncio.create_task(queuePersist.get(CustomMicroUser(pk=1), after_get)))
await asyncio.gather(
*users_to_get
)
await STOP.wait()
await client.disconnect()
class MQTTClient(Client):
''' Client for subscribing to a broker and pipe incomming messages to configured producer. '''
def __init__(
self,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self._inner_client = GMQTTClient(None)
self._inner_client.on_connect = self.on_connect
self._inner_client.on_message = self.on_message
self._inner_client.on_disconnect = self.on_disconnect
self._inner_client.on_subscribe = self.on_subscribe
self._inner_client.pipe_message = self.pipe_message
if not self.username is None:
self._inner_client.set_auth_credentials(self.username, self.password)
async def connect(self, topics: Tuple[str, int]) -> None:
''' Connects to broker. '''
LOG.debug('Connecting to MQTT broker.')
try:
await self._inner_client.connect(self.uri, 1883, keepalive=60, version=MQTTv311)
subscriptions = [Subscription(t[0], qos=t[1]) for t in topics]
self._inner_client.subscribe(subscriptions, subscription_identifier=1)
await self.producer.connect()
self.connected = True
except:
self.connected = False
async def disconnect(self) -> None:
await self._inner_client.disconnect()
await self.producer.disconnect()
self.connected = False
class Deconz2acp():
###################
# Sync class init
###################
def __init__(self, settings):
self.settings = settings
print("{} Deconz2acp __init__() DEBUG={}".format(
self.ts_string(), DEBUG),
file=sys.stderr,
flush=True)
#####################################
# Signal handler for SIGINT, SIGTERM
#####################################
def ask_exit(self, *args):
self.STOP.set()
#####################################
# Return current timestamp as string
#####################################
def ts_string(self):
return '{:.6f}'.format(time.time())
###############################################################
# Async initialization
###############################################################
async def start(self, zigbee_data):
print("{} Deconz2acp start()".format(self.ts_string()),
file=sys.stderr,
flush=True)
self.zigbee_data = zigbee_data
# Define async events for exit and reload (will set via signals)
self.STOP = asyncio.Event()
self.RELOAD = asyncio.Event()
# connect output MQTT broker
await self.connect_output_mqtt()
# Connect input WebSocket
asyncio.ensure_future(self.subscribe_input_ws())
# Connect to input websocket
async def subscribe_input_ws(self):
ws_url = self.settings["input_ws"]["url"]
connected = False
while True:
try:
async with websockets.connect(ws_url) as ws:
connected = True
print("{} Deconz2acp connected to {}".format(
self.ts_string(), ws_url),
flush=True)
while connected:
try:
if DEBUG:
print("{} Deconz2acp awaiting msg from {}".
format(self.ts_string(), ws_url),
flush=True)
# Here we await & receive any websocket message
msg = await ws.recv()
if DEBUG:
pretty_msg = json.dumps(json.loads(msg),
indent=4)
print(
"{} Deconz2acp msg received from {}:\n{}".
format(self.ts_string(), ws_url,
pretty_msg),
flush=True)
#debug we're stuffing in a fake "zigbee" topic
self.handle_input_message(msg)
except websockets.exceptions.ConnectionClosedError:
connected = False
print("{} Deconz2acp disconnected from {}".format(
self.ts_string(), ws_url),
flush=True)
print("{} Deconz2acp websocket read loop ended".format(
self.ts_string()),
flush=True)
except ConnectionRefusedError:
print("{} Deconz2acp websocket connection refused from {}".
format(self.ts_string(), ws_url),
flush=True)
await asyncio.sleep(2) # sleep 2 seconds and retry
print("{} Deconz2acp websocket connect loop ended".format(
self.ts_string()),
flush=True)
async def connect_output_mqtt(self):
self.output_client = MQTTClient(None) # auto-generate client id
self.output_client.on_connect = self.output_on_connect
self.output_client.on_message = self.output_on_message
self.output_client.on_disconnect = self.output_on_disconnect
self.output_client.on_subscribe = self.output_on_subscribe
user = self.settings["output_mqtt"]["user"]
password = self.settings["output_mqtt"]["password"]
host = self.settings["output_mqtt"]["host"]
port = self.settings["output_mqtt"]["port"]
self.output_client.set_auth_credentials(user, password)
try:
await self.output_client.connect(host,
port,
keepalive=60,
version=MQTTv311)
except Exception as e:
if hasattr(e, 'args') and e.args[0] == 5:
print(
"{}\033[1;31m FAIL: Connect output_mqtt auth (as {} )\033[0;0m"
.format(self.ts_string(), user),
file=sys.stderr,
flush=True)
else:
print(
"{}\033[1;31m FAIL gmqtt connect exception\n{}\n{}\033[0;0m"
.format(self.ts_string(), e),
file=sys.stderr,
flush=True)
self.ask_exit()
###############################################################
# Sensor data message handler
###############################################################
def handle_input_message(self, msg_bytes):
msg_dict = json.loads(msg_bytes)
# Add required zigbee properties by updating msg_dict
# send_data will be True if zigbee_data.decode decides this message should be sent via MQTT.
send_data = self.zigbee_data.handle_ws_message(msg_dict)
if send_data:
topic = ""
if "acp_id" in msg_dict:
topic += msg_dict["acp_id"]
self.send_output_message(topic, msg_dict)
else:
print("{} Incoming message not sent to MQTT\n{}\n".format(
self.ts_string(), msg_bytes),
flush=True)
def send_output_message(self, topic, msg_dict):
msg_bytes = json.dumps(msg_dict)
#print("publishing {}".format(msg_bytes), flush=True)
output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic
if DEBUG:
pretty_msg = json.dumps(msg_dict, indent=4)
print("{} MQTT publish disabled by DEBUG setting:\n{}".format(
self.ts_string(), pretty_msg),
flush=True)
else:
self.output_client.publish(output_topic, msg_bytes, qos=0)
###############################################################
# WS INPUT
###############################################################
def input_ws_connected(self, uri):
print('{} INPUT Connected to {}'.format(self.ts_string(), uri),
flush=True)
###############################################################
# MQTT OUTPUT
###############################################################
def output_on_connect(self, client, flags, rc, properties):
print('{} OUTPUT Connected to {} as {}'.format(
self.ts_string(), self.settings["output_mqtt"]["host"],
self.settings["output_mqtt"]["user"]),
flush=True)
def output_on_disconnect(self, client, packet, exc=None):
print("{} OUTPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
# These GMQTT methods here for completeness although not used
def output_on_message(self, client, topic, msg_bytes, qos, properties):
print('OUTPUT RECV MSG?:', msg_bytes, flush=True)
def output_on_subscribe(self, client, mid, qos, properties):
print('OUTPUT SUBSCRIBED? to {}', flush=True)
###############################################################
# CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM)
###############################################################
async def finish(self):
await self.STOP.wait()
print("\n{} Deconz2acp interrupted - disconnecting\n".format(
self.ts_string()),
file=sys.stderr,
flush=True)
await self.output_client.disconnect()
class MQTTClient(Entity):
"""
A helper class for MQTT. Handles all the connection details. Returned to the library or module
that calls self._MQTTYombo.new().
.. code-block:: python
self.my_mqtt = self._MQTT.new(on_message_callback=self.mqtt_incoming, client_id="my_client_name")
self.my_mqtt.subscribe("yombo/devices/+/get") # subscribe to a topic. + is a wildcard for a single section.
"""
def __init__(self,
parent,
hostname: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
use_ssl: Optional[bool] = None,
version: Optional[str] = None,
keepalive: Optional[int] = None,
session_expiry: Optional[int] = None,
receive_maximum: Optional[int] = None,
user_property: Optional[Union[tuple, List[tuple]]] = None,
last_will: Optional = None,
maximum_packet_size: Optional[int] = None,
on_message_callback: Callable = None,
subscribe_callback: Callable = None,
unsubscribe_callback: Callable = None,
connected_callback: Optional[Callable] = None,
disconnected_callback: Optional[Callable] = None,
error_callback: Optional[Callable] = None,
client_id: Optional[str] = None,
password2: Optional[str] = None):
"""
Creates a new client connection to an MQTT broker.
:param parent: A reference to the MQTT library.
:param hostname: IP address or hostname to connect to.
:param port: Port number to connect to.
:param username: Username to connect as. Use "" to not use a username & password.
:param password: Password to to connect with. Use "" to not use a password.
:param use_ssl: Use SSL when attempting to connect to server, default is True.
:param version: MQTT version to use, default: MQTTv50. Other: MQTTv311
:param keepalive: How often the connection should be checked that it's still alive.
:param session_expiry: How many seconds the session should be valid. Defaults to 0.
:param receive_maximum: The Client uses this value to limit the number of QoS 1 and QoS 2 publications that it
is willing to process concurrently.
:param user_property: Connection user_property. A tuple or list of tuples.
:param last_will: Last will message generated by 'will()'.
:param maximum_packet_size: The maximum size the mqtt payload should be, in size.
:param on_message_callback: (required) method - Method to send messages to.
:param connected_callback: method - If you want a function called when connected to server.
:param disconnected_callback: method - If you want a function called when disconnected from server.
:param subscribe_callback: method - This method will be called when successfully subscribed to topic.
:param unsubscribe_callback: method - This method will be called when successfully unsubscribed from topic.
:param error_callback: method - A function to call if something goes wrong.
:param client_id: (default - random) - A client id to use for logging.
:param password2: A second password to try. Used by MQTTYombo.
:return:
"""
self._Entity_type: str = "MQTTClient"
self._Entity_label_attribute: str = "client_id"
super().__init__(parent)
self.connected = False
self.incoming_duplicates = deque([], 150)
self.send_queue = deque()
self.subscriptions = {}
self.unsubscriptions = {}
self.topics = {} # Store topics to resubscribe to
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.password2 = password2
self.use_ssl = use_ssl
self.version = version
self.keepalive = keepalive
self.session_expiry = session_expiry
self.receive_maximum = receive_maximum
self.user_property = user_property
self.last_will = last_will
self.maximum_packet_size = maximum_packet_size
self.on_message_callback = on_message_callback
self.subscribe_callback = subscribe_callback
self.unsubscribe_callback = unsubscribe_callback
self.connected_callback = connected_callback
self.disconnected_callback = disconnected_callback
self.error_callback = error_callback
self.client_id = client_id
client_options = {
"receive_maximum": receive_maximum,
"session_expiry_interval": session_expiry,
"maximum_packet_size": maximum_packet_size,
"user_property": user_property,
}
self.client = QClient(
client_id,
**{k: v
for k, v in client_options.items() if v is not None})
self.client.set_auth_credentials(username, password.encode())
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message_callback
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
@inlineCallbacks
def connect(self):
"""Connects to the mqtt broker."""
d = self.as_deferred(self.do_connect())
yield d
def as_deferred(self, f):
return Deferred.fromFuture(asyncio.ensure_future(f))
async def do_connect(self):
"""Connects to the mqtt broker."""
await asyncio.create_task(
self.client.connect(host=self.hostname, port=self.port))
def on_connect(self, client, flags, rc, properties):
"""Received a message."""
self.connected = True
# Do subscribes
for topic, kwargs in self.subscriptions.items():
self.client.subscribe(topic, **kwargs)
for topic, kwargs in self.unsubscriptions.items():
self.client.unsubscribe(topic, **kwargs)
# Do messages
for message in self.send_queue:
self.client.publish(message["topic"], **message["kwargs"])
if callable(self.connected_callback):
self.connected_callback(properties=properties)
def on_disconnect(self, client, packet, exc=None):
"""Disconnected notification."""
self.connected = False
if callable(self.disconnected_callback):
self.disconnected_callback(client=client, packet=packet)
def on_message(self, client, topic, body, qos, properties):
"""Received a message."""
if callable(self.on_message_callback):
self.on_message_callback(client=client,
topic=topic,
body=body,
qos=qos,
properties=properties)
def on_subscribe(self, client, mid, qos, properties):
"""Received subscribe confirmation."""
if callable(self.subscribe_callback):
self.subscribe_callback(client=client,
mid=mid,
qos=qos,
properties=properties)
def on_unsubscribe(self, client, mid, qos):
"""Received unsubscribe confirmation."""
if callable(self.unsubscribe_callback):
self.unsubscribe_callback(client=client, mid=mid, qos=qos)
def subscribe(self, topic: str, **kwargs):
"""
Subscribe to a topic.
:param topic:
:param kwargs:
:return:
"""
if "qos" not in kwargs:
kwargs["qos"] = 1
if self.session_expiry == 0:
self.subscriptions[topic] = kwargs
if self.connected is True:
self.client.subscribe(topic, **kwargs)
def unsubscribe(self, topic: str, **kwargs):
"""
Unsubscribe from topic.
:param topic: Topic to unsubscribe from.
:param kwargs:
:return:
"""
if "qos" not in kwargs:
kwargs["qos"] = 1
if self.connected is True:
self.client.unsubscribe(topic, **kwargs)
if self.session_expiry == 0:
self.unsubscriptions[topic] = kwargs
def publish(self,
topic: str,
message: Optional[str] = None,
qos: Optional[int] = None,
**kwargs):
"""
Publish a message to the MQTT broker. If not connected yet, will hold in a queue for later.
:param topic: Topic to publish too.
:param message: Message to send.
:param qos: quality of service.
:param kwargs: Any additional items to send to the qmqtt publish command.
:return:
"""
if qos is None:
qos = 1
if self.connected is True:
self.client.publish(topic, payload=message, qos=qos, **kwargs)
else:
kwargs["message"] = message
kwargs["qos"] = qos
self.send_queue.append({"topic": topic, "kwargs": kwargs})
class MQTT:
"""MQTT Class"""
def __init__(self):
# Secret can be set in either an environment variable (used first) or config.json
self.MQTT_USERNAME = getenv("MQTT_USER")
self.MQTT_PASS = getenv("MQTT_KEY")
# The connection handle for making calls
self.__client = None
self.last_sent_time = dict()
self.mqtt_cooldown = dict()
self.hostname = "mqtt"
self.client_id = getenv("WEB_HOSTNAME") + "-twitchbot-" + str(
datetime.now().timestamp())
# Default to not connected
self.MQTT_CONNECTION_STATE = False
self.ATTN_ENABLE = True
self.Topics = MqttTopics
async def connect_to_mqtt(self):
"""Connect to MQTT Server"""
# Create an instance of the REST client.
if self.MQTT_USERNAME is None or self.MQTT_PASS is None:
print("MQTT keys not found, aborting connection")
return False
try:
print("Atempting to connect to MQTT as " + self.MQTT_USERNAME)
self.__client = Client(client_id=self.client_id)
self.__client.set_auth_credentials(self.MQTT_USERNAME,
self.MQTT_PASS)
await self.__client.connect(self.hostname, port=1883)
print("Connected to MQTT")
self.MQTT_CONNECTION_STATE = True
return True
except Exception as e:
print("Failed to connect to MQTT, disabling it")
print(e)
self.MQTT_CONNECTION_STATE = False
return False
def get_cooldown(self, feed: str):
"""
Returns the cooldown
Loads it from the database,
or returns 0 if one is not set.
"""
if feed not in self.mqtt_cooldown:
q = session.query(Settings.value).filter(
Settings.key == f"mqtt_cooldown_{feed}").one_or_none()
if q is None:
# Value wasn't in the database, lets insert it.
insert = Settings(key=f"mqtt_cooldown_{feed}", value=0)
session.add(insert)
self.mqtt_cooldown[feed] = 0
else:
self.mqtt_cooldown[feed] = int(q[0])
return self.mqtt_cooldown[feed]
def set_cooldown(self, feed: str, cooldown: int) -> None:
"""
Sets the MQTT cooldown
Updates or inserts the value into the database
Exception handling should be done in the calling function
"""
q = session.query(Settings.id).filter(
Settings.key == f"mqtt_cooldown_{feed}").one_or_none()
if q is None:
# Value wasn't in the database, lets insert it.
insert = Settings(key=f"mqtt_cooldown_{feed}", value=cooldown)
session.add(insert)
self.mqtt_cooldown[feed] = cooldown
else:
session.query(Settings).filter(
Settings.key == f"mqtt_cooldown_{feed}").update(
{"value": cooldown})
self.mqtt_cooldown[feed] = cooldown
session.commit()
async def send(self,
feed,
value: Union[str, int] = 1,
retain: bool = False):
"""Send to an MQTT topic"""
last_sent = self.last_sent_time.get(feed, datetime.min)
cooldown = self.get_cooldown(feed)
now = datetime.now()
if (last_sent + timedelta(seconds=cooldown)) > now:
print(
f"MQTT {feed} on cooldown for {(last_sent + timedelta(seconds=cooldown)) - now}."
)
return False
if self.MQTT_CONNECTION_STATE is False:
try:
if not await self.connect_to_mqtt():
return False
except Exception as e:
print(e)
return False
try:
self.__client.publish(feed, value, retain=retain)
self.last_sent_time[feed] = now
return True
except Exception as e:
print(e)
return False
class LinkGMQTT(object):
def __init__(self, settings=None):
print("LinkGMQTT __init__()")
self.settings = settings
self.client = MQTTClient(None) # None => autogenerated client id
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
self.subscription_queue = asyncio.Queue()
print("LinkGMQTT __init__ completed")
async def start(self, server_settings):
"""
Connects to broker
"""
print('LinkGMQTT.start() connecting as user {}'.format(
server_settings["user"]))
self.client.set_auth_credentials(server_settings["user"],
server_settings["password"])
try:
await self.client.connect(server_settings["host"],
keepalive=60,
version=MQTTv311)
except Exception as e:
print("LinkGMQTT connect exception: {}".format(e))
return
print('LinkGMQTT.start() connected {}'.format(server_settings["host"]))
async def put(self, sensor_id, event):
"""
Sends sensor_id/event to MQTT broker.
sensor_id is string, used as MQTT topic
event is dictionary which will be converted to bytes for MQTT message
"""
#print('LinkGMQTT.put() sending {}'.format(sensor_id))
message = json.dumps(event)
self.client.publish(sensor_id, message, qos=0)
print("LinkGMQTT.put() published {} {}".format(sensor_id, message))
async def subscribe(self, subscribe_settings):
"""
Subscribes to sensor events.
"""
try:
self.client.subscribe(subscribe_settings["topic"], qos=0)
except Exception as e:
print("LinkGMQTT subscribe exception: {}".format(e))
return
print("LinkGMQTT.subscribed() {}".format(subscribe_settings["topic"]))
async def get(self):
print("LinkGMQTT get requested from client, awaiting queue")
message = await self.subscription_queue.get()
print("LinkGMQTT get returned from queue")
return message
def on_connect(self, client, flags, rc, properties):
print('LinkGMQTT Connected')
def on_message(self, client, topic, payload, qos, properties):
print('LinkGMQTT RECV MSG:', topic, payload)
message = payload.decode('utf-8')
message_dict = {}
try:
message_dict = json.loads(message)
except JSONDecodeError:
message_dict["message"] = message
print("remote_sensors() json msg error: {} => {}".format(
topic, message))
message_dict["topic"] = topic
self.subscription_queue.put_nowait(message_dict)
def on_disconnect(self, client, packet, exc=None):
print('LinkGMQTT Disconnected')
def on_subscribe(self, client, mid, qos, properties):
print('LinkGMQTT Subscribed')
async def finish(self):
await self.client.disconnect()
class MqttClient:
def __init__(self,
host: str,
port: int,
user: str,
password: typing.Optional[str] = None):
self.subscriptions: typing.Dict[str, typing.List[ValueCallback]] = {}
self.host = host
self.port = port
self.client = Client('sorokdva-dialogs')
self.client.set_auth_credentials(user, password)
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
async def _on_message(
self,
client: Client,
topic: str,
payload: bytes,
qos,
properties,
) -> constants.PubRecReasonCode:
log = logging.getLogger('mqtt')
futures = []
value = payload.decode()
for cb in self.subscriptions.get(topic, []):
log.info('passing (%r, %r) to %s', topic, value, cb)
futures.append(cb(topic, value))
if futures:
await asyncio.wait(futures, return_when=asyncio.ALL_COMPLETED)
return constants.PubRecReasonCode.SUCCESS
def _on_connect(self, client: Client, flags: int, result: int,
properties) -> None:
# FIXME make base path configurable
self.client.subscribe('/devices/#')
def subscribe(self, topic: str, callback: ValueCallback) -> None:
self.subscriptions.setdefault(topic, []).append(callback)
def send(self, topic: str, message):
self.client.publish(topic, message)
async def run(self):
await self.client.connect(self.host,
self.port,
version=constants.MQTTv311,
keepalive=30)
while True:
self.client.publish('smarthome', b'ping')
await asyncio.sleep(10)
@classmethod
def from_config(cls, cfg: dict) -> "MqttClient":
return cls(
host=cfg.get('host', 'localhost'),
port=cfg.get('port', 1883),
user=cfg.get('login', ''),
password=cfg.get('password', None),
)
class MQTTPublisher: # pylint: disable=too-many-instance-attributes
_APP_NAME = "call_detector"
_LOGGER = logging.getLogger(f"{__name__}.{__qualname__}")
_UPDATE_INTERVAL = 60
def __init__( # pylint: disable=too-many-arguments
self,
queue,
host="localhost",
port=8333,
username=None,
password=None,
ssl=False,
retry=False,
topic=f"call_detector/{socket.gethostname()}",
):
self._client = MQTTClient(self._APP_NAME)
if username is not None:
self._client.set_auth_credentials(username, password)
self._host = host
self._port = port
self._queue = queue
self._topic = topic
self._retry = retry
self._ssl = ssl
self._state = {"call": False}
async def run(self):
self._LOGGER.info("Running.")
while True:
try:
await self._client.connect(self._host, port=self._port, ssl=self._ssl, version=MQTTv311, keepalive=10)
self._LOGGER.info("Connected.")
break
except Exception: # pylint: disable=broad-except
if not self._retry:
raise
self._LOGGER.exception("Error occured during connecting")
await asyncio.sleep(5)
while True:
try:
try:
with async_timeout.timeout(self._UPDATE_INTERVAL):
msg = await self._queue.get()
self._update_state(msg)
except asyncio.exceptions.TimeoutError:
pass
await self._publish_state()
except Exception: # pylint: disable=broad-except
if not self._retry:
raise
self._LOGGER.exception("Error occured during publishing")
await asyncio.sleep(5)
def _update_state(self, msg):
del self._state["call"]
self._state[msg["source"]] = msg["apps"]
apps = reduce(lambda a, b: a + len(b), self._state.values(), 0)
self._state["call"] = apps > 0
self._LOGGER.info("State updated: %s", self._state)
@throttle(0.5)
async def _publish_state(self):
self._LOGGER.info("Publishing state %s to topic %s", self._state, self._topic)
self._client.publish(
self._topic,
json.dumps(self._state),
qos=1,
)
class DecoderManager():
###################
# Sync class init
###################
def __init__(self):
print("DecoderManager __init__", flush=True)
print("{} acp_decoders started\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
self.settings = {}
self.settings["decoders"] = []
#####################################
# Signal handler for SIGINT, SIGTERM
#####################################
def ask_exit(self, *args):
self.STOP.set()
#####################################
# Signal handler for SIGALRM
#####################################
def reload(self, *args):
self.load_decoders_file()
#####################################
# Return current timestamp as string
#####################################
def ts_string(self):
return '{:.6f}'.format(time.time())
###############################################################
# Async initialization
###############################################################
async def start(self):
# Define async events for exit and reload (will set via signals)
self.STOP = asyncio.Event()
self.RELOAD = asyncio.Event()
# load settings.json into self.settings
self.read_settings()
# Connect input and output MQTT brokers (which can be same or different)
await self.connect_input_mqtt()
# debug testing timeout, disabling start of publisher
await self.connect_output_mqtt()
async def connect_input_mqtt(self):
self.input_client = MQTTClient(None) # auto-generate client id
self.input_client.on_connect = self.input_on_connect
self.input_client.on_message = self.input_on_message
self.input_client.on_disconnect = self.input_on_disconnect
self.input_client.on_subscribe = self.input_on_subscribe
user = self.settings["input_mqtt"]["user"]
password = self.settings["input_mqtt"]["password"]
host = self.settings["input_mqtt"]["host"]
port = self.settings["input_mqtt"]["port"]
self.input_client.set_auth_credentials(user, password)
await self.input_client.connect(host,
port,
keepalive=20,
version=MQTTv311)
async def connect_output_mqtt(self):
self.output_client = MQTTClient(None) # auto-generate client id
self.output_client.on_connect = self.output_on_connect
self.output_client.on_message = self.output_on_message
self.output_client.on_disconnect = self.output_on_disconnect
self.output_client.on_subscribe = self.output_on_subscribe
user = self.settings["output_mqtt"]["user"]
password = self.settings["output_mqtt"]["password"]
host = self.settings["output_mqtt"]["host"]
port = self.settings["output_mqtt"]["port"]
self.output_client.set_auth_credentials(user, password)
await self.output_client.connect(host,
port,
keepalive=60,
version=MQTTv311)
###############################################################
# Settings, including loading enabled decoders
#
# Builds self.settings from file "settings.json"
# Then loads decoders listed in the setting "decoders_file"
###############################################################
def read_settings(self):
with open('settings.json', 'r') as sf:
settings_data = sf.read()
# parse file
self.settings = json.loads(settings_data)
self.load_decoders_file()
def load_decoders_file(self):
# getting settings filename for decoders list (json)
decoders_file = self.settings["decoders_file"]
# read the json file
with open(decoders_file, 'r') as df:
decoders_data = df.read()
# parse to a python dictionary
decoders_obj = json.loads(decoders_data)
# store the new list of decoders as settings["decoders"]
self.settings["decoders"] = decoders_obj["decoders"]
# import/reload the decoders
self.import_decoders(self.settings["decoders"])
# import a list of decoder names
def import_decoders(self, new_decoders):
self.decoders = []
for decoder_name in new_decoders:
self.import_decoder(decoder_name)
# import a decoder, given name
# Will add { "name": , "decoder": } to self.decoders list
def import_decoder(self, decoder_name):
print("loading Decoder {}".format(decoder_name), flush=True)
module_name = 'decoders.' + decoder_name
# A new module can be imported with importlib.import_module()
# BUT an already loaded module must use importlib.reload for update to work.
if module_name in sys.modules:
module = sys.modules[module_name]
importlib.reload(module)
else:
module = importlib.import_module(module_name)
# now we have the refreshed/new module, so put Decoder on list self.decoders
decoder = module.Decoder(self.settings)
print(" loaded Decoder {}".format(decoder_name), flush=True)
self.decoders.append({"name": decoder_name, "decoder": decoder})
###############################################################
# Sensor data message handler
###############################################################
def handle_input_message(self, topic, msg_bytes):
acp_ts = self.ts_string()
msg_is_decoded = False
for decoder in self.decoders:
if decoder["decoder"].test(topic, msg_bytes):
decoded = decoder["decoder"].decode(topic, msg_bytes)
# If no acp_ts from decoder, insert from server time
if not "acp_ts" in decoded:
decoded["acp_ts"] = acp_ts
print("{} {} decoded by {}".format(acp_ts, decoded["acp_id"],
decoder["name"]),
flush=True)
#debug testing timeout, disabled send:
#self.send_output_message(topic, decoded)
msg_is_decoded = True
break # terminate the loop through decoders when first is found
if msg_is_decoded:
self.send_output_message(topic, decoded)
else:
print("{} Incoming message not decoded\n{}\n".format(
acp_ts, msg_bytes),
flush=True)
def send_output_message(self, topic, decoded):
msg_bytes = json.dumps(decoded)
#print("publishing {}".format(msg_bytes), flush=True)
output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic
self.output_client.publish(output_topic, msg_bytes, qos=0)
###############################################################
# MQTT INPUT
###############################################################
def input_on_connect(self, client, flags, rc, properties):
print('INPUT Connected to {} as {}'.format(
self.settings["input_mqtt"]["host"],
self.settings["input_mqtt"]["user"]),
flush=True)
client.subscribe('#', qos=0)
def input_on_message(self, client, topic, msg_bytes, qos, properties):
# IMPORTANT! We avoid a loop by ignoring input messages with the output prefix
if not topic.startswith(self.settings["output_mqtt"]["topic_prefix"]):
if DEBUG:
print('INPUT RECV MSG:', msg_bytes, flush=True)
self.handle_input_message(topic, msg_bytes)
else:
if DEBUG:
print('INPUT RECV COOKED MSG SKIPPED', flush=True)
def input_on_disconnect(self, client, packet, exc=None):
print('INPUT Disconnected', flush=True)
print("{} INPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
def input_on_subscribe(self, client, mid, qos, properties):
print('INPUT SUBSCRIBED to {}'.format(
self.settings["input_mqtt"]["topic"]),
flush=True)
###############################################################
# MQTT OUTPUT
###############################################################
def output_on_connect(self, client, flags, rc, properties):
print('OUTPUT Connected to {} as {}'.format(
self.settings["output_mqtt"]["host"],
self.settings["output_mqtt"]["user"]),
flush=True)
def output_on_disconnect(self, client, packet, exc=None):
print('OUTPUT Disconnected', flush=True)
print("{} OUTPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
# These GMQTT methods here for completeness although not used
def output_on_message(self, client, topic, msg_bytes, qos, properties):
print('OUTPUT RECV MSG?:', msg_bytes, flush=True)
def output_on_subscribe(self, client, mid, qos, properties):
print('OUTPUT SUBSCRIBED? to {}', flush=True)
###############################################################
# CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM)
###############################################################
async def finish(self):
await self.STOP.wait()
print("\nDecoderManager interrupted, closing MQTT clients", flush=True)
print("{} DecoderManager interrupted - disconnecting\n".format(
self.ts_string()),
file=sys.stderr,
flush=True)
await self.input_client.disconnect()
await self.output_client.disconnect()
class GmqttClient(Client):
counter = 0
def __init__(self, config: BrokerConfig):
self.config: BrokerConfig = config
self.client = None
self.thread = None
self.all_listeners: Dict[str, Subscriber] = {}
self.active_listeners: Set = set()
self.loop = None
self.ready = threading.Event()
@property
def is_connected(self):
return self.client.is_connected
def publish(self, topic: str, payload):
# At most once (0)
# At least once (1)
# Exactly once (2).
self.ready.wait()
assert self.client, 'you need to activate connection'
self.client.publish(topic, payload, qos=0, message_expiry_interval=10)
def subscribe(self, topic: str, listener: Subscriber):
if self.all_listeners.get(topic, None):
raise Exception(f'Topic {topic} already registered')
self.all_listeners[topic] = listener
self.ready.wait()
self.PROCESS.set()
def unsubscribe(self, topic):
listener = self.all_listeners.get(topic, None)
if not listener:
raise Exception(f'Topic {topic} was not registered')
self.all_listeners.pop(topic)
self.active_listeners.discard(topic)
def connect(self) -> Client:
if self.client:
return self
self.thread = Thread(target=self._loop, daemon=True)
self.thread.start()
# print('ready received')
return self
def _loop(self):
try:
self.loop = asyncio.new_event_loop()
self.loop.run_until_complete(self._connect())
# asyncio.run(self._connect())
except KeyboardInterrupt:
if self.client:
self.client.disconnect()
# print("Received exit, exiting")
async def _connect(self):
GmqttClient.counter += 1
client_id = f'client-id/{platform.node()}/pid_{os.getpid()}/{uuid.getnode()}/{GmqttClient.counter}'
self.client = MQTTClient(client_id)
self.PROCESS = asyncio.Event()
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
# print('set')
self.client.set_auth_credentials(self.config.username,
self.config.password)
while True:
try:
await self.client.connect(self.config.hostname,
port=self.config.port,
ssl=self.config.ssl,
version=self.config.mqtt_version)
break
except Exception as ex:
print('connect failed', ex)
time.sleep(1)
self.ready.set()
while True:
await self.PROCESS.wait()
self.PROCESS.clear()
# print('process signaled')
self._sync_subscriptions()
# await self.STOP.wait()
await self.client.disconnect()
def on_connect(self, client, flags, rc, properties):
self.connected = True
print('Connected', id(self))
self.PROCESS.set()
self._sync_subscriptions()
def _sync_subscriptions(self):
if not self.client.is_connected:
return
unregistered = self.all_listeners.keys() - self.active_listeners
for topic in unregistered:
self.client.subscribe(topic, qos=1)
self.active_listeners.add(topic)
def _no_subscriber(self, message):
print('RECV MSG with no subscriber:', message.topic, message.payload)
def on_message(self, client, topic, payload, qos, properties):
s = self.all_listeners.get(topic, self._no_subscriber)
s(topic, payload)
def on_disconnect(self, client, packet, exc=None):
self.connected = False
self.active_listeners.clear()
print('Disconnected', id(self))
self.PROCESS.set()
def on_subscribe(self, client, mid, qos, *args, **kwargs):
# print('SUBSCRIBED')
pass
class EcovacsMqtt:
"""Handle mqtt connections."""
def __init__(self, *, continent: str, country: str):
self._subscribers: MutableMapping[str, VacuumBot] = {}
self._port = 443
self._hostname = f"mq-{continent}.ecouser.net"
if country.lower() == "cn":
self._hostname = "mq.ecouser.net"
self._client: Optional[Client] = None
self._received_set_commands: MutableMapping[str,
SetCommand] = TTLCache(
maxsize=60 * 60,
ttl=60)
# pylint: disable=unused-argument
async def _on_message(client: Client, topic: str, payload: bytes,
qos: int, properties: Dict) -> None:
_LOGGER.debug("Got message: topic=%s; payload=%s;", topic,
payload.decode())
topic_split = topic.split("/")
if topic.startswith("iot/atr"):
await self._handle_atr(topic_split, payload)
elif topic.startswith("iot/p2p"):
self._handle_p2p(topic_split, payload)
else:
_LOGGER.debug("Got unsupported topic: %s", topic)
self.__on_message = _on_message
async def initialize(self, auth: RequestAuth) -> None:
"""Initialize MQTT."""
if self._client is not None:
self.disconnect()
client_id = f"{auth.user_id}@ecouser/{auth.resource}"
self._client = Client(client_id)
self._client.on_message = self.__on_message
self._client.set_auth_credentials(auth.user_id, auth.token)
ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
await self._client.connect(self._hostname,
self._port,
ssl=ssl_ctx,
version=MQTTv311)
async def subscribe(self, vacuum_bot: VacuumBot) -> None:
"""Subscribe for messages for given vacuum."""
if self._client is None:
raise NotInitializedError
vacuum = vacuum_bot.vacuum
self._client.subscribe(_get_subscriptions(vacuum))
self._subscribers[vacuum.did] = vacuum_bot
def unsubscribe(self, vacuum_bot: VacuumBot) -> None:
"""Unsubscribe given vacuum."""
vacuum = vacuum_bot.vacuum
if self._subscribers.pop(vacuum.did, None) and self._client:
for subscription in _get_subscriptions(vacuum):
self._client.unsubscribe(subscription.topic)
def disconnect(self) -> None:
"""Disconnect from MQTT."""
if self._client:
self._client.disconnect()
self._subscribers.clear()
async def _handle_atr(self, topic_split: List[str],
payload: bytes) -> None:
try:
bot = self._subscribers.get(topic_split[3])
if bot:
data = json.loads(payload)
await bot.handle(topic_split[2], data)
except Exception: # pylint: disable=broad-except
_LOGGER.error("An exception occurred during handling atr message",
exc_info=True)
def _handle_p2p(self, topic_split: List[str], payload: bytes) -> None:
try:
command_name = topic_split[2]
if command_name not in SET_COMMAND_NAMES:
# command doesn't need special treatment or is not supported yet
return
is_request = topic_split[9] == "q"
request_id = topic_split[10]
if is_request:
payload_json = json.loads(payload)
try:
data = payload_json["body"]["data"]
except KeyError:
_LOGGER.warning(
"Could not parse p2p payload: topic=%s; payload=%s",
"/".join(topic_split),
payload_json,
)
return
command_class = COMMANDS.get(command_name)
if command_class and issubclass(command_class, SetCommand):
self._received_set_commands[request_id] = command_class(
**data)
else:
command = self._received_set_commands.get(request_id, None)
if not command:
_LOGGER.debug(
"Response to setCommand came in probably to late. requestId=%s, commandName=%s",
request_id,
command_name,
)
return
bot = self._subscribers.get(topic_split[3])
if bot:
data = json.loads(payload)
if command.handle(bot.events, data) and isinstance(
command.args, dict):
command.get_command.handle(bot.events, command.args)
except Exception: # pylint: disable=broad-except
_LOGGER.error("An exception occurred during handling p2p message",
exc_info=True)
class MyMQTT(object):
def __init__(
self,
broker: str,
port: int = 1883,
keepalive: int = 60,
auth: List[str] = [None, None],
event_functions=None,
client_id: str = "client_id",
event_loop=None,
**kwargs,
) -> None:
default_events = {
"on_connect": self.on_connect,
"on_message": self.on_message,
"on_disconnect": self.on_disconnect,
"on_subscribe": self.on_subscribe,
}
if event_functions is None:
event_functions = {}
self.event_functions = default_events
self.event_functions.update(event_functions)
# self.client = MQTTClient(client_id)
self.auth = auth
self.broker = broker
self.port = port
self.keepalive = keepalive
self.stop = asyncio.Event()
if event_loop is None:
loop = asyncio.get_event_loop()
else:
loop = event_loop
loop.add_signal_handler(signal.SIGINT, self.ask_exit)
loop.add_signal_handler(signal.SIGTERM, self.ask_exit)
# passing any extra kwargs to the call to be used for e.g. on_connect
self.client = MQTTClient(client_id, **kwargs)
for _ in [k for k, v in self.event_functions.items() if v is None]:
logger.warning(f"mqtt no function assigned to {_}")
self.event_functions.pop(k)
for k, v in self.event_functions.items():
setattr(self.client, k, v)
if any(self.auth):
self.client.set_auth_credentials(*self.auth)
loop.create_task(self.start(self.broker))
async def start(self, broker):
logger.info("starting mqtt")
try:
await self.client.connect(
broker, port=self.port, keepalive=self.keepalive, version=MQTTv311
)
except OSError as e:
logger.error(f"unable to connect to mqtt with following error {e}")
else:
await self.stop.wait()
await self.client.disconnect()
def publish(self, topic, message, qos=1):
self.client.publish(topic, message, qos=qos)
def on_connect(self, client, flags, rc, properties):
logger.info("MQTT Connected")
client.subscribe("#")
def on_message(self, client, topic, payload, qos, properties):
logger.info(f"MQTT RECV MSG: {topic} {payload}")
def on_disconnect(self, client, packet, exc=None):
logger.info("MQTT Disconnected")
@staticmethod
def on_subscribe(*args):
logger.info("MQTT SUBSCRIBED")
def ask_exit(self, *args):
asyncio.Event().set()
class Pipeline:
"""Object detection and tracking pipeline"""
def __init__(self, args):
self.args = args
# Initialise camera & camera viewport
self.init_camera()
# Initialise output
self.init_output(self.args.output)
# Initialise object detector (for some reason it has to happen
# here & not within detect_objects(), or else the inference engine
# gets upset and starts throwing NaNs at me. Thanks, Python.)
self.object_detector = SSD_MOBILENET(wanted_label='person',
model_file=self.args.model,
label_file=self.args.labels,
num_threads=self.args.num_threads,
edgetpu=self.args.edgetpu)
# Initialise feature encoder
if self.args.encoder_model is None:
model_filename = '{}/mars-64x32x3.pb'.format(
self.args.deepsorthome)
else:
model_filename = self.args.encoder_model
self.encoder = gdet.create_box_encoder(
model_filename, batch_size=self.args.encoder_batch_size)
self.background_subtraction = not self.args.disable_background_subtraction
# Initialise tracker
nn_budget = None
metric = nn_matching.NearestNeighborDistanceMetric(
"cosine", self.args.max_cosine_distance, nn_budget)
self.tracker = Tracker(metric,
max_iou_distance=self.args.max_iou_distance,
max_age=self.args.max_age)
# Initialise database
self.db = {}
self.delcount = 0
self.intcount = 0
self.poscount = 0
self.negcount = 0
self.mqtt = None
self.topic = self.args.mqtt_topic
self.mqtt_acp_id = self.args.mqtt_acp_id
self.heartbeat_delay_secs = self.args.heartbeat_delay_secs
self.loop = asyncio.get_event_loop()
async def init_mqtt(self):
if self.args.mqtt_broker is not None:
self.mqtt = MQTTClient('deepdish')
if self.args.mqtt_user is not None:
self.mqtt.set_auth_credentials(self.args.mqtt_user,
self.args.mqtt_pass)
await self.mqtt.connect(self.args.mqtt_broker)
if self.topic is None:
self.topic = 'default/topic'
def init_camera(self):
self.input = self.args.input
if self.input is None:
self.input = self.args.camera
# Allow live camera frames to be dropped
self.everyframe = None
else:
# Capture every frame from the video file
self.everyframe = asyncio.Event()
self.cap = cv2.VideoCapture(self.input)
# Configure the 'counting line' in the camera viewport
if self.args.line is None:
w, h = self.args.camera_width, self.args.camera_height
self.countline = np.array([[w / 2, 0], [w / 2, h]], dtype=int)
else:
self.countline = np.array(list(
map(int,
self.args.line.strip().split(','))),
dtype=int).reshape(2, 2)
self.cameracountline = self.countline.astype(float)
def init_output(self, output):
self.color_mode = None # fixme
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
fps = self.cap.get(cv2.CAP_PROP_FPS)
(w, h) = (self.args.camera_width, self.args.camera_height)
self.backbuf = Image.new("RGBA", (w, h), (0, 0, 0, 0))
self.draw = ImageDraw.Draw(self.backbuf)
self.output = cv2.VideoWriter(self.args.output, fourcc, fps, (w, h))
if self.args.no_framebuffer:
self.framebufdev = None
else:
self.framebufdev = self.args.framebuffer
fbX = self.framebufdev[-3:]
vsizefile = '/sys/class/graphics/{}/virtual_size'.format(fbX)
if not os.path.exists(
self.framebufdev) or not os.path.exists(vsizefile):
#raise Error('Invalid framebuffer device: {}'.format(self.framebufdev))
print('Invalid framebuffer device: {}'.format(
self.framebufdev))
self.framebufdev = None
if self.framebufdev is not None:
(w, h) = (self.args.framebuffer_width,
self.args.framebuffer_height)
if w is None or h is None:
nums = re.findall('(.*),(.*)', open(vsizefile).read())[0]
if w is None:
w = int(nums[0])
if h is None:
h = int(nums[1])
self.framebufres = (w, h)
def read_frame(self):
ret, frame = self.cap.read()
return (frame, time())
def shutdown(self):
self.running = False
for p in asyncio.Task.all_tasks():
p.cancel()
async def capture(self, q):
try:
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
frame = None
# Fetch next frame
(frame, t_frame) = await self.loop.run_in_executor(
pool, self.read_frame)
if frame is None:
print('No more frames.')
self.shutdown()
break
if self.args.camera_flip:
# If we need to flip the image vertically
frame = cv2.flip(frame, 0)
# Ensure frame is proper size
frame = cv2.resize(
frame,
(self.args.camera_width, self.args.camera_height))
await q.put((frame, t_frame, time()))
# If we are ensuring every frame is processed then wait for
# synchronising event to be triggered
if self.everyframe is not None:
await self.everyframe.wait()
self.everyframe.clear()
finally:
self.cap.release()
def run_object_detector(self, image):
t1 = time()
boxes = self.object_detector.detect_image(image)
t2 = time()
return (boxes, t2 - t1)
async def detect_objects(self, q_in, q_out):
# Initialise background subtractor
backSub = cv2.createBackgroundSubtractorMOG2()
frameCount = 0
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
frameCount += 1
# Obtain next video frame
(frame, t_frame, t_prev) = await q_in.get()
t_frame_recv = time()
# Apply background subtraction to find image-mask of areas of motion
if self.background_subtraction:
fgMask = backSub.apply(frame)
# Convert to PIL Image
image = Image.fromarray(
cv2.cvtColor(frame, cv2.COLOR_BGRA2RGBA))
t_backsub = time()
# Run object detection engine within a Thread Pool
(boxes0, delta_t) = await self.loop.run_in_executor(
pool, self.run_object_detector, image)
# Filter object detection boxes, including only those with areas of motion
t1 = time()
boxes = []
max_x, max_y = self.args.camera_width, self.args.camera_height
for (x, y, w, h) in boxes0:
if np.any(np.isnan(boxes0)):
# Drop any rubbish results
continue
x, y = int(np.clip(x, 0, max_x)), int(np.clip(y, 0, max_y))
w, h = int(np.clip(w, 0, max_x - x)), int(
np.clip(h, 0, max_y - y))
# Check if the box is almost as large as the camera viewport
if w * h > 0.9 * max_x * max_y:
# reject as spurious
continue
# Check if the box includes any detected motion
if not self.background_subtraction or np.any(
fgMask[x:x + w, y:y + h]):
boxes.append((x, y, w, h))
t2 = time()
# Send results to next step in pipeline
elements = [
FrameInfo(t_frame, frameCount),
CameraImage(image),
CameraCountLine(self.cameracountline),
TimingInfo('Frame processing latency', 'fram',
t_prev - t_frame),
TimingInfo('Frame / Q1 item received latency', 'q1',
t_frame_recv - t_prev),
TimingInfo('Background subtraction latency', 'bsub',
t_backsub - t_frame_recv),
TimingInfo('Object detection latency', 'objd',
delta_t + (t2 - t1))
]
await q_out.put((frame, boxes, elements, time()))
async def encode_features(self, q_in, q_out):
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
# Obtain next video frame and object detection boxes
(frame, boxes, elements, t_prev) = await q_in.get()
t1 = time()
# Run feature encoder within a Thread Pool
features = await self.loop.run_in_executor(
pool, self.encoder, frame, boxes)
t2 = time()
# Build list of 'Detection' objects and send them to next step in pipeline
detections = [
Detection(bbox, 1.0, feature)
for bbox, feature in zip(boxes, features)
]
elements.append(
TimingInfo('Q1 / Q2 latency', 'q2', (t1 - t_prev)))
elements.append(
TimingInfo('Feature encoder latency', 'feat', (t2 - t1)))
await q_out.put((detections, elements, time()))
async def track_objects(self, q_in, q_out):
while self.running:
(detections, elements, t_prev) = await q_in.get()
t1 = time()
boxes = np.array([d.tlwh for d in detections])
scores = np.array([d.confidence for d in detections])
indices = preprocessing.non_max_suppression(
boxes, self.args.nms_max_overlap, scores)
detections = [detections[i] for i in indices]
self.tracker.predict()
self.tracker.update(detections)
t2 = time()
elements.append(TimingInfo('Q2 / Q3 latency', 'q3', (t1 - t_prev)))
elements.append(TimingInfo('Tracker latency', 'trak', (t2 - t1)))
await q_out.put((detections, elements, time()))
async def process_results(self, q_in, q_out):
while self.running:
(detections, elements, t_prev) = await (q_in.get())
t1 = time()
for track in self.tracker.deleted_tracks:
i = track.track_id
if track.is_deleted():
self.check_deleted_track(track.track_id)
for track in self.tracker.tracks:
i = track.track_id
if not track.is_confirmed() or track.time_since_update > 1:
continue
if i not in self.db:
self.db[i] = []
bbox = track.to_tlbr()
# Find the bottom-centre of the bounding box & add it to the tracking database
bottomCentre = np.array([(bbox[0] + bbox[2]) / 2.0, bbox[3]])
self.db[i].append(bottomCentre)
if len(self.db[i]) > 1:
# If we have more than one datapoint for this tracked object
pts = (np.array(self.db[i]).reshape(
(-1, 1, 2))).reshape(-1)
elements.append(TrackedPath(pts))
p1 = self.cameracountline[0]
q1 = self.cameracountline[1]
p2 = np.array(self.db[i][-1])
q2 = np.array(self.db[i][-2])
cp = np.cross(q1 - p1, q2 - p2)
if intersection(p1, q1, p2, q2):
self.intcount += 1
print(
"track_id={} just intersected camera countline; cross-prod={}; intcount={}"
.format(i, cp, self.intcount))
elements.append(TrackedPathIntersection(pts[-4:]))
if cp >= 0:
self.poscount += 1
crossing_type = 'pos'
else:
self.negcount += 1
crossing_type = 'neg'
await self.publish_crossing_event_to_mqtt(
elements, crossing_type)
elements.append(TrackedObject(bbox, str(track.track_id)))
for det in detections:
bbox = det.to_tlbr()
elements.append(DetectedObject(bbox))
elements.append(CountingStats(self.negcount, self.poscount))
t2 = time()
elements.append(TimingInfo('Q3 / Q4 latency', 'q4', (t1 - t_prev)))
elements.append(
TimingInfo('Results processing latency', 'proc', (t2 - t1)))
await q_out.put((elements, time()))
async def publish_crossing_event_to_mqtt(self, elements, crossing_type):
if self.mqtt is not None:
for e in elements:
if isinstance(e, FrameInfo):
t_frame = e.t_frame
break
payload = json.dumps({
'acp_ts': str(t_frame),
'acp_id': self.mqtt_acp_id,
'acp_event': 'crossing',
'acp_event_value': crossing_type,
'poscount': self.poscount,
'negcount': self.negcount,
'diff': self.poscount - self.negcount
})
self.mqtt.publish(self.topic, payload)
async def periodic_mqtt_heartbeat(self):
if self.mqtt is not None:
while True:
payload = json.dumps({
'acp_ts': str(time()),
'acp_id': self.mqtt_acp_id,
'poscount': self.poscount,
'negcount': self.negcount,
'diff': self.poscount - self.negcount
})
self.mqtt.publish(self.topic, payload)
await asyncio.sleep(self.heartbeat_delay_secs)
async def graphical_output(self, render: RenderInfo, elements,
output_wh: (int, int)):
(output_w, output_h) = output_wh
# Clear screen
self.draw.rectangle([0, 0, output_w, output_h], fill=0, outline=0)
# Sort elements by display priority
elements.sort(key=lambda e: e.priority)
# Draw elements
for e in elements:
if hasattr(e, 'do_render'):
e.do_render(render)
# Copy backbuf to output
backarray = np.array(self.backbuf)
if self.color_mode is not None:
outputbgra = cv2.cvtColor(backarray, self.color_mode)
else:
outputbgra = backarray
outputrgb = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGB)
if self.output is not None:
self.output.write(outputrgb)
if self.framebufdev is not None:
outputrgba = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGBA)
outputfbuf = cv2.resize(outputrgba, self.framebufres)
try:
with open(self.framebufdev, 'wb') as buf:
buf.write(outputfbuf)
except:
print(
'failed to write to framebuffer device {} ...disabling it.'
.format(self.framebufdev))
self.framebufdev = None
await streaminfo.set_frame(outputrgb)
#cv2.imshow('main', outputrgb)
def text_output(self, handle, elements):
# Sort elements by priority
elements.sort(key=lambda e: e.priority)
for e in elements:
if hasattr(e, 'do_text'):
e.do_text(handle, elements)
async def render_output(self, q_in):
(output_w, output_h) = (self.args.camera_width,
self.args.camera_height)
ratio = 1 #fixme
render = RenderInfo(ratio, FontLib(output_w), self.draw, self.backbuf)
try:
while self.running:
(elements, t_prev) = await q_in.get()
t1 = time()
await self.graphical_output(render, elements,
(output_w, output_h))
for e in elements:
if isinstance(e, FrameInfo):
t_frame = e.t_frame
break
elements.append(
TimingInfo('Q4 / Q5 latency', 'q5', t1 - t_prev))
elements.append(
TimingInfo('Graphical display latency', 'disp',
time() - t1))
t_sum = 0
for e in elements:
if isinstance(e, TimingInfo):
t_sum += e.delta_t
elements.append(TimingInfo('Latency sum', 'sum', t_sum))
t_e2e = time() - t_frame
elements.append(TimingInfo('End to end latency', 'e2e', t_e2e))
elements.append(TimingInfo('Missing', 'miss', t_e2e - t_sum))
self.text_output(sys.stdout, elements)
if self.everyframe is not None:
# Notify other side that this frame is completely processed
self.everyframe.set()
finally:
self.output.release()
def check_deleted_track(self, i):
if i in self.db and len(self.db[i]) > 1:
if any_intersection(self.cameracountline[0],
self.cameracountline[1], np.array(self.db[i])):
self.delcount += 1
print("delcount={}".format(self.delcount))
self.db[i] = []
async def start(self):
self.running = True
cameraQueue = FreshQueue()
objectQueue = asyncio.Queue(maxsize=1)
detectionQueue = asyncio.Queue(maxsize=1)
resultQueue = asyncio.Queue(maxsize=1)
drawQueue = asyncio.Queue(maxsize=1)
asyncio.ensure_future(self.render_output(drawQueue))
asyncio.ensure_future(self.process_results(resultQueue, drawQueue))
asyncio.ensure_future(self.track_objects(detectionQueue, resultQueue))
asyncio.ensure_future(self.encode_features(objectQueue,
detectionQueue))
asyncio.ensure_future(self.detect_objects(cameraQueue, objectQueue))
await self.capture(cameraQueue)
self.running = False
class GMQTT_Client(MQTT_Base):
def __init__(self, mqtt_settings):
MQTT_Base.__init__(self, mqtt_settings)
self.mqtt_client = None
def connect(self):
MQTT_Base.connect(self)
self.mqtt_client = MQTTClient(
'gmqtt' #self.mqtt_settings["MQTT_CLIENT_ID"]
)
self.mqtt_client.on_connect = self._on_connect
self.mqtt_client.on_message = self._on_message
self.mqtt_client.on_disconnect = self._on_disconnect
if self.mqtt_settings["MQTT_USERNAME"]:
self.mqtt_client.set_auth_credentials(
self.mqtt_settings["MQTT_USERNAME"],
self.mqtt_settings["MQTT_PASSWORD"],
)
def start():
try:
logger.warning('Connecting to MQTT')
asyncio.set_event_loop(self.event_loop)
# self.event_loop.run_until_complete(
# self.mqtt_client.connect(self.mqtt_settings["MQTT_BROKER"], self.mqtt_settings["MQTT_PORT"],keepalive=self.mqtt_settings["MQTT_KEEPALIVE"], version=MQTTv311)
# )
logger.warning('Looping forever')
self.event_loop.run_forever()
logger.warning('Event loop stopped')
#self.session.close()
except Exception as e:
logger.error('Error in event loop {}'.format(e))
self.event_loop = asyncio.new_event_loop()
logger.warning("Starting MQTT thread")
self._ws_thread = threading.Thread(target=start, args=())
self._ws_thread.daemon = True
self._ws_thread.start()
future = asyncio.run_coroutine_threadsafe(
self.mqtt_client.connect(
self.mqtt_settings["MQTT_BROKER"],
self.mqtt_settings["MQTT_PORT"],
keepalive=self.mqtt_settings["MQTT_KEEPALIVE"],
version=MQTTv311), self.event_loop)
def publish(self, topic, payload, retain, qos):
MQTT_Base.publish(self, topic, payload, retain, qos)
if self.mqtt_connected is True:
wrapped = functools.partial(self.mqtt_client.publish,
topic,
payload,
retain=retain,
qos=qos)
self.event_loop.call_soon_threadsafe(wrapped)
else:
logger.warning(
"Device MQTT publish NOT CONNECTED: {}, retain {}, qos {}, payload: {}"
.format(topic, retain, qos, payload))
# future = asyncio.run_coroutine_threadsafe(
# self.mqtt_client.publish(topic, payload, retain=retain, qos=qos),
# self.event_loop
# )
def subscribe(self, topic, qos): # subclass to provide
MQTT_Base.subscribe(self, topic, qos)
self.mqtt_client.subscribe(topic, qos)
def unsubscribe(self, topic): # subclass to provide
MQTT_Base.unsubscribe(self, topic)
self.mqtt_client.unsubscribe(topic)
def set_will(self, will, topic, retain, qos):
MQTT_Base.set_will(self, will, topic, retain, qos)
#self.mqtt_client.will_set(will, topic, retain, qos)
def _on_connect(self, client, flags, rc, properties):
logger.info("MQTT On Connect: {}".format(rc))
self.mqtt_connected = rc == 0
def _on_message(self, client, topic, payload, qos, properties):
#topic = msg.topic
#payload = msg.payload.decode("utf-8")
MQTT_Base._on_message(self, topic, payload, False, qos)
def _on_disconnect(self, client, packet, exc=None):
self.mqtt_connected = False # note, change this uses the property setter, do not really need to catch this in the base class
logger.warning("MQTT Disconnection {} {} {}".format(
client, packet, exc))
MQTT_Base._on_disconnect(self, 0)
class GMQTTConnector(BaseConnector):
"""GMQTTConnector uses gmqtt library for connectors
running over MQTT.
"""
def __init__(self, host, port, subscribe_topic, publish_topic, **kwargs):
self.host = host
self.port = port
# topics
self.subscribe_topic = subscribe_topic
self.publish_topic = publish_topic
# connection
self.connection_id = uuid.uuid4().hex[:8]
self.is_connected = False
self.client = MQTTClient(self.connection_id)
# callbacks
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
self.STOP = asyncio.Event()
# options
self.ack_topic = kwargs.get('ack_topic')
self.enable_ssl = kwargs.get('enable_ssl', False)
self.enable_auth = kwargs.get('enable_auth', False)
self.username = kwargs.get('username')
self.password = kwargs.get('password')
self.client_cert = kwargs.get('client_cert')
self.client_key = kwargs.get('client_key')
self.qos = kwargs.get('qos', 2)
def get_connection_details(self):
"""get_connection_details returns the details
about the current MQTT connection.
"""
return dict(
connection_id=self.connection_id,
host=self.host,
port=self.port,
is_connected=self.is_connected,
subscribe_topic=self.subscribe_topic,
publish_topic=self.publish_topic
)
def on_connect(self, *args):
"""on_connect is a callback that gets exectued after the
connection is made.
Arguments:
client {MQTTClient} -- gmqtt.MQTTClient
flags {int} -- connection flags
rc {int} -- connection result code
properties {dict} -- config of the current connection
"""
logger.info("Connected with result code %s", str(args[2]))
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
# client.subscribe("$SYS/#", qos=0)
if isinstance(self.subscribe_topic, str):
self.client.subscribe(self.subscribe_topic, qos=self.qos)
elif isinstance(self.subscribe_topic, list):
for topic in self.subscribe_topic:
self.client.subscribe(topic, qos=self.qos)
else:
logger.warning('subscribe_topic is either None or an unknown data type.'
' Currently subscribed to 0 topics.')
async def on_message(self, *args):
"""on_message callback gets executed when the connection receives
a message.
Arguments:
client {MQTTClient} -- gmqtt.MQTTClient
topic {string} -- topic from which message was received
payload {bytes} -- actual message bytes received
qos {string} -- message QOS level (0,1,2)
properties {dict} -- message properties
"""
logger.info("%s %s", args[1], str(args[2]))
return 0
@staticmethod
def on_disconnect(*args):
"""on_disconnect is a callback that gets executed
after a disconnection occurs"""
logger.info('Disconnected')
@staticmethod
def on_subscribe(*args):
"""on_subscribe is a callback that gets executed
after a subscription is succesful"""
logger.info('Subscribed')
def ask_exit(self):
"""sets the STOP variable so that a signal gets sent
to disconnect the client
"""
self.STOP.set()
async def start(self):
"""starts initiates the connnection with the broker
Raises:
DestinationNotAvailable: If broker is not available
ConnectionFailed: If connection failed due to any other reason
"""
try:
conn_kwargs = dict(host=self.host, port=self.port)
if self.enable_auth:
self.client.set_auth_credentials(self.username, self.password)
if self.enable_ssl:
assert self.client_cert and self.client_key, \
"Cannot enable ssl without specifying client_cert and client_key"
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.load_cert_chain(self.client_cert,
keyfile=self.client_key)
conn_kwargs.update(dict(ssl=ssl_context))
await self.client.connect(**conn_kwargs)
self.is_connected = True
except ConnectionRefusedError as e:
# raising from None suppresses the exception chain
raise DestinationNotAvailable(
f'Connection Failed: Error connecting to'
f' {self.host}:{self.port} - {e}'
) from None
except Exception as e:
raise ConnectionFailed(e)
async def publish(self, *args, **kwargs):
"""publishes the message to the topic using client.publish"""
self.client.publish(*args, **kwargs)
async def stop(self):
"""force stop the connection with the MQTT broker."""
await self.client.disconnect()
self.is_connected = False
