async def main(broker_host, username, password):
client = MQTTClient("client-id-mitch", user_property=('hello', 'there'))
# client = MQTTClient("client-id-mitch")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(username, password.encode())
await client.connect(host=broker_host, port=1883)
# await client.connect()
print("connected, now ready to send...")
data = f"This is a test! {str(time.time())}"
hash = hashlib.sha256(data.encode()).hexdigest()
client.publish('test/time1',
"hello test/time1..",
qos=1,
message_expiry_interval=5,
content_type="json",
response_topic='RESPONSE/TOPIC2',
user_property=[('hash', hash), ('time', str(time.time()))])
client.publish('test/time2',
"hello test/time2..",
qos=1,
message_expiry_interval=5,
content_type="json",
response_topic='RESPONSE/TOPIC',
user_property=[('hash', hash), ('time', str(time.time()))])
await STOP.wait()
await client.disconnect()
class Mqtt(Connector):
def __init__(self, config, asm=None):
self.name = "mqtt"
self.config = config
self.asm = asm
self.default_room = "MyDefaultRoom"
self.client = None
async def connect(self):
self.client = MQTTClient(self.asm.name)
self.client.on_message = on_message
self.client.set_auth_credentials(
os.getenv('MQTT_USER', "arcus"),
os.getenv('MQTT_PASSWORD', "arcusarcus"))
await self.client.connect(os.getenv('MQTT_HOST', "mqtt"),
1883,
keepalive=60,
version=MQTTv311)
_LOGGER.info("Connected to MQTT")
async def listen(self):
self.client.subscribe(self.asm.name + "/#", qos=1)
stop = asyncio.Event()
await stop.wait()
@register_event(Message)
async def respond(self, message):
self.client.publish(self.asm.name,
'Message payload',
response_topic='RESPONSE/TOPIC')
async def disconnect(self):
# Disconnect from the service
await self.client.disconnect()
async def main(broker_host, token):
print("main started")
client = MQTTClient(None)
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
print("connecting")
# client.set_auth_credentials(token, None)
await client.connect(broker_host, version=MQTTv311)
print("connected... publishing")
client.publish('foo', str(time.time()), qos=1)
print("published")
await STOP.wait()
await client.disconnect()
print("finished")
async def main(broker_host):
print("creating client2")
client = MQTTClient("client2")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.on_publish = on_publish
await client.connect(broker_host)
client.subscribe("org/responses/client2")
await asyncio.sleep(5)
client.publish('org/common',"client2 message",\
response_topic="org/responses/client2")
await asyncio.sleep(50) #wait to receive message
await client.disconnect()
async def main(broker_host):
print("creating client")
client = MQTTClient("client1")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.on_publish = on_publish
await client.connect(broker_host)
client.subscribe('org/common', no_local=True)
await asyncio.sleep(5)
print("Publish response topic")
msg_out1 = "test message"
client.publish('org/common', "aa", response_topic="org/responses/client1")
await asyncio.sleep(50) #wait to receive message
if len(messages) == 0:
print("test failed")
else:
msg = messages.pop()
if msg == msg_out1:
print("test succeeded")
await client.disconnect()
async def main(broker_host, username, password):
client = MQTTClient(client_id="pub-client-id", receive_maximum=24000)
client.on_disconnect = on_disconnect
client.set_auth_credentials(username=username, password=password)
await client.connect(broker_host)
for i in range(10000):
client.publish(message_or_topic='TEST/T1', payload=str(i), qos=1, retain=True, message_expiry_interval=60)
async def publish_mqtt():
client = MQTTClient("client-id")
await client.connect("maggie-kafka-1.thenetcircle.lab",
port=1883,
version=MQTTv50)
client.publish("1972", str(time.time()), qos=1, message_expiry_interval=10)
await client.disconnect()
async def main(broker_host):
client = MQTTClient('client-id')
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
await client.connect(broker_host, 1883)
for x in range(100):
client.publish('TEST/TIME', time.time(), qos=1)
await STOP.wait()
await client.disconnect()
async def main(broker_host, token):
client = MQTTClient("client-id")
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
client.on_subscribe = on_subscribe
client.set_auth_credentials(token, None)
await client.connect(broker_host)
client.publish('TEST/TIME', str(time.time()), qos=0)
await STOP.wait()
await client.disconnect()
async def main(host):
client = MQTTClient("smart-agri-data-publisher")
# client.on_connect = on_connect
# client.on_message = on_message
client.on_disconnect = on_disconnect
# client.on_subscribe = on_subscribe
#client.set_auth_credentials(token, None)
#await client.connect(broker_host)
await client.connect(host)
with open('./20200701_0830_Outdoor-utf8.csv') as fo:
with open('./20200701_0830_South-utf8.csv') as fs:
readero = csv.reader(fo)
readerio = iter(readero)
next(readerio)
headero = next(readerio)
print(headero)
next(readerio)
readers = csv.reader(fs)
readeris = iter(readers)
next(readeris)
headers = next(readeris)
print(headers)
next(readeris)
while True:
try:
for rowo, rows in zip(readero, readers):
print(rowo)
for ent in (1, 6, 21, 26, 31, 36, 41, 46, 51):
pubtopic = TOPIC+'outdoor/'+headero[ent]
pubdata = str(rowo[ent])
print(pubtopic, pubdata)
client.publish(pubtopic, pubdata, qos=1)
for ent in (1, 6, 11, 16, 21, 26):
pubtopic = TOPIC+'south/'+headers[ent]
pubdata = str(rows[ent])
print(pubtopic, pubdata)
client.publish(pubtopic, pubdata, qos=1)
time.sleep(1)
except StopIteration:
fo.close()
fs.close()
return
# await STOP.wait()
await client.disconnect()
async def main(broker_host):
client = MQTTClient(client_id)
assign_callbacks_to_client(client)
await client.connect(broker_host)
print('Subscribe topic for the Last Will and Testament (LWT)...')
client.subscribe(status_topic)
print('Subscribe response topic...')
client.subscribe(res_topic)
print('Publish request message with response topic...')
client.publish(teds_topic, req_msg, response_topic=res_topic)
await STOP.wait()
await client.disconnect()
async def main():
client = MQTTClient(client_id=None)
client.on_connect = on_connect
client.on_message = on_message
await client.connect('localhost', version=MQTTv311)
# simulate sending a command on CMD_TOPIC
client.publish(CMD_TOPIC,
'{"jsonrpc": "2.0", "method": "ping", "id": "1" }',
qos=0)
client.publish(CMD_TOPIC,
'{"jsonrpc": "2.0", "method": "ping", "id": "2" }',
qos=0)
await STOP.wait()
await client.disconnect()
async def main(broker_host, actor_name, loop):
actor = load_actor(actor_name).Actor()
client_name = f'{os.getpid()}:{actor_name}'
client = MQTTClient(client_name)
aa = ActorAdapter(actor, client)
client.on_connect = aa.on_connect
client.on_message = aa.on_message
client.on_disconnect = aa.on_disconnect
client.on_subscribe = aa.on_subscribe
await client.connect(broker_host)
await actor._hook_loop(loop)
client.publish('TEST/TIME', str(time.time()), qos=1)
await STOP.wait()
await client.disconnect()
class DIDCommTeeMQTTClient:
def __init__(self, their_vk, endpoint):
self.client = MQTTClient("client-id")
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
self.did_conn = StaticConnection(crypto.create_keypair(),
their_vk=their_vk,
endpoint=endpoint)
async def mqtt_connect(self, broker_host):
await self.client.connect(broker_host)
async def mqtt_disconnect(self):
await self.client.disconnect()
def on_connect(self, client, flags, rc, properties):
print('Connected')
async def on_message(self, client, topic, payload, qos, properties):
await self.did_conn.send_async({
'@type': 'https://didcomm.dbluhm.com/mqtt/0.1/msg',
'topic': topic,
'payload': payload.decode('ascii'),
'properties': properties
})
print('RECV MSG:', payload)
def on_disconnect(self, client, packet, exc=None):
print('Disconnected')
def on_subscribe(self, client, mid, qos):
print('SUBSCRIBED')
def publish(self, *args, **kwargs):
self.client.publish(*args, **kwargs)
def subscribe(self, *args, **kwargs):
self.client.subscribe(*args, **kwargs)
class DecoderManager():
###################
# Sync class init
###################
def __init__(self):
print("DecoderManager __init__", flush=True)
print("{} acp_decoders started\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
self.settings = {}
self.settings["decoders"] = []
#####################################
# Signal handler for SIGINT, SIGTERM
#####################################
def ask_exit(self, *args):
self.STOP.set()
#####################################
# Signal handler for SIGALRM
#####################################
def reload(self, *args):
self.load_decoders_file()
#####################################
# Return current timestamp as string
#####################################
def ts_string(self):
return '{:.6f}'.format(time.time())
###############################################################
# Async initialization
###############################################################
async def start(self):
# Define async events for exit and reload (will set via signals)
self.STOP = asyncio.Event()
self.RELOAD = asyncio.Event()
# load settings.json into self.settings
self.read_settings()
# Connect input and output MQTT brokers (which can be same or different)
await self.connect_input_mqtt()
# debug testing timeout, disabling start of publisher
await self.connect_output_mqtt()
async def connect_input_mqtt(self):
self.input_client = MQTTClient(None) # auto-generate client id
self.input_client.on_connect = self.input_on_connect
self.input_client.on_message = self.input_on_message
self.input_client.on_disconnect = self.input_on_disconnect
self.input_client.on_subscribe = self.input_on_subscribe
user = self.settings["input_mqtt"]["user"]
password = self.settings["input_mqtt"]["password"]
host = self.settings["input_mqtt"]["host"]
port = self.settings["input_mqtt"]["port"]
self.input_client.set_auth_credentials(user, password)
await self.input_client.connect(host,
port,
keepalive=20,
version=MQTTv311)
async def connect_output_mqtt(self):
self.output_client = MQTTClient(None) # auto-generate client id
self.output_client.on_connect = self.output_on_connect
self.output_client.on_message = self.output_on_message
self.output_client.on_disconnect = self.output_on_disconnect
self.output_client.on_subscribe = self.output_on_subscribe
user = self.settings["output_mqtt"]["user"]
password = self.settings["output_mqtt"]["password"]
host = self.settings["output_mqtt"]["host"]
port = self.settings["output_mqtt"]["port"]
self.output_client.set_auth_credentials(user, password)
await self.output_client.connect(host,
port,
keepalive=60,
version=MQTTv311)
###############################################################
# Settings, including loading enabled decoders
#
# Builds self.settings from file "settings.json"
# Then loads decoders listed in the setting "decoders_file"
###############################################################
def read_settings(self):
with open('settings.json', 'r') as sf:
settings_data = sf.read()
# parse file
self.settings = json.loads(settings_data)
self.load_decoders_file()
def load_decoders_file(self):
# getting settings filename for decoders list (json)
decoders_file = self.settings["decoders_file"]
# read the json file
with open(decoders_file, 'r') as df:
decoders_data = df.read()
# parse to a python dictionary
decoders_obj = json.loads(decoders_data)
# store the new list of decoders as settings["decoders"]
self.settings["decoders"] = decoders_obj["decoders"]
# import/reload the decoders
self.import_decoders(self.settings["decoders"])
# import a list of decoder names
def import_decoders(self, new_decoders):
self.decoders = []
for decoder_name in new_decoders:
self.import_decoder(decoder_name)
# import a decoder, given name
# Will add { "name": , "decoder": } to self.decoders list
def import_decoder(self, decoder_name):
print("loading Decoder {}".format(decoder_name), flush=True)
module_name = 'decoders.' + decoder_name
# A new module can be imported with importlib.import_module()
# BUT an already loaded module must use importlib.reload for update to work.
if module_name in sys.modules:
module = sys.modules[module_name]
importlib.reload(module)
else:
module = importlib.import_module(module_name)
# now we have the refreshed/new module, so put Decoder on list self.decoders
decoder = module.Decoder(self.settings)
print(" loaded Decoder {}".format(decoder_name), flush=True)
self.decoders.append({"name": decoder_name, "decoder": decoder})
###############################################################
# Sensor data message handler
###############################################################
def handle_input_message(self, topic, msg_bytes):
acp_ts = self.ts_string()
msg_is_decoded = False
for decoder in self.decoders:
if decoder["decoder"].test(topic, msg_bytes):
decoded = decoder["decoder"].decode(topic, msg_bytes)
# If no acp_ts from decoder, insert from server time
if not "acp_ts" in decoded:
decoded["acp_ts"] = acp_ts
print("{} {} decoded by {}".format(acp_ts, decoded["acp_id"],
decoder["name"]),
flush=True)
#debug testing timeout, disabled send:
#self.send_output_message(topic, decoded)
msg_is_decoded = True
break # terminate the loop through decoders when first is found
if msg_is_decoded:
self.send_output_message(topic, decoded)
else:
print("{} Incoming message not decoded\n{}\n".format(
acp_ts, msg_bytes),
flush=True)
def send_output_message(self, topic, decoded):
msg_bytes = json.dumps(decoded)
#print("publishing {}".format(msg_bytes), flush=True)
output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic
self.output_client.publish(output_topic, msg_bytes, qos=0)
###############################################################
# MQTT INPUT
###############################################################
def input_on_connect(self, client, flags, rc, properties):
print('INPUT Connected to {} as {}'.format(
self.settings["input_mqtt"]["host"],
self.settings["input_mqtt"]["user"]),
flush=True)
client.subscribe('#', qos=0)
def input_on_message(self, client, topic, msg_bytes, qos, properties):
# IMPORTANT! We avoid a loop by ignoring input messages with the output prefix
if not topic.startswith(self.settings["output_mqtt"]["topic_prefix"]):
if DEBUG:
print('INPUT RECV MSG:', msg_bytes, flush=True)
self.handle_input_message(topic, msg_bytes)
else:
if DEBUG:
print('INPUT RECV COOKED MSG SKIPPED', flush=True)
def input_on_disconnect(self, client, packet, exc=None):
print('INPUT Disconnected', flush=True)
print("{} INPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
def input_on_subscribe(self, client, mid, qos, properties):
print('INPUT SUBSCRIBED to {}'.format(
self.settings["input_mqtt"]["topic"]),
flush=True)
###############################################################
# MQTT OUTPUT
###############################################################
def output_on_connect(self, client, flags, rc, properties):
print('OUTPUT Connected to {} as {}'.format(
self.settings["output_mqtt"]["host"],
self.settings["output_mqtt"]["user"]),
flush=True)
def output_on_disconnect(self, client, packet, exc=None):
print('OUTPUT Disconnected', flush=True)
print("{} OUTPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
# These GMQTT methods here for completeness although not used
def output_on_message(self, client, topic, msg_bytes, qos, properties):
print('OUTPUT RECV MSG?:', msg_bytes, flush=True)
def output_on_subscribe(self, client, mid, qos, properties):
print('OUTPUT SUBSCRIBED? to {}', flush=True)
###############################################################
# CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM)
###############################################################
async def finish(self):
await self.STOP.wait()
print("\nDecoderManager interrupted, closing MQTT clients", flush=True)
print("{} DecoderManager interrupted - disconnecting\n".format(
self.ts_string()),
file=sys.stderr,
flush=True)
await self.input_client.disconnect()
await self.output_client.disconnect()
class MQTTPublisher: # pylint: disable=too-many-instance-attributes
_APP_NAME = "call_detector"
_LOGGER = logging.getLogger(f"{__name__}.{__qualname__}")
_UPDATE_INTERVAL = 60
def __init__( # pylint: disable=too-many-arguments
self,
queue,
host="localhost",
port=8333,
username=None,
password=None,
ssl=False,
retry=False,
topic=f"call_detector/{socket.gethostname()}",
):
self._client = MQTTClient(self._APP_NAME)
if username is not None:
self._client.set_auth_credentials(username, password)
self._host = host
self._port = port
self._queue = queue
self._topic = topic
self._retry = retry
self._ssl = ssl
self._state = {"call": False}
async def run(self):
self._LOGGER.info("Running.")
while True:
try:
await self._client.connect(self._host, port=self._port, ssl=self._ssl, version=MQTTv311, keepalive=10)
self._LOGGER.info("Connected.")
break
except Exception: # pylint: disable=broad-except
if not self._retry:
raise
self._LOGGER.exception("Error occured during connecting")
await asyncio.sleep(5)
while True:
try:
try:
with async_timeout.timeout(self._UPDATE_INTERVAL):
msg = await self._queue.get()
self._update_state(msg)
except asyncio.exceptions.TimeoutError:
pass
await self._publish_state()
except Exception: # pylint: disable=broad-except
if not self._retry:
raise
self._LOGGER.exception("Error occured during publishing")
await asyncio.sleep(5)
def _update_state(self, msg):
del self._state["call"]
self._state[msg["source"]] = msg["apps"]
apps = reduce(lambda a, b: a + len(b), self._state.values(), 0)
self._state["call"] = apps > 0
self._LOGGER.info("State updated: %s", self._state)
@throttle(0.5)
async def _publish_state(self):
self._LOGGER.info("Publishing state %s to topic %s", self._state, self._topic)
self._client.publish(
self._topic,
json.dumps(self._state),
qos=1,
)
class MqttClient:
def __init__(self,
host: str,
port: int,
user: str,
password: typing.Optional[str] = None):
self.subscriptions: typing.Dict[str, typing.List[ValueCallback]] = {}
self.host = host
self.port = port
self.client = Client('sorokdva-dialogs')
self.client.set_auth_credentials(user, password)
self.client.on_connect = self._on_connect
self.client.on_message = self._on_message
async def _on_message(
self,
client: Client,
topic: str,
payload: bytes,
qos,
properties,
) -> constants.PubRecReasonCode:
log = logging.getLogger('mqtt')
futures = []
value = payload.decode()
for cb in self.subscriptions.get(topic, []):
log.info('passing (%r, %r) to %s', topic, value, cb)
futures.append(cb(topic, value))
if futures:
await asyncio.wait(futures, return_when=asyncio.ALL_COMPLETED)
return constants.PubRecReasonCode.SUCCESS
def _on_connect(self, client: Client, flags: int, result: int,
properties) -> None:
# FIXME make base path configurable
self.client.subscribe('/devices/#')
def subscribe(self, topic: str, callback: ValueCallback) -> None:
self.subscriptions.setdefault(topic, []).append(callback)
def send(self, topic: str, message):
self.client.publish(topic, message)
async def run(self):
await self.client.connect(self.host,
self.port,
version=constants.MQTTv311,
keepalive=30)
while True:
self.client.publish('smarthome', b'ping')
await asyncio.sleep(10)
@classmethod
def from_config(cls, cfg: dict) -> "MqttClient":
return cls(
host=cfg.get('host', 'localhost'),
port=cfg.get('port', 1883),
user=cfg.get('login', ''),
password=cfg.get('password', None),
)
class MQTTClient(Entity):
"""
A helper class for MQTT. Handles all the connection details. Returned to the library or module
that calls self._MQTTYombo.new().
.. code-block:: python
self.my_mqtt = self._MQTT.new(on_message_callback=self.mqtt_incoming, client_id="my_client_name")
self.my_mqtt.subscribe("yombo/devices/+/get") # subscribe to a topic. + is a wildcard for a single section.
"""
def __init__(self,
parent,
hostname: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
use_ssl: Optional[bool] = None,
version: Optional[str] = None,
keepalive: Optional[int] = None,
session_expiry: Optional[int] = None,
receive_maximum: Optional[int] = None,
user_property: Optional[Union[tuple, List[tuple]]] = None,
last_will: Optional = None,
maximum_packet_size: Optional[int] = None,
on_message_callback: Callable = None,
subscribe_callback: Callable = None,
unsubscribe_callback: Callable = None,
connected_callback: Optional[Callable] = None,
disconnected_callback: Optional[Callable] = None,
error_callback: Optional[Callable] = None,
client_id: Optional[str] = None,
password2: Optional[str] = None):
"""
Creates a new client connection to an MQTT broker.
:param parent: A reference to the MQTT library.
:param hostname: IP address or hostname to connect to.
:param port: Port number to connect to.
:param username: Username to connect as. Use "" to not use a username & password.
:param password: Password to to connect with. Use "" to not use a password.
:param use_ssl: Use SSL when attempting to connect to server, default is True.
:param version: MQTT version to use, default: MQTTv50. Other: MQTTv311
:param keepalive: How often the connection should be checked that it's still alive.
:param session_expiry: How many seconds the session should be valid. Defaults to 0.
:param receive_maximum: The Client uses this value to limit the number of QoS 1 and QoS 2 publications that it
is willing to process concurrently.
:param user_property: Connection user_property. A tuple or list of tuples.
:param last_will: Last will message generated by 'will()'.
:param maximum_packet_size: The maximum size the mqtt payload should be, in size.
:param on_message_callback: (required) method - Method to send messages to.
:param connected_callback: method - If you want a function called when connected to server.
:param disconnected_callback: method - If you want a function called when disconnected from server.
:param subscribe_callback: method - This method will be called when successfully subscribed to topic.
:param unsubscribe_callback: method - This method will be called when successfully unsubscribed from topic.
:param error_callback: method - A function to call if something goes wrong.
:param client_id: (default - random) - A client id to use for logging.
:param password2: A second password to try. Used by MQTTYombo.
:return:
"""
self._Entity_type: str = "MQTTClient"
self._Entity_label_attribute: str = "client_id"
super().__init__(parent)
self.connected = False
self.incoming_duplicates = deque([], 150)
self.send_queue = deque()
self.subscriptions = {}
self.unsubscriptions = {}
self.topics = {} # Store topics to resubscribe to
self.hostname = hostname
self.port = port
self.username = username
self.password = password
self.password2 = password2
self.use_ssl = use_ssl
self.version = version
self.keepalive = keepalive
self.session_expiry = session_expiry
self.receive_maximum = receive_maximum
self.user_property = user_property
self.last_will = last_will
self.maximum_packet_size = maximum_packet_size
self.on_message_callback = on_message_callback
self.subscribe_callback = subscribe_callback
self.unsubscribe_callback = unsubscribe_callback
self.connected_callback = connected_callback
self.disconnected_callback = disconnected_callback
self.error_callback = error_callback
self.client_id = client_id
client_options = {
"receive_maximum": receive_maximum,
"session_expiry_interval": session_expiry,
"maximum_packet_size": maximum_packet_size,
"user_property": user_property,
}
self.client = QClient(
client_id,
**{k: v
for k, v in client_options.items() if v is not None})
self.client.set_auth_credentials(username, password.encode())
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message_callback
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
@inlineCallbacks
def connect(self):
"""Connects to the mqtt broker."""
d = self.as_deferred(self.do_connect())
yield d
def as_deferred(self, f):
return Deferred.fromFuture(asyncio.ensure_future(f))
async def do_connect(self):
"""Connects to the mqtt broker."""
await asyncio.create_task(
self.client.connect(host=self.hostname, port=self.port))
def on_connect(self, client, flags, rc, properties):
"""Received a message."""
self.connected = True
# Do subscribes
for topic, kwargs in self.subscriptions.items():
self.client.subscribe(topic, **kwargs)
for topic, kwargs in self.unsubscriptions.items():
self.client.unsubscribe(topic, **kwargs)
# Do messages
for message in self.send_queue:
self.client.publish(message["topic"], **message["kwargs"])
if callable(self.connected_callback):
self.connected_callback(properties=properties)
def on_disconnect(self, client, packet, exc=None):
"""Disconnected notification."""
self.connected = False
if callable(self.disconnected_callback):
self.disconnected_callback(client=client, packet=packet)
def on_message(self, client, topic, body, qos, properties):
"""Received a message."""
if callable(self.on_message_callback):
self.on_message_callback(client=client,
topic=topic,
body=body,
qos=qos,
properties=properties)
def on_subscribe(self, client, mid, qos, properties):
"""Received subscribe confirmation."""
if callable(self.subscribe_callback):
self.subscribe_callback(client=client,
mid=mid,
qos=qos,
properties=properties)
def on_unsubscribe(self, client, mid, qos):
"""Received unsubscribe confirmation."""
if callable(self.unsubscribe_callback):
self.unsubscribe_callback(client=client, mid=mid, qos=qos)
def subscribe(self, topic: str, **kwargs):
"""
Subscribe to a topic.
:param topic:
:param kwargs:
:return:
"""
if "qos" not in kwargs:
kwargs["qos"] = 1
if self.session_expiry == 0:
self.subscriptions[topic] = kwargs
if self.connected is True:
self.client.subscribe(topic, **kwargs)
def unsubscribe(self, topic: str, **kwargs):
"""
Unsubscribe from topic.
:param topic: Topic to unsubscribe from.
:param kwargs:
:return:
"""
if "qos" not in kwargs:
kwargs["qos"] = 1
if self.connected is True:
self.client.unsubscribe(topic, **kwargs)
if self.session_expiry == 0:
self.unsubscriptions[topic] = kwargs
def publish(self,
topic: str,
message: Optional[str] = None,
qos: Optional[int] = None,
**kwargs):
"""
Publish a message to the MQTT broker. If not connected yet, will hold in a queue for later.
:param topic: Topic to publish too.
:param message: Message to send.
:param qos: quality of service.
:param kwargs: Any additional items to send to the qmqtt publish command.
:return:
"""
if qos is None:
qos = 1
if self.connected is True:
self.client.publish(topic, payload=message, qos=qos, **kwargs)
else:
kwargs["message"] = message
kwargs["qos"] = qos
self.send_queue.append({"topic": topic, "kwargs": kwargs})
class Deconz2acp():
###################
# Sync class init
###################
def __init__(self, settings):
self.settings = settings
print("{} Deconz2acp __init__() DEBUG={}".format(
self.ts_string(), DEBUG),
file=sys.stderr,
flush=True)
#####################################
# Signal handler for SIGINT, SIGTERM
#####################################
def ask_exit(self, *args):
self.STOP.set()
#####################################
# Return current timestamp as string
#####################################
def ts_string(self):
return '{:.6f}'.format(time.time())
###############################################################
# Async initialization
###############################################################
async def start(self, zigbee_data):
print("{} Deconz2acp start()".format(self.ts_string()),
file=sys.stderr,
flush=True)
self.zigbee_data = zigbee_data
# Define async events for exit and reload (will set via signals)
self.STOP = asyncio.Event()
self.RELOAD = asyncio.Event()
# connect output MQTT broker
await self.connect_output_mqtt()
# Connect input WebSocket
asyncio.ensure_future(self.subscribe_input_ws())
# Connect to input websocket
async def subscribe_input_ws(self):
ws_url = self.settings["input_ws"]["url"]
connected = False
while True:
try:
async with websockets.connect(ws_url) as ws:
connected = True
print("{} Deconz2acp connected to {}".format(
self.ts_string(), ws_url),
flush=True)
while connected:
try:
if DEBUG:
print("{} Deconz2acp awaiting msg from {}".
format(self.ts_string(), ws_url),
flush=True)
# Here we await & receive any websocket message
msg = await ws.recv()
if DEBUG:
pretty_msg = json.dumps(json.loads(msg),
indent=4)
print(
"{} Deconz2acp msg received from {}:\n{}".
format(self.ts_string(), ws_url,
pretty_msg),
flush=True)
#debug we're stuffing in a fake "zigbee" topic
self.handle_input_message(msg)
except websockets.exceptions.ConnectionClosedError:
connected = False
print("{} Deconz2acp disconnected from {}".format(
self.ts_string(), ws_url),
flush=True)
print("{} Deconz2acp websocket read loop ended".format(
self.ts_string()),
flush=True)
except ConnectionRefusedError:
print("{} Deconz2acp websocket connection refused from {}".
format(self.ts_string(), ws_url),
flush=True)
await asyncio.sleep(2) # sleep 2 seconds and retry
print("{} Deconz2acp websocket connect loop ended".format(
self.ts_string()),
flush=True)
async def connect_output_mqtt(self):
self.output_client = MQTTClient(None) # auto-generate client id
self.output_client.on_connect = self.output_on_connect
self.output_client.on_message = self.output_on_message
self.output_client.on_disconnect = self.output_on_disconnect
self.output_client.on_subscribe = self.output_on_subscribe
user = self.settings["output_mqtt"]["user"]
password = self.settings["output_mqtt"]["password"]
host = self.settings["output_mqtt"]["host"]
port = self.settings["output_mqtt"]["port"]
self.output_client.set_auth_credentials(user, password)
try:
await self.output_client.connect(host,
port,
keepalive=60,
version=MQTTv311)
except Exception as e:
if hasattr(e, 'args') and e.args[0] == 5:
print(
"{}\033[1;31m FAIL: Connect output_mqtt auth (as {} )\033[0;0m"
.format(self.ts_string(), user),
file=sys.stderr,
flush=True)
else:
print(
"{}\033[1;31m FAIL gmqtt connect exception\n{}\n{}\033[0;0m"
.format(self.ts_string(), e),
file=sys.stderr,
flush=True)
self.ask_exit()
###############################################################
# Sensor data message handler
###############################################################
def handle_input_message(self, msg_bytes):
msg_dict = json.loads(msg_bytes)
# Add required zigbee properties by updating msg_dict
# send_data will be True if zigbee_data.decode decides this message should be sent via MQTT.
send_data = self.zigbee_data.handle_ws_message(msg_dict)
if send_data:
topic = ""
if "acp_id" in msg_dict:
topic += msg_dict["acp_id"]
self.send_output_message(topic, msg_dict)
else:
print("{} Incoming message not sent to MQTT\n{}\n".format(
self.ts_string(), msg_bytes),
flush=True)
def send_output_message(self, topic, msg_dict):
msg_bytes = json.dumps(msg_dict)
#print("publishing {}".format(msg_bytes), flush=True)
output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic
if DEBUG:
pretty_msg = json.dumps(msg_dict, indent=4)
print("{} MQTT publish disabled by DEBUG setting:\n{}".format(
self.ts_string(), pretty_msg),
flush=True)
else:
self.output_client.publish(output_topic, msg_bytes, qos=0)
###############################################################
# WS INPUT
###############################################################
def input_ws_connected(self, uri):
print('{} INPUT Connected to {}'.format(self.ts_string(), uri),
flush=True)
###############################################################
# MQTT OUTPUT
###############################################################
def output_on_connect(self, client, flags, rc, properties):
print('{} OUTPUT Connected to {} as {}'.format(
self.ts_string(), self.settings["output_mqtt"]["host"],
self.settings["output_mqtt"]["user"]),
flush=True)
def output_on_disconnect(self, client, packet, exc=None):
print("{} OUTPUT Disconnected\n".format(self.ts_string()),
file=sys.stderr,
flush=True)
# These GMQTT methods here for completeness although not used
def output_on_message(self, client, topic, msg_bytes, qos, properties):
print('OUTPUT RECV MSG?:', msg_bytes, flush=True)
def output_on_subscribe(self, client, mid, qos, properties):
print('OUTPUT SUBSCRIBED? to {}', flush=True)
###############################################################
# CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM)
###############################################################
async def finish(self):
await self.STOP.wait()
print("\n{} Deconz2acp interrupted - disconnecting\n".format(
self.ts_string()),
file=sys.stderr,
flush=True)
await self.output_client.disconnect()
class Pipeline:
"""Object detection and tracking pipeline"""
def __init__(self, args):
self.args = args
# Initialise camera & camera viewport
self.init_camera()
# Initialise output
self.init_output(self.args.output)
# Initialise object detector (for some reason it has to happen
# here & not within detect_objects(), or else the inference engine
# gets upset and starts throwing NaNs at me. Thanks, Python.)
self.object_detector = SSD_MOBILENET(wanted_label='person',
model_file=self.args.model,
label_file=self.args.labels,
num_threads=self.args.num_threads,
edgetpu=self.args.edgetpu)
# Initialise feature encoder
if self.args.encoder_model is None:
model_filename = '{}/mars-64x32x3.pb'.format(
self.args.deepsorthome)
else:
model_filename = self.args.encoder_model
self.encoder = gdet.create_box_encoder(
model_filename, batch_size=self.args.encoder_batch_size)
self.background_subtraction = not self.args.disable_background_subtraction
# Initialise tracker
nn_budget = None
metric = nn_matching.NearestNeighborDistanceMetric(
"cosine", self.args.max_cosine_distance, nn_budget)
self.tracker = Tracker(metric,
max_iou_distance=self.args.max_iou_distance,
max_age=self.args.max_age)
# Initialise database
self.db = {}
self.delcount = 0
self.intcount = 0
self.poscount = 0
self.negcount = 0
self.mqtt = None
self.topic = self.args.mqtt_topic
self.mqtt_acp_id = self.args.mqtt_acp_id
self.heartbeat_delay_secs = self.args.heartbeat_delay_secs
self.loop = asyncio.get_event_loop()
async def init_mqtt(self):
if self.args.mqtt_broker is not None:
self.mqtt = MQTTClient('deepdish')
if self.args.mqtt_user is not None:
self.mqtt.set_auth_credentials(self.args.mqtt_user,
self.args.mqtt_pass)
await self.mqtt.connect(self.args.mqtt_broker)
if self.topic is None:
self.topic = 'default/topic'
def init_camera(self):
self.input = self.args.input
if self.input is None:
self.input = self.args.camera
# Allow live camera frames to be dropped
self.everyframe = None
else:
# Capture every frame from the video file
self.everyframe = asyncio.Event()
self.cap = cv2.VideoCapture(self.input)
# Configure the 'counting line' in the camera viewport
if self.args.line is None:
w, h = self.args.camera_width, self.args.camera_height
self.countline = np.array([[w / 2, 0], [w / 2, h]], dtype=int)
else:
self.countline = np.array(list(
map(int,
self.args.line.strip().split(','))),
dtype=int).reshape(2, 2)
self.cameracountline = self.countline.astype(float)
def init_output(self, output):
self.color_mode = None # fixme
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
fps = self.cap.get(cv2.CAP_PROP_FPS)
(w, h) = (self.args.camera_width, self.args.camera_height)
self.backbuf = Image.new("RGBA", (w, h), (0, 0, 0, 0))
self.draw = ImageDraw.Draw(self.backbuf)
self.output = cv2.VideoWriter(self.args.output, fourcc, fps, (w, h))
if self.args.no_framebuffer:
self.framebufdev = None
else:
self.framebufdev = self.args.framebuffer
fbX = self.framebufdev[-3:]
vsizefile = '/sys/class/graphics/{}/virtual_size'.format(fbX)
if not os.path.exists(
self.framebufdev) or not os.path.exists(vsizefile):
#raise Error('Invalid framebuffer device: {}'.format(self.framebufdev))
print('Invalid framebuffer device: {}'.format(
self.framebufdev))
self.framebufdev = None
if self.framebufdev is not None:
(w, h) = (self.args.framebuffer_width,
self.args.framebuffer_height)
if w is None or h is None:
nums = re.findall('(.*),(.*)', open(vsizefile).read())[0]
if w is None:
w = int(nums[0])
if h is None:
h = int(nums[1])
self.framebufres = (w, h)
def read_frame(self):
ret, frame = self.cap.read()
return (frame, time())
def shutdown(self):
self.running = False
for p in asyncio.Task.all_tasks():
p.cancel()
async def capture(self, q):
try:
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
frame = None
# Fetch next frame
(frame, t_frame) = await self.loop.run_in_executor(
pool, self.read_frame)
if frame is None:
print('No more frames.')
self.shutdown()
break
if self.args.camera_flip:
# If we need to flip the image vertically
frame = cv2.flip(frame, 0)
# Ensure frame is proper size
frame = cv2.resize(
frame,
(self.args.camera_width, self.args.camera_height))
await q.put((frame, t_frame, time()))
# If we are ensuring every frame is processed then wait for
# synchronising event to be triggered
if self.everyframe is not None:
await self.everyframe.wait()
self.everyframe.clear()
finally:
self.cap.release()
def run_object_detector(self, image):
t1 = time()
boxes = self.object_detector.detect_image(image)
t2 = time()
return (boxes, t2 - t1)
async def detect_objects(self, q_in, q_out):
# Initialise background subtractor
backSub = cv2.createBackgroundSubtractorMOG2()
frameCount = 0
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
frameCount += 1
# Obtain next video frame
(frame, t_frame, t_prev) = await q_in.get()
t_frame_recv = time()
# Apply background subtraction to find image-mask of areas of motion
if self.background_subtraction:
fgMask = backSub.apply(frame)
# Convert to PIL Image
image = Image.fromarray(
cv2.cvtColor(frame, cv2.COLOR_BGRA2RGBA))
t_backsub = time()
# Run object detection engine within a Thread Pool
(boxes0, delta_t) = await self.loop.run_in_executor(
pool, self.run_object_detector, image)
# Filter object detection boxes, including only those with areas of motion
t1 = time()
boxes = []
max_x, max_y = self.args.camera_width, self.args.camera_height
for (x, y, w, h) in boxes0:
if np.any(np.isnan(boxes0)):
# Drop any rubbish results
continue
x, y = int(np.clip(x, 0, max_x)), int(np.clip(y, 0, max_y))
w, h = int(np.clip(w, 0, max_x - x)), int(
np.clip(h, 0, max_y - y))
# Check if the box is almost as large as the camera viewport
if w * h > 0.9 * max_x * max_y:
# reject as spurious
continue
# Check if the box includes any detected motion
if not self.background_subtraction or np.any(
fgMask[x:x + w, y:y + h]):
boxes.append((x, y, w, h))
t2 = time()
# Send results to next step in pipeline
elements = [
FrameInfo(t_frame, frameCount),
CameraImage(image),
CameraCountLine(self.cameracountline),
TimingInfo('Frame processing latency', 'fram',
t_prev - t_frame),
TimingInfo('Frame / Q1 item received latency', 'q1',
t_frame_recv - t_prev),
TimingInfo('Background subtraction latency', 'bsub',
t_backsub - t_frame_recv),
TimingInfo('Object detection latency', 'objd',
delta_t + (t2 - t1))
]
await q_out.put((frame, boxes, elements, time()))
async def encode_features(self, q_in, q_out):
with concurrent.futures.ThreadPoolExecutor() as pool:
while self.running:
# Obtain next video frame and object detection boxes
(frame, boxes, elements, t_prev) = await q_in.get()
t1 = time()
# Run feature encoder within a Thread Pool
features = await self.loop.run_in_executor(
pool, self.encoder, frame, boxes)
t2 = time()
# Build list of 'Detection' objects and send them to next step in pipeline
detections = [
Detection(bbox, 1.0, feature)
for bbox, feature in zip(boxes, features)
]
elements.append(
TimingInfo('Q1 / Q2 latency', 'q2', (t1 - t_prev)))
elements.append(
TimingInfo('Feature encoder latency', 'feat', (t2 - t1)))
await q_out.put((detections, elements, time()))
async def track_objects(self, q_in, q_out):
while self.running:
(detections, elements, t_prev) = await q_in.get()
t1 = time()
boxes = np.array([d.tlwh for d in detections])
scores = np.array([d.confidence for d in detections])
indices = preprocessing.non_max_suppression(
boxes, self.args.nms_max_overlap, scores)
detections = [detections[i] for i in indices]
self.tracker.predict()
self.tracker.update(detections)
t2 = time()
elements.append(TimingInfo('Q2 / Q3 latency', 'q3', (t1 - t_prev)))
elements.append(TimingInfo('Tracker latency', 'trak', (t2 - t1)))
await q_out.put((detections, elements, time()))
async def process_results(self, q_in, q_out):
while self.running:
(detections, elements, t_prev) = await (q_in.get())
t1 = time()
for track in self.tracker.deleted_tracks:
i = track.track_id
if track.is_deleted():
self.check_deleted_track(track.track_id)
for track in self.tracker.tracks:
i = track.track_id
if not track.is_confirmed() or track.time_since_update > 1:
continue
if i not in self.db:
self.db[i] = []
bbox = track.to_tlbr()
# Find the bottom-centre of the bounding box & add it to the tracking database
bottomCentre = np.array([(bbox[0] + bbox[2]) / 2.0, bbox[3]])
self.db[i].append(bottomCentre)
if len(self.db[i]) > 1:
# If we have more than one datapoint for this tracked object
pts = (np.array(self.db[i]).reshape(
(-1, 1, 2))).reshape(-1)
elements.append(TrackedPath(pts))
p1 = self.cameracountline[0]
q1 = self.cameracountline[1]
p2 = np.array(self.db[i][-1])
q2 = np.array(self.db[i][-2])
cp = np.cross(q1 - p1, q2 - p2)
if intersection(p1, q1, p2, q2):
self.intcount += 1
print(
"track_id={} just intersected camera countline; cross-prod={}; intcount={}"
.format(i, cp, self.intcount))
elements.append(TrackedPathIntersection(pts[-4:]))
if cp >= 0:
self.poscount += 1
crossing_type = 'pos'
else:
self.negcount += 1
crossing_type = 'neg'
await self.publish_crossing_event_to_mqtt(
elements, crossing_type)
elements.append(TrackedObject(bbox, str(track.track_id)))
for det in detections:
bbox = det.to_tlbr()
elements.append(DetectedObject(bbox))
elements.append(CountingStats(self.negcount, self.poscount))
t2 = time()
elements.append(TimingInfo('Q3 / Q4 latency', 'q4', (t1 - t_prev)))
elements.append(
TimingInfo('Results processing latency', 'proc', (t2 - t1)))
await q_out.put((elements, time()))
async def publish_crossing_event_to_mqtt(self, elements, crossing_type):
if self.mqtt is not None:
for e in elements:
if isinstance(e, FrameInfo):
t_frame = e.t_frame
break
payload = json.dumps({
'acp_ts': str(t_frame),
'acp_id': self.mqtt_acp_id,
'acp_event': 'crossing',
'acp_event_value': crossing_type,
'poscount': self.poscount,
'negcount': self.negcount,
'diff': self.poscount - self.negcount
})
self.mqtt.publish(self.topic, payload)
async def periodic_mqtt_heartbeat(self):
if self.mqtt is not None:
while True:
payload = json.dumps({
'acp_ts': str(time()),
'acp_id': self.mqtt_acp_id,
'poscount': self.poscount,
'negcount': self.negcount,
'diff': self.poscount - self.negcount
})
self.mqtt.publish(self.topic, payload)
await asyncio.sleep(self.heartbeat_delay_secs)
async def graphical_output(self, render: RenderInfo, elements,
output_wh: (int, int)):
(output_w, output_h) = output_wh
# Clear screen
self.draw.rectangle([0, 0, output_w, output_h], fill=0, outline=0)
# Sort elements by display priority
elements.sort(key=lambda e: e.priority)
# Draw elements
for e in elements:
if hasattr(e, 'do_render'):
e.do_render(render)
# Copy backbuf to output
backarray = np.array(self.backbuf)
if self.color_mode is not None:
outputbgra = cv2.cvtColor(backarray, self.color_mode)
else:
outputbgra = backarray
outputrgb = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGB)
if self.output is not None:
self.output.write(outputrgb)
if self.framebufdev is not None:
outputrgba = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGBA)
outputfbuf = cv2.resize(outputrgba, self.framebufres)
try:
with open(self.framebufdev, 'wb') as buf:
buf.write(outputfbuf)
except:
print(
'failed to write to framebuffer device {} ...disabling it.'
.format(self.framebufdev))
self.framebufdev = None
await streaminfo.set_frame(outputrgb)
#cv2.imshow('main', outputrgb)
def text_output(self, handle, elements):
# Sort elements by priority
elements.sort(key=lambda e: e.priority)
for e in elements:
if hasattr(e, 'do_text'):
e.do_text(handle, elements)
async def render_output(self, q_in):
(output_w, output_h) = (self.args.camera_width,
self.args.camera_height)
ratio = 1 #fixme
render = RenderInfo(ratio, FontLib(output_w), self.draw, self.backbuf)
try:
while self.running:
(elements, t_prev) = await q_in.get()
t1 = time()
await self.graphical_output(render, elements,
(output_w, output_h))
for e in elements:
if isinstance(e, FrameInfo):
t_frame = e.t_frame
break
elements.append(
TimingInfo('Q4 / Q5 latency', 'q5', t1 - t_prev))
elements.append(
TimingInfo('Graphical display latency', 'disp',
time() - t1))
t_sum = 0
for e in elements:
if isinstance(e, TimingInfo):
t_sum += e.delta_t
elements.append(TimingInfo('Latency sum', 'sum', t_sum))
t_e2e = time() - t_frame
elements.append(TimingInfo('End to end latency', 'e2e', t_e2e))
elements.append(TimingInfo('Missing', 'miss', t_e2e - t_sum))
self.text_output(sys.stdout, elements)
if self.everyframe is not None:
# Notify other side that this frame is completely processed
self.everyframe.set()
finally:
self.output.release()
def check_deleted_track(self, i):
if i in self.db and len(self.db[i]) > 1:
if any_intersection(self.cameracountline[0],
self.cameracountline[1], np.array(self.db[i])):
self.delcount += 1
print("delcount={}".format(self.delcount))
self.db[i] = []
async def start(self):
self.running = True
cameraQueue = FreshQueue()
objectQueue = asyncio.Queue(maxsize=1)
detectionQueue = asyncio.Queue(maxsize=1)
resultQueue = asyncio.Queue(maxsize=1)
drawQueue = asyncio.Queue(maxsize=1)
asyncio.ensure_future(self.render_output(drawQueue))
asyncio.ensure_future(self.process_results(resultQueue, drawQueue))
asyncio.ensure_future(self.track_objects(detectionQueue, resultQueue))
asyncio.ensure_future(self.encode_features(objectQueue,
detectionQueue))
asyncio.ensure_future(self.detect_objects(cameraQueue, objectQueue))
await self.capture(cameraQueue)
self.running = False
class MQTT:
"""MQTT Class"""
def __init__(self):
# Secret can be set in either an environment variable (used first) or config.json
self.MQTT_USERNAME = getenv("MQTT_USER")
self.MQTT_PASS = getenv("MQTT_KEY")
# The connection handle for making calls
self.__client = None
self.last_sent_time = dict()
self.mqtt_cooldown = dict()
self.hostname = "mqtt"
self.client_id = getenv("WEB_HOSTNAME") + "-twitchbot-" + str(
datetime.now().timestamp())
# Default to not connected
self.MQTT_CONNECTION_STATE = False
self.ATTN_ENABLE = True
self.Topics = MqttTopics
async def connect_to_mqtt(self):
"""Connect to MQTT Server"""
# Create an instance of the REST client.
if self.MQTT_USERNAME is None or self.MQTT_PASS is None:
print("MQTT keys not found, aborting connection")
return False
try:
print("Atempting to connect to MQTT as " + self.MQTT_USERNAME)
self.__client = Client(client_id=self.client_id)
self.__client.set_auth_credentials(self.MQTT_USERNAME,
self.MQTT_PASS)
await self.__client.connect(self.hostname, port=1883)
print("Connected to MQTT")
self.MQTT_CONNECTION_STATE = True
return True
except Exception as e:
print("Failed to connect to MQTT, disabling it")
print(e)
self.MQTT_CONNECTION_STATE = False
return False
def get_cooldown(self, feed: str):
"""
Returns the cooldown
Loads it from the database,
or returns 0 if one is not set.
"""
if feed not in self.mqtt_cooldown:
q = session.query(Settings.value).filter(
Settings.key == f"mqtt_cooldown_{feed}").one_or_none()
if q is None:
# Value wasn't in the database, lets insert it.
insert = Settings(key=f"mqtt_cooldown_{feed}", value=0)
session.add(insert)
self.mqtt_cooldown[feed] = 0
else:
self.mqtt_cooldown[feed] = int(q[0])
return self.mqtt_cooldown[feed]
def set_cooldown(self, feed: str, cooldown: int) -> None:
"""
Sets the MQTT cooldown
Updates or inserts the value into the database
Exception handling should be done in the calling function
"""
q = session.query(Settings.id).filter(
Settings.key == f"mqtt_cooldown_{feed}").one_or_none()
if q is None:
# Value wasn't in the database, lets insert it.
insert = Settings(key=f"mqtt_cooldown_{feed}", value=cooldown)
session.add(insert)
self.mqtt_cooldown[feed] = cooldown
else:
session.query(Settings).filter(
Settings.key == f"mqtt_cooldown_{feed}").update(
{"value": cooldown})
self.mqtt_cooldown[feed] = cooldown
session.commit()
async def send(self,
feed,
value: Union[str, int] = 1,
retain: bool = False):
"""Send to an MQTT topic"""
last_sent = self.last_sent_time.get(feed, datetime.min)
cooldown = self.get_cooldown(feed)
now = datetime.now()
if (last_sent + timedelta(seconds=cooldown)) > now:
print(
f"MQTT {feed} on cooldown for {(last_sent + timedelta(seconds=cooldown)) - now}."
)
return False
if self.MQTT_CONNECTION_STATE is False:
try:
if not await self.connect_to_mqtt():
return False
except Exception as e:
print(e)
return False
try:
self.__client.publish(feed, value, retain=retain)
self.last_sent_time[feed] = now
return True
except Exception as e:
print(e)
return False
async def handle_message(self, topic, payload, properties,
mqtt_client: MQTTClient):
payload = json.loads(payload.decode())
response_topic = payload[protocol.PAYLOAD_RESPONSE_TOPIC_FIELD]
mqtt_client.publish(response_topic, self.get_generic_response())
class MQTTGateway(GatewayBase):
"""Gateway application for MQTT nodes on a network."""
PROTOCOL = "MQTT"
def __init__(self, keys, options):
self.host = options.mqtt_host
self.port = options.mqtt_port
self.max_time = options.max_time
self.options = options
self.node_mapping = {} # map node id to its uuid (TODO: FIXME)
super().__init__(keys, options)
# Connect to the MQTT broker
self.mqtt_client = MQTTClient("client-id")
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_message = self.on_message
self.mqtt_client.on_disconnect = self.on_disconnect
self.mqtt_client.on_subscribe = self.on_subscribe
asyncio.get_event_loop().create_task(self.start())
# Start the node cleanup task
PeriodicCallback(self.check_dead_nodes, 1000).start()
PeriodicCallback(self.request_alive, 30000).start()
logger.info('MQTT gateway application started')
async def start(self):
await self.mqtt_client.connect('{}:{}'.format(self.host, self.port))
def on_connect(self, client, flags, rc, properties):
self.mqtt_client.subscribe('node/check', 1)
def on_message(self, client, topic, payload, qos, properties):
try:
data = json.loads(payload)
except Exception:
# Skip data if not valid
return
logger.debug("Received message from node: {} => {}".format(
topic, data))
if topic.endswith("/check"):
asyncio.get_event_loop().create_task(self.handle_node_check(data))
elif topic.endswith("/resources"):
asyncio.get_event_loop().create_task(
self.handle_node_resources(topic, data))
else:
self.handle_node_update(topic, data)
def on_disconnect(self, client, packet, exc=None):
print('Disconnected')
def on_subscribe(self, client, mid, qos, properties):
print('SUBSCRIBED')
def close(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._disconnect())
async def _disconnect(self):
for node in self.nodes:
await self._disconnect_from_node(node)
await self.mqtt_client.disconnect()
async def discover_node(self, node):
discover_topic = 'gateway/{}/discover'.format(node.resources['id'])
await self.mqtt_client.publish(discover_topic, "resources", qos=1)
logger.debug("Published '{}' to topic: {}".format(
"resources", discover_topic))
def update_node_resource(self, node, endpoint, payload):
node_id = node.resources['id']
asyncio.get_event_loop().create_task(
self.mqtt_client.publish('gateway/{}/{}/set'.format(
node_id, endpoint),
payload,
qos=1))
async def handle_node_check(self, data):
"""Handle alive message received from coap node."""
node_id = data['id']
if node_id not in self.node_mapping:
node = Node(str(uuid.uuid4()), id=node_id)
self.node_mapping.update({node_id: node.uid})
resources_topic = 'node/{}/resources'.format(node_id)
await self.mqtt_client.subscribe(resources_topic, 1)
logger.debug("Subscribed to topic: {}".format(resources_topic))
self.add_node(node)
else:
# The node simply sent a check message to notify that it's still
# online.
node = self.get_node(self.node_mapping[node_id])
node.update_last_seen()
async def handle_node_resources(self, topic, data):
"""Process resources published by a node."""
node_id = topic.split("/")[1]
if node_id not in self.node_mapping:
return
for resource in data:
await self.mqtt_client.subscribe(
'node/{}/{}'.format(node_id, resource), 1)
await self.mqtt_client.publish('gateway/{}/discover'.format(node_id),
"values",
qos=1)
def handle_node_update(self, topic_name, data):
"""Handle CoAP post message sent from coap node."""
_, node_id, resource = topic_name.split("/")
value = data['value']
if self.node_mapping[node_id] not in self.nodes:
return
node = self.get_node(self.node_mapping[node_id])
self.forward_data_from_node(node, resource, value)
def request_alive(self):
"""Publish a request to trigger a check publish from nodes."""
logger.debug("Request check message from all MQTT nodes")
asyncio.get_event_loop().create_task(
self.mqtt_client.publish('gateway/check', '', qos=1))
def check_dead_nodes(self):
"""Check and remove nodes that are not alive anymore."""
to_remove = [
node for node in self.nodes.values()
if int(time.time()) > node.last_seen + self.max_time
]
for node in to_remove:
logger.info("Removing inactive node {}".format(node.uid))
asyncio.get_event_loop().create_task(
self._disconnect_from_node(node))
self.node_mapping.pop(node.resources['id'])
self.remove_node(node)
async def _disconnect_from_node(self, node):
node_id = node.resources['id']
await self.mqtt_client.unsubscribe(
['node/{}/resource'.format(node_id)])
for resource in node.resources:
await self.mqtt_client.unsubscribe(
['node/{}/{}'.format(node_id, resource)])
class MyMQTT(object):
def __init__(
self,
broker: str,
port: int = 1883,
keepalive: int = 60,
auth: List[str] = [None, None],
event_functions=None,
client_id: str = "client_id",
event_loop=None,
**kwargs,
) -> None:
default_events = {
"on_connect": self.on_connect,
"on_message": self.on_message,
"on_disconnect": self.on_disconnect,
"on_subscribe": self.on_subscribe,
}
if event_functions is None:
event_functions = {}
self.event_functions = default_events
self.event_functions.update(event_functions)
# self.client = MQTTClient(client_id)
self.auth = auth
self.broker = broker
self.port = port
self.keepalive = keepalive
self.stop = asyncio.Event()
if event_loop is None:
loop = asyncio.get_event_loop()
else:
loop = event_loop
loop.add_signal_handler(signal.SIGINT, self.ask_exit)
loop.add_signal_handler(signal.SIGTERM, self.ask_exit)
# passing any extra kwargs to the call to be used for e.g. on_connect
self.client = MQTTClient(client_id, **kwargs)
for _ in [k for k, v in self.event_functions.items() if v is None]:
logger.warning(f"mqtt no function assigned to {_}")
self.event_functions.pop(k)
for k, v in self.event_functions.items():
setattr(self.client, k, v)
if any(self.auth):
self.client.set_auth_credentials(*self.auth)
loop.create_task(self.start(self.broker))
async def start(self, broker):
logger.info("starting mqtt")
try:
await self.client.connect(
broker, port=self.port, keepalive=self.keepalive, version=MQTTv311
)
except OSError as e:
logger.error(f"unable to connect to mqtt with following error {e}")
else:
await self.stop.wait()
await self.client.disconnect()
def publish(self, topic, message, qos=1):
self.client.publish(topic, message, qos=qos)
def on_connect(self, client, flags, rc, properties):
logger.info("MQTT Connected")
client.subscribe("#")
def on_message(self, client, topic, payload, qos, properties):
logger.info(f"MQTT RECV MSG: {topic} {payload}")
def on_disconnect(self, client, packet, exc=None):
logger.info("MQTT Disconnected")
@staticmethod
def on_subscribe(*args):
logger.info("MQTT SUBSCRIBED")
def ask_exit(self, *args):
asyncio.Event().set()
async def main(args):
"""
Main function of the program. Initiates the publishing process of the Client.
:param args: arguments provided via CLI
"""
logging.info(
f"Connecting you to {args.host} on Port {args.port}. Your clientID: '{args.client_id}'. "
f"Multilateral Security for all messages {'is' if args.multilateral else 'is not'} enabled."
)
user_property = ('multilateral', '1') if args.multilateral else None
if user_property:
client = MQTTClient(args.client_id, user_property=user_property)
else:
client = MQTTClient(args.client_id)
client.on_connect = on_connect
client.on_disconnect = on_disconnect
# if both, cert and key, are specified, try to establish TLS connection to broker
if args.cert and args.key:
try:
context = create_tls_context(args.cert, args.key)
await client.connect(host=args.host, port=args.port, ssl=context)
except FileNotFoundError as e:
logging.error(e)
exit(0)
except ssl.SSLError:
logging.error(
f"SSL Error. Either your key/cert is not valid or the Broker does not support TLS on Port {args.port}."
)
exit(0)
# if both are not specified, then connect via insecure channel
elif not args.cert and not args.key:
await client.connect(host=args.host, port=args.port)
# if only one of them is specified, print error and exit
else:
logging.error(
f"Client certificate and client private key must be specified if connection should be secure. You have only specified {'the certificate' if args.cert else 'the private key'}."
)
exit(0)
# Once connected, publish the specified message to the specified topic with specified user properties (multilateral)
if len(args.multilateral_message) > 0:
for i, multilateral_security_index in enumerate(
args.multilateral_message):
i += 1
if multilateral_security_index:
logging.info(
f"Message {i} sent with enforced Multilateral Security")
user_property_message = ('multilateral', '1')
client.publish(args.topic,
args.message + f" {i}",
user_property=user_property_message,
qos=0)
else:
logging.info(f"Message {i} sent without Multilateral Security")
client.publish(args.topic, args.message + f" {i}", qos=0)
else:
logging.info(
f"Publishing '{args.topic}:{args.message}', Multilateral Security: {'on' if args.multilateral else 'off'}"
)
client.publish(args.topic, args.message, qos=0)
await STOP.wait()
try:
await client.disconnect(session_expiry_interval=0)
except ConnectionResetError:
logging.info("Broker successfully closed the connection.")
class LinkGMQTT(object):
def __init__(self, settings=None):
print("LinkGMQTT __init__()")
self.settings = settings
self.client = MQTTClient(None) # None => autogenerated client id
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
self.subscription_queue = asyncio.Queue()
print("LinkGMQTT __init__ completed")
async def start(self, server_settings):
"""
Connects to broker
"""
print('LinkGMQTT.start() connecting as user {}'.format(
server_settings["user"]))
self.client.set_auth_credentials(server_settings["user"],
server_settings["password"])
try:
await self.client.connect(server_settings["host"],
keepalive=60,
version=MQTTv311)
except Exception as e:
print("LinkGMQTT connect exception: {}".format(e))
return
print('LinkGMQTT.start() connected {}'.format(server_settings["host"]))
async def put(self, sensor_id, event):
"""
Sends sensor_id/event to MQTT broker.
sensor_id is string, used as MQTT topic
event is dictionary which will be converted to bytes for MQTT message
"""
#print('LinkGMQTT.put() sending {}'.format(sensor_id))
message = json.dumps(event)
self.client.publish(sensor_id, message, qos=0)
print("LinkGMQTT.put() published {} {}".format(sensor_id, message))
async def subscribe(self, subscribe_settings):
"""
Subscribes to sensor events.
"""
try:
self.client.subscribe(subscribe_settings["topic"], qos=0)
except Exception as e:
print("LinkGMQTT subscribe exception: {}".format(e))
return
print("LinkGMQTT.subscribed() {}".format(subscribe_settings["topic"]))
async def get(self):
print("LinkGMQTT get requested from client, awaiting queue")
message = await self.subscription_queue.get()
print("LinkGMQTT get returned from queue")
return message
def on_connect(self, client, flags, rc, properties):
print('LinkGMQTT Connected')
def on_message(self, client, topic, payload, qos, properties):
print('LinkGMQTT RECV MSG:', topic, payload)
message = payload.decode('utf-8')
message_dict = {}
try:
message_dict = json.loads(message)
except JSONDecodeError:
message_dict["message"] = message
print("remote_sensors() json msg error: {} => {}".format(
topic, message))
message_dict["topic"] = topic
self.subscription_queue.put_nowait(message_dict)
def on_disconnect(self, client, packet, exc=None):
print('LinkGMQTT Disconnected')
def on_subscribe(self, client, mid, qos, properties):
print('LinkGMQTT Subscribed')
async def finish(self):
await self.client.disconnect()
class GMQTTConnector(BaseConnector):
"""GMQTTConnector uses gmqtt library for connectors
running over MQTT.
"""
def __init__(self, host, port, subscribe_topic, publish_topic, **kwargs):
self.host = host
self.port = port
# topics
self.subscribe_topic = subscribe_topic
self.publish_topic = publish_topic
# connection
self.connection_id = uuid.uuid4().hex[:8]
self.is_connected = False
self.client = MQTTClient(self.connection_id)
# callbacks
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_disconnect = self.on_disconnect
self.client.on_subscribe = self.on_subscribe
self.STOP = asyncio.Event()
# options
self.ack_topic = kwargs.get('ack_topic')
self.enable_ssl = kwargs.get('enable_ssl', False)
self.enable_auth = kwargs.get('enable_auth', False)
self.username = kwargs.get('username')
self.password = kwargs.get('password')
self.client_cert = kwargs.get('client_cert')
self.client_key = kwargs.get('client_key')
self.qos = kwargs.get('qos', 2)
def get_connection_details(self):
"""get_connection_details returns the details
about the current MQTT connection.
"""
return dict(
connection_id=self.connection_id,
host=self.host,
port=self.port,
is_connected=self.is_connected,
subscribe_topic=self.subscribe_topic,
publish_topic=self.publish_topic
)
def on_connect(self, *args):
"""on_connect is a callback that gets exectued after the
connection is made.
Arguments:
client {MQTTClient} -- gmqtt.MQTTClient
flags {int} -- connection flags
rc {int} -- connection result code
properties {dict} -- config of the current connection
"""
logger.info("Connected with result code %s", str(args[2]))
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
# client.subscribe("$SYS/#", qos=0)
if isinstance(self.subscribe_topic, str):
self.client.subscribe(self.subscribe_topic, qos=self.qos)
elif isinstance(self.subscribe_topic, list):
for topic in self.subscribe_topic:
self.client.subscribe(topic, qos=self.qos)
else:
logger.warning('subscribe_topic is either None or an unknown data type.'
' Currently subscribed to 0 topics.')
async def on_message(self, *args):
"""on_message callback gets executed when the connection receives
a message.
Arguments:
client {MQTTClient} -- gmqtt.MQTTClient
topic {string} -- topic from which message was received
payload {bytes} -- actual message bytes received
qos {string} -- message QOS level (0,1,2)
properties {dict} -- message properties
"""
logger.info("%s %s", args[1], str(args[2]))
return 0
@staticmethod
def on_disconnect(*args):
"""on_disconnect is a callback that gets executed
after a disconnection occurs"""
logger.info('Disconnected')
@staticmethod
def on_subscribe(*args):
"""on_subscribe is a callback that gets executed
after a subscription is succesful"""
logger.info('Subscribed')
def ask_exit(self):
"""sets the STOP variable so that a signal gets sent
to disconnect the client
"""
self.STOP.set()
async def start(self):
"""starts initiates the connnection with the broker
Raises:
DestinationNotAvailable: If broker is not available
ConnectionFailed: If connection failed due to any other reason
"""
try:
conn_kwargs = dict(host=self.host, port=self.port)
if self.enable_auth:
self.client.set_auth_credentials(self.username, self.password)
if self.enable_ssl:
assert self.client_cert and self.client_key, \
"Cannot enable ssl without specifying client_cert and client_key"
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.load_cert_chain(self.client_cert,
keyfile=self.client_key)
conn_kwargs.update(dict(ssl=ssl_context))
await self.client.connect(**conn_kwargs)
self.is_connected = True
except ConnectionRefusedError as e:
# raising from None suppresses the exception chain
raise DestinationNotAvailable(
f'Connection Failed: Error connecting to'
f' {self.host}:{self.port} - {e}'
) from None
except Exception as e:
raise ConnectionFailed(e)
async def publish(self, *args, **kwargs):
"""publishes the message to the topic using client.publish"""
self.client.publish(*args, **kwargs)
async def stop(self):
"""force stop the connection with the MQTT broker."""
await self.client.disconnect()
self.is_connected = False
