def test_initialize_insecure(self, tls_set_mock, *mocks):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost',
secure=MQTTBrokerSettings.INSECURE)
self.assertFalse(tls_set_mock.called)
dsmr_mqtt.services.broker.initialize()
self.assertFalse(tls_set_mock.called)
def test_initialize_secure_cert_none(self, tls_set_mock, *mocks):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(
hostname='localhost', secure=MQTTBrokerSettings.SECURE_CERT_NONE)
self.assertFalse(tls_set_mock.called)
dsmr_mqtt.services.broker.initialize()
tls_set_mock.assert_called_with(cert_reqs=ssl.CERT_NONE)
def test_dsmr_mqtt_disabled(self, sleep_mock):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname=None)
self.assertFalse(sleep_mock.called)
with self.assertRaises(StopInfiniteRun):
self._intercept_command_stdout('dsmr_mqtt', run_once=True)
self.assertTrue(sleep_mock.called)
def test_dsmr_mqtt_restart_required(self, stop_mock, shutdown_mock, *mocks):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(restart_required=True)
self.assertFalse(shutdown_mock.called)
self.assertFalse(stop_mock.called)
self._intercept_command_stdout('dsmr_mqtt', run_once=True)
self.assertTrue(shutdown_mock.called)
self.assertTrue(stop_mock.called)
def test_initialize_enabled(self, connect_mock):
""" MQTT support enabled. """
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost')
self.assertFalse(connect_mock.called)
mqtt_client = dsmr_mqtt.services.broker.initialize()
self.assertIsNotNone(mqtt_client)
self.assertTrue(connect_mock.called)
def test_on_settings_update(self, send_robust_mock):
""" Test whether restart required signal is triggered. """
self.assertFalse(send_robust_mock.called)
broker_settings = MQTTBrokerSettings.get_solo()
broker_settings.hostname = 'xxx'
broker_settings.save()
self.assertTrue(send_robust_mock.called)
# Should do nothing when created.
send_robust_mock.reset_mock()
MQTTBrokerSettings.objects.all().delete()
MQTTBrokerSettings.get_solo()
self.assertFalse(send_robust_mock.called)
def test_dsmr_mqtt_with_client(self, connect_mock, *mocks):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost')
self.assertFalse(connect_mock.called)
try:
self._intercept_command_stdout('dsmr_mqtt', run_once=True)
except TypeError:
# Weird mock issue.
pass
self.assertTrue(connect_mock.called)
def test_initialize_connection_refused(self, connect_mock, sleep_mock):
""" Whenever the broker is unavailable. """
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost')
connect_mock.side_effect = ConnectionRefusedError() # Fail.
self.assertFalse(sleep_mock.called)
self.assertFalse(connect_mock.called)
with self.assertRaises(StopInfiniteRun):
dsmr_mqtt.services.broker.initialize()
self.assertTrue(connect_mock.called)
self.assertTrue(sleep_mock.called)
def run(mqtt_client):
""" Reads any messages from the queue and publishing them to the MQTT broker. """
broker_settings = MQTTBrokerSettings.get_solo()
# Keep batches small, only send the latest X messages. The rest will be purged (in case of delay).
message_queue = queue.Message.objects.all().order_by(
'-pk')[0:settings.DSMRREADER_MQTT_MAX_MESSAGES_IN_QUEUE]
if not message_queue:
return
logger.info('MQTT: Processing %d message(s)', len(message_queue))
for current in message_queue:
mqtt_client.publish(topic=current.topic,
payload=current.payload,
qos=broker_settings.qos,
retain=True)
current.delete()
# Delete any overflow in messages.
queue.Message.objects.all().delete()
# Networking.
mqtt_client.loop(0.1)
# We cannot raise any exception in callbacks, this is our check point. This MUST be called AFTER the first loop().
if not mqtt_client.is_connected():
signal_reconnect()
raise RuntimeError('MQTT: Client no longer connected')
def test_initialize_debug(self, *mocks):
""" Debug enabled. """
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost')
mqtt_client = dsmr_mqtt.services.broker.initialize()
self.assertIsNone(mqtt_client.on_log)
self.assertIsNone(mqtt_client.on_publish)
MQTTBrokerSettings.objects.update(debug=True)
mqtt_client = dsmr_mqtt.services.broker.initialize()
# Check callbacks set.
self.assertEqual(mqtt_client.on_log, dsmr_mqtt.services.broker.on_log)
self.assertEqual(mqtt_client.on_publish,
dsmr_mqtt.services.broker.on_publish)
def test_initialize_no_hostname(self, connect_mock):
MQTTBrokerSettings.objects.update(hostname='')
self.assertFalse(connect_mock.called)
with self.assertRaises(RuntimeError):
dsmr_mqtt.services.broker.initialize_client()
self.assertFalse(connect_mock.called)
self.assertFalse(MQTTBrokerSettings.get_solo().enabled)
def _on_broker_settings_updated_signal(self, instance, created, raw, update_fields, **kwargs):
# Skip new or imported (fixture) instances. And do not update if this hook has just updated it.
if created or raw or (update_fields and 'restart_required' in update_fields):
return
from dsmr_mqtt.models.settings.broker import MQTTBrokerSettings
broker_settings = MQTTBrokerSettings.get_solo()
broker_settings.restart_required = True
broker_settings.save(update_fields=['restart_required']) # DO NOT CHANGE: Keep this save() + update_fields.
def test_initialize_credentials(self, *mocks):
""" User/password set. """
USER = '******'
PASS = '******'
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(hostname='localhost')
mqtt_client = dsmr_mqtt.services.broker.initialize()
self.assertIsNone(mqtt_client._username)
self.assertIsNone(mqtt_client._password)
MQTTBrokerSettings.objects.update(username=USER, password=PASS)
mqtt_client = dsmr_mqtt.services.broker.initialize()
# Check credentials set.
self.assertEqual(mqtt_client._username, USER.encode('utf-8'))
self.assertEqual(mqtt_client._password, PASS.encode('utf-8'))
def test_initialize_connection_refused(self, connect_mock):
""" Whenever the broker is unavailable. """
connect_mock.side_effect = ConnectionRefusedError() # Fail.
self.assertFalse(connect_mock.called)
with self.assertRaises(RuntimeError):
dsmr_mqtt.services.broker.initialize_client()
self.assertTrue(connect_mock.called)
self.assertTrue(MQTTBrokerSettings.get_solo().enabled) # Different from above.
def run(self, **options):
""" InfiniteManagementCommandMixin listens to handle() and calls run() in a loop. """
# Check on each run. In case MQTT was either disabled, enabled or settings were changed.
if MQTTBrokerSettings.get_solo().restart_required:
MQTTBrokerSettings.objects.update(restart_required=False)
logger.warning(
'MQTT | --- Detected settings change, requiring process restart, stopping...'
)
raise StopInfiniteRun()
dsmr_mqtt.services.broker.run(mqtt_client=self.mqtt_client)
def test_restart_required_on_save(self, send_robust_mock):
""" Any change should flag a restart. """
broker_settings = MQTTBrokerSettings.get_solo()
self.assertFalse(send_robust_mock.called)
broker_settings.hostname = 'xxx'
broker_settings.save()
self.assertTrue(send_robust_mock.called)
send_robust_mock.reset_mock()
broker_settings.enabled = True
broker_settings.save()
self.assertTrue(send_robust_mock.called)
def initialize_client() -> Optional[paho.Client]:
""" Initializes the MQTT client and returns client instance. """
broker_settings = MQTTBrokerSettings.get_solo()
if not broker_settings.enabled:
logger.debug(
'MQTT: Integration disabled in settings (or it was disabled due to a configuration error)'
)
return None
if not broker_settings.hostname:
logger.error('MQTT: No hostname found in settings, disabling MQTT')
broker_settings.update(enabled=False)
raise RuntimeError('No hostname found in settings')
logger.debug('MQTT: Initializing MQTT client for "%s:%d" (QoS level %d)',
broker_settings.hostname, broker_settings.port,
settings.DSMRREADER_MQTT_QOS_LEVEL)
mqtt_client = paho.Client(client_id=broker_settings.client_id)
mqtt_client.on_connect = on_connect
mqtt_client.on_disconnect = on_disconnect
mqtt_client.on_log = on_log
if broker_settings.username:
mqtt_client.username_pw_set(broker_settings.username,
broker_settings.password)
# SSL/TLS.
if broker_settings.secure == MQTTBrokerSettings.SECURE_CERT_NONE:
logger.debug('MQTT: Using secure connection (ssl.CERT_NONE)')
mqtt_client.tls_set(cert_reqs=ssl.CERT_NONE)
elif broker_settings.secure == MQTTBrokerSettings.SECURE_CERT_REQUIRED:
logger.debug('MQTT: Using secure connection (ssl.CERT_REQUIRED)')
mqtt_client.tls_set(cert_reqs=ssl.CERT_REQUIRED)
else:
logger.debug('MQTT: Using insecure connection (no TLS)')
try:
mqtt_client.connect(host=broker_settings.hostname,
port=broker_settings.port)
except Exception as error:
logger.error('MQTT: Failed to connect to broker (%s:%d): %s',
broker_settings.hostname, broker_settings.port, error)
signal_reconnect()
raise RuntimeError('MQTT: Failed to connect to broker') from error
return mqtt_client
def run(mqtt_client):
""" Reads any messages from the queue and publishing them to the MQTT broker. """
mqtt_client.loop()
broker_settings = MQTTBrokerSettings.get_solo()
# Keep batches small, only send the latest X messages. The rest will be purged (in case of delay).
message_queue = queue.Message.objects.all().order_by('-pk')[0:settings.DSMRREADER_MQTT_MAX_MESSAGES_IN_QUEUE]
for current in message_queue:
mqtt_client.publish(
topic=current.topic,
payload=current.payload,
qos=broker_settings.qos
)
current.delete()
# Delete any overflow in messages.
queue.Message.objects.all().delete()
def initialize():
""" Initializes the MQTT client and returns client instance. """
broker_settings = MQTTBrokerSettings.get_solo()
if not broker_settings.hostname:
logger.warning(
'MQTT: No hostname found in settings, restarting in a minute...')
time.sleep(60)
raise StopInfiniteRun()
mqtt_client = paho.Client(client_id=broker_settings.client_id)
mqtt_client.on_connect = on_connect
mqtt_client.on_disconnect = on_disconnect
mqtt_client.on_log = on_log
mqtt_client.on_publish = on_publish
if broker_settings.username:
mqtt_client.username_pw_set(broker_settings.username,
broker_settings.password)
# SSL/TLS.
if broker_settings.secure == MQTTBrokerSettings.SECURE_CERT_NONE:
logger.debug('MQTT: Initializing secure connection (ssl.CERT_NONE)')
mqtt_client.tls_set(cert_reqs=ssl.CERT_NONE)
elif broker_settings.secure == MQTTBrokerSettings.SECURE_CERT_REQUIRED:
logger.debug(
'MQTT: Initializing secure connection (ssl.CERT_REQUIRED)')
mqtt_client.tls_set(cert_reqs=ssl.CERT_REQUIRED)
else:
logger.debug('MQTT: Initializing insecure connection (no TLS)')
try:
mqtt_client.connect(host=broker_settings.hostname,
port=broker_settings.port)
except Exception as error:
logger.error(
'MQTT: Failed to connect to broker, restarting in a minute: %s',
error)
time.sleep(60)
raise StopInfiniteRun()
return mqtt_client
def get_context_data(self, **kwargs):
context_data = super(Configuration, self).get_context_data(**kwargs)
# 20+ queries, we should cache this at some point.
context_data.update(
dict(
api_settings=APISettings.get_solo(),
backend_settings=BackendSettings.get_solo(),
backup_settings=BackupSettings.get_solo(),
consumption_settings=ConsumptionSettings.get_solo(),
datalogger_settings=DataloggerSettings.get_solo(),
dropbox_settings=DropboxSettings.get_solo(),
email_settings=EmailSettings.get_solo(),
frontend_settings=FrontendSettings.get_solo(),
mindergas_settings=MinderGasSettings.get_solo(),
mqtt_broker_settings=MQTTBrokerSettings.get_solo(),
mqtt_jsondaytotals_settings=JSONDayTotalsMQTTSettings.get_solo(
),
mqtt_splittopicdaytotals_settings=
SplitTopicDayTotalsMQTTSettings.get_solo(),
mqtt_jsoncurrentperiodtotals_settings=
JSONCurrentPeriodTotalsMQTTSettings.get_solo(),
mqtt_splittopiccurrentperiodtotals_settings=
SplitTopicCurrentPeriodTotalsMQTTSettings.get_solo(),
mqtt_jsongasconsumption_settings=JSONGasConsumptionMQTTSettings
.get_solo(),
mqtt_splittopicgasconsumption_settings=
SplitTopicGasConsumptionMQTTSettings.get_solo(),
mqtt_splittopicmeterstatistics_settings=
SplitTopicMeterStatisticsMQTTSettings.get_solo(),
mqtt_jsontelegram_settings=JSONTelegramMQTTSettings.get_solo(),
mqtt_rawtelegram_settings=RawTelegramMQTTSettings.get_solo(),
mqtt_splittopictelegram_settings=SplitTopicTelegramMQTTSettings
.get_solo(),
notification_settings=NotificationSetting.get_solo(),
pvoutput_api_settings=PVOutputAPISettings.get_solo(),
pvoutput_addstatus_settings=PVOutputAddStatusSettings.get_solo(
),
retention_settings=RetentionSettings.get_solo(),
weather_settings=WeatherSettings.get_solo(),
influxdb_settings=InfluxdbIntegrationSettings.get_solo(),
))
return context_data
def setUp(self):
MQTTBrokerSettings.get_solo()
MQTTBrokerSettings.objects.update(
enabled=True,
hostname='localhost'
)
def initialize(self):
""" Set up persistent MQTT client. """
self.sleep_time = MQTTBrokerSettings.get_solo().process_sleep
self.mqtt_client = dsmr_mqtt.services.broker.initialize()