def test_dsmr_version_3(self):
""" Test connection parameters for DSMR v2/3. """
datalogger_settings = DataloggerSettings.get_solo()
datalogger_settings.dsmr_version = DataloggerSettings.DSMR_VERSION_3
datalogger_settings.save()
self.assertEqual(DataloggerSettings.get_solo().dsmr_version, DataloggerSettings.DSMR_VERSION_3)
connection_parameters = dsmr_datalogger.services.get_dsmr_connection_parameters()
self.assertEqual(connection_parameters['baudrate'], 9600)
self.assertEqual(connection_parameters['bytesize'], serial.SEVENBITS)
self.assertEqual(connection_parameters['parity'], serial.PARITY_EVEN)
def test_dsmr_version_4(self):
""" Test connection parameters for DSMR v4. """
self.assertEqual(DataloggerSettings.get_solo().dsmr_version, DataloggerSettings.DSMR_VERSION_4)
connection_parameters = dsmr_datalogger.services.get_dsmr_connection_parameters()
self.assertEqual(connection_parameters['baudrate'], 115200)
self.assertEqual(connection_parameters['bytesize'], serial.EIGHTBITS)
self.assertEqual(connection_parameters['parity'], serial.PARITY_NONE)
def test_tracking_disabled(self):
""" Test whether datalogger can bij stopped by changing track setting. """
datalogger_settings = DataloggerSettings.get_solo()
datalogger_settings.track = False
datalogger_settings.save()
# Datalogger should crash with error.
with self.assertRaisesMessage(CommandError, 'Datalogger tracking is DISABLED!'):
self._intercept_command_stdout('dsmr_datalogger')
def get_context_data(self, **kwargs):
context_data = super(Configuration, self).get_context_data(**kwargs)
context_data['api_settings'] = APISettings.get_solo()
context_data['consumption_settings'] = ConsumptionSettings.get_solo()
context_data['datalogger_settings'] = DataloggerSettings.get_solo()
context_data['frontend_settings'] = FrontendSettings.get_solo()
context_data['weather_settings'] = WeatherSettings.get_solo()
context_data['backup_settings'] = BackupSettings.get_solo()
context_data['dropbox_settings'] = DropboxSettings.get_solo()
context_data['mindergas_settings'] = MinderGasSettings.get_solo()
return context_data
def run(self, **options):
""" InfiniteManagementCommandMixin listens to handle() and calls run() in a loop. """
datalogger_settings = DataloggerSettings.get_solo()
# This should only by disabled when performing huge migrations.
if not datalogger_settings.track:
raise CommandError("Datalogger tracking is DISABLED!")
telegram = dsmr_datalogger.services.read_telegram()
# Reflect output to STDOUT for logging and convenience.
self.stdout.write(telegram)
dsmr_datalogger.services.telegram_to_reading(data=telegram)
def get_dsmr_connection_parameters():
""" Returns the communication settings required for the DSMR version set. """
DSMR_VERSION_MAPPING = {
DataloggerSettings.DSMR_VERSION_3: {
'baudrate': 9600,
'bytesize': serial.SEVENBITS,
'parity': serial.PARITY_EVEN,
},
DataloggerSettings.DSMR_VERSION_4: {
'baudrate': 115200,
'bytesize': serial.EIGHTBITS,
'parity': serial.PARITY_NONE,
},
}
datalogger_settings = DataloggerSettings.get_solo()
connection_parameters = DSMR_VERSION_MAPPING[datalogger_settings.dsmr_version]
connection_parameters['com_port'] = datalogger_settings.com_port
return connection_parameters
def get_context_data(self, **kwargs):
context_data = super(Statistics, self).get_context_data(**kwargs)
context_data['capabilities'] = dsmr_backend.services.get_capabilities()
try:
context_data['latest_reading'] = DsmrReading.objects.all().order_by('-pk')[0]
except IndexError:
pass
today = timezone.localtime(timezone.now()).date()
context_data['datalogger_settings'] = DataloggerSettings.get_solo()
context_data['meter_statistics'] = MeterStatistics.get_solo()
try:
context_data['energy_prices'] = EnergySupplierPrice.objects.by_date(today)
except EnergySupplierPrice.DoesNotExist:
pass
# Use stats
context_data['slumber_consumption_watt'] = dsmr_consumption.services.calculate_slumber_consumption_watt()
context_data['min_max_consumption_watt'] = dsmr_consumption.services.calculate_min_max_consumption_watt()
return context_data
def setUp(self):
datalogger_settings = DataloggerSettings.get_solo()
datalogger_settings.dsmr_version = DataloggerSettings.DSMR_VERSION_2
datalogger_settings.save()
def telegram_to_reading(data): # noqa: C901
"""
Converts a P1 telegram to a DSMR reading, which will be stored in database.
"""
READING_FIELDS = [x.name for x in DsmrReading._meta.get_fields() if x.name not in ('id', 'processed')]
STATISTICS_FIELDS = [
x.name for x in MeterStatistics._meta.get_fields() if x.name not in ('id', 'rejected_telegrams')
]
# We will log the telegrams in base64 for convenience and debugging 'n stuff.
base64_data = base64.b64encode(data.encode())
datalogger_settings = DataloggerSettings.get_solo()
# Discard CRC check when any support is lacking anyway. Or when it's disabled.
connection_parameters = get_dsmr_connection_parameters()
if connection_parameters['crc'] and datalogger_settings.verify_telegram_crc:
try:
# Verify telegram by checking its CRC.
verify_telegram_checksum(data=data)
except InvalidTelegramError as error:
# Hook to keep track of failed readings count.
MeterStatistics.objects.all().update(rejected_telegrams=F('rejected_telegrams') + 1)
dsmrreader_logger.warning('Rejected telegram (base64 encoded): {}'.format(base64_data))
dsmrreader_logger.exception(error)
raise
# Defaults all fields to NULL.
parsed_reading = {k: None for k in READING_FIELDS + STATISTICS_FIELDS}
field_splitter = re.compile(r'([^(]+)\((.+)\)')
lines_read = data.split("\r\n")
for index, current_line in enumerate(lines_read):
result = field_splitter.search(current_line)
if not result:
continue
code = result.group(1)
# M-bus (0-n:24.1) cannot identify the type of device, see issue #92.
if code in ('0-2:24.2.1', '0-3:24.2.1', '0-4:24.2.1'):
code = '0-1:24.2.1'
# DSMR 2.x emits gas readings in different format.
if code == '0-1:24.3.0':
parsed_reading = _convert_legacy_dsmr_gas_line(
parsed_reading, current_line, lines_read[index + 1]
)
continue
try:
field = DSMR_MAPPING[code]
except KeyError:
continue
value = result.group(2)
# Drop units, as the database does not care for them.
value = value.replace('*kWh', '').replace('*kW', '').replace('*m3', '')
# Extra device parameters are placed on a single line, meh.
if code == "0-1:24.2.1":
timestamp_value, gas_usage = value.split(")(")
parsed_reading[field[0]] = reading_timestamp_to_datetime(string=timestamp_value)
parsed_reading[field[1]] = gas_usage
else:
if field == "timestamp":
value = reading_timestamp_to_datetime(string=value)
parsed_reading[field] = value
# Hack for DSMR 2.x legacy, which lacks timestamp info..
if parsed_reading['timestamp'] is None:
parsed_reading['timestamp'] = timezone.now()
# For some reason, there are telegrams generated with a timestamp in the far future. We should disallow that.
discard_timestamp = timezone.now() + timezone.timedelta(hours=24)
if parsed_reading['timestamp'] > discard_timestamp or (
parsed_reading['extra_device_timestamp'] is not None and
parsed_reading['extra_device_timestamp'] > discard_timestamp):
error_message = 'Discarded telegram with future timestamp: {}'.format(data)
django_logger.error(error_message)
raise InvalidTelegramError(error_message)
# Optional tracking of phases, but since we already mapped this above, just remove it again... :]
if not datalogger_settings.track_phases:
parsed_reading.update({
'phase_currently_delivered_l1': None,
'phase_currently_delivered_l2': None,
'phase_currently_delivered_l3': None,
'phase_currently_returned_l1': None,
'phase_currently_returned_l2': None,
'phase_currently_returned_l3': None,
})
# Now we need to split reading & statistics. So we split the dict here.
reading_kwargs = {k: parsed_reading[k] for k in READING_FIELDS}
statistics_kwargs = {k: parsed_reading[k] for k in STATISTICS_FIELDS}
# Reading will be processed later.
new_reading = DsmrReading.objects.create(**reading_kwargs)
# There should already be one in database, created when migrating.
statistics_kwargs['latest_telegram'] = data
MeterStatistics.objects.all().update(**statistics_kwargs)
# Broadcast this telegram as signal.
dsmr_datalogger.signals.raw_telegram.send_robust(sender=None, data=data)
if settings.DSMRREADER_LOG_TELEGRAMS:
dsmrreader_logger.info('Received telegram (base64 encoded): {}'.format(base64_data))
return new_reading
def get(self, request):
data = {}
data['capabilities'] = dsmr_backend.services.get_capabilities()
form = DashboardGraphForm(request.GET)
if not form.is_valid():
return HttpResponseBadRequest(form.errors)
# Optimize queries for large datasets by restricting the data to the last week in the first place.
base_timestamp = timezone.now() - timezone.timedelta(days=7)
electricity = ElectricityConsumption.objects.filter(read_at__gt=base_timestamp).order_by('-read_at')
gas = GasConsumption.objects.filter(read_at__gt=base_timestamp).order_by('-read_at')
temperature = TemperatureReading.objects.filter(read_at__gt=base_timestamp).order_by('-read_at')
# Apply any offset requested by the user.
electricity_offset = form.cleaned_data.get('electricity_offset')
electricity = electricity[electricity_offset:electricity_offset + self.MAX_ITEMS]
gas_offset = form.cleaned_data.get('gas_offset')
gas = gas[gas_offset:gas_offset + self.MAX_ITEMS]
temperature = temperature[:self.MAX_ITEMS]
# Reverse all sets gain.
electricity = electricity[::-1]
gas = gas[::-1]
temperature = temperature[::-1]
# By default we only display the time, scrolling should enable a more verbose x-axis.
graph_x_format_electricity = 'DSMR_GRAPH_SHORT_TIME_FORMAT'
graph_x_format_gas = 'DSMR_GRAPH_SHORT_TIME_FORMAT'
if electricity_offset > 0:
graph_x_format_electricity = 'DSMR_GRAPH_LONG_TIME_FORMAT'
if gas_offset > 0:
graph_x_format_gas = 'DSMR_GRAPH_LONG_TIME_FORMAT'
data['electricity_x'] = [
formats.date_format(
timezone.localtime(x.read_at), graph_x_format_electricity
)
for x in electricity
]
data['electricity_y'] = [float(x.currently_delivered * 1000) for x in electricity]
data['electricity_returned_y'] = [float(x.currently_returned * 1000) for x in electricity]
data['gas_x'] = [
formats.date_format(
timezone.localtime(x.read_at), graph_x_format_gas
) for x in gas
]
data['gas_y'] = [float(x.currently_delivered) for x in gas]
# Some users have multiple phases installed.
if DataloggerSettings.get_solo().track_phases and data['capabilities']['multi_phases']:
data['phases_l1_y'] = self._parse_phases_data(electricity, 'phase_currently_delivered_l1')
data['phases_l2_y'] = self._parse_phases_data(electricity, 'phase_currently_delivered_l2')
data['phases_l3_y'] = self._parse_phases_data(electricity, 'phase_currently_delivered_l3')
if WeatherSettings.get_solo().track:
data['temperature_x'] = [
formats.date_format(
timezone.localtime(x.read_at), 'DSMR_GRAPH_SHORT_TIME_FORMAT'
)
for x in temperature
]
data['temperature_y'] = [float(x.degrees_celcius) for x in temperature]
return HttpResponse(json.dumps(data), content_type='application/json')
def setUp(self):
DataloggerSettings.get_solo()
DataloggerSettings.objects.all().update(dsmr_version=DataloggerSettings.DSMR_BELGIUM_FLUVIUS)
def setUp(self):
self.instance = DataloggerSettings().get_solo()
def setUp(self):
DataloggerSettings.get_solo()
DataloggerSettings.objects.all().update(
dsmr_version=DataloggerSettings.DSMR_LUXEMBOURG_SMARTY)
def test_track_meter_statistics(self):
datalogger_settings = DataloggerSettings.get_solo()
datalogger_settings.track_meter_statistics = False
datalogger_settings.save()
fake_telegram = ''.join([
"/XMX5LGBBFFB123456789\r\n",
"\r\n",
"1-3:0.2.8(40)\r\n",
"0-0:1.0.0(151110192959W)\r\n",
"0-0:96.1.1(xxxxxxxxxxxxx)\r\n",
"1-0:1.8.1(000510.747*kWh)\r\n",
"1-0:2.8.1(000000.123*kWh)\r\n",
"1-0:1.8.2(000500.013*kWh)\r\n",
"1-0:2.8.2(000123.456*kWh)\r\n",
"0-0:96.14.0(0001)\r\n",
"1-0:1.7.0(00.192*kW)\r\n",
"1-0:2.7.0(00.123*kW)\r\n",
"0-0:17.0.0(999.9*kW)\r\n",
"0-0:96.3.10(1)\r\n",
"0-0:96.7.21(00003)\r\n",
"0-0:96.7.9(00000)\r\n",
"1-0:99.97.0(0)(0-0:96.7.19)\r\n",
"1-0:32.32.0(00002)\r\n",
"1-0:52.32.0(00002)\r\n",
"1-0:72.32.0(00000)\r\n",
"1-0:32.36.0(00000)\r\n",
"1-0:52.36.0(00000)\r\n",
"1-0:72.36.0(00000)\r\n",
"0-0:96.13.1()\r\n",
"0-0:96.13.0()\r\n",
"1-0:31.7.0(000*A)\r\n",
"1-0:51.7.0(000*A)\r\n",
"1-0:71.7.0(001*A)\r\n",
"1-0:21.7.0(00.000*kW)\r\n",
"1-0:41.7.0(00.000*kW)\r\n",
"1-0:61.7.0(00.192*kW)\r\n",
"1-0:22.7.0(00.000*kW)\r\n",
"1-0:42.7.0(00.000*kW)\r\n",
"1-0:62.7.0(00.000*kW)\r\n",
"0-1:24.1.0(003)\r\n",
"0-1:96.1.0(xxxxxxxxxxxxx)\r\n",
"0-1:24.2.1(151110190000W)(00845.206*m3)\r\n",
"0-1:24.4.0(1)\r\n",
"!D19A\n",
])
self.assertIsNone(MeterStatistics.get_solo().electricity_tariff) # Empty model in DB.
dsmr_datalogger.services.telegram_to_reading(data=fake_telegram)
self.assertIsNone(MeterStatistics.get_solo().electricity_tariff) # Unaffected
# Try again, but now with tracking settings enabled.
datalogger_settings = DataloggerSettings.get_solo()
datalogger_settings.track_meter_statistics = True
datalogger_settings.save()
self.assertIsNone(MeterStatistics.get_solo().electricity_tariff) # Empty model in DB.
dsmr_datalogger.services.telegram_to_reading(data=fake_telegram)
# Should be populated now.
meter_statistics = MeterStatistics.get_solo()
self.assertIsNotNone(meter_statistics.electricity_tariff)
self.assertEqual(meter_statistics.electricity_tariff, 1)
self.assertEqual(meter_statistics.power_failure_count, 3)
self.assertEqual(meter_statistics.voltage_sag_count_l1, 2)
self.assertEqual(meter_statistics.voltage_sag_count_l2, 2)
def telegram_to_reading(data):
"""
Converts a P1 telegram to a DSMR reading, which will be stored in database.
"""
def _get_reading_fields():
reading_fields = DsmrReading._meta.get_all_field_names()
reading_fields.remove('id')
reading_fields.remove('processed')
return reading_fields
def _get_statistics_fields():
reading_fields = MeterStatistics._meta.get_all_field_names()
reading_fields.remove('id')
return reading_fields
def _convert_legacy_dsmr_gas_line(parsed_reading, current_line, next_line):
""" Legacy support for DSMR 2.x gas. """
legacy_gas_line = current_line
if next_line.startswith('('):
legacy_gas_line = current_line + next_line
legacy_gas_result = re.search(
r'[^(]+\((\d+)\)\(\d+\)\(\d+\)\(\d+\)\([0-9-.:]+\)\(m3\)\(([0-9.]+)\)',
legacy_gas_line
)
gas_timestamp = legacy_gas_result.group(1)
if timezone.now().dst() != timezone.timedelta(0):
gas_timestamp += 'S'
else:
gas_timestamp += 'W'
parsed_reading['extra_device_timestamp'] = reading_timestamp_to_datetime(
string=gas_timestamp
)
parsed_reading['extra_device_delivered'] = legacy_gas_result.group(2)
return parsed_reading
# Defaults all fields to NULL.
parsed_reading = {k: None for k in _get_reading_fields() + _get_statistics_fields()}
field_splitter = re.compile(r'([^(]+)\((.+)\)')
lines_read = data.split("\r\n")
for index, current_line in enumerate(lines_read):
result = field_splitter.search(current_line)
if not result:
continue
code = result.group(1)
# M-bus (0-n:24.1) cannot identify the type of device, see issue #92.
if code in ('0-2:24.2.1', '0-3:24.2.1', '0-4:24.2.1'):
code = '0-1:24.2.1'
# DSMR 2.x emits gas readings in different format.
if code == '0-1:24.3.0':
parsed_reading = _convert_legacy_dsmr_gas_line(
parsed_reading, current_line, lines_read[index + 1]
)
continue
try:
field = DSMR_MAPPING[code]
except KeyError:
continue
value = result.group(2)
# Drop units, as the database does not care for them.
value = value.replace('*kWh', '').replace('*kW', '').replace('*m3', '')
# Extra device parameters are placed on a single line, meh.
if code == "0-1:24.2.1":
timestamp_value, gas_usage = value.split(")(")
parsed_reading[field[0]] = reading_timestamp_to_datetime(string=timestamp_value)
parsed_reading[field[1]] = gas_usage
else:
if field == "timestamp":
value = reading_timestamp_to_datetime(string=value)
parsed_reading[field] = value
# Hack for DSMR 2.x legacy, which lacks timestamp info..
if parsed_reading['timestamp'] is None:
parsed_reading['timestamp'] = timezone.now()
# Now we need to split reading & statistics. So we split the dict here.
reading_kwargs = {k: parsed_reading[k] for k in _get_reading_fields()}
new_reading = DsmrReading.objects.create(**reading_kwargs)
# Optional feature.
if DataloggerSettings.get_solo().track_meter_statistics:
statistics_kwargs = {k: parsed_reading[k] for k in _get_statistics_fields()}
# There should already be one in database, created when migrating.
MeterStatistics.objects.all().update(**statistics_kwargs)
return new_reading
def _map_telegram_to_model(parsed_telegram, data):
""" Maps parsed telegram to the fields. """
READING_FIELDS = [
x.name for x in DsmrReading._meta.get_fields()
if x.name not in ('id', 'processed')
]
STATISTICS_FIELDS = [
x.name for x in MeterStatistics._meta.get_fields()
if x.name not in ('id', 'rejected_telegrams', 'latest_telegram')
]
model_fields = {k: None for k in READING_FIELDS + STATISTICS_FIELDS}
mapping = _get_dsmrreader_mapping(
DataloggerSettings.get_solo().dsmr_version)
for obis_ref, obis_data in parsed_telegram.items():
try:
# Skip any fields we're not storing in our system.
target_field = mapping[obis_ref]
except KeyError:
continue
if isinstance(target_field, dict):
model_fields[target_field['value']] = obis_data.value
model_fields[target_field['datetime']] = obis_data.datetime
else:
model_fields[target_field] = obis_data.value
# Defaults for telegrams with missing data.
model_fields['timestamp'] = model_fields['timestamp'] or timezone.now()
model_fields['electricity_delivered_2'] = model_fields[
'electricity_delivered_2'] or 0
model_fields[
'electricity_returned_2'] = model_fields['electricity_returned_2'] or 0
# For some reason, there are telegrams generated with a timestamp in the far future. We should disallow that.
discard_timestamp = timezone.now() + timezone.timedelta(hours=24)
if model_fields['timestamp'] > discard_timestamp or (
model_fields['extra_device_timestamp'] is not None
and model_fields['extra_device_timestamp'] > discard_timestamp):
error_message = 'Discarded telegram with future timestamp(s): {} / {}'.format(
model_fields['timestamp'], model_fields['extra_device_timestamp'])
django_logger.error(error_message)
raise InvalidTelegramError(error_message)
# Now we need to split reading & statistics. So we split the dict here.
reading_kwargs = {k: model_fields[k] for k in READING_FIELDS}
statistics_kwargs = {k: model_fields[k] for k in STATISTICS_FIELDS}
# Reading will be processed later.
new_instance = DsmrReading.objects.create(**reading_kwargs)
# There should already be one in database, created when migrating.
statistics_kwargs['latest_telegram'] = data
MeterStatistics.objects.all().update(**statistics_kwargs)
# Broadcast this telegram as signal.
dsmr_datalogger.signals.raw_telegram.send_robust(sender=None, data=data)
return new_instance
def initialize(self):
self.sleep_time = DataloggerSettings.get_solo().process_sleep
