def setUp(self): self.assertEqual(DsmrReading.objects.all().count(), 7) self.assertTrue(DsmrReading.objects.unprocessed().exists()) ConsumptionSettings.get_solo() MeterStatistics.get_solo() MeterStatistics.objects.all().update(dsmr_version='50') self.schedule_process = ScheduledProcess.objects.get(module=settings.DSMRREADER_MODULE_GENERATE_CONSUMPTION) self.schedule_process.update(active=True, planned=timezone.make_aware(timezone.datetime(2000, 1, 1)))
def setUp(self): self.support_gas_readings = True self.assertEqual(DsmrReading.objects.all().count(), 3) if self.support_gas_readings: self.assertTrue(DsmrReading.objects.unprocessed().exists()) else: self.assertFalse(DsmrReading.objects.unprocessed().exists()) ConsumptionSettings.get_solo()
def setUp(self): self.support_gas_readings = True self.assertEqual(DsmrReading.objects.all().count(), 3) if self.support_gas_readings: self.assertTrue(DsmrReading.objects.unprocessed().exists()) else: self.assertFalse(DsmrReading.objects.unprocessed().exists()) ConsumptionSettings.get_solo()
def setUp(self): self.support_gas_readings = True self.assertEqual(DsmrReading.objects.all().count(), 3) MeterStatistics.get_solo() MeterStatistics.objects.all().update(dsmr_version='42') if self.support_gas_readings: self.assertTrue(DsmrReading.objects.unprocessed().exists()) else: self.assertFalse(DsmrReading.objects.unprocessed().exists()) ConsumptionSettings.get_solo()
def test_processing(self): """ Test fixed data parse outcome. """ # Default is grouping by minute, so make sure to revert that here. consumption_settings = ConsumptionSettings.get_solo() consumption_settings.compactor_grouping_type = ConsumptionSettings.COMPACTOR_GROUPING_BY_READING consumption_settings.save() self.assertFalse( ElectricityConsumption.objects.filter( phase_currently_delivered_l2__isnull=False, phase_currently_delivered_l3__isnull=False).exists()) dsmr_consumption.services.compact_all() self.assertTrue(DsmrReading.objects.processed().exists()) self.assertFalse(DsmrReading.objects.unprocessed().exists()) self.assertEqual(ElectricityConsumption.objects.count(), 3) if self.support_gas_readings: self.assertEqual(GasConsumption.objects.count(), 2) else: self.assertEqual(GasConsumption.objects.count(), 0) self.assertTrue( ElectricityConsumption.objects.filter( phase_currently_delivered_l2__isnull=False, phase_currently_delivered_l3__isnull=False).exists())
def compact(dsmr_reading): """ Compacts/converts DSMR readings to consumption data. Optionally groups electricity by minute. """ grouping_type = ConsumptionSettings.get_solo().compactor_grouping_type # Grouping by minute requires some distinction and history checking. reading_start = timezone.datetime.combine( dsmr_reading.timestamp.date(), time(hour=dsmr_reading.timestamp.hour, minute=dsmr_reading.timestamp.minute), ).replace(tzinfo=pytz.UTC) if grouping_type == ConsumptionSettings.COMPACTOR_GROUPING_BY_MINUTE: system_time_past_minute = timezone.now() >= reading_start + timezone.timedelta(minutes=1) reading_past_minute_exists = DsmrReading.objects.filter( timestamp__gte=reading_start + timezone.timedelta(minutes=1) ).exists() # Postpone until the minute has passed on the system time. And when there are (new) readings beyond this minute. if not system_time_past_minute or not reading_past_minute_exists: return # Create consumption records. _compact_electricity(dsmr_reading=dsmr_reading, grouping_type=grouping_type, reading_start=reading_start) _compact_gas(dsmr_reading=dsmr_reading, grouping_type=grouping_type, reading_start=reading_start) dsmr_reading.processed = True dsmr_reading.save(update_fields=['processed']) # For backend logging in Supervisor. logger.debug(' - Processed reading: %s', dsmr_reading)
def compact(dsmr_reading): """ Compacts/converts DSMR readings to consumption data. Optionally groups electricity by minute. """ grouping_type = ConsumptionSettings.get_solo().compactor_grouping_type # Grouping by minute requires some distinction and history checking. reading_start = timezone.datetime.combine( dsmr_reading.timestamp.date(), time(hour=dsmr_reading.timestamp.hour, minute=dsmr_reading.timestamp.minute), ).replace(tzinfo=pytz.UTC) if grouping_type == ConsumptionSettings.COMPACTOR_GROUPING_BY_MINUTE: # Postpone when current minute hasn't passed yet. if timezone.now() <= reading_start + timezone.timedelta(minutes=1): return # Create consumption records. _compact_electricity(dsmr_reading=dsmr_reading, grouping_type=grouping_type, reading_start=reading_start) _compact_gas(dsmr_reading=dsmr_reading, grouping_type=grouping_type, reading_start=reading_start) dsmr_reading.processed = True dsmr_reading.save(update_fields=['processed']) # For backend logging in Supervisor. print(' - Processed reading: {}'.format(dsmr_reading))
def get_context_data(self, **kwargs): context_data = super(Configuration, self).get_context_data(**kwargs) context_data['api_settings'] = APISettings.get_solo() context_data['consumption_settings'] = ConsumptionSettings.get_solo() context_data['datalogger_settings'] = DataloggerSettings.get_solo() context_data['frontend_settings'] = FrontendSettings.get_solo() context_data['weather_settings'] = WeatherSettings.get_solo() context_data['backup_settings'] = BackupSettings.get_solo() context_data['dropbox_settings'] = DropboxSettings.get_solo() context_data['mindergas_settings'] = MinderGasSettings.get_solo() return context_data
def test_processing(self): """ Test fixed data parse outcome. """ # Default is grouping by minute, so make sure to revert that here. consumption_settings = ConsumptionSettings.get_solo() consumption_settings.compactor_grouping_type = ConsumptionSettings.COMPACTOR_GROUPING_BY_READING consumption_settings.save() dsmr_consumption.services.compact_all() self.assertTrue(DsmrReading.objects.processed().exists()) self.assertFalse(DsmrReading.objects.unprocessed().exists()) self.assertEqual(ElectricityConsumption.objects.count(), 3) if self.support_gas_readings: self.assertEqual(GasConsumption.objects.count(), 2) else: self.assertEqual(GasConsumption.objects.count(), 0)
def get_context_data(self, **kwargs): context_data = super(Configuration, self).get_context_data(**kwargs) # 20+ queries, we should cache this at some point. context_data.update( dict( api_settings=APISettings.get_solo(), backend_settings=BackendSettings.get_solo(), backup_settings=BackupSettings.get_solo(), consumption_settings=ConsumptionSettings.get_solo(), datalogger_settings=DataloggerSettings.get_solo(), dropbox_settings=DropboxSettings.get_solo(), email_settings=EmailSettings.get_solo(), frontend_settings=FrontendSettings.get_solo(), mindergas_settings=MinderGasSettings.get_solo(), mqtt_broker_settings=MQTTBrokerSettings.get_solo(), mqtt_jsondaytotals_settings=JSONDayTotalsMQTTSettings.get_solo( ), mqtt_splittopicdaytotals_settings= SplitTopicDayTotalsMQTTSettings.get_solo(), mqtt_jsoncurrentperiodtotals_settings= JSONCurrentPeriodTotalsMQTTSettings.get_solo(), mqtt_splittopiccurrentperiodtotals_settings= SplitTopicCurrentPeriodTotalsMQTTSettings.get_solo(), mqtt_jsongasconsumption_settings=JSONGasConsumptionMQTTSettings .get_solo(), mqtt_splittopicgasconsumption_settings= SplitTopicGasConsumptionMQTTSettings.get_solo(), mqtt_splittopicmeterstatistics_settings= SplitTopicMeterStatisticsMQTTSettings.get_solo(), mqtt_jsontelegram_settings=JSONTelegramMQTTSettings.get_solo(), mqtt_rawtelegram_settings=RawTelegramMQTTSettings.get_solo(), mqtt_splittopictelegram_settings=SplitTopicTelegramMQTTSettings .get_solo(), notification_settings=NotificationSetting.get_solo(), pvoutput_api_settings=PVOutputAPISettings.get_solo(), pvoutput_addstatus_settings=PVOutputAddStatusSettings.get_solo( ), retention_settings=RetentionSettings.get_solo(), weather_settings=WeatherSettings.get_solo(), influxdb_settings=InfluxdbIntegrationSettings.get_solo(), )) return context_data
def test_processing(self): """ Test fixed data parse outcome. """ # Default is grouping by minute, so make sure to revert that here. consumption_settings = ConsumptionSettings.get_solo() consumption_settings.electricity_grouping_type = ConsumptionSettings.ELECTRICITY_GROUPING_BY_READING consumption_settings.save() self.assertFalse( ElectricityConsumption.objects.filter( phase_currently_delivered_l2__isnull=False, phase_currently_delivered_l3__isnull=False ).exists() ) dsmr_consumption.services.run(self.schedule_process) self.assertTrue(DsmrReading.objects.processed().exists()) self.assertFalse(DsmrReading.objects.unprocessed().exists()) self.assertEqual(ElectricityConsumption.objects.count(), 3) if self.support_gas_readings: self.assertEqual(GasConsumption.objects.count(), 2) self.assertEqual( [x.read_at for x in GasConsumption.objects.all()], [ # Asume a one hour backtrack. timezone.make_aware(timezone.datetime(2015, 11, 10, hour=18), timezone.utc), timezone.make_aware(timezone.datetime(2015, 11, 10, hour=19), timezone.utc) ] ) else: self.assertEqual(GasConsumption.objects.count(), 0) self.assertTrue( ElectricityConsumption.objects.filter( phase_currently_delivered_l2__isnull=False, phase_currently_delivered_l3__isnull=False ).exists() )
def test_duplicate_processing(self): """ Duplicate readings should not crash the compactor when not grouping. """ # Default is grouping by minute, so make sure to revert that here. consumption_settings = ConsumptionSettings.get_solo() consumption_settings.compactor_grouping_type = ConsumptionSettings.COMPACTOR_GROUPING_BY_READING consumption_settings.save() # Just duplicate one, as it will cause: IntegrityError UNIQUE constraint failed: ElectricityConsumption.read_at duplicate_reading = DsmrReading.objects.all()[0] duplicate_reading.pk = None duplicate_reading.save() dsmr_consumption.services.compact_all() self.assertTrue(DsmrReading.objects.processed().exists()) self.assertFalse(DsmrReading.objects.unprocessed().exists()) self.assertEqual(ElectricityConsumption.objects.count(), 3) if self.support_gas_readings: self.assertEqual(GasConsumption.objects.count(), 2) else: self.assertEqual(GasConsumption.objects.count(), 0)
def compact(dsmr_reading: DsmrReading) -> None: """ Compacts/converts DSMR readings to consumption data. Optionally groups electricity by minute. """ consumption_settings = ConsumptionSettings.get_solo() # Grouping by minute requires some distinction and history checking. reading_start = timezone.datetime.combine( dsmr_reading.timestamp.date(), time(hour=dsmr_reading.timestamp.hour, minute=dsmr_reading.timestamp.minute), ).replace(tzinfo=pytz.UTC) if consumption_settings.electricity_grouping_type == ConsumptionSettings.ELECTRICITY_GROUPING_BY_MINUTE: system_time_past_minute = timezone.now( ) >= reading_start + timezone.timedelta(minutes=1) reading_past_minute_exists = DsmrReading.objects.filter( timestamp__gte=reading_start + timezone.timedelta(minutes=1)).exists() # Postpone until the minute has passed on the system time. And when there are (new) readings beyond this minute. if not system_time_past_minute or not reading_past_minute_exists: logger.debug( 'Compact: Waiting for newer readings before grouping data...') raise CompactorNotReadyError() # Create consumption records. _compact_electricity(dsmr_reading=dsmr_reading, electricity_grouping_type=consumption_settings. electricity_grouping_type, reading_start=reading_start) _compact_gas(dsmr_reading=dsmr_reading, gas_grouping_type=consumption_settings.gas_grouping_type) dsmr_reading.processed = True dsmr_reading.save(update_fields=['processed']) logger.debug('Compact: Processed reading: %s', dsmr_reading)
def setUp(self): self.assertEqual(DsmrReading.objects.all().count(), 6) self.assertTrue(DsmrReading.objects.unprocessed().exists()) ConsumptionSettings.get_solo() MeterStatistics.get_solo() MeterStatistics.objects.all().update(dsmr_version='50')
def compact(dsmr_reading): """ Compacts/converts DSMR readings to consumption data. Optionally groups electricity by minute. """ grouping_type = ConsumptionSettings.get_solo().compactor_grouping_type # Electricity should be unique, because it's the reading with the lowest interval anyway. if grouping_type == ConsumptionSettings.COMPACTOR_GROUPING_BY_READING: ElectricityConsumption.objects.get_or_create( read_at=dsmr_reading.timestamp, delivered_1=dsmr_reading.electricity_delivered_1, returned_1=dsmr_reading.electricity_returned_1, delivered_2=dsmr_reading.electricity_delivered_2, returned_2=dsmr_reading.electricity_returned_2, currently_delivered=dsmr_reading.electricity_currently_delivered, currently_returned=dsmr_reading.electricity_currently_returned, phase_currently_delivered_l1=dsmr_reading. phase_currently_delivered_l1, phase_currently_delivered_l2=dsmr_reading. phase_currently_delivered_l2, phase_currently_delivered_l3=dsmr_reading. phase_currently_delivered_l3, ) # Grouping by minute requires some distinction and history checking. else: minute_start = timezone.datetime.combine( dsmr_reading.timestamp.date(), time(hour=dsmr_reading.timestamp.hour, minute=dsmr_reading.timestamp.minute), ).replace(tzinfo=pytz.UTC) minute_end = minute_start + timezone.timedelta(minutes=1) # Postpone when current minute hasn't passed yet. if timezone.now() <= minute_end: return # We might have six readings per minute, so there is a chance we already parsed it. if not ElectricityConsumption.objects.filter( read_at=minute_end).exists(): grouped_reading = DsmrReading.objects.filter( timestamp__gte=minute_start, timestamp__lt=minute_end).aggregate( avg_delivered=Avg('electricity_currently_delivered'), avg_returned=Avg('electricity_currently_returned'), max_delivered_1=Max('electricity_delivered_1'), max_delivered_2=Max('electricity_delivered_2'), max_returned_1=Max('electricity_returned_1'), max_returned_2=Max('electricity_returned_2'), avg_phase_delivered_l1=Avg('phase_currently_delivered_l1'), avg_phase_delivered_l2=Avg('phase_currently_delivered_l2'), avg_phase_delivered_l3=Avg('phase_currently_delivered_l3'), ) # This instance is the average/max and combined result. ElectricityConsumption.objects.create( read_at=minute_end, delivered_1=grouped_reading['max_delivered_1'], returned_1=grouped_reading['max_returned_1'], delivered_2=grouped_reading['max_delivered_2'], returned_2=grouped_reading['max_returned_2'], currently_delivered=grouped_reading['avg_delivered'], currently_returned=grouped_reading['avg_returned'], phase_currently_delivered_l1=grouped_reading[ 'avg_phase_delivered_l1'], phase_currently_delivered_l2=grouped_reading[ 'avg_phase_delivered_l2'], phase_currently_delivered_l3=grouped_reading[ 'avg_phase_delivered_l3'], ) # Gas is optional. if dsmr_reading.extra_device_timestamp and dsmr_reading.extra_device_delivered: # Gas however is only read (or updated) once every hour, so we should check for any duplicates # as they will exist at some point. passed_hour_start = dsmr_reading.extra_device_timestamp - timezone.timedelta( hours=1) if not GasConsumption.objects.filter( read_at=passed_hour_start).exists(): # DSMR does not expose current gas rate, so we have to calculate # it ourselves, relative to the previous gas consumption, if any. try: previous_gas_consumption = GasConsumption.objects.get( # Compare to reading before, if any. read_at=passed_hour_start - timezone.timedelta(hours=1)) except GasConsumption.DoesNotExist: gas_diff = 0 else: gas_diff = dsmr_reading.extra_device_delivered - previous_gas_consumption.delivered GasConsumption.objects.create( # Gas consumption is aligned to start of the hour. read_at=passed_hour_start, delivered=dsmr_reading.extra_device_delivered, currently_delivered=gas_diff) dsmr_reading.processed = True dsmr_reading.save(update_fields=['processed']) # For backend logging in Supervisor. print(' - Processed reading: {}.'.format( timezone.localtime(dsmr_reading.timestamp)))
def setUp(self): self.instance = ConsumptionSettings().get_solo()
def compact(dsmr_reading): """ Compacts/converts DSMR readings to consumption data. Optionally groups electricity by minute. """ grouping_type = ConsumptionSettings.get_solo().compactor_grouping_type # Electricity should be unique, because it's the reading with the lowest interval anyway. if grouping_type == ConsumptionSettings.COMPACTOR_GROUPING_BY_READING: ElectricityConsumption.objects.create( read_at=dsmr_reading.timestamp, delivered_1=dsmr_reading.electricity_delivered_1, returned_1=dsmr_reading.electricity_returned_1, delivered_2=dsmr_reading.electricity_delivered_2, returned_2=dsmr_reading.electricity_returned_2, currently_delivered=dsmr_reading.electricity_currently_delivered, currently_returned=dsmr_reading.electricity_currently_returned, ) # Grouping by minute requires some distinction and history checking. else: minute_start = timezone.datetime.combine( dsmr_reading.timestamp.date(), time(hour=dsmr_reading.timestamp.hour, minute=dsmr_reading.timestamp.minute), ).replace(tzinfo=pytz.UTC) minute_end = minute_start + timezone.timedelta(minutes=1) # Postpone when current minute hasn't passed yet. if timezone.now() <= minute_end: return # We might have six readings per minute, so there is a chance we already parsed it. if not ElectricityConsumption.objects.filter(read_at=minute_end).exists(): grouped_reading = DsmrReading.objects.filter( timestamp__gte=minute_start, timestamp__lt=minute_end ).aggregate( avg_delivered=Avg('electricity_currently_delivered'), avg_returned=Avg('electricity_currently_returned'), max_delivered_1=Max('electricity_delivered_1'), max_delivered_2=Max('electricity_delivered_2'), max_returned_1=Max('electricity_returned_1'), max_returned_2=Max('electricity_returned_2') ) # This instance is the average/max and combined result. ElectricityConsumption.objects.create( read_at=minute_end, delivered_1=grouped_reading['max_delivered_1'], returned_1=grouped_reading['max_returned_1'], delivered_2=grouped_reading['max_delivered_2'], returned_2=grouped_reading['max_returned_2'], currently_delivered=grouped_reading['avg_delivered'], currently_returned=grouped_reading['avg_returned'], ) # Gas is optional. if dsmr_reading.extra_device_timestamp and dsmr_reading.extra_device_delivered: # Gas however is only read (or updated) once every hour, so we should check for any duplicates # as they will exist at some point. passed_hour_start = dsmr_reading.extra_device_timestamp - timezone.timedelta(hours=1) if not GasConsumption.objects.filter(read_at=passed_hour_start).exists(): # DSMR does not expose current gas rate, so we have to calculate # it ourselves, relative to the previous gas consumption, if any. try: previous_gas_consumption = GasConsumption.objects.get( # Compare to reading before, if any. read_at=passed_hour_start - timezone.timedelta(hours=1) ) except GasConsumption.DoesNotExist: gas_diff = 0 else: gas_diff = dsmr_reading.extra_device_delivered - previous_gas_consumption.delivered GasConsumption.objects.create( # Gas consumption is aligned to start of the hour. read_at=passed_hour_start, delivered=dsmr_reading.extra_device_delivered, currently_delivered=gas_diff ) dsmr_reading.processed = True dsmr_reading.save(update_fields=['processed'])