def run(scheduled_process: ScheduledProcess) -> None:
""" Checks whether a new backup should be created. Creates one if needed as well. """
# Create a partial, minimal backup first. Since it will grow and take disk space, only create one weekly.
today = timezone.localtime(timezone.now()).date()
if today.isoweekday() == 1:
create_partial(folder=os.path.join(get_backup_directory(), 'archive',
formats.date_format(today, 'Y'),
formats.date_format(today, 'm')),
models_to_backup=(DayStatistics, ))
# Now create full.
create_full(folder=get_backup_directory())
# Schedule tomorrow, for the time specified.
backup_settings = BackupSettings.get_solo()
next_backup_timestamp = timezone.now() + timezone.timedelta(days=1)
next_backup_timestamp = timezone.localtime(next_backup_timestamp)
next_backup_timestamp = next_backup_timestamp.replace(
hour=backup_settings.backup_time.hour,
minute=backup_settings.backup_time.minute,
second=0,
microsecond=0)
scheduled_process.reschedule(next_backup_timestamp)
def get_dropbox_client(scheduled_process: ScheduledProcess) -> dropbox.Dropbox:
"""
The Dropbox refresh token we locally store (after linking the user's Dropbox account) never expires. This method
returns a new and authenticated instance of the Dropbox client that can be used for its (short) duration.
"""
dropbox_settings = DropboxSettings.get_solo()
dbx = dropbox.Dropbox(oauth2_refresh_token=dropbox_settings.refresh_token,
app_key=settings.DSMRREADER_DROPBOX_APP_KEY)
try:
dbx.refresh_access_token()
dbx.check_user() # No-op, just to verify the client/session.
except Exception as error:
logger.error(' - Dropbox error: %s', error)
# Network errors should NOT reset the client side app token (see further below). Only API errors should do so.
if not isinstance(error, dropbox.exceptions.DropboxException):
scheduled_process.delay(minutes=1)
raise
logger.error(' - Removing Dropbox credentials due to API failure')
message = _(
"Unable to authenticate with Dropbox, removing credentials. Error: {}"
.format(error))
dsmr_frontend.services.display_dashboard_message(message=message)
DropboxSettings.objects.update(refresh_token=None,
) # Does not trigger auto disable
scheduled_process.disable()
raise
logger.info('Dropbox: Auth/user check OK')
return dbx
def run(scheduled_process: ScheduledProcess) -> None:
""" Compacts all unprocessed readings, capped by a max to prevent hanging backend. """
for current_reading in DsmrReading.objects.unprocessed(
)[0:settings.DSMRREADER_COMPACT_MAX]:
try:
compact(dsmr_reading=current_reading)
except CompactorNotReadyError:
# Try again in a while, since we can't do anything now anyway.
scheduled_process.delay(seconds=15)
return
scheduled_process.delay(seconds=1)
def sync_file(scheduled_process: ScheduledProcess,
dropbox_client: dropbox.Dropbox, local_root_dir: str,
abs_file_path: str) -> None:
# The path we use in our Dropbox app folder.
relative_file_path = abs_file_path.replace(local_root_dir, '')
try:
# Check whether the file is already at Dropbox, if so, check its hash.
dropbox_meta = dropbox_client.files_get_metadata(relative_file_path)
except dropbox.exceptions.ApiError as exception:
error_message = str(exception.error)
dropbox_meta = None
# Unexpected.
if 'not_found' not in error_message:
logger.error(' - Dropbox error: %s', error_message)
return
# Calculate local hash and compare with remote. Ignore if the remote file is exactly the same.
if dropbox_meta and calculate_content_hash(
abs_file_path) == dropbox_meta.content_hash:
logger.debug('Dropbox: Content hash is the same, skipping: %s',
relative_file_path)
return
try:
upload_chunked(dropbox_client=dropbox_client,
local_file_path=abs_file_path,
remote_file_path=relative_file_path)
except dropbox.exceptions.DropboxException as exception:
error_message = str(exception.error)
logger.error('Dropbox: %s', error_message)
if 'insufficient_space' in error_message:
message = _(
"Unable to upload files to Dropbox due to {}. Ignoring new files for the next {} hours..."
.format(error_message,
settings.DSMRREADER_DROPBOX_ERROR_INTERVAL))
dsmr_frontend.services.display_dashboard_message(message=message)
scheduled_process.delay(
hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL)
raise # pragma: no cover
def run(scheduled_process: ScheduledProcess) -> None:
""" Checks for new updates. If one is available, it's displayed on the Dashboard. """
try:
is_latest_version = dsmr_backend.services.backend.is_latest_version()
except Exception as error:
logger.error('Update checker: Error %s', error)
scheduled_process.delay(hours=1)
return
if not is_latest_version:
logger.debug('Update checker: Newer version of DSMR-reader available')
dsmr_frontend.services.display_dashboard_message(
message=_(
'There is a newer version of DSMR-reader available. See the changelog for more information.'
),
redirect_to='frontend:changelog-redirect'
)
scheduled_process.delay(days=7)
def run(scheduled_process: ScheduledProcess) -> None:
dropbox_settings = DropboxSettings.get_solo()
if not dropbox_settings.refresh_token:
# Should not happen, safe fallback
scheduled_process.disable()
return
dropbox_client = get_dropbox_client(scheduled_process=scheduled_process)
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Sync each file, recursively.
for current_file in list_files_in_dir(directory=backup_directory):
if not should_sync_file(current_file):
continue
sync_file(scheduled_process=scheduled_process,
dropbox_client=dropbox_client,
local_root_dir=backup_directory,
abs_file_path=current_file)
scheduled_process.delay(hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL)
def run(scheduled_process: ScheduledProcess) -> None:
mindergas_settings = MinderGasSettings.get_solo()
# Only when enabled and token set.
if not mindergas_settings.auth_token:
mindergas_settings.update(export=False) # Should also disable SP.
return
# Nonsense when having no data.
if not dsmr_backend.services.backend.get_capability(Capability.GAS):
scheduled_process.delay(hours=1)
return
try:
export()
except Exception as error:
logger.exception(error)
scheduled_process.delay(hours=1)
dsmr_frontend.services.display_dashboard_message(message=_(
'Failed to export to MinderGas: {}'.format(error)
))
return
# Reschedule between 3 AM and 6 AM next day.
midnight = timezone.localtime(timezone.make_aware(
timezone.datetime.combine(timezone.now(), time.min)
))
next_midnight = midnight + timezone.timedelta(
hours=dsmr_backend.services.backend.hours_in_day(
day=timezone.now().date()
)
)
scheduled_process.reschedule(
next_midnight + timezone.timedelta(
hours=random.randint(3, 5),
minutes=random.randint(15, 59)
)
)
def run(scheduled_process: ScheduledProcess) -> None:
retention_settings = RetentionSettings.get_solo()
if retention_settings.data_retention_in_hours == RetentionSettings.RETENTION_NONE:
scheduled_process.disable() # Changing the retention settings in the admin will re-activate it again.
return
# These models should be rotated with retention. Dict value is the datetime field used.
ITEM_COUNT_PER_HOUR = 2
MODELS_TO_CLEANUP = {
DsmrReading.objects.processed(): 'timestamp',
ElectricityConsumption.objects.all(): 'read_at',
GasConsumption.objects.all(): 'read_at',
}
retention_date = timezone.now() - timezone.timedelta(hours=retention_settings.data_retention_in_hours)
data_to_clean_up = False
# We need to force UTC here, to avoid AmbiguousTimeError's on DST changes.
timezone.activate(pytz.UTC)
for base_queryset, datetime_field in MODELS_TO_CLEANUP.items():
hours_to_cleanup = base_queryset.filter(
**{'{}__lt'.format(datetime_field): retention_date}
).annotate(
item_hour=TruncHour(datetime_field)
).values('item_hour').annotate(
item_count=Count('id')
).order_by().filter(
item_count__gt=ITEM_COUNT_PER_HOUR
).order_by('item_hour').values_list(
'item_hour', flat=True
)[:settings.DSMRREADER_RETENTION_MAX_CLEANUP_HOURS_PER_RUN]
hours_to_cleanup = list(hours_to_cleanup) # Force evaluation.
if not hours_to_cleanup:
continue
data_to_clean_up = True
for current_hour in hours_to_cleanup:
# Fetch all data per hour.
data_set = base_queryset.filter(
**{
'{}__gte'.format(datetime_field): current_hour,
'{}__lt'.format(datetime_field): current_hour + timezone.timedelta(hours=1),
}
)
# Extract the first/last item, so we can exclude it.
# NOTE: Want to alter this? Please update ITEM_COUNT_PER_HOUR above as well!
keeper_pks = [
data_set.order_by(datetime_field)[0].pk,
data_set.order_by('-{}'.format(datetime_field))[0].pk
]
# Now drop all others.
logger.debug('Retention: Cleaning up: %s (%s)', current_hour, data_set[0].__class__.__name__)
data_set.exclude(pk__in=keeper_pks).delete()
timezone.deactivate()
# Delay for a bit, as there is nothing to do.
if not data_to_clean_up:
scheduled_process.delay(hours=12)
def run_quarter_hour_peaks(scheduled_process: ScheduledProcess) -> None:
""" Calculates the quarter-hour peak consumption. For background info see issue #1084 ."""
MINUTE_INTERVAL = 15
# Just start with whatever time this process was scheduled.
# As it's incremental and will fix data gaps (see further below).
fuzzy_start = scheduled_process.planned.replace(second=0, microsecond=0)
# The fuzzy start should be just beyond whatever we target. E.g. fuzzy start = currently 14:34
logger.debug('Quarter hour peaks: Using %s as fuzzy start',
timezone.localtime(fuzzy_start))
# Rewind at least 15 minutes. E.g. currently 14:34 -> 14:19 (rewind_minutes = 15)
rewind_minutes = MINUTE_INTERVAL
# Map to xx:00, xx:15, xx:30 or xx:45. E.g. 14:19 -> 14:15. Makes 19 % 15 = 4 (rewind_minutes = 15 + 4)
rewind_minutes += (fuzzy_start - timezone.timedelta(minutes=rewind_minutes)
).minute % MINUTE_INTERVAL
# E.g. Fuzzy start was 14:34. Now we start/end at 14:15/14:30.
start = fuzzy_start - timezone.timedelta(minutes=rewind_minutes)
end = start + timezone.timedelta(minutes=MINUTE_INTERVAL)
# Do NOT continue until we've received new readings AFTER the targeted end. Ensuring we do not miss any and it also
# blocks the "self-healing" implementation when having data gaps.
# Only happens for data gaps or directly after new installations (edge cases). This will keep pushing forward.
if not DsmrReading.objects.filter(timestamp__gte=end).exists():
logger.debug(
'Quarter hour peaks: Ready but awaiting any new readings after %s, postponing for a bit...',
timezone.localtime(end),
)
# Assumes new readings will arrive shortly (for most users/setups)
scheduled_process.postpone(seconds=5)
return
quarter_hour_readings = DsmrReading.objects.filter(timestamp__gte=start,
timestamp__lte=end)
# Only happens for data gaps or directly after new installations (edge cases). This will keep pushing forward.
if len(quarter_hour_readings) < 2:
logger.warning(
'Quarter hour peaks: Ready but not enough readings found between %s - %s, skipping quarter...',
timezone.localtime(start),
timezone.localtime(end),
)
scheduled_process.postpone(minutes=MINUTE_INTERVAL)
return
first_reading = quarter_hour_readings.first()
last_reading = quarter_hour_readings.last()
logger.debug(
'Quarter hour peaks: Quarter %s - %s resulted in readings %s - %s',
timezone.localtime(start),
timezone.localtime(end),
timezone.localtime(first_reading.timestamp),
timezone.localtime(last_reading.timestamp),
)
# Do not create duplicate data.
existing_data = QuarterHourPeakElectricityConsumption.objects.filter(
read_at_start__gte=start,
read_at_start__lte=end,
).exists()
if existing_data:
logger.debug(
'Quarter hour peaks: Ready but quarter already processed, rescheduling for next quarter...'
)
scheduled_process.reschedule(
planned_at=end + timezone.timedelta(minutes=MINUTE_INTERVAL))
return
# Calculate quarter data.
total_delivered_start = first_reading.electricity_delivered_1 + first_reading.electricity_delivered_2
total_delivered_end = last_reading.electricity_delivered_1 + last_reading.electricity_delivered_2
avg_delivered_in_quarter = total_delivered_end - total_delivered_start
logger.debug(
'Quarter hour peaks: Calculating for %s - %s',
timezone.localtime(first_reading.timestamp),
timezone.localtime(last_reading.timestamp),
)
new_instance = QuarterHourPeakElectricityConsumption.objects.create(
# Using the reading timestamps used to ensure we can indicate gaps or lag in reading input.
# E.g. due backend/datalogger process sleep or simply v4 meters emitting a reading only once per 10 seconds.
read_at_start=first_reading.timestamp,
read_at_end=last_reading.timestamp,
# avg_delivered_in_quarter = kW QUARTER peak during 15 minutes... x 4 maps it to avg per hour for kW HOUR peak
average_delivered=avg_delivered_in_quarter * 4)
logger.debug('Quarter hour peaks: Created %s', new_instance)
# Reschedule around the next moment we can expect to process the next quarter. Also works retroactively/with gaps.
scheduled_process.reschedule(planned_at=new_instance.read_at_end +
timezone.timedelta(minutes=MINUTE_INTERVAL))
def run(scheduled_process: ScheduledProcess) -> None:
""" Analyzes daily consumption and statistics to determine whether new analysis is required. """
if not is_data_available():
logger.debug('Stats: No data available')
scheduled_process.delay(hours=1)
return
now = timezone.localtime(timezone.now())
target_day = get_next_day_to_generate()
next_day = target_day + timezone.timedelta(days=1)
# Skip current day, wait until midnight.
if target_day >= now.date():
logger.debug('Stats: Waiting for day to pass: %s', target_day)
scheduled_process.reschedule(
timezone.make_aware(timezone.datetime.combine(next_day, time.min)))
return
# All readings of the day must be processed.
unprocessed_readings = DsmrReading.objects.unprocessed().filter(
timestamp__date=target_day).exists()
if unprocessed_readings:
logger.debug('Stats: Found unprocessed readings for: %s', target_day)
scheduled_process.delay(minutes=5)
return
# Ensure we have any consumption.
consumption_found = ElectricityConsumption.objects.filter(
read_at__date=target_day).exists()
if not consumption_found:
logger.debug('Stats: Missing consumption data for: %s', target_day)
scheduled_process.delay(hours=1)
return
# If we recently supported gas, make sure we've received a gas reading on the next day (or later).
recently_gas_read = GasConsumption.objects.filter(
read_at__date__gte=target_day - timezone.timedelta(days=1)).exists()
# Unless it was disabled.
gas_capability = dsmr_backend.services.backend.get_capability(
Capability.GAS)
if gas_capability and recently_gas_read and not GasConsumption.objects.filter(
read_at__date__gte=next_day).exists():
logger.debug('Stats: Waiting for first gas reading on the next day...')
scheduled_process.delay(minutes=5)
return
create_statistics(target_day=target_day)
# We keep trying until we've caught on to the current day (which will then delay it for a day above).
scheduled_process.delay(seconds=1)
return