def get_dropbox_client(scheduled_process: ScheduledProcess) -> dropbox.Dropbox:
"""
The Dropbox refresh token we locally store (after linking the user's Dropbox account) never expires. This method
returns a new and authenticated instance of the Dropbox client that can be used for its (short) duration.
"""
dropbox_settings = DropboxSettings.get_solo()
dbx = dropbox.Dropbox(oauth2_refresh_token=dropbox_settings.refresh_token,
app_key=settings.DSMRREADER_DROPBOX_APP_KEY)
try:
dbx.refresh_access_token()
dbx.check_user() # No-op, just to verify the client/session.
except Exception as error:
logger.error(' - Dropbox error: %s', error)
# Network errors should NOT reset the client side app token (see further below). Only API errors should do so.
if not isinstance(error, dropbox.exceptions.DropboxException):
scheduled_process.delay(minutes=1)
raise
logger.error(' - Removing Dropbox credentials due to API failure')
message = _(
"Unable to authenticate with Dropbox, removing credentials. Error: {}"
.format(error))
dsmr_frontend.services.display_dashboard_message(message=message)
DropboxSettings.objects.update(refresh_token=None,
) # Does not trigger auto disable
scheduled_process.disable()
raise
logger.info('Dropbox: Auth/user check OK')
return dbx
def run(scheduled_process: ScheduledProcess) -> None:
""" Analyzes daily consumption and statistics to determine whether new analysis is required. """
if not is_data_available():
logger.debug('Stats: No data available')
scheduled_process.delay(hours=1)
return
now = timezone.localtime(timezone.now())
target_day = get_next_day_to_generate()
next_day = target_day + timezone.timedelta(days=1)
# Skip current day, wait until midnight.
if target_day >= now.date():
logger.debug('Stats: Waiting for day to pass: %s', target_day)
scheduled_process.reschedule(
timezone.make_aware(timezone.datetime.combine(next_day, time.min)))
return
# All readings of the day must be processed.
unprocessed_readings = DsmrReading.objects.unprocessed().filter(
timestamp__date=target_day).exists()
if unprocessed_readings:
logger.debug('Stats: Found unprocessed readings for: %s', target_day)
scheduled_process.delay(minutes=5)
return
# Ensure we have any consumption.
consumption_found = ElectricityConsumption.objects.filter(
read_at__date=target_day).exists()
if not consumption_found:
logger.debug('Stats: Missing consumption data for: %s', target_day)
scheduled_process.delay(hours=1)
return
# If we recently supported gas, make sure we've received a gas reading on the next day (or later).
recently_gas_read = GasConsumption.objects.filter(
read_at__date__gte=target_day - timezone.timedelta(days=1)).exists()
# Unless it was disabled.
gas_capability = dsmr_backend.services.backend.get_capability(
Capability.GAS)
if gas_capability and recently_gas_read and not GasConsumption.objects.filter(
read_at__date__gte=next_day).exists():
logger.debug('Stats: Waiting for first gas reading on the next day...')
scheduled_process.delay(minutes=5)
return
create_statistics(target_day=target_day)
# We keep trying until we've caught on to the current day (which will then delay it for a day above).
scheduled_process.delay(seconds=1)
return
def run(scheduled_process: ScheduledProcess) -> None:
""" Compacts all unprocessed readings, capped by a max to prevent hanging backend. """
for current_reading in DsmrReading.objects.unprocessed(
)[0:settings.DSMRREADER_COMPACT_MAX]:
try:
compact(dsmr_reading=current_reading)
except CompactorNotReadyError:
# Try again in a while, since we can't do anything now anyway.
scheduled_process.delay(seconds=15)
return
scheduled_process.delay(seconds=1)
def sync_file(scheduled_process: ScheduledProcess,
dropbox_client: dropbox.Dropbox, local_root_dir: str,
abs_file_path: str) -> None:
# The path we use in our Dropbox app folder.
relative_file_path = abs_file_path.replace(local_root_dir, '')
try:
# Check whether the file is already at Dropbox, if so, check its hash.
dropbox_meta = dropbox_client.files_get_metadata(relative_file_path)
except dropbox.exceptions.ApiError as exception:
error_message = str(exception.error)
dropbox_meta = None
# Unexpected.
if 'not_found' not in error_message:
logger.error(' - Dropbox error: %s', error_message)
return
# Calculate local hash and compare with remote. Ignore if the remote file is exactly the same.
if dropbox_meta and calculate_content_hash(
abs_file_path) == dropbox_meta.content_hash:
logger.debug('Dropbox: Content hash is the same, skipping: %s',
relative_file_path)
return
try:
upload_chunked(dropbox_client=dropbox_client,
local_file_path=abs_file_path,
remote_file_path=relative_file_path)
except dropbox.exceptions.DropboxException as exception:
error_message = str(exception.error)
logger.error('Dropbox: %s', error_message)
if 'insufficient_space' in error_message:
message = _(
"Unable to upload files to Dropbox due to {}. Ignoring new files for the next {} hours..."
.format(error_message,
settings.DSMRREADER_DROPBOX_ERROR_INTERVAL))
dsmr_frontend.services.display_dashboard_message(message=message)
scheduled_process.delay(
hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL)
raise # pragma: no cover
def run(scheduled_process: ScheduledProcess) -> None:
""" Checks for new updates. If one is available, it's displayed on the Dashboard. """
try:
is_latest_version = dsmr_backend.services.backend.is_latest_version()
except Exception as error:
logger.error('Update checker: Error %s', error)
scheduled_process.delay(hours=1)
return
if not is_latest_version:
logger.debug('Update checker: Newer version of DSMR-reader available')
dsmr_frontend.services.display_dashboard_message(
message=_(
'There is a newer version of DSMR-reader available. See the changelog for more information.'
),
redirect_to='frontend:changelog-redirect'
)
scheduled_process.delay(days=7)
def run(scheduled_process: ScheduledProcess) -> None:
mindergas_settings = MinderGasSettings.get_solo()
# Only when enabled and token set.
if not mindergas_settings.auth_token:
mindergas_settings.update(export=False) # Should also disable SP.
return
# Nonsense when having no data.
if not dsmr_backend.services.backend.get_capability(Capability.GAS):
scheduled_process.delay(hours=1)
return
try:
export()
except Exception as error:
logger.exception(error)
scheduled_process.delay(hours=1)
dsmr_frontend.services.display_dashboard_message(message=_(
'Failed to export to MinderGas: {}'.format(error)
))
return
# Reschedule between 3 AM and 6 AM next day.
midnight = timezone.localtime(timezone.make_aware(
timezone.datetime.combine(timezone.now(), time.min)
))
next_midnight = midnight + timezone.timedelta(
hours=dsmr_backend.services.backend.hours_in_day(
day=timezone.now().date()
)
)
scheduled_process.reschedule(
next_midnight + timezone.timedelta(
hours=random.randint(3, 5),
minutes=random.randint(15, 59)
)
)
def run(scheduled_process: ScheduledProcess) -> None:
dropbox_settings = DropboxSettings.get_solo()
if not dropbox_settings.refresh_token:
# Should not happen, safe fallback
scheduled_process.disable()
return
dropbox_client = get_dropbox_client(scheduled_process=scheduled_process)
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Sync each file, recursively.
for current_file in list_files_in_dir(directory=backup_directory):
if not should_sync_file(current_file):
continue
sync_file(scheduled_process=scheduled_process,
dropbox_client=dropbox_client,
local_root_dir=backup_directory,
abs_file_path=current_file)
scheduled_process.delay(hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL)
def run(scheduled_process: ScheduledProcess) -> None:
retention_settings = RetentionSettings.get_solo()
if retention_settings.data_retention_in_hours == RetentionSettings.RETENTION_NONE:
scheduled_process.disable() # Changing the retention settings in the admin will re-activate it again.
return
# These models should be rotated with retention. Dict value is the datetime field used.
ITEM_COUNT_PER_HOUR = 2
MODELS_TO_CLEANUP = {
DsmrReading.objects.processed(): 'timestamp',
ElectricityConsumption.objects.all(): 'read_at',
GasConsumption.objects.all(): 'read_at',
}
retention_date = timezone.now() - timezone.timedelta(hours=retention_settings.data_retention_in_hours)
data_to_clean_up = False
# We need to force UTC here, to avoid AmbiguousTimeError's on DST changes.
timezone.activate(pytz.UTC)
for base_queryset, datetime_field in MODELS_TO_CLEANUP.items():
hours_to_cleanup = base_queryset.filter(
**{'{}__lt'.format(datetime_field): retention_date}
).annotate(
item_hour=TruncHour(datetime_field)
).values('item_hour').annotate(
item_count=Count('id')
).order_by().filter(
item_count__gt=ITEM_COUNT_PER_HOUR
).order_by('item_hour').values_list(
'item_hour', flat=True
)[:settings.DSMRREADER_RETENTION_MAX_CLEANUP_HOURS_PER_RUN]
hours_to_cleanup = list(hours_to_cleanup) # Force evaluation.
if not hours_to_cleanup:
continue
data_to_clean_up = True
for current_hour in hours_to_cleanup:
# Fetch all data per hour.
data_set = base_queryset.filter(
**{
'{}__gte'.format(datetime_field): current_hour,
'{}__lt'.format(datetime_field): current_hour + timezone.timedelta(hours=1),
}
)
# Extract the first/last item, so we can exclude it.
# NOTE: Want to alter this? Please update ITEM_COUNT_PER_HOUR above as well!
keeper_pks = [
data_set.order_by(datetime_field)[0].pk,
data_set.order_by('-{}'.format(datetime_field))[0].pk
]
# Now drop all others.
logger.debug('Retention: Cleaning up: %s (%s)', current_hour, data_set[0].__class__.__name__)
data_set.exclude(pk__in=keeper_pks).delete()
timezone.deactivate()
# Delay for a bit, as there is nothing to do.
if not data_to_clean_up:
scheduled_process.delay(hours=12)