def test_sync_insufficient_space(self, now_mock, _, __,
upload_chunked_mock):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2000, 1, 1))
# Crash the party, no more space available!
upload_chunked_mock.side_effect = dropbox.exceptions.ApiError(
12345,
"UploadError('path', UploadWriteFailed(reason=WriteError('insufficient_space', None), ...)",
'x', 'y')
Notification.objects.all().delete()
self.assertEqual(Notification.objects.count(), 0)
with self.assertRaises(dropbox.exceptions.ApiError):
dsmr_dropbox.services.sync_file(
dropbox_settings=DropboxSettings.get_solo(),
local_root_dir='/var/tmp/',
abs_file_path='/var/tmp/fake')
# Warning message should be created and next sync should be skipped ahead.
self.assertEqual(Notification.objects.count(), 1)
self.assertGreater(
DropboxSettings.get_solo().next_sync,
timezone.now() + timezone.timedelta(
hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL - 1))
def setUp(self):
DropboxSettings.get_solo()
DropboxSettings.objects.all().update(refresh_token='FAKE')
self.schedule_process = ScheduledProcess.objects.get(
module=settings.DSMRREADER_MODULE_DROPBOX_EXPORT)
self.schedule_process.update(active=True,
planned=timezone.make_aware(
timezone.datetime(2000, 1, 1)))
def test_reschedule_dropbox_sync(self, now_mock):
URL = reverse('admin:dsmr_backup_dropboxsettings_changelist')
now_mock.return_value = timezone.make_aware(timezone.datetime(2019, 1, 1)) # Lock time
DropboxSettings.get_solo()
DropboxSettings.objects.all().update(next_sync=timezone.now() + timezone.timedelta(hours=1))
self.assertFalse(DropboxSettings.objects.filter(next_sync__lte=timezone.now()).exists())
# Just posting should reset it.
response = self.client.post(URL)
self.assertEqual(response.status_code, 302)
self.assertTrue(DropboxSettings.objects.filter(next_sync__lte=timezone.now()).exists())
def test_sync(self, now_mock, upload_chunked_mock):
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
old_next_sync = timezone.now() - timezone.timedelta(weeks=1)
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.next_sync = old_next_sync
dropbox_settings.save()
self.assertFalse(upload_chunked_mock.called)
self.assertIsNotNone(DropboxSettings.get_solo().access_token)
dsmr_backup.services.dropbox.sync()
self.assertTrue(upload_chunked_mock.called)
self.assertNotEqual(DropboxSettings.get_solo().next_sync, old_next_sync)
def test_sync(self, now_mock, upload_chunked_mock):
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
old_latest_sync = timezone.now() - timezone.timedelta(weeks=1)
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.latest_sync = old_latest_sync
dropbox_settings.save()
self.assertFalse(upload_chunked_mock.called)
self.assertIsNotNone(DropboxSettings.get_solo().access_token)
dsmr_backup.services.dropbox.sync()
self.assertTrue(upload_chunked_mock.called)
self.assertNotEqual(DropboxSettings.get_solo().latest_sync, old_latest_sync)
def sync():
dropbox_settings = DropboxSettings.get_solo()
# Skip when either no token was entered.
if not dropbox_settings.access_token:
return
if dropbox_settings.next_sync and dropbox_settings.next_sync > timezone.now(
):
return
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Sync each file, recursively.
for current_file in list_files_in_dir(directory=backup_directory):
if not should_sync_file(current_file):
continue
sync_file(dropbox_settings=dropbox_settings,
local_root_dir=backup_directory,
abs_file_path=current_file)
# Try again in a while.
DropboxSettings.objects.update(
latest_sync=timezone.now(),
next_sync=timezone.now() +
timezone.timedelta(hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL))
def get_dropbox_client(scheduled_process: ScheduledProcess) -> dropbox.Dropbox:
"""
The Dropbox refresh token we locally store (after linking the user's Dropbox account) never expires. This method
returns a new and authenticated instance of the Dropbox client that can be used for its (short) duration.
"""
dropbox_settings = DropboxSettings.get_solo()
dbx = dropbox.Dropbox(oauth2_refresh_token=dropbox_settings.refresh_token,
app_key=settings.DSMRREADER_DROPBOX_APP_KEY)
try:
dbx.refresh_access_token()
dbx.check_user() # No-op, just to verify the client/session.
except Exception as error:
logger.error(' - Dropbox error: %s', error)
# Network errors should NOT reset the client side app token (see further below). Only API errors should do so.
if not isinstance(error, dropbox.exceptions.DropboxException):
scheduled_process.delay(minutes=1)
raise
logger.error(' - Removing Dropbox credentials due to API failure')
message = _(
"Unable to authenticate with Dropbox, removing credentials. Error: {}"
.format(error))
dsmr_frontend.services.display_dashboard_message(message=message)
DropboxSettings.objects.update(refresh_token=None,
) # Does not trigger auto disable
scheduled_process.disable()
raise
logger.info('Dropbox: Auth/user check OK')
return dbx
def sync():
dropbox_settings = DropboxSettings.get_solo()
# Skip when either no token was entered.
if not dropbox_settings.access_token:
return
if dropbox_settings.next_sync and dropbox_settings.next_sync > timezone.now(
):
return
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Just check for modified files since the last sync.
for (_, _, filenames) in os.walk(backup_directory):
for current_file in filenames:
current_file_path = os.path.join(backup_directory, current_file)
check_synced_file(file_path=current_file_path,
dropbox_settings=dropbox_settings)
# Try again in a while.
DropboxSettings.objects.update(
latest_sync=timezone.now(),
next_sync=timezone.now() +
timezone.timedelta(hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL))
def test_sync_insufficient_space(self, now_mock, upload_chunked_mock):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2000, 1, 1))
# Crash the party, no more space available!
upload_chunked_mock.side_effect = dropbox.exceptions.ApiError(
12345,
"UploadError('path', UploadWriteFailed(reason=WriteError('insufficient_space', None), ...)",
'x', 'y')
Notification.objects.all().delete()
self.assertEqual(Notification.objects.count(), 0)
with self.assertRaises(dropbox.exceptions.ApiError):
dsmr_backup.services.dropbox.sync()
# Warning message should be created and next sync should be skipped ahead.
self.assertEqual(Notification.objects.count(), 1)
self.assertGreater(
DropboxSettings.get_solo().latest_sync,
timezone.now() + timezone.timedelta(
hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL - 1))
# Test alternate path.
DropboxSettings.objects.update(latest_sync=timezone.now() -
timezone.timedelta(hours=24))
upload_chunked_mock.side_effect = dropbox.exceptions.ApiError(
12345,
"UploadError('path', UploadWriteFailed(reason=WriteError('other_error', None), ...)",
'x', 'y')
with self.assertRaises(dropbox.exceptions.ApiError):
dsmr_backup.services.dropbox.sync()
self.assertEqual(Notification.objects.count(), 1)
def upload_chunked(file_path):
""" Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """
dropbox_settings = DropboxSettings.get_solo()
file_name = os.path.split(file_path)[-1]
# From Dropbox docs.
retries = 3
client = DropboxClient(dropbox_settings.access_token)
size = os.stat(file_path).st_size
file_handle = open(file_path, 'rb')
uploader = client.get_chunked_uploader(file_handle, size)
while uploader.offset < size:
try:
uploader.upload_chunked(chunk_size=1 * 1024 * 1024)
except rest.ErrorResponse: # pragma: no cover
retries -= 1 # pragma: no cover
if retries == 0: # pragma: no cover
raise IOError("Failed to upload to dropbox") # pragma: no cover
# This will commit the file and persist it in Dropbox. Due to rotating backups we MUST override.
uploader.finish(file_name, overwrite=True)
def status_info():
""" Returns the status info of the application. """
capabilities = get_capabilities()
status = {
'capabilities': capabilities,
'electricity': get_electricity_status(capabilities),
'gas': get_gas_status(capabilities),
'readings': get_reading_status(),
'statistics': get_statistics_status(),
'tools': {
'backup': {
'enabled': False,
'latest_backup': None,
},
'dropbox': {
'enabled': False,
'latest_sync': None,
},
'pvoutput': {
'enabled': False,
'latest_sync': None,
},
'mindergas': {
'enabled': False,
'latest_sync': None,
},
'mqtt': get_mqtt_status(),
}
}
# (External) tools below.
backup_settings = BackupSettings.get_solo()
if backup_settings.daily_backup:
status['tools']['backup']['enabled'] = True
status['tools']['backup'][
'latest_backup'] = backup_settings.latest_backup
dropbox_settings = DropboxSettings.get_solo()
if dropbox_settings.access_token:
status['tools']['dropbox']['enabled'] = True
status['tools']['dropbox'][
'latest_sync'] = dropbox_settings.latest_sync
pvoutput_settings = PVOutputAddStatusSettings.get_solo()
if pvoutput_settings.export:
status['tools']['pvoutput']['enabled'] = True
status['tools']['pvoutput'][
'latest_sync'] = pvoutput_settings.latest_sync
mindergas_settings = MinderGasSettings.get_solo()
if mindergas_settings.export:
status['tools']['mindergas']['enabled'] = True
status['tools']['mindergas'][
'latest_sync'] = mindergas_settings.latest_sync
return status
def test_sync(self, now_mock, _, should_mock, sync_file_mock):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2016, 1, 1))
should_mock.side_effect = [
False, True, True, True, True, True, True, True, True, True, True
] # Both branches.
old_next_sync = timezone.now() - timezone.timedelta(weeks=1)
DropboxSettings.objects.all().update(next_sync=old_next_sync)
self.assertFalse(sync_file_mock.called)
self.assertIsNotNone(DropboxSettings.get_solo().access_token)
dsmr_dropbox.services.sync()
self.assertTrue(sync_file_mock.called)
self.assertNotEqual(DropboxSettings.get_solo().next_sync,
old_next_sync)
def test_sync_disabled(self, upload_chunked_mock):
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.access_token = None
dropbox_settings.save()
self.assertFalse(upload_chunked_mock.called)
dsmr_backup.services.dropbox.sync()
self.assertFalse(upload_chunked_mock.called)
def test_sync_disabled(self, upload_chunked_mock):
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.access_token = None
dropbox_settings.save()
self.assertFalse(upload_chunked_mock.called)
dsmr_backup.services.dropbox.sync()
self.assertFalse(upload_chunked_mock.called)
def test_sync_initial(self, now_mock, upload_chunked_mock):
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
# Initial project state.
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.next_sync = None
dropbox_settings.save()
dsmr_backup.services.dropbox.sync()
self.assertTrue(upload_chunked_mock.called)
def get_context_data(self, **kwargs):
context_data = super(Configuration, self).get_context_data(**kwargs)
context_data['api_settings'] = APISettings.get_solo()
context_data['consumption_settings'] = ConsumptionSettings.get_solo()
context_data['datalogger_settings'] = DataloggerSettings.get_solo()
context_data['frontend_settings'] = FrontendSettings.get_solo()
context_data['weather_settings'] = WeatherSettings.get_solo()
context_data['backup_settings'] = BackupSettings.get_solo()
context_data['dropbox_settings'] = DropboxSettings.get_solo()
context_data['mindergas_settings'] = MinderGasSettings.get_solo()
return context_data
def test_sync_latest_sync(self, now_mock, get_backup_directory_mock):
""" Test whether syncs are limited to intervals. """
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.latest_sync = timezone.now() + timezone.timedelta(minutes=1)
dropbox_settings.save()
self.assertFalse(get_backup_directory_mock.called)
dsmr_backup.services.dropbox.sync()
self.assertFalse(get_backup_directory_mock.called)
def test_sync_latest_sync(self, now_mock, get_backup_directory_mock):
""" Test whether syncs are limited to intervals. """
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.next_sync = timezone.now() + timezone.timedelta(minutes=1)
dropbox_settings.save()
self.assertFalse(get_backup_directory_mock.called)
dsmr_backup.services.dropbox.sync()
self.assertFalse(get_backup_directory_mock.called)
def test_sync_invalid_access_token(self, now_mock, _, __,
upload_chunked_mock):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2000, 1, 1))
# Crash the party, no more space available!
upload_chunked_mock.side_effect = dropbox.exceptions.AuthError(
12345, "AuthError('xxx', AuthError('invalid_access_token', None))")
Notification.objects.all().delete()
self.assertEqual(Notification.objects.count(), 0)
with self.assertRaises(dropbox.exceptions.AuthError):
dsmr_dropbox.services.sync_file(
dropbox_settings=DropboxSettings.get_solo(),
local_root_dir='/var/tmp/',
abs_file_path='/var/tmp/fake')
# Warning message should be created and credentials should be wiped.
dropbox_settings = DropboxSettings.get_solo()
self.assertEqual(Notification.objects.count(), 1)
self.assertIsNone(dropbox_settings.access_token)
self.assertEqual(dropbox_settings.latest_sync, timezone.now())
self.assertIsNone(dropbox_settings.next_sync)
def upload_chunked(file_path):
""" Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """
# For backend logging in Supervisor.
logger.info(' - Uploading file to Dropbox: %s', file_path)
dropbox_settings = DropboxSettings.get_solo()
file_name = os.path.split(file_path)[-1]
dest_path = '/{}'.format(
file_name
) # The slash indicates it's relative to the root of app folder.
dbx = dropbox.Dropbox(dropbox_settings.access_token)
write_mode = dropbox.files.WriteMode.overwrite
file_handle = open(file_path, 'rb')
file_size = os.path.getsize(file_path)
# Many thanks to https://stackoverflow.com/documentation/dropbox-api/409/uploading-a-file/1927/uploading-a-file-usin
# g-the-dropbox-python-sdk#t=201610181733061624381
CHUNK_SIZE = 2 * 1024 * 1024
# Small uploads should be transfers at one go.
if file_size <= CHUNK_SIZE:
dbx.files_upload(file_handle.read(), dest_path, mode=write_mode)
# Large uploads can be sent in chunks, by creating a session allowing multiple separate uploads.
else:
upload_session_start_result = dbx.files_upload_session_start(
file_handle.read(CHUNK_SIZE))
cursor = dropbox.files.UploadSessionCursor(
session_id=upload_session_start_result.session_id,
offset=file_handle.tell())
commit = dropbox.files.CommitInfo(path=dest_path, mode=write_mode)
# We keep sending the data in chunks, until we reach the last one, then we instruct Dropbox to finish the upload
# by combining all the chunks sent previously.
while file_handle.tell() < file_size:
if (file_size - file_handle.tell()) <= CHUNK_SIZE:
dbx.files_upload_session_finish(file_handle.read(CHUNK_SIZE),
cursor, commit)
else:
dbx.files_upload_session_append(file_handle.read(CHUNK_SIZE),
cursor.session_id,
cursor.offset)
cursor.offset = file_handle.tell()
file_handle.close()
def test_sync(self, now_mock, _, should_mock, sync_file_mock,
list_files_in_dir_mock, *mocks):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2016, 1, 1))
should_mock.side_effect = [False, True] # Both branches.
list_files_in_dir_mock.return_value = ['/tmp/fake1', '/tmp/fake2']
self.assertFalse(sync_file_mock.called)
self.assertIsNotNone(DropboxSettings.get_solo().refresh_token)
dsmr_dropbox.services.run(self.schedule_process)
self.assertTrue(sync_file_mock.called)
self.schedule_process.refresh_from_db()
self.assertEqual(
self.schedule_process.planned,
timezone.make_aware(timezone.datetime(2016, 1, 1, hour=1)))
def check_dropbox_sync(**kwargs):
from dsmr_backup.models.settings import DropboxSettings
dropbox_settings = DropboxSettings.get_solo()
if not dropbox_settings.access_token:
return
offset = timezone.now() - timezone.timedelta(
minutes=settings.
DSMRREADER_STATUS_ALLOWED_SCHEDULED_PROCESS_LAGG_IN_MINUTES)
if dropbox_settings.next_sync > offset:
return
return MonitoringStatusIssue(
__name__, _('Waiting for the next Dropbox sync to be executed'),
dropbox_settings.next_sync)
def get_context_data(self, **kwargs):
context_data = super(Configuration, self).get_context_data(**kwargs)
# 20+ queries, we should cache this at some point.
context_data.update(
dict(
api_settings=APISettings.get_solo(),
backend_settings=BackendSettings.get_solo(),
backup_settings=BackupSettings.get_solo(),
consumption_settings=ConsumptionSettings.get_solo(),
datalogger_settings=DataloggerSettings.get_solo(),
dropbox_settings=DropboxSettings.get_solo(),
email_settings=EmailSettings.get_solo(),
frontend_settings=FrontendSettings.get_solo(),
mindergas_settings=MinderGasSettings.get_solo(),
mqtt_broker_settings=MQTTBrokerSettings.get_solo(),
mqtt_jsondaytotals_settings=JSONDayTotalsMQTTSettings.get_solo(
),
mqtt_splittopicdaytotals_settings=
SplitTopicDayTotalsMQTTSettings.get_solo(),
mqtt_jsoncurrentperiodtotals_settings=
JSONCurrentPeriodTotalsMQTTSettings.get_solo(),
mqtt_splittopiccurrentperiodtotals_settings=
SplitTopicCurrentPeriodTotalsMQTTSettings.get_solo(),
mqtt_jsongasconsumption_settings=JSONGasConsumptionMQTTSettings
.get_solo(),
mqtt_splittopicgasconsumption_settings=
SplitTopicGasConsumptionMQTTSettings.get_solo(),
mqtt_splittopicmeterstatistics_settings=
SplitTopicMeterStatisticsMQTTSettings.get_solo(),
mqtt_jsontelegram_settings=JSONTelegramMQTTSettings.get_solo(),
mqtt_rawtelegram_settings=RawTelegramMQTTSettings.get_solo(),
mqtt_splittopictelegram_settings=SplitTopicTelegramMQTTSettings
.get_solo(),
notification_settings=NotificationSetting.get_solo(),
pvoutput_api_settings=PVOutputAPISettings.get_solo(),
pvoutput_addstatus_settings=PVOutputAddStatusSettings.get_solo(
),
retention_settings=RetentionSettings.get_solo(),
weather_settings=WeatherSettings.get_solo(),
influxdb_settings=InfluxdbIntegrationSettings.get_solo(),
))
return context_data
def get_redirect_url(self, *args, **kwargs):
dropbox_settings = DropboxSettings.get_solo()
auth_flow = dropbox.DropboxOAuth2FlowNoRedirect( # noqa: S106
settings.DSMRREADER_DROPBOX_APP_KEY,
use_pkce=True,
token_access_type='offline',
timeout=settings.DSMRREADER_CLIENT_TIMEOUT)
authorize_url = auth_flow.start()
# We need the data generated by the SDK (auth challenge) on oauth completion. So we'll serialize it.
serialized_auth_flow = pickle.dumps(auth_flow)
dropbox_settings.update(
serialized_auth_flow=serialized_auth_flow,
# Reset all other fields as well.
refresh_token=None,
one_time_authorization_code=None)
return authorize_url
def test_sync_last_modified(self, now_mock, upload_chunked_mock, get_backup_directory_mock):
""" Test whether syncs are skipped when file was not modified. """
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.latest_sync = timezone.now() - timezone.timedelta(weeks=1)
dropbox_settings.save()
with tempfile.TemporaryDirectory() as temp_dir:
get_backup_directory_mock.return_value = temp_dir
temp_file = tempfile.NamedTemporaryFile(dir=temp_dir, delete=False)
temp_file.write(b'Meh.')
temp_file.flush()
# 1420070400: 01 Jan 2015 00:00:00 GMT
os.utime(temp_file.name, times=(1420070400, 1420070400))
self.assertFalse(upload_chunked_mock.called)
# File should be ignored, as its modification timestamp is before latest sync.
dsmr_backup.services.dropbox.sync()
self.assertFalse(upload_chunked_mock.called)
def test_sync_last_modified(self, now_mock, upload_chunked_mock, get_backup_directory_mock):
""" Test whether syncs are skipped when file was not modified. """
now_mock.return_value = timezone.make_aware(timezone.datetime(2016, 1, 1))
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.latest_sync = timezone.now() - timezone.timedelta(weeks=1)
dropbox_settings.save()
with tempfile.TemporaryDirectory() as temp_dir:
get_backup_directory_mock.return_value = temp_dir
temp_file = tempfile.NamedTemporaryFile(dir=temp_dir, delete=False)
temp_file.write(b'Meh.')
temp_file.flush()
# 1420070400: 01 Jan 2015 00:00:00 GMT
os.utime(temp_file.name, times=(1420070400, 1420070400))
self.assertFalse(upload_chunked_mock.called)
# File should be ignored, as it's modification timestamp is before latest sync.
dsmr_backup.services.dropbox.sync()
self.assertFalse(upload_chunked_mock.called)
def test_check_access_token(self, now_mock, users_get_space_usage_mock):
now_mock.return_value = timezone.make_aware(
timezone.datetime(2020, 1, 1))
for current_class in (dropbox.exceptions.BadInputError,
dropbox.exceptions.AuthError):
DropboxSettings.objects.all().update(next_sync=timezone.now(),
access_token='invalid-token')
users_get_space_usage_mock.side_effect = current_class(
12345, "Some error")
Notification.objects.all().delete()
self.assertEqual(Notification.objects.count(), 0)
with self.assertRaises(current_class):
dsmr_dropbox.services.check_access_token('dummy')
# Warning message should be created and next sync should be skipped ahead.
self.assertEqual(Notification.objects.count(), 1)
self.assertIsNone(DropboxSettings.get_solo().next_sync)
def sync():
dropbox_settings = DropboxSettings.get_solo()
# Skip when either no token was entered.
if not dropbox_settings.access_token:
return
# Or when we already synced within the last hour.
next_sync_interval = None
if dropbox_settings.latest_sync:
next_sync_interval = dropbox_settings.latest_sync + timezone.timedelta(
hours=settings.DSMR_DROPBOX_SYNC_INTERVAL
)
if next_sync_interval and timezone.now() < next_sync_interval:
return
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Just check for modified files since the last sync.
for (_, _, filenames) in os.walk(backup_directory):
for current_file in filenames:
current_file_path = os.path.join(backup_directory, current_file)
file_stats = os.stat(current_file_path)
# Ignore empty files.
if file_stats.st_size == 0:
continue
last_modified = timezone.datetime.fromtimestamp(file_stats.st_mtime)
last_modified = timezone.make_aware(last_modified)
# Ignore when file was not altered since last sync.
if dropbox_settings.latest_sync and last_modified < dropbox_settings.latest_sync:
continue
upload_chunked(file_path=current_file_path)
DropboxSettings.objects.update(latest_sync=timezone.now())
def sync():
dropbox_settings = DropboxSettings.get_solo()
# Skip when either no token was entered.
if not dropbox_settings.access_token:
return
# Or when we already synced within the last hour.
next_sync_interval = None
if dropbox_settings.latest_sync:
next_sync_interval = dropbox_settings.latest_sync + timezone.timedelta(
hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL)
if next_sync_interval and timezone.now() < next_sync_interval:
return
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Just check for modified files since the last sync.
for (_, _, filenames) in os.walk(backup_directory):
for current_file in filenames:
current_file_path = os.path.join(backup_directory, current_file)
file_stats = os.stat(current_file_path)
# Ignore empty files.
if file_stats.st_size == 0:
continue
last_modified = timezone.datetime.fromtimestamp(
file_stats.st_mtime)
last_modified = timezone.make_aware(last_modified)
# Ignore when file was not altered since last sync.
if dropbox_settings.latest_sync and last_modified < dropbox_settings.latest_sync:
continue
upload_chunked(file_path=current_file_path)
DropboxSettings.objects.update(latest_sync=timezone.now())
def test_upload_chunked(self, session_finish_mock, session_append_mock,
session_start_mock, files_upload_mock):
dropbox_settings = DropboxSettings.get_solo()
DATA = b'Lots of data.'
session_start_result = mock.MagicMock()
type(session_start_result).session_id = mock.PropertyMock(
side_effect=['session-xxxxx'])
session_start_mock.return_value = session_start_result
self.assertFalse(files_upload_mock.called)
self.assertFalse(session_start_mock.called)
self.assertFalse(session_append_mock.called)
self.assertFalse(session_finish_mock.called)
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(DATA)
temp_file.flush()
dsmr_dropbox.services.upload_chunked(dropbox_settings,
temp_file.name,
'/remote-path.ext')
# Only small file upload should be called.
self.assertTrue(files_upload_mock.called)
self.assertFalse(session_start_mock.called)
self.assertFalse(session_append_mock.called)
self.assertFalse(session_finish_mock.called)
# Large file upload (> 2 MB chunks).
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(DATA * 2 * 1024 * 1024)
temp_file.flush()
dsmr_dropbox.services.upload_chunked(dropbox_settings,
temp_file.name,
'/remote-path.ext')
self.assertTrue(session_start_mock.called)
self.assertTrue(session_append_mock.called)
self.assertTrue(session_finish_mock.called)
def run(scheduled_process: ScheduledProcess) -> None:
dropbox_settings = DropboxSettings.get_solo()
if not dropbox_settings.refresh_token:
# Should not happen, safe fallback
scheduled_process.disable()
return
dropbox_client = get_dropbox_client(scheduled_process=scheduled_process)
backup_directory = dsmr_backup.services.backup.get_backup_directory()
# Sync each file, recursively.
for current_file in list_files_in_dir(directory=backup_directory):
if not should_sync_file(current_file):
continue
sync_file(scheduled_process=scheduled_process,
dropbox_client=dropbox_client,
local_root_dir=backup_directory,
abs_file_path=current_file)
scheduled_process.delay(hours=settings.DSMRREADER_DROPBOX_SYNC_INTERVAL)
def test_check_dropbox_sync_disabled(self):
DropboxSettings.get_solo().update(
access_token=None,
next_sync=timezone.now() - timezone.timedelta(minutes=1)
)
self.assertIsNone(check_dropbox_sync())
def setUp(self):
DropboxSettings.get_solo().update(
access_token='1234',
next_sync=timezone.now()
)
def setUp(self):
DropboxSettings.get_solo()
DropboxSettings.objects.all().update(access_token='FAKE')
def setUp(self):
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.access_token = 'FAKE'
dropbox_settings.save()
def setUp(self):
dropbox_settings = DropboxSettings.get_solo()
dropbox_settings.access_token = 'FAKE'
dropbox_settings.save()
