class CloudDropbox(Wrapper):
""" Wraps a Dropbox connection client.
"""
wrapper_type = 'Dropbox connection'
required_secret_attr = 'secret'
required_secret_label = 'an OAuth 2 access token'
def __init__(self, *args, **kwargs):
super(CloudDropbox, self).__init__(*args, **kwargs)
self._impl = None # type: DropboxClient
# ################################################################################################################################
def _init_impl(self):
with self.update_lock:
# Create a pool of at most that many connections
session = create_session(50)
scope = as_list(self.config.default_scope, ',')
config = {
'session': session,
'user_agent': self.config.user_agent,
'oauth2_access_token': self.server.decrypt(self.config.secret),
'oauth2_access_token_expiration': int(self.config.oauth2_access_token_expiration or 0),
'scope': scope,
'max_retries_on_error': int(self.config.max_retries_on_error or 0),
'max_retries_on_rate_limit': int(self.config.max_retries_on_rate_limit or 0),
'timeout': int(self.config.timeout),
'headers': parse_extra_into_dict(self.config.http_headers),
}
# Create the actual connection object
self._impl = DropboxClient(**config)
# Confirm the connection was established
self.ping()
# We can assume we are connected now
self.is_connected = True
# ################################################################################################################################
def _delete(self):
if self._impl:
self._impl.close()
# ################################################################################################################################
def _ping(self):
self._impl.check_user()
class Dropbox(BlackboxStorage):
"""Storage handler that uploads backups to Dropbox."""
required_fields = ("access_token", )
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.upload_base = self.config.get("upload_directory") or "/"
self.client = DropboxClient(self.config["access_token"])
self.valid = self._validate_token()
def _validate_token(self):
"""Check if dropbox token is valid."""
try:
return self.client.check_user("test").result == "test"
except AuthError:
return False
def sync(self, file_path: Path) -> None:
"""Sync a file to Dropbox."""
# Check if Dropbox token is valid.
if self.valid is False:
error = "Dropbox token is invalid!"
self.success = False
self.output = error
log.error(error)
return None
# This is size what can be uploaded as one chunk.
# When file is bigger than that, this will be uploaded
# in multiple parts.
chunk_size = 4 * 1024 * 1024
temp_file, recompressed = self.compress(file_path)
upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}"
try:
with temp_file as f:
file_size = os.stat(f.name).st_size
log.debug(file_size)
if file_size <= chunk_size:
self.client.files_upload(f.read(), upload_path,
WriteMode.overwrite)
else:
session_start = self.client.files_upload_session_start(
f.read(chunk_size))
cursor = UploadSessionCursor(session_start.session_id,
offset=f.tell())
# Commit contains path in Dropbox and write mode about file
commit = CommitInfo(upload_path, WriteMode.overwrite)
while f.tell() < file_size:
if (file_size - f.tell()) <= chunk_size:
self.client.files_upload_session_finish(
f.read(chunk_size), cursor, commit)
else:
self.client.files_upload_session_append(
f.read(chunk_size), cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
self.success = True
except (ApiError, HttpError) as e:
log.error(e)
self.success = False
self.output = str(e)
def rotate(self, database_id: str) -> None:
"""
Rotate the files in the Dropbox directory.
All files in base directory of backups will be deleted when they
are older than `retention_days`, and because of this,
it's better to have backups in isolated folder.
"""
# Check if Dropbox token is valid.
if self.valid is False:
log.error("Dropbox token is invalid - Can't delete old backups!")
return None
# Let's rotate only this type of database
db_type_regex = rf"{database_id}_blackbox_\d{{2}}_\d{{2}}_\d{{4}}.+"
# Receive first batch of files.
files_result = self.client.files_list_folder(
self.upload_base if self.upload_base != "/" else "")
entries = [
entry for entry in files_result.entries
if self._is_backup_file(entry, db_type_regex)
]
# If there is more files, receive all of them.
while files_result.has_more:
cursor = files_result.cursor
files_result = self.client.files_list_folder_continue(cursor)
entries += [
entry for entry in files_result.entries
if self._is_backup_file(entry, db_type_regex)
]
retention_days = 7
if Blackbox.retention_days:
retention_days = Blackbox.retention_days
# Find all old files and delete them.
for item in entries:
last_modified = item.server_modified
now = datetime.now(tz=last_modified.tzinfo)
delta = now - last_modified
if delta.days >= retention_days:
self.client.files_delete(item.path_lower)
@staticmethod
def _is_backup_file(entry, db_type_regex) -> bool:
"""Check if file is actually this kind of database backup."""
return isinstance(entry, FileMetadata) and re.match(
db_type_regex, entry.name)
