class DropboxWriter(FilebaseBaseWriter):
"""
Writes items to dropbox folder.
options available
- access_token (str)
Oauth access token for Dropbox api.
- filebase (str)
Base path to store the items in the share.
"""
supported_options = {
'access_token': {'type': six.string_types, 'env_fallback': 'EXPORTERS_DROPBOXWRITER_TOKEN'},
}
def __init__(self, *args, **kw):
from dropbox import Dropbox
super(DropboxWriter, self).__init__(*args, **kw)
access_token = self.read_option('access_token')
self.set_metadata('files_counter', Counter())
self.client = Dropbox(access_token)
def write(self, dump_path, group_key=None, file_name=False):
if group_key is None:
group_key = []
self._write_file(dump_path, group_key, file_name)
@retry_long
def _upload_file(self, input_file, filepath):
from dropbox import files
session_id = self.client.files_upload_session_start('')
current_offset = 0
while True:
data = input_file.read(2**20)
if not data:
break
self.client.files_upload_session_append(data, session_id.session_id, current_offset)
current_offset += len(data)
cursor = files.UploadSessionCursor(session_id.session_id, current_offset)
self.client.files_upload_session_finish(
'', cursor, files.CommitInfo(path='{}'.format(filepath)))
def _write_file(self, dump_path, group_key, file_name=None):
filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name)
with open(dump_path, 'r') as f:
self._upload_file(f, '{}/{}'.format(filebase_path, file_name))
self.get_metadata('files_counter')[filebase_path] += 1
def get_file_suffix(self, path, prefix):
number_of_keys = self.get_metadata('files_counter').get(path, 0)
suffix = '{}'.format(str(number_of_keys))
return suffix
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None):
dbx = Dropbox(access_token)
with open(file_path, 'rb') as file:
chunk = file.read(CHUNK_SIZE)
offset = len(chunk)
upload_session = dbx.files_upload_session_start(chunk)
progress_callback and progress_callback(offset)
while True:
chunk = file.read(CHUNK_SIZE)
if not chunk:
break
dbx.files_upload_session_append_v2(
chunk,
UploadSessionCursor(
upload_session.session_id,
offset,
),
)
offset += len(chunk)
progress_callback and progress_callback(offset)
file_metadata = dbx.files_upload_session_finish(
b'',
UploadSessionCursor(
upload_session.session_id,
offset=offset,
),
CommitInfo(
dropbox_path,
# When writing the file it won't overwrite an existing file, just add
# another file like "filename (2).txt"
WriteMode('add'),
),
)
progress_callback and progress_callback(offset)
return file_metadata.path_display
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException('dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
access_token, _ = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' % e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info("dpbx: Authentication failed. Trying to obtain new access token")
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception. So this line should not be reached
raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
progress.report_transfer(0, file_size)
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed chunk
while (f.tell() < file_size) or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if len(buf) != 0:
log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset)
else:
log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
# Server reports that we should send another chunk. Most likely this is caused by
# network error during previous upload attempt. In such case we'll get expected offset
# from server and it's enough to just seek() and retry again
new_offset = error.get_incorrect_offset().correct_offset
log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something strange and no safe way to recover
raise BackendException("dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if retry_number == 0:
raise
# We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size))
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata again.
# Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
res = []
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn('Warning! It looks like there are automatically renamed files on backend')
log.Warn('They were probably created when using older version of duplicity.')
log.Warn('')
log.Warn('Please check your backup consistency. Most likely you will need to choose')
log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.')
log.Warn('')
log.Warn('These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
class DropboxStorage(Storage):
"""Dropbox Storage class for Django pluggable storage system."""
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = oauth2_access_token or setting(
'DROPBOX_OAUTH2_TOKEN')
self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def _full_path(self, name):
if name == '/':
name = ''
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
metadata = self.client.files_get_metadata(full_path)
for entry in metadata['contents']:
entry['path'] = entry['path'].replace(full_path, '', 1)
entry['path'] = entry['path'].replace('/', '', 1)
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
def _open(self, name, mode='rb'):
remote_file = DropboxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
content.open()
if content.size <= self.CHUNK_SIZE:
self.client.files_upload(content.read(), self._full_path(name))
else:
self._chunked_upload(content, self._full_path(name))
content.close()
return name
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE))
cursor = UploadSessionCursor(session_id=upload_session.session_id,
offset=content.tell())
commit = CommitInfo(path=dest_path)
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor)
cursor.offset = content.tell()
def upload(dropbox_helper_id, access_token, size, max_retries):
from .models import DropboxUploadHelper
helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id)
dbx = Dropbox(access_token)
try:
with open(helper.src, 'rb') as f:
chunk = f.read(CHUNK_SIZE)
offset = len(chunk)
upload_session = dbx.files_upload_session_start(chunk)
while True:
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
helper.progress = offset / size
helper.save()
dbx.files_upload_session_append_v2(
chunk,
UploadSessionCursor(
upload_session.session_id,
offset,
),
)
offset += len(chunk)
file_metadata = dbx.files_upload_session_finish(
b'',
UploadSessionCursor(
upload_session.session_id,
offset=offset,
),
CommitInfo(
'/{}'.format(os.path.basename(helper.src)),
# When writing the file it won't overwrite an existing file, just add
# another file like "filename (2).txt"
WriteMode('add'),
),
)
except Exception as e:
helper.failure_reason = str(e)
helper.save()
couch_user = CouchUser.get_by_username(helper.user.username)
if helper.failure_reason is None:
path_link_metadata = dbx.sharing_create_shared_link_with_settings(
file_metadata.path_display,
SharedLinkSettings(
requested_visibility=RequestedVisibility.team_only,
),
)
context = {
'share_url': path_link_metadata.url,
'path': os.path.join(
u'Apps',
settings.DROPBOX_APP_NAME,
path_link_metadata.name,
)
}
with localize(couch_user.get_language_code()):
subject = _(u'{} has been uploaded to dropbox!'.format(helper.dest))
html_content = render_to_string('dropbox/emails/upload_success.html', context)
text_content = render_to_string('dropbox/emails/upload_success.txt', context)
else:
context = {
'reason': helper.failure_reason,
'path': helper.dest
}
with localize(couch_user.get_language_code()):
subject = _(u'{} has failed to upload to dropbox'.format(helper.dest))
html_content = render_to_string('dropbox/emails/upload_error.html', context)
text_content = render_to_string('dropbox/emails/upload_error.txt', context)
send_HTML_email(
subject,
helper.user.email,
html_content,
text_content=text_content,
)
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
location = setting('DROPBOX_ROOT_PATH', '/')
oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN')
timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE)
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout,
write_mode=write_mode):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure an auth token at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.root_path = root_path
self.write_mode = write_mode
self.client = Dropbox(oauth2_access_token, timeout=timeout)
def _full_path(self, name):
if name == '/':
name = ''
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
if full_path == '/':
full_path = ''
metadata = self.client.files_list_folder(full_path)
for entry in metadata.entries:
if isinstance(entry, FolderMetadata):
directories.append(entry.name)
else:
files.append(entry.name)
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.size
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.server_modified
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.client_modified
def url(self, name):
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
content.open()
if content.size <= self.CHUNK_SIZE:
self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode))
else:
self._chunked_upload(content, self._full_path(name))
content.close()
return name
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE)
)
cursor = UploadSessionCursor(
session_id=upload_session.session_id,
offset=content.tell()
)
commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode))
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit
)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor
)
cursor.offset = content.tell()
def get_available_name(self, name, max_length=None):
"""Overwrite existing file with the same name."""
name = self._full_path(name)
if self.write_mode == 'overwrite':
return get_available_overwrite_name(name, max_length)
return super(DropBoxStorage, self).get_available_name(name, max_length)
class DPBXBackend(duplicity.backend.Backend):
u"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
global Dropbox
global AuthError, BadInputError, ApiError
global UploadSessionCursor, CommitInfo
global WriteMode, GetMetadataError
global DeleteError, UploadSessionLookupError
global ListFolderError
global DropboxOAuth2FlowNoRedirect
try:
from dropbox import Dropbox
from dropbox.exceptions import AuthError, BadInputError, ApiError
from dropbox.files import (UploadSessionCursor, CommitInfo,
WriteMode, GetMetadataError,
DeleteError, UploadSessionLookupError,
ListFolderError)
from dropbox.oauth import DropboxOAuth2FlowNoRedirect
except ImportError as e:
raise BackendException(u"""\
This backend requires the dropbox package version 6.9.0
To install use "sudo pip install dropbox==6.9.0"
Exception: %s""" % str(e))
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def user_authenticated(self):
try:
account = self.api_client.users_get_current_account()
log.Debug(u"User authenticated as ,%s" % account)
return True
except:
log.Debug(u'User not authenticated')
return False
def load_access_token(self):
return os.environ.get(u'DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException(
u'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
access_token)
def obtain_access_token(self):
log.Info(u"dpbx: trying to obtain access token")
for env_var in [u'DPBX_APP_KEY', u'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException(
u'dpbx: %s environment variable not set' % env_var)
app_key = os.environ[u'DPBX_APP_KEY']
app_secret = os.environ[u'DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError(
u'dpbx error: cannot interact, but need human attention',
log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug(u'dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print()
print(u'-' * 72)
print(u"1. Go to: " + authorize_url)
print(u"2. Click \"Allow\" (you might have to log in first).")
print(u"3. Copy the authorization code.")
print(u'-' * 72)
auth_code = input(u"Enter the authorization code here: ").strip()
try:
log.Debug(u'dpbx,auth_flow.finish(%s)' % auth_code)
authresult = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException(u'dpbx: Unable to obtain access token: %s' %
e)
log.Info(u"dpbx: Authentication successfull")
self.save_access_token(authresult.access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug(u'dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug(u"dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug(u'dpbx,exception: %s' % e)
log.Info(
u"dpbx: Authentication failed. Trying to obtain new access token"
)
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception.
# So this line should not be reached
raise BackendException(
u"dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info(u"dpbx: Successfully authenticated as %s" %
self.api_account.name.display_name)
def _error_code(self, operation, e): # pylint: disable=unused-argument
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.parse.unquote(self.parsed_url.path.lstrip(u'/'))
remote_path = u'/' + os.path.join(remote_dir,
remote_filename.decode()).rstrip()
file_size = os.path.getsize(source_path.name)
progress.report_transfer(0, file_size)
if file_size < DPBX_UPLOAD_CHUNK_SIZE:
# Upload whole file at once to avoid extra server request
res_metadata = self.put_file_small(source_path, remote_path)
else:
res_metadata = self.put_file_chunked(source_path, remote_path)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException(
u'dpbx: result path mismatch: %s (expected: %s)' %
(res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException(
u'dpbx: result size mismatch: %s (expected: %s)' %
(res_metadata.size, file_size))
def put_file_small(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open(u'rb')
try:
log.Debug(u'dpbx,files_upload(%s, [%d bytes])' %
(remote_path, file_size))
res_metadata = self.api_client.files_upload(
f.read(),
remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
log.Debug(u'dpbx,files_upload(): %s' % res_metadata)
progress.report_transfer(file_size, file_size)
return res_metadata
finally:
f.close()
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open(u'rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug(
u'dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug(u'dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id,
f.tell())
commit_info = CommitInfo(remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = config.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = config.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug(
u'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(
buf, upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug(
u'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(
buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug(u'progress: %d of %d' %
(upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError
) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset(
).correct_offset
log.Debug(
u'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
% (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException(
u"dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug(u'dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info(u'dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException(u'dpbx: something wrong')
log.Debug(u'dpbx,files_upload_sesssion_finish(): %s' %
res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
if not self.user_authenticated():
self.login()
remote_dir = urllib.parse.unquote(self.parsed_url.path.lstrip(u'/'))
remote_path = u'/' + os.path.join(remote_dir,
remote_filename.decode()).rstrip()
log.Debug(u'dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug(u'dpbx,files_download(%s): %s, %s' %
(remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open(u'wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata
# again. Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException(
u"dpbx: wrong file size: %d (expected: %d)" %
(local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
if not self.user_authenticated():
self.login()
remote_dir = u'/' + urllib.parse.unquote(
self.parsed_url.path.lstrip(u'/')).rstrip()
log.Debug(u'dpbx.files_list_folder(%s)' % remote_dir)
res = []
try:
resp = self.api_client.files_list_folder(remote_dir)
log.Debug(u'dpbx.list(%s): %s' % (remote_dir, resp))
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
except ApiError as e:
if (isinstance(e.error, ListFolderError) and e.error.is_path()
and e.error.get_path().is_not_found()):
log.Debug(u'dpbx.list(%s): ignore missing folder (%s)' %
(remote_dir, e))
else:
raise
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.parse.unquote(self.parsed_url.path.lstrip(u'/'))
remote_path = u'/' + os.path.join(remote_dir,
filename.decode()).rstrip()
log.Debug(u'dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
u"""close backend session? no! just "flush" the data"""
log.Debug(u'dpbx.close():')
@command()
def _query(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.parse.unquote(self.parsed_url.path.lstrip(u'/'))
remote_path = u'/' + os.path.join(remote_dir,
filename.decode()).rstrip()
log.Debug(u'dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug(u'dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {u'size': info.size}
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [
x for x in file_list
if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
]
if len(bad_list) == 0:
return
log.Warn(u'-' * 72)
log.Warn(
u'Warning! It looks like there are automatically renamed files on backend'
)
log.Warn(
u'They were probably created when using older version of duplicity.'
)
log.Warn(u'')
log.Warn(
u'Please check your backup consistency. Most likely you will need to choose'
)
log.Warn(
u'largest file from duplicity-* (number).gpg and remove brackets from its name.'
)
log.Warn(u'')
log.Warn(
u'These files are not managed by duplicity at all and will not be')
log.Warn(u'removed/rotated automatically.')
log.Warn(u'')
log.Warn(u'Affected files:')
for x in bad_list:
log.Warn(u'\t%s' % x)
log.Warn(u'')
log.Warn(u'In any case it\'s better to create full backup.')
log.Warn(u'-' * 72)