class DropboxWriter(FilebaseBaseWriter): """ Writes items to dropbox folder. options available - access_token (str) Oauth access token for Dropbox api. - filebase (str) Base path to store the items in the share. """ supported_options = { 'access_token': { 'type': six.string_types, 'env_fallback': 'EXPORTERS_DROPBOXWRITER_TOKEN' }, } def __init__(self, *args, **kw): from dropbox import Dropbox super(DropboxWriter, self).__init__(*args, **kw) access_token = self.read_option('access_token') self.set_metadata('files_counter', Counter()) self.client = Dropbox(access_token) def write(self, dump_path, group_key=None, file_name=False): if group_key is None: group_key = [] self._write_file(dump_path, group_key, file_name) @retry_long def _upload_file(self, input_file, filepath): from dropbox import files session_id = self.client.files_upload_session_start('') current_offset = 0 while True: data = input_file.read(2**20) if not data: break self.client.files_upload_session_append(data, session_id.session_id, current_offset) current_offset += len(data) cursor = files.UploadSessionCursor(session_id.session_id, current_offset) self.client.files_upload_session_finish( '', cursor, files.CommitInfo(path='{}'.format(filepath))) def _write_file(self, dump_path, group_key, file_name=None): filebase_path, file_name = self.create_filebase_name( group_key, file_name=file_name) with open(dump_path, 'r') as f: self._upload_file(f, '{}/{}'.format(filebase_path, file_name)) self.get_metadata('files_counter')[filebase_path] += 1 def get_file_suffix(self, path, prefix): number_of_keys = self.get_metadata('files_counter').get(path, 0) suffix = '{}'.format(str(number_of_keys)) return suffix
class DropboxWriter(FilebaseBaseWriter): """ Writes items to dropbox folder. options available - access_token (str) Oauth access token for Dropbox api. - filebase (str) Base path to store the items in the share. """ supported_options = { 'access_token': {'type': six.string_types, 'env_fallback': 'EXPORTERS_DROPBOXWRITER_TOKEN'}, } def __init__(self, *args, **kw): from dropbox import Dropbox super(DropboxWriter, self).__init__(*args, **kw) access_token = self.read_option('access_token') self.set_metadata('files_counter', Counter()) self.client = Dropbox(access_token) def write(self, dump_path, group_key=None, file_name=False): if group_key is None: group_key = [] self._write_file(dump_path, group_key, file_name) @retry_long def _upload_file(self, input_file, filepath): from dropbox import files session_id = self.client.files_upload_session_start('') current_offset = 0 while True: data = input_file.read(2**20) if not data: break self.client.files_upload_session_append(data, session_id.session_id, current_offset) current_offset += len(data) cursor = files.UploadSessionCursor(session_id.session_id, current_offset) self.client.files_upload_session_finish( '', cursor, files.CommitInfo(path='{}'.format(filepath))) def _write_file(self, dump_path, group_key, file_name=None): filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name) with open(dump_path, 'r') as f: self._upload_file(f, '{}/{}'.format(filebase_path, file_name)) self.get_metadata('files_counter')[filebase_path] += 1 def get_file_suffix(self, path, prefix): number_of_keys = self.get_metadata('files_counter').get(path, 0) suffix = '{}'.format(str(number_of_keys)) return suffix
def upload_dropbox_file_chucks(file_path, file_size): dbx = Dropbox(DROPBOX_API_KEY) f = open(file_path, 'rb') session = dbx.files_upload_session_start(f.read(FILE_CHUNK_SIZE)) cursor = dbx_files.UploadSessionCursor(session_id=session.session_id, offset=f.tell()) commit = dbx_files.CommitInfo(path=file_path) while f.tell() < file_size: if ((file_size - f.tell()) <= FILE_CHUNK_SIZE): dbx.files_upload_session_finish(f.read(FILE_CHUNK_SIZE), cursor, commit) else: dbx.files_upload_session_append(f.read(FILE_CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell()
class DPBXBackend(duplicity.backend.Backend): """Connect to remote store using Dr*pB*x service""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) self.api_account = None self.api_client = None self.auth_flow = None self.login() def user_authenticated(self): try: account = self.api_client.users_get_current_account() log.Debug("User authenticated as ,%s" % account) return True except: log.Debug('User not authenticated') return False def load_access_token(self): return os.environ.get('DPBX_ACCESS_TOKEN', None) def save_access_token(self, access_token): raise BackendException( 'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token) def obtain_access_token(self): log.Info("dpbx: trying to obtain access token") for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']: if env_var not in os.environ: raise BackendException( 'dpbx: %s environment variable not set' % env_var) app_key = os.environ['DPBX_APP_KEY'] app_secret = os.environ['DPBX_APP_SECRET'] if not sys.stdout.isatty() or not sys.stdin.isatty(): log.FatalError( 'dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error) auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret) log.Debug('dpbx,auth_flow.start()') authorize_url = auth_flow.start() print print '-' * 72 print "1. Go to: " + authorize_url print "2. Click \"Allow\" (you might have to log in first)." print "3. Copy the authorization code." print '-' * 72 auth_code = raw_input("Enter the authorization code here: ").strip() try: log.Debug('dpbx,auth_flow.finish(%s)' % auth_code) authresult = auth_flow.finish(auth_code) except Exception as e: raise BackendException('dpbx: Unable to obtain access token: %s' % e) log.Info("dpbx: Authentication successfull") self.save_access_token(authresult.access_token) def login(self): if self.load_access_token() is None: self.obtain_access_token() self.api_client = Dropbox(self.load_access_token()) self.api_account = None try: log.Debug('dpbx,users_get_current_account([token])') self.api_account = self.api_client.users_get_current_account() log.Debug("dpbx,%s" % self.api_account) except (BadInputError, AuthError) as e: log.Debug('dpbx,exception: %s' % e) log.Info( "dpbx: Authentication failed. Trying to obtain new access token" ) self.obtain_access_token() # We're assuming obtain_access_token will throw exception. # So this line should not be reached raise BackendException( "dpbx: Please update DPBX_ACCESS_TOKEN and try again") log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name) def _error_code(self, operation, e): if isinstance(e, ApiError): err = e.error if isinstance(err, GetMetadataError) and err.is_path(): if err.get_path().is_not_found(): return log.ErrorCode.backend_not_found elif isinstance(err, DeleteError) and err.is_path_lookup(): lookup = e.error.get_path_lookup() if lookup.is_not_found(): return log.ErrorCode.backend_not_found @command() def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) progress.report_transfer(0, file_size) if file_size < DPBX_UPLOAD_CHUNK_SIZE: # Upload whole file at once to avoid extra server request res_metadata = self.put_file_small(source_path, remote_path) else: res_metadata = self.put_file_chunked(source_path, remote_path) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException( 'dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException( 'dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) def put_file_small(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: log.Debug('dpbx,files_upload(%s, [%d bytes])' % (remote_path, file_size)) res_metadata = self.api_client.files_upload( f.read(), remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) log.Debug('dpbx,files_upload(): %s' % res_metadata) progress.report_transfer(file_size, file_size) return res_metadata finally: f.close() def put_file_chunked(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug( 'dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries is_eof = False # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed # chunk while not is_eof or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) is_eof = f.tell() >= file_size if not is_eof and len(buf) == 0: continue # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if not is_eof: assert len(buf) != 0 log.Debug( 'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append( buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug( 'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish( buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError ) and error.is_incorrect_offset(): # Server reports that we should send another chunk. # Most likely this is caused by network error during # previous upload attempt. In such case we'll get # expected offset from server and it's enough to just # seek() and retry again new_offset = error.get_incorrect_offset( ).correct_offset log.Debug( 'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something # strange and no safe way to recover raise BackendException( "dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if not self.user_authenticated(): self.login() if retry_number == 0: raise # We don't know for sure, was partial upload successful or # not. So it's better to retry smaller amount to avoid extra # reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) return res_metadata finally: f.close() @command() def _get(self, remote_filename, local_path): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() log.Debug('dpbx,files_download(%s)' % remote_path) res_metadata, http_fd = self.api_client.files_download(remote_path) log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd)) file_size = res_metadata.size to_fd = None progress.report_transfer(0, file_size) try: to_fd = local_path.open('wb') for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE): to_fd.write(c) progress.report_transfer(to_fd.tell(), file_size) finally: if to_fd: to_fd.close() http_fd.close() # It's different from _query() check because we're not querying metadata # again. Since this check is free, it's better to have it here local_size = os.path.getsize(local_path.name) if local_size != file_size: raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size)) local_path.setdata() @command() def _list(self): # Do a long listing to avoid connection reset if not self.user_authenticated(): self.login() remote_dir = '/' + urllib.unquote( self.parsed_url.path.lstrip('/')).rstrip() log.Debug('dpbx.files_list_folder(%s)' % remote_dir) res = [] try: resp = self.api_client.files_list_folder(remote_dir) log.Debug('dpbx.list(%s): %s' % (remote_dir, resp)) while True: res.extend([entry.name for entry in resp.entries]) if not resp.has_more: break resp = self.api_client.files_list_folder_continue(resp.cursor) except ApiError as e: if (isinstance(e.error, ListFolderError) and e.error.is_path() and e.error.get_path().is_not_found()): log.Debug('dpbx.list(%s): ignore missing folder (%s)' % (remote_dir, e)) else: raise # Warn users of old version dpbx about automatically renamed files self.check_renamed_files(res) return res @command() def _delete(self, filename): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_delete(%s)' % remote_path) self.api_client.files_delete(remote_path) # files_permanently_delete seems to be better for backup purpose # but it's only available for Business accounts # self.api_client.files_permanently_delete(remote_path) @command() def _close(self): """close backend session? no! just "flush" the data""" log.Debug('dpbx.close():') @command() def _query(self, filename): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_get_metadata(%s)' % remote_path) info = self.api_client.files_get_metadata(remote_path) log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info)) return {'size': info.size} def check_renamed_files(self, file_list): if not self.user_authenticated(): self.login() bad_list = [ x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None ] if len(bad_list) == 0: return log.Warn('-' * 72) log.Warn( 'Warning! It looks like there are automatically renamed files on backend' ) log.Warn( 'They were probably created when using older version of duplicity.' ) log.Warn('') log.Warn( 'Please check your backup consistency. Most likely you will need to choose' ) log.Warn( 'largest file from duplicity-* (number).gpg and remove brackets from its name.' ) log.Warn('') log.Warn( 'These files are not managed by duplicity at all and will not be') log.Warn('removed/rotated automatically.') log.Warn('') log.Warn('Affected files:') for x in bad_list: log.Warn('\t%s' % x) log.Warn('') log.Warn('In any case it\'s better to create full backup.') log.Warn('-' * 72)
class DPBXBackend(duplicity.backend.Backend): """Connect to remote store using Dr*pB*x service""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) self.api_account = None self.api_client = None self.auth_flow = None self.login() def load_access_token(self): return os.environ.get('DPBX_ACCESS_TOKEN', None) def save_access_token(self, access_token): raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token) def obtain_access_token(self): log.Info("dpbx: trying to obtain access token") for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']: if env_var not in os.environ: raise BackendException('dpbx: %s environment variable not set' % env_var) app_key = os.environ['DPBX_APP_KEY'] app_secret = os.environ['DPBX_APP_SECRET'] if not sys.stdout.isatty() or not sys.stdin.isatty(): log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error) auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret) log.Debug('dpbx,auth_flow.start()') authorize_url = auth_flow.start() print print '-' * 72 print "1. Go to: " + authorize_url print "2. Click \"Allow\" (you might have to log in first)." print "3. Copy the authorization code." print '-' * 72 auth_code = raw_input("Enter the authorization code here: ").strip() try: log.Debug('dpbx,auth_flow.finish(%s)' % auth_code) access_token, _ = auth_flow.finish(auth_code) except Exception as e: raise BackendException('dpbx: Unable to obtain access token: %s' % e) log.Info("dpbx: Authentication successfull") self.save_access_token(access_token) def login(self): if self.load_access_token() is None: self.obtain_access_token() self.api_client = Dropbox(self.load_access_token()) self.api_account = None try: log.Debug('dpbx,users_get_current_account([token])') self.api_account = self.api_client.users_get_current_account() log.Debug("dpbx,%s" % self.api_account) except (BadInputError, AuthError) as e: log.Debug('dpbx,exception: %s' % e) log.Info("dpbx: Authentication failed. Trying to obtain new access token") self.obtain_access_token() # We're assuming obtain_access_token will throw exception. So this line should not be reached raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again") log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name) def _error_code(self, operation, e): if isinstance(e, ApiError): err = e.error if isinstance(err, GetMetadataError) and err.is_path(): if err.get_path().is_not_found(): return log.ErrorCode.backend_not_found elif isinstance(err, DeleteError) and err.is_path_lookup(): lookup = e.error.get_path_lookup() if lookup.is_not_found(): return log.ErrorCode.backend_not_found @command() def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: progress.report_transfer(0, file_size) buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed chunk while (f.tell() < file_size) or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if len(buf) != 0: log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset(): # Server reports that we should send another chunk. Most likely this is caused by # network error during previous upload attempt. In such case we'll get expected offset # from server and it's enough to just seek() and retry again new_offset = error.get_incorrect_offset().correct_offset log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something strange and no safe way to recover raise BackendException("dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if retry_number == 0: raise # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) finally: f.close() @command() def _get(self, remote_filename, local_path): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() log.Debug('dpbx,files_download(%s)' % remote_path) res_metadata, http_fd = self.api_client.files_download(remote_path) log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd)) file_size = res_metadata.size to_fd = None progress.report_transfer(0, file_size) try: to_fd = local_path.open('wb') for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE): to_fd.write(c) progress.report_transfer(to_fd.tell(), file_size) finally: if to_fd: to_fd.close() http_fd.close() # It's different from _query() check because we're not querying metadata again. # Since this check is free, it's better to have it here local_size = os.path.getsize(local_path.name) if local_size != file_size: raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size)) local_path.setdata() @command() def _list(self): # Do a long listing to avoid connection reset remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip() log.Debug('dpbx.files_list_folder(%s)' % remote_dir) resp = self.api_client.files_list_folder(remote_dir) log.Debug('dpbx.list(%s): %s' % (remote_dir, resp)) res = [] while True: res.extend([entry.name for entry in resp.entries]) if not resp.has_more: break resp = self.api_client.files_list_folder_continue(resp.cursor) # Warn users of old version dpbx about automatically renamed files self.check_renamed_files(res) return res @command() def _delete(self, filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_delete(%s)' % remote_path) self.api_client.files_delete(remote_path) # files_permanently_delete seems to be better for backup purpose # but it's only available for Business accounts # self.api_client.files_permanently_delete(remote_path) @command() def _close(self): """close backend session? no! just "flush" the data""" log.Debug('dpbx.close():') @command() def _query(self, filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_get_metadata(%s)' % remote_path) info = self.api_client.files_get_metadata(remote_path) log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info)) return {'size': info.size} def check_renamed_files(self, file_list): bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None] if len(bad_list) == 0: return log.Warn('-' * 72) log.Warn('Warning! It looks like there are automatically renamed files on backend') log.Warn('They were probably created when using older version of duplicity.') log.Warn('') log.Warn('Please check your backup consistency. Most likely you will need to choose') log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.') log.Warn('') log.Warn('These files are not managed by duplicity at all and will not be') log.Warn('removed/rotated automatically.') log.Warn('') log.Warn('Affected files:') for x in bad_list: log.Warn('\t%s' % x) log.Warn('') log.Warn('In any case it\'s better to create full backup.') log.Warn('-' * 72)
def push_to_dropbox(branch_name, symbol, gui): global settings # Read saves_path = settings["SAVES_DIR"] temp_dir = settings["TEMP_DIR"] if settings["OAUTH"] == 'null': return "Please type in /login to use this feature" # clear temp_dir for path_temp in listdir(temp_dir): remove(path.join(temp_dir, path_temp)) # archive worlds starting with 'symbol' to temp_dir for path_save in listdir(saves_path): file_path = path.join(saves_path, path_save) if path.isdir(file_path) and path_save[0] == symbol: make_archive(path.join(temp_dir, path_save), 'zip', file_path) dbx = Dropbox(settings["OAUTH"].access_token) # clear branch_directory in dropbox try: if settings["CONFIRM"]: confirm = simpledialog.askstring( "Confirm", "Type in 'YES' if you wish to proceed. This will delete the current '{0}'" " branch if it already exists in dropbox".format(branch_name)) if not confirm == "YES": return "Action Not " dbx.files_delete("/" + branch_name) except Exception: pass println("Starting upload... ", gui) println( "Do not close the app until 'done uploading' message is shown on the console", gui) # upload every zip file to dropbox in temp_dir for path_temp in listdir(temp_dir): zip_file = path.join(temp_dir, path_temp) destination = "/" + branch_name + "/" + path_temp with open(zip_file, "rb") as f: file_size = path.getsize(zip_file) if file_size < CHUNK_SIZE: dbx.files_upload(f.read(), destination) else: # upload_session_start_result upload_ssr = dbx.files_upload_session_start(f.read(CHUNK_SIZE)) cursor = files.UploadSessionCursor( session_id=upload_ssr.session_id, offset=f.tell()) commit = files.CommitInfo(path=destination) while f.tell() < file_size: percent = str(f.tell() / file_size * 100) + "%" print(percent) if (file_size - f.tell()) <= CHUNK_SIZE: dbx.files_upload_session_finish( f.read(CHUNK_SIZE), cursor, commit) else: dbx.files_upload_session_append( f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell() # clear temp_dir for path_temp in listdir(temp_dir): remove(path.join(temp_dir, path_temp)) save(settings) return "Done Uploading"
class Dropbox(BlackboxStorage): """Storage handler that uploads backups to Dropbox.""" required_fields = ("access_token", ) def __init__(self, **kwargs): super().__init__(**kwargs) self.upload_base = self.config.get("upload_directory") or "/" self.client = DropboxClient(self.config["access_token"]) self.valid = self._validate_token() def _validate_token(self): """Check if dropbox token is valid.""" try: return self.client.check_user("test").result == "test" except AuthError: return False def sync(self, file_path: Path) -> None: """Sync a file to Dropbox.""" # Check if Dropbox token is valid. if self.valid is False: error = "Dropbox token is invalid!" self.success = False self.output = error log.error(error) return None # This is size what can be uploaded as one chunk. # When file is bigger than that, this will be uploaded # in multiple parts. chunk_size = 4 * 1024 * 1024 temp_file, recompressed = self.compress(file_path) upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}" try: with temp_file as f: file_size = os.stat(f.name).st_size log.debug(file_size) if file_size <= chunk_size: self.client.files_upload(f.read(), upload_path, WriteMode.overwrite) else: session_start = self.client.files_upload_session_start( f.read(chunk_size)) cursor = UploadSessionCursor(session_start.session_id, offset=f.tell()) # Commit contains path in Dropbox and write mode about file commit = CommitInfo(upload_path, WriteMode.overwrite) while f.tell() < file_size: if (file_size - f.tell()) <= chunk_size: self.client.files_upload_session_finish( f.read(chunk_size), cursor, commit) else: self.client.files_upload_session_append( f.read(chunk_size), cursor.session_id, cursor.offset) cursor.offset = f.tell() self.success = True except (ApiError, HttpError) as e: log.error(e) self.success = False self.output = str(e) def rotate(self, database_id: str) -> None: """ Rotate the files in the Dropbox directory. All files in base directory of backups will be deleted when they are older than `retention_days`, and because of this, it's better to have backups in isolated folder. """ # Check if Dropbox token is valid. if self.valid is False: log.error("Dropbox token is invalid - Can't delete old backups!") return None # Let's rotate only this type of database db_type_regex = rf"{database_id}_blackbox_\d{{2}}_\d{{2}}_\d{{4}}.+" # Receive first batch of files. files_result = self.client.files_list_folder( self.upload_base if self.upload_base != "/" else "") entries = [ entry for entry in files_result.entries if self._is_backup_file(entry, db_type_regex) ] # If there is more files, receive all of them. while files_result.has_more: cursor = files_result.cursor files_result = self.client.files_list_folder_continue(cursor) entries += [ entry for entry in files_result.entries if self._is_backup_file(entry, db_type_regex) ] retention_days = 7 if Blackbox.retention_days: retention_days = Blackbox.retention_days # Find all old files and delete them. for item in entries: last_modified = item.server_modified now = datetime.now(tz=last_modified.tzinfo) delta = now - last_modified if delta.days >= retention_days: self.client.files_delete(item.path_lower) @staticmethod def _is_backup_file(entry, db_type_regex) -> bool: """Check if file is actually this kind of database backup.""" return isinstance(entry, FileMetadata) and re.match( db_type_regex, entry.name)