def file_exists_remotelly(dbx: dropbox.Dropbox, remote_report_file_path: str) -> bool: try: dbx.files_get_metadata(remote_report_file_path) return True except: return False
def sync_file(scheduled_process: ScheduledProcess, dropbox_client: dropbox.Dropbox, local_root_dir: str, abs_file_path: str) -> None: # The path we use in our Dropbox app folder. relative_file_path = abs_file_path.replace(local_root_dir, '') try: # Check whether the file is already at Dropbox, if so, check its hash. dropbox_meta = dropbox_client.files_get_metadata(relative_file_path) except dropbox.exceptions.ApiError as exception: error_message = str(exception.error) dropbox_meta = None # Unexpected. if 'not_found' not in error_message: logger.error(' - Dropbox error: %s', error_message) return # Calculate local hash and compare with remote. Ignore if the remote file is exactly the same. if dropbox_meta and calculate_content_hash( abs_file_path) == dropbox_meta.content_hash: logger.debug('Dropbox: Content hash is the same, skipping: %s', relative_file_path) return try: upload_chunked(dropbox_client=dropbox_client, local_file_path=abs_file_path, remote_file_path=relative_file_path) except dropbox.exceptions.DropboxException as exception: error_message = str(exception.error) logger.error('Dropbox: %s', error_message) if 'insufficient_space' in error_message: message = _( "Unable to upload files to Dropbox due to {}. Ignoring new files for the next {} hours..." .format(error_message, settings.DSMRREADER_DROPBOX_ERROR_INTERVAL)) dsmr_frontend.services.display_dashboard_message(message=message) scheduled_process.delay( hours=settings.DSMRREADER_DROPBOX_ERROR_INTERVAL) raise # pragma: no cover
class DropboxConnector: def __init__(self, account): self.client = Dropbox(account["access_token"]) # Dropbox does not like the "/" path for some stupid reason... # "" refers to the root directory @staticmethod def convert_to_dropbox_path(path): if path == "/": return "" else: return path def make_directory(self, path): self.client.files_create_folder(path) def change_mode(self, path, mode): pass def change_owner(self, path, user_id, group_id): pass def getattr(self, path, file_handle=None): dropbox_path = DropboxConnector.convert_to_dropbox_path(path) metadata = self.client.files_get_metadata(dropbox_path) file_status = FileStatus() # I'm going to change this hard coded stuff later file_status.group_id = 20 file_status.user_id = 501 @property def total_storage(self): space_usage = self.client.users_get_space_usage() return space_usage.allocation.get_individual().allocated @property def free_storage(self): space_usage = self.client.users_get_space_usage() return space_usage.used
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: entry['path'] = entry['path'].replace(full_path, '', 1) entry['path'] = entry['path'].replace('/', '', 1) if entry['is_dir']: directories.append(entry['path']) else: files.append(entry['path']) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE) ) cursor = UploadSessionCursor( session_id=upload_session.session_id, offset=content.tell() ) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit ) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor ) cursor.offset = content.tell()
class dropboxClient(object): ''' a class of methods to manage file storage on Dropbox API ''' # https://www.dropbox.com/developers/documentation/http/documentation _class_fields = { 'schema': { 'access_token': '', 'collection_name': 'labPack', 'record_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.ogg', 'record_key_path': '/home/user/.config/collective-acuity-labpack/user-data/obs/terminal', 'record_key_comp': 'obs', 'previous_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.yaml', 'secret_key': '6tZ0rUexOiBcOse2-dgDkbeY', 'prefix': 'obs/terminal', 'delimiter': '2016-03-17T17-24-51-687845Z.yaml', 'max_results': 1 }, 'components': { '.collection_name': { 'max_length': 255, 'must_not_contain': ['/', '^\\.'] }, '.record_key': { 'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ] }, '.record_key_path': { 'max_length': 32767 }, '.record_key_comp': { 'max_length': 255 }, '.secret_key': { 'must_not_contain': [ '[\\t\\n\\r]' ] }, '.max_results': { 'min_value': 1, 'integer_data': True }, '.previous_key': { 'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ] }, '.prefix': { 'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ] } }, 'metadata': { 'record_optimal_bytes': 10000 * 1024, 'record_max_bytes': 150000 * 1024 } } def __init__(self, access_token, collection_name=''): ''' a method to initialize the dropboxClient class :param access_token: string with oauth2 access token for users account ''' title = '%s.__init__' % self.__class__.__name__ # construct input validation model self.fields = jsonModel(self._class_fields) # validate inputs input_fields = { 'access_token': access_token, 'collection_name': collection_name } for key, value in input_fields.items(): object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # workaround for module namespace conflict from sys import path as sys_path sys_path.append(sys_path.pop(0)) from dropbox import Dropbox from dropbox.files import FileMetadata, WriteMode, DeleteArg from dropbox.exceptions import ApiError sys_path.insert(0, sys_path.pop()) # construct dropbox client from labpack.compilers.objects import _method_constructor self.dropbox = Dropbox(oauth2_access_token=access_token) # construct dropbox objects self.objects = _method_constructor({ 'FileMetadata': FileMetadata, 'ApiError': ApiError, 'WriteMode': WriteMode, 'DeleteArg': DeleteArg }) # construct collection name self.collection_name = collection_name def _import(self, record_key, record_data, overwrite=True, last_modified=0.0, **kwargs): ''' a helper method for other storage clients to import into appdata :param record_key: string with key for record :param record_data: byte data for body of record :param overwrite: [optional] boolean to overwrite existing records :param last_modified: [optional] float to record last modified date :param kwargs: [optional] keyword arguments from other import methods :return: boolean indicating whether record was imported ''' title = '%s._import' % self.__class__.__name__ # check overwrite if not overwrite: if self.exists(record_key): return False # check max size import sys record_max = self.fields.metadata['record_max_bytes'] record_size = sys.getsizeof(record_data) error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key) if record_size > record_max: raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max)) # TODO: apply session upload for files greater than record_max # construct upload kwargs upload_kwargs = { 'f': record_data, 'path': '/%s' % record_key, 'mute': True, 'mode': self.objects.WriteMode.overwrite } # modify file time import re if re.search('\\.drep$', record_key): from labpack.records.time import labDT drep_time = labDT.fromEpoch(1) upload_kwargs['client_modified'] = drep_time elif last_modified: from labpack.records.time import labDT mod_time = labDT.fromEpoch(last_modified) upload_kwargs['client_modified'] = mod_time # send upload request try: self.dropbox.files_upload(**upload_kwargs) except: raise DropboxConnectionError(title) return True def _walk(self, root_path=''): ''' an iterator method which walks the file structure of the dropbox collection ''' title = '%s._walk' % self.__class__.__name__ if root_path: root_path = '/%s' % root_path try: response = self.dropbox.files_list_folder(path=root_path, recursive=True) for record in response.entries: if not isinstance(record, self.objects.FileMetadata): continue yield record.path_display[1:] if response.has_more: while response.has_more: response = self.dropbox.files_list_folder_continue(response.cursor) for record in response.entries: if not isinstance(record, self.objects.FileMetadata): continue yield record.path_display[1:] except: raise DropboxConnectionError(title) def exists(self, record_key): ''' a method to determine if a record exists in collection :param record_key: string with key of record :return: boolean reporting status ''' title = '%s.exists' % self.__class__.__name__ # validate inputs input_fields = { 'record_key': record_key } for key, value in input_fields.items(): object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # send get metadata request file_path = '/%s' % record_key try: self.dropbox.files_get_metadata(file_path) except Exception as err: if str(err).find("LookupError('not_found'") > -1: return False else: raise DropboxConnectionError(title) return True def save(self, record_key, record_data, overwrite=True, secret_key=''): ''' a method to create a record in the collection folder :param record_key: string with name to assign to record (see NOTES below) :param record_data: byte data for record body :param overwrite: [optional] boolean to overwrite records with same name :param secret_key: [optional] string with key to encrypt data :return: string with name of record NOTE: record_key may only contain alphanumeric, /, _, . or - characters and may not begin with the . or / character. NOTE: using one or more / characters splits the key into separate segments. these segments will appear as a sub directories inside the record collection and each segment is used as a separate index for that record when using the list method eg. lab/unittests/1473719695.2165067.json is indexed: [ 'lab', 'unittests', '1473719695.2165067', '.json' ] ''' title = '%s.save' % self.__class__.__name__ # validate inputs input_fields = { 'record_key': record_key, 'secret_key': secret_key } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # validate byte data if not isinstance(record_data, bytes): raise ValueError('%s(record_data=b"...") must be byte data.' % title) # construct and validate file path file_root, file_name = os.path.split(record_key) self.fields.validate(file_name, '.record_key_comp') while file_root: file_root, path_node = os.path.split(file_root) self.fields.validate(path_node, '.record_key_comp') # check overwrite exception if not overwrite: if self.exists(record_key): raise Exception('%s(record_key="%s") already exists. To overwrite, set overwrite=True' % (title, record_key)) # check size of file import sys record_optimal = self.fields.metadata['record_optimal_bytes'] record_max = self.fields.metadata['record_max_bytes'] record_size = sys.getsizeof(record_data) error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key) if record_size > record_max: raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max)) elif record_size > record_optimal: print('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal)) # TODO add upload session for support of files over 150MB # http://dropbox-sdk-python.readthedocs.io/en/latest/moduledoc.html#dropbox.dropbox.Dropbox.files_upload_session_start # encrypt data if secret_key: from labpack.encryption import cryptolab record_data, secret_key = cryptolab.encrypt(record_data, secret_key) # construct upload kwargs upload_kwargs = { 'f': record_data, 'path': '/%s' % record_key, 'mute': True, 'mode': self.objects.WriteMode.overwrite } # modify file time import re if re.search('\\.drep$', file_name): from labpack.records.time import labDT drep_time = labDT.fromEpoch(1) upload_kwargs['client_modified'] = drep_time # send upload request try: self.dropbox.files_upload(**upload_kwargs) except: raise DropboxConnectionError(title) return record_key def load(self, record_key, secret_key=''): ''' a method to retrieve byte data of appdata record :param record_key: string with name of record :param secret_key: [optional] string used to decrypt data :return: byte data for record body ''' title = '%s.load' % self.__class__.__name__ # validate inputs input_fields = { 'record_key': record_key, 'secret_key': secret_key } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # construct file path file_path = '/%s' % record_key # request file data try: metadata, response = self.dropbox.files_download(file_path) except Exception as err: if str(err).find("LookupError('not_found'") > -1: raise Exception('%s(record_key=%s) does not exist.' % (title, record_key)) else: raise DropboxConnectionError(title) record_data = response.content # decrypt (if necessary) if secret_key: from labpack.encryption import cryptolab record_data = cryptolab.decrypt(record_data, secret_key) return record_data def conditional_filter(self, path_filters): ''' a method to construct a conditional filter function for list method :param path_filters: dictionary or list of dictionaries with query criteria :return: filter_function object path_filters: [ { 0: { conditional operators }, 1: { conditional_operators }, ... } ] conditional operators: "byte_data": false, "discrete_values": [ "" ], "excluded_values": [ "" ], "greater_than": "", "less_than": "", "max_length": 0, "max_value": "", "min_length": 0, "min_value": "", "must_contain": [ "" ], "must_not_contain": [ "" ], "contains_either": [ "" ] ''' title = '%s.conditional_filter' % self.__class__.__name__ from labpack.compilers.filters import positional_filter filter_function = positional_filter(path_filters, title) return filter_function def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''): ''' a method to list keys in the dropbox collection :param prefix: string with prefix value to filter results :param delimiter: string with value which results must not contain (after prefix) :param filter_function: (positional arguments) function used to filter results :param max_results: integer with maximum number of results to return :param previous_key: string with key in collection to begin search after :return: list of key strings NOTE: each key string can be divided into one or more segments based upon the / characters which occur in the key string as well as its file extension type. if the key string represents a file path, then each directory in the path, the file name and the file extension are all separate indexed values. eg. lab/unittests/1473719695.2165067.json is indexed: [ 'lab', 'unittests', '1473719695.2165067', '.json' ] it is possible to filter the records in the collection according to one or more of these path segments using a filter_function. NOTE: the filter_function must be able to accept an array of positional arguments and return a value that can evaluate to true or false. while searching the records, list produces an array of strings which represent the directory structure in relative path of each key string. if a filter_function is provided, this list of strings is fed to the filter function. if the function evaluates this input and returns a true value the file will be included in the list results. ''' title = '%s.list' % self.__class__.__name__ # validate input input_fields = { 'prefix': prefix, 'delimiter': delimiter, 'max_results': max_results, 'previous_key': previous_key } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # validate filter function if filter_function: try: path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ] filter_function(*path_segments) except: err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__) raise TypeError('%s must accept positional arguments.' % err_msg) # construct empty results list results_list = [] check_key = True if previous_key: check_key = False # determine root path root_path = '' if prefix: from os import path root_path, file_name = path.split(prefix) # iterate over dropbox files for file_path in self._walk(root_path): path_segments = file_path.split(os.sep) record_key = os.path.join(*path_segments) record_key = record_key.replace('\\','/') if record_key == previous_key: check_key = True # find starting point if not check_key: continue # apply prefix filter partial_key = record_key if prefix: if record_key.find(prefix) == 0: partial_key = record_key[len(prefix):] else: continue # apply delimiter filter if delimiter: if partial_key.find(delimiter) > -1: continue # apply filter function if filter_function: if filter_function(*path_segments): results_list.append(record_key) else: results_list.append(record_key) # return results list if len(results_list) == max_results: return results_list return results_list def delete(self, record_key): ''' a method to delete a file :param record_key: string with name of file :return: string reporting outcome ''' title = '%s.delete' % self.__class__.__name__ # validate inputs input_fields = { 'record_key': record_key } for key, value in input_fields.items(): object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # validate existence of file if not self.exists(record_key): exit_msg = '%s does not exist.' % record_key return exit_msg # remove file current_dir = os.path.split(record_key)[0] try: file_path = '/%s' % record_key self.dropbox.files_delete(file_path) except: raise DropboxConnectionError(title) # remove empty directories in path to file try: while current_dir: folder_path = '/%s' % current_dir response = self.dropbox.files_list_folder(folder_path) if not response.entries: self.dropbox.files_delete(folder_path) current_dir = os.path.split(current_dir)[0] else: break except: raise DropboxConnectionError(title) exit_msg = '%s has been deleted.' % record_key return exit_msg def remove(self): ''' a method to remove all records in the collection NOTE: this method removes all the files in the collection, but the collection folder itself created by oauth2 cannot be removed. only the user can remove the app folder :return: string with confirmation of deletion ''' title = '%s.remove' % self.__class__.__name__ # get contents in root try: response = self.dropbox.files_list_folder(path='') except: raise DropboxConnectionError(title) # populate delete list delete_list = [] for file in response.entries: delete_list.append(self.objects.DeleteArg(path=file.path_display)) # continue retrieval if folder is large if response.has_more: try: while response.has_more: response = self.dropbox.files_list_folder_continue(response.cursor) for file in response.entries: delete_list.append(self.objects.DeleteArg(path=file.path_display)) except: raise DropboxConnectionError(title) # send batch delete request try: self.dropbox.files_delete_batch(delete_list) except: raise DropboxConnectionError(title) # return outcome insert = 'collection' if self.collection_name: insert = self.collection_name exit_msg = 'Contents of %s will been removed from Dropbox.' % insert return exit_msg def export(self, storage_client, overwrite=True): ''' a method to export all the records in collection to another platform :param storage_client: class object with storage client methods :return: string with exit message ''' title = '%s.export' % self.__class__.__name__ # validate storage client method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ] for method in method_list: if not getattr(storage_client, method, None): from labpack.parsing.grammar import join_words raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list))) # walk collection folder to find files import os count = 0 skipped = 0 for file_path in self._walk(): path_segments = file_path.split(os.sep) record_key = os.path.join(*path_segments) record_key = record_key.replace('\\','/') file_path = '/%s' % file_path # retrieve data and metadata try: metadata, response = self.dropbox.files_download(file_path) except: raise DropboxConnectionError(title) record_data = response.content client_modified = metadata.client_modified # import record into storage client last_modified = 0.0 if client_modified: from dateutil.tz import tzutc from labpack.records.time import labDT last_modified = labDT.fromPython(client_modified.replace(tzinfo=tzutc())).epoch() outcome = storage_client._import(record_key, record_data, overwrite=overwrite, last_modified=last_modified) if outcome: count += 1 else: skipped += 1 # report outcome plural = '' skip_insert = '' new_folder = storage_client.collection_name if count != 1: plural = 's' if skipped > 0: skip_plural = '' if skipped > 1: skip_plural = 's' skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural) exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert) return exit_msg
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE) CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, write_mode=write_mode): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path self.write_mode = write_mode self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): if name == '/': name = '' # If the machine is windows do not append the drive letter to file path if os.name == 'nt': final_path = os.path.join(self.root_path, name).replace('\\', '/') # Separator on linux system sep = '//' base_path = self.root_path if (not os.path.normcase(final_path).startswith(os.path.normcase(base_path + sep)) and os.path.normcase(final_path) != os.path.normcase(base_path) and os.path.dirname(os.path.normcase(base_path)) != os.path.normcase(base_path)): raise SuspiciousFileOperation( 'The joined path ({}) is located outside of the base path ' 'component ({})'.format(final_path, base_path)) return final_path else: return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) if full_path == '/': full_path = '' metadata = self.client.files_list_folder(full_path) for entry in metadata.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.client_modified def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE) ) cursor = UploadSessionCursor( session_id=upload_session.session_id, offset=content.tell() ) commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode)) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit ) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor ) cursor.offset = content.tell() def get_available_name(self, name, max_length=None): """Overwrite existing file with the same name.""" name = self._full_path(name) if self.write_mode == 'overwrite': return get_available_overwrite_name(name, max_length) return super().get_available_name(name, max_length)
class DropboxConnection(object): class Parameters(object): def __init__(self, auth_token, remote_dir_path): self._auth_token = auth_token self._remote_dir_path = remote_dir_path @property def auth_token(self): return self._auth_token @property def remote_dir_path(self): return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path def __str__(self): return """ remote_dir_path: {} """.format(self.remote_dir_path) @classmethod def get_client_from_params(cls, dropbox_parameters): #Dropbox connection placeholder dropbox = None if dropbox_parameters: dropbox_params = DropboxConnection.Parameters( dropbox_parameters[0], dropbox_parameters[1]) dropbox = DropboxConnection(dropbox_params) return dropbox @classmethod def get_client(cls, auth_token, remote_dir_path): #Initialize the paramters params = DropboxConnection.Parameters(auth_token, remote_dir_path) #Create dropbox client client = DropboxConnection(params) return client def __init__(self, params): #Required parameters self._params = params #Derived parameters self._client = Dropbox(self._params.auth_token) #Logging self._logger = logging.get_logger(__name__) def upload(self, source_file_path): """It upload the source files to the dropbox. Arguments: source_file_path {string} -- The source file path. Raises: ValueError -- It raise value error for invalid files. """ #Validate parameters if not source_file_path: raise ValueError("Invalid source file path") with open(source_file_path, 'rb') as handle: #Source file name source_file_name = Path(source_file_path).name #Remote path remote_file_path = (self._params.remote_dir_path / source_file_name).as_posix() #File size upload_size = path.getsize(source_file_path) #Upload the files based on the upload size if upload_size <= constants.DROPBOX_CHUNK_SIZE: self._logger.info( 'Preparing to upload small file: %s with size: %d to: %s', source_file_path, upload_size, remote_file_path) self._upload_small_file(handle, remote_file_path) else: self._logger.info( 'Preparing to upload large file: %s with size: %d to: %s', source_file_path, upload_size, remote_file_path) self._upload_large_file(handle, upload_size, remote_file_path) self._logger.info('Uploaded: %s', source_file_path) def _upload_small_file(self, handle, remote_file_path): """It uploads a small source files to the dropbox. Arguments: handle {A File handle} -- The source file handle. remote_file_path {string} -- The destination path of the file. """ self._client.files_upload(handle.read(), remote_file_path, mode=Dropbox_WriteMode.overwrite) def _upload_large_file(self, handle, upload_size, remote_file_path): """It uploads a large source files to the dropbox. Arguments: handle {A File handle} -- The source file handle. upload_size {int} -- The number of bytes to be uploaded. remote_file_path {string} -- The destination path of the file. """ #Upload session session = self._client.files_upload_session_start( handle.read(constants.DROPBOX_CHUNK_SIZE)) cursor = Dropbox_UploadSessionCursor(session_id=session.session_id, offset=handle.tell()) #Upload look with tqdm(desc='Uploading: {}'.format(remote_file_path), total=upload_size) as pbar: #Update the progress bar for the session start reads pbar.update(handle.tell()) while handle.tell() < upload_size: #Calculate remaining bytes remaining_bytes = upload_size - handle.tell() #If it is the last chunk, finalize the upload if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE: #Commit info commit = Dropbox_CommitInfo( path=remote_file_path, mode=Dropbox_WriteMode.overwrite) #Finish upload self._client.files_upload_session_finish( handle.read(remaining_bytes), cursor, commit) #Update progress pbar.update(remaining_bytes) #More than chunk size remaining to upload else: self._client.files_upload_session_append_v2( handle.read(constants.DROPBOX_CHUNK_SIZE), cursor) #Update the cursor cursor.offset = handle.tell() #Update the progress pbar.update(constants.DROPBOX_CHUNK_SIZE) #Refresh the progress bar pbar.refresh() def download(self, remote_file_path): """It downloads the remote files from the dropbox. Arguments: remote_file_path {Path} -- The path to the remote file. Raises: ValueError -- It raise value error for invalid file name. """ #Validate parameters if not remote_file_path: raise ValueError("Invalid remote file path") #Destination file path dest_file_path = remote_file_path #Full remote file path remote_file_path = self._params.remote_dir_path / remote_file_path #Download file size placeholder download_size = 0 try: download_size = self._client.files_get_metadata( remote_file_path.as_posix()).size except ApiError as e: raise FileNotFoundError( 'File: {} is not found'.format(remote_file_path.as_posix()), e) self._logger.info('Preparing file download: %s with size: %d to: %s', remote_file_path, download_size, dest_file_path) #Download the file self._download_file(dest_file_path, remote_file_path, download_size) self._logger.info('Completed the file download: %s to: %s', remote_file_path, dest_file_path) def _download_file(self, dest_file_path, remote_file_path, download_size): """It downloads the remote files from the dropbox. Arguments: remote_file_path {A Path object} -- The path of the remote file. dest_file_path {string} -- The destination file path. download_size {int} -- The number of bytes to be downloaded. """ #Download _, result = self._client.files_download(remote_file_path.as_posix()) #Temporary dest_file_name tmp_dest_file_path = "{}.tmp".format(dest_file_path) with open(tmp_dest_file_path, 'wb') as handle: with tqdm(desc='Downloading: {}'.format( remote_file_path.as_posix()), total=download_size) as pbar: for bytes_read in result.iter_content( constants.DROPBOX_CHUNK_SIZE): handle.write(bytes_read) #Update the progress pbar.update(len(bytes_read)) if Path(tmp_dest_file_path).exists(): rename(tmp_dest_file_path, dest_file_path) def list(self, dir_path=Path(), file_name_prefix=''): """It lists the files in the dropbox folder that starts with the given prefix. Arguments: file_name_prefix {string} -- The prefix to filter the results. """ #Candidate directory whose contents are to be listed candidate_dir_path = self._params.remote_dir_path / dir_path self._logger.info('Enumerating: %s with file_name_prefix: %s', candidate_dir_path, file_name_prefix) #Call the downstream API response = self._client.files_list_folder( candidate_dir_path.as_posix()) #Output list placeholder files = [] sizes = [] if response.entries: #Log the response summary self._logger.info('Got %d files in: %s', len(response.entries), candidate_dir_path) #Extract the name of files satisfying the input criteria from the response entries. file_infos = [(dir_path / entry.name, entry.size) for entry in response.entries if entry.name.startswith(file_name_prefix)] files, sizes = zip(*file_infos) return files, sizes
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: entry['path'] = entry['path'].replace(full_path, '', 1) entry['path'] = entry['path'].replace('/', '', 1) if entry['is_dir']: directories.append(entry['path']) else: files.append(entry['path']) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): self.client.files_upload(content, self._full_path(name)) return name
class DropBoxStorage(Storage): """ The default Storage base for all file storing of the Chat app """ def __init__(self, oauth2_access_token: str = None, root_path: str = None): """ The access token and root path may be provided here as well, if they are not provided here, program will check settings for environment variables :param oauth2_access_token: The OAUTH2 access token for the DropBox api :param root_path: The root path for storing the files in the DropBox storage, defaults '/' """ oauth2_access_token = oauth2_access_token or settings.DROPBOX_OAUTH2_TOKEN self.root_path = root_path or settings.DROPBOX_ROOT_PATH or '/' if oauth2_access_token is None: raise ImproperlyConfigured( "You must configure an OATH2 access token ENV named " "'DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def delete(self, name: str): """ Deletes the specified file from the storage system. """ self.client.files_delete_v2(join(self.root_path, basename(name))) def exists(self, name: str): """ Returns True if a file referenced by the given name already exists in the storage system, or False if the name is available for a new file. """ try: return bool( self.client.files_get_metadata( join(self.root_path, basename(name)))) except ApiError: return False def url(self, name: str): """ Returns an absolute URL where the file's contents can be accessed directly by a Web browser. """ media = self.client.files_get_temporary_link( join(self.root_path, basename(name))) return media.link def _open(self, name: str, mode: str = 'rb'): """ Call DropBoxStorage.open(...) instead """ file = DropBoxFile(join(self.root_path, basename(name)), self.client) return file def _save(self, name: str, content: File): """ Call DropBoxStorage.save(...) instead """ self.client.files_upload(content.read(), join(self.root_path, basename(name))) return name
class DPBXBackend(duplicity.backend.Backend): """Connect to remote store using Dr*pB*x service""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) self.api_account = None self.api_client = None self.auth_flow = None self.login() def load_access_token(self): return os.environ.get('DPBX_ACCESS_TOKEN', None) def save_access_token(self, access_token): raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token) def obtain_access_token(self): log.Info("dpbx: trying to obtain access token") for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']: if env_var not in os.environ: raise BackendException('dpbx: %s environment variable not set' % env_var) app_key = os.environ['DPBX_APP_KEY'] app_secret = os.environ['DPBX_APP_SECRET'] if not sys.stdout.isatty() or not sys.stdin.isatty(): log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error) auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret) log.Debug('dpbx,auth_flow.start()') authorize_url = auth_flow.start() print print '-' * 72 print "1. Go to: " + authorize_url print "2. Click \"Allow\" (you might have to log in first)." print "3. Copy the authorization code." print '-' * 72 auth_code = raw_input("Enter the authorization code here: ").strip() try: log.Debug('dpbx,auth_flow.finish(%s)' % auth_code) access_token, _ = auth_flow.finish(auth_code) except Exception as e: raise BackendException('dpbx: Unable to obtain access token: %s' % e) log.Info("dpbx: Authentication successfull") self.save_access_token(access_token) def login(self): if self.load_access_token() is None: self.obtain_access_token() self.api_client = Dropbox(self.load_access_token()) self.api_account = None try: log.Debug('dpbx,users_get_current_account([token])') self.api_account = self.api_client.users_get_current_account() log.Debug("dpbx,%s" % self.api_account) except (BadInputError, AuthError) as e: log.Debug('dpbx,exception: %s' % e) log.Info("dpbx: Authentication failed. Trying to obtain new access token") self.obtain_access_token() # We're assuming obtain_access_token will throw exception. So this line should not be reached raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again") log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name) def _error_code(self, operation, e): if isinstance(e, ApiError): err = e.error if isinstance(err, GetMetadataError) and err.is_path(): if err.get_path().is_not_found(): return log.ErrorCode.backend_not_found elif isinstance(err, DeleteError) and err.is_path_lookup(): lookup = e.error.get_path_lookup() if lookup.is_not_found(): return log.ErrorCode.backend_not_found @command() def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: progress.report_transfer(0, file_size) buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed chunk while (f.tell() < file_size) or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if len(buf) != 0: log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset(): # Server reports that we should send another chunk. Most likely this is caused by # network error during previous upload attempt. In such case we'll get expected offset # from server and it's enough to just seek() and retry again new_offset = error.get_incorrect_offset().correct_offset log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something strange and no safe way to recover raise BackendException("dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if retry_number == 0: raise # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) finally: f.close() @command() def _get(self, remote_filename, local_path): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() log.Debug('dpbx,files_download(%s)' % remote_path) res_metadata, http_fd = self.api_client.files_download(remote_path) log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd)) file_size = res_metadata.size to_fd = None progress.report_transfer(0, file_size) try: to_fd = local_path.open('wb') for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE): to_fd.write(c) progress.report_transfer(to_fd.tell(), file_size) finally: if to_fd: to_fd.close() http_fd.close() # It's different from _query() check because we're not querying metadata again. # Since this check is free, it's better to have it here local_size = os.path.getsize(local_path.name) if local_size != file_size: raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size)) local_path.setdata() @command() def _list(self): # Do a long listing to avoid connection reset remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip() log.Debug('dpbx.files_list_folder(%s)' % remote_dir) resp = self.api_client.files_list_folder(remote_dir) log.Debug('dpbx.list(%s): %s' % (remote_dir, resp)) res = [] while True: res.extend([entry.name for entry in resp.entries]) if not resp.has_more: break resp = self.api_client.files_list_folder_continue(resp.cursor) # Warn users of old version dpbx about automatically renamed files self.check_renamed_files(res) return res @command() def _delete(self, filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_delete(%s)' % remote_path) self.api_client.files_delete(remote_path) # files_permanently_delete seems to be better for backup purpose # but it's only available for Business accounts # self.api_client.files_permanently_delete(remote_path) @command() def _close(self): """close backend session? no! just "flush" the data""" log.Debug('dpbx.close():') @command() def _query(self, filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_get_metadata(%s)' % remote_path) info = self.api_client.files_get_metadata(remote_path) log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info)) return {'size': info.size} def check_renamed_files(self, file_list): bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None] if len(bad_list) == 0: return log.Warn('-' * 72) log.Warn('Warning! It looks like there are automatically renamed files on backend') log.Warn('They were probably created when using older version of duplicity.') log.Warn('') log.Warn('Please check your backup consistency. Most likely you will need to choose') log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.') log.Warn('') log.Warn('These files are not managed by duplicity at all and will not be') log.Warn('removed/rotated automatically.') log.Warn('') log.Warn('Affected files:') for x in bad_list: log.Warn('\t%s' % x) log.Warn('') log.Warn('In any case it\'s better to create full backup.') log.Warn('-' * 72)
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting( 'DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: entry['path'] = entry['path'].replace(full_path, '', 1) entry['path'] = entry['path'].replace('/', '', 1) if entry['is_dir']: directories.append(entry['path']) else: files.append(entry['path']) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media['link'] def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): self.client.files_upload(content, self._full_path(name)) return name
class DropboxFS(FS): _meta = { "case_insensitive": False, "invalid_path_chars": "\0", "network": True, "read_only": False, "thread_safe": True, "unicode_paths": True, "virtual": False, } def __init__(self, accessToken, session=None): super(DropboxFS, self).__init__() self._lock = threading.RLock() self.dropbox = Dropbox(accessToken, session=session) def fix_path(self, path): if isinstance(path, bytes): try: path = path.decode("utf-8") except AttributeError: pass if not path.startswith("/"): path = "/" + path if path == "." or path == "./": path = "/" path = self.validatepath(path) return path def __repr__(self): return "<DropboxDriveFS>" def _infoFromMetadata(self, metadata): rawInfo = { "basic": { "name": metadata.name, "is_dir": isinstance(metadata, FolderMetadata), } } if isinstance(metadata, FileMetadata): rawInfo.update( {"details": {"size": metadata.size, "type": ResourceType.file}} ) else: rawInfo.update({"details": {"type": ResourceType.directory}}) return Info(rawInfo) def getinfo(self, path, namespaces=None): _path = self.fix_path(path) if _path == "/": info_dict = { "basic": {"name": "", "is_dir": True}, "details": {"type": ResourceType.directory}, } return Info(info_dict) try: metadata = self.dropbox.files_get_metadata( _path, include_media_info=True ) except ApiError as e: raise errors.ResourceNotFound(path=path, exc=e) return self._infoFromMetadata(metadata) def setinfo(self, path, info): if not self.exists(path): raise errors.ResourceNotFound(path) def listdir(self, path): _path = self.fix_path(path) if _path == "/": _path = "" if not self.exists(_path): raise errors.ResourceNotFound(path) meta = self.getinfo(_path) if meta.is_file: raise errors.DirectoryExpected(path) result = self.dropbox.files_list_folder(_path, include_media_info=True) allEntries = result.entries while result.has_more: result = self.dropbox.files_list_folder_continue(result.cursor) allEntries += result.entries return [x.name for x in allEntries] def makedir(self, path, permissions=None, recreate=False): path = self.fix_path(path) if self.exists(path) and not recreate: raise errors.DirectoryExists(path) if path == "/": return SubFS(self, path) if self.exists(path): meta = self.getinfo(path) if meta.is_dir: if recreate == False: raise errors.DirectoryExists(path) else: return SubFS(self, path) if meta.is_file: raise errors.DirectoryExpected(path) ppath = self.get_parent(path) if not self.exists(ppath): raise errors.ResourceNotFound(ppath) try: folderMetadata = self.dropbox.files_create_folder_v2(path) except ApiError as e: raise errors.DirectoryExpected(path=path) return SubFS(self, path) def openbin(self, path, mode="r", buffering=-1, **options): path = self.fix_path(path) _mode = Mode(mode) mode = _mode _mode.validate_bin() _path = self.validatepath(path) log.debug("openbin: %s, %s", path, mode) with self._lock: try: info = self.getinfo(_path) log.debug("Info: %s", info) except errors.ResourceNotFound: if not _mode.create: raise errors.ResourceNotFound(path) # Check the parent is an existing directory if not self.getinfo(self.get_parent(_path)).is_dir: raise errors.DirectoryExpected(path) else: if info.is_dir: raise errors.FileExpected(path) if _mode.exclusive: raise errors.FileExists(path) return DropboxFile(self.dropbox, path, mode) def remove(self, path): _path = self.fix_path(path) try: info = self.getinfo(path) if info.is_dir: raise errors.FileExpected(path=path) self.dropbox.files_delete_v2(_path) except ApiError as e: if isinstance(e.error._value, LookupError): raise errors.ResourceNotFound(path=path) log.debug(e) raise errors.FileExpected(path=path, exc=e) def removedir(self, path): _path = self.fix_path(path) if _path == "/": raise errors.RemoveRootError() try: info = self.getinfo(path) if not info.is_dir: raise errors.DirectoryExpected(path=path) if len(self.listdir(path)) > 0: raise errors.DirectoryNotEmpty(path=path) self.dropbox.files_delete_v2(_path) except ApiError as e: if isinstance(e.error._value, LookupError): raise errors.ResourceNotFound(path=path) raise errors.FileExpected(path=path, exc=e) def copy(self, src_path, dst_path, overwrite=False): src_path = self.fix_path(src_path) dst_path = self.fix_path(dst_path) try: src_meta = self.getinfo(src_path) if src_meta.is_dir: raise errors.FileExpected(src_path) except ApiError as e: raise errors.ResourceNotFound dst_meta = None try: dst_meta = self.getinfo(dst_path) except Exception as e: pass if dst_meta is not None: if overwrite == True: self.remove(dst_path) else: raise errors.DestinationExists(dst_path) parent_path = self.get_parent(dst_path) if not self.exists(parent_path): raise errors.ResourceNotFound(dst_path) self.dropbox.files_copy_v2(src_path, dst_path) def get_parent(self, dst_path): import os parent_path = os.path.abspath(os.path.join(dst_path, "..")) return parent_path def exists(self, path): path = self.fix_path(path) try: self.getinfo(path) return True except Exception as e: return False def move(self, src_path, dst_path, overwrite=False): _src_path = self.fix_path(src_path) _dst_path = self.fix_path(dst_path) if not self.getinfo(_src_path).is_file: raise errors.FileExpected(src_path) if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) if "/" in dst_path and not self.exists(self.get_parent(_dst_path)): raise errors.ResourceNotFound(src_path) with self._lock: try: if overwrite: try: # remove file anyways self.dropbox.files_delete_v2(_dst_path) except Exception as e: pass self.dropbox.files_move_v2(_src_path, _dst_path) except ApiError as e: raise errors.ResourceNotFound(src_path, exc=e) def apierror_map(self, error): log.debug(error) def geturl(self, path, purpose='download'): url = self.dropbox.sharing_create_shared_link(path).url url = url.replace('?dl=0', '?raw=1') return url
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = DROPBOX_OAUTH2_TOKEN self.root_path = DROPBOX_ROOT_PATH if oauth2_access_token is None: raise ImproperlyConfigured("Você deve configurar um token em DROPBOX_OAUTH2_TOKEN ou em settings.py") self.dbx = Dropbox(oauth2_access_token) def user_profile(self): self.dt = self.dbx.users_get_current_account() print(self.dt) def list_dir_and_files_all(self): try: self.dt = self.dbx.files_list_folder(self.root_path) print('DIRETÓRIOS\n') self.list_subdirs(self.dt) except: self.dt = self.dbx.files_get_metadata(self.root_path) print('ARQUIVOS\n') if (isinstance(self.dt, dropbox.files.FileMetadata)): print('FUNCIONA') file = self.list_files(self.dt) return file def list_subdirs(self, dt): for entry in self.dt.entries: i = entry.path_display print(i) def list_files(self): self.dt = self.dbx.files_get_metadata(self.root_path) file = self.dt.path_display print(file) return file def upload_file(self): #print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW) time = datetime.datetime.now() time = time.strftime("%Y%m%d%H%M%S") FILEPATH = self.simple_backup() with open(FILEPATH, 'rb') as f: self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump') link = self.dbx.sharing_create_shared_link_with_settings(DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump') url = link.url dl_url = re.sub(r"\?dl\=0", "?dl=1", url) return dl_url def upload_file_compress(self, filename=''): print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW) t = datetime.datetime.now() t.strftime("%Y%m%d%H%M%S") new_basename = os.path.basename(filename) with open(filename, 'rb') as f: self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + new_basename) link = self.dbx.sharing_create_shared_link_with_settings( DROPBOX_ROOT_PATH_NEW + '/' + new_basename) url = link.url dl_url = re.sub(r"\?dl\=0", "?dl=1", url) return dl_url def download_file(self, file=''): self.file = (DROPBOX_ROOT_PATH + '/' + file) file_name = self.file.replace('/sistemaweb/backup/', '') print('\nDownloading... /data/backup/' + file_name) try: metadata, res = self.dbx.files_download(self.file) except: pass metadata, res = self.dbx.files_download(file) final_path = ROOT_DIR + '/data/backup/' + file_name f = open(final_path, "wb") f.write(res.content) f.close() if '.zip' in final_path or '.gz' in final_path: print('Arquivo compactado...efetuando descompressão de dados.') self.uncompress_file(final_path) new_basename = os.path.basename(final_path).replace('.gz', '') return new_basename return final_path def list_dirs_root_path(self): self.dt = self.dbx.files_list_folder(self.root_path) #print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n') dir = self.download_file(self.dt.entries[-1].path_display) #self.list_files_root_path(self.dt) return dir def list_files_all(self): self.dt = self.dbx.files_list_folder(self.root_path) #print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n') self.data = [] for entry in self.dt.entries: data = {} data['backup_link'] data['client_modified'] = entry.client_modified data['size'] = str(entry.size)+" bytes" t = entry.client_modified time = datetime.timedelta(hours=2) hora = datetime.datetime.strptime(str(t), '%Y-%m-%d %H:%M:%S') now = hora - time size = entry.size size = str(size)+' bytes' display = entry.path_display print(display, now , size) self.data.append(data) return self.data def simple_backup(self): g = dbbackup.get_connector() execute_from_command_line(["manage.py", "dbbackup", "-v", "1"]) filename = (ROOT_DIR + '/data/backup/' + g.generate_filename()) #print(filename) return filename def compress_file(self, filename='', n=''): if n == '': self.compress_all(filename) elif n == '1': file = self.compress_all(filename) action = self.upload_file_compress(file) return action def compress_all(self, filename): if 'C:' in filename: new_basename = os.path.basename(filename) file = (ROOT_DIR + '/data/backup/' + new_basename + '.gz') f = open(filename, 'rb') data = f.read() f = gzip.open(file, 'wb') f.write(data) f.close() #print('Arquivo compactado com sucesso!!!') return file else: filepath = (ROOT_DIR + '/data/backup/' + filename) file = (ROOT_DIR + '/data/backup/' + filename + '.gz') f = open(filepath, 'rb') data = f.read() f = gzip.open(file, 'wb') f.write(data) f.close() #print('Arquivo compactado com sucesso!!!') return file def uncompress_file(self, filename): if 'C:' in filename: new_basename = os.path.basename(filename).replace('.gz', '') local_file = (ROOT_DIR + '/data/backup/' + new_basename) f = gzip.open(filename, 'rb') data = f.read() f.close() f = open(local_file, 'wb') f.write(data) f.close() print('Arquivo descomprimido com sucesso!!!') else: local_file = (ROOT_DIR + '/data/backup/' + filename) new_basename = os.path.basename(filename).replace('.gz', '') file = (ROOT_DIR + '/data/backup/' + new_basename) f = gzip.open(local_file, 'rb') data = f.read() f.close() f = open(file, 'wb') f.write(data) f.close() print('Arquivo descomprimido com sucesso!!!') def restore_db(self, filepath=''): print("VEJA O FILEPATH: ",filepath) execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", filepath]) def restore(self): # faz a restauração do banco de dados a partir de um backup salvo na dropbox. file_name = DropBoxStorage().list_dirs_root_path() new_basename = file_name if '/' in file_name: basename = shutil.copy(file_name, ROOT_DIR + '/data/backup/') new_basename = os.path.basename(basename) DropBoxStorage().restore_db(new_basename) from django.core.management import call_command import django django.setup() call_command('dbrestore', '-v', '1', "--noinput", "-i", new_basename) #execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", new_basename]) """
class ServiceDropbox(Service): def __init__(self, access_token): """ starts a new connection to dropbox :param str access_token: dbx access token """ self._access_token = access_token self._session = session() self._dbx = Dropbox(oauth2_access_token=self._access_token, session=self._session) def __del__(self): self._session.close() def exists(self, path): """ check whether file exists or not :param str path: file_path :return bool : True, if file exists or False if not :raise: ApiError on other dbx errors """ try: self._dbx.files_get_metadata(path) return True except ApiError as e: if e.error.get_path().is_not_found(): return False else: raise def delete(self, path): """ delete some dir or file :param str path: path to delete """ self._dbx.files_delete_v2(path) def dirs(self, path, recursive=True): """ return files and dirs in current folder :param str path: current folder :param bool recursive: go deeper? :return: list with files/dirs """ r = [] for entry in self._dbx.files_list_folder(path, recursive).entries: file_name_len = len(entry.name) if path.split("/")[-2] == entry.name: continue if isinstance(entry, FileMetadata): r.append(File(entry.name, entry.path_lower[0:-file_name_len].lstrip("/"), entry.client_modified, entry.client_modified)) elif isinstance(entry, FolderMetadata): r.append(Folder(entry.name, entry.path_lower[0:-file_name_len].lstrip("/"))) return r def chunk(self, path, filename, size, offset=0): """ return one chunk of file :param str path: path on server :param str filename: name of file :param int size: chunk-size :param int offset: bits from the beginning :return: tuple(File obj, content) """ p_session = session() dbx_p = Dropbox(oauth2_access_token=self._access_token, headers={ "Range": "bytes=" + str(offset) + "-" + str(offset + size - 1)}, session=p_session) # fetch chunks from dropbox meta, response = dbx_p.files_download(path+"/"+filename) f = File(meta.name, meta.path_lower, meta.client_modified, meta.client_modified) p_session.close() return f, response.content def file(self, path): """ imitates open-method of python by using context-manager, so you can use "with" statement :return: interateable object """ class _OpenStreamSession(ContextDecorator): def __init__(self, dbx): self._dbx = dbx self._data_offset = 0 self._cur = None def __enter__(self): self._sess_id = self._dbx.files_upload_session_start(b'').session_id return self def write(self, content): cur = UploadSessionCursor(self._sess_id, self._data_offset) self._dbx.files_upload_session_append_v2(content, cur) self._data_offset += len(content) def __exit__(self, exc_type, exc_val, exc_tb): commit = CommitInfo(path=path, mute=True) cur = UploadSessionCursor(self._sess_id, self._data_offset) self._dbx.files_upload_session_finish(b'', cur, commit) return _OpenStreamSession(self._dbx) def create_dir(self, path): """ create dir at specific location :param path: location to create dir """ self._dbx.files_create_folder_v2(path)
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox Public folder. """ def __init__(self, location='/Public'): self.client = Dropbox(ACCESS_TOKEN) self.account_info = self.client.users_get_current_account() self.location = location self.base_url = 'https://dl.dropboxusercontent.com/' def _get_abs_path(self, name): return os.path.realpath(os.path.join(self.location, name)) def _open(self, name, mode='rb'): name = self._get_abs_path(name) remote_file = DropboxFile(name, self, mode=mode) return remote_file def _save(self, name, content): name = self._get_abs_path(name) directory = os.path.dirname(name) if not self.exists(directory) and directory: self.client.files_create_folder(directory) # response = self.client.files_get_metadata(directory) # if not response['is_dir']: # raise IOError("%s exists and is not a directory." % directory) abs_name = os.path.realpath(os.path.join(self.location, name)) foo = self.client.files_upload(content.read(), abs_name) return name def delete(self, name): name = self._get_abs_path(name) self.client.files_delete(name) def exists(self, name): name = self._get_abs_path(name) try: self.client.files_get_metadata(name) except ApiError as e: if e.error.is_path() and e.error.get_path().is_not_found( ): # not found return False raise e return True def listdir(self, path): path = self._get_abs_path(path) response = self.client.files_list_folder(path) directories = [] files = [] for entry in response.entries: if type(entry) == FolderMetadata: directories.append(os.path.basename(entry.path_display)) elif type(entry) == FileMetadata: files.append(os.path.basename(entry.path_display)) return directories, files def size(self, name): cache_key = 'django-dropbox-size:{}'.format(filepath_to_uri(name)) size = cache.get(cache_key) if not size: size = self.client.files_get_metadata(name).size cache.set(cache_key, size, CACHE_TIMEOUT) return size def url(self, name): cache_key = 'django-dropbox-size:{}'.format(filepath_to_uri(name)) url = cache.get(cache_key) if not url: url = self.client.files_get_temporary_link(name).link cache.set(cache_key, url, SHARE_LINK_CACHE_TIMEOUT) return url def get_available_name(self, name): """ Returns a filename that's free on the target storage system, and available for new content to be written to. """ name = self._get_abs_path(name) dir_name, file_name = os.path.split(name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a number (before # the file extension, if one exists) to the filename until the generated # filename doesn't exist. count = itertools.count(1) while self.exists(name): # file_ext includes the dot. name = os.path.join( dir_name, "%s_%s%s" % (file_root, count.next(), file_ext)) return name
class Client(ApiConnection): def __init__(self, secrets=None, base=None, **kwargs): # update token from the environment name = self._update_secrets() super(ApiConnection, self).__init__(**kwargs) def _speak(self): '''if you want to add an extra print (of a parameter, for example) for the user when the client initalizes, write it here, eg: bot.info('[setting] value') ''' if hasattr(self, 'account'): bot.info('connected to %s' %self.account.name.display_name) def _get_metadata(self, image_file=None, dbx_metadata=None): '''this is a wrapper around the main client.get_metadata to first parse a Dropbox FileMetadata into a dicionary, then pass it on to the primary get_metadata function. Parameters ========== image_file: the full path to the image file that had metadata extracted metadata: the Dropbox FileMetadata to parse. ''' metadata = dict() if dbx_metadata is not None: for key in dbx_metadata.__dir__(): value = getattr(dbx_metadata, key) if type(value) in [str, datetime.datetime, bool, int, float]: metadata[key.strip('_')] = value return self.get_metadata(image_file, names=metadata) def _update_secrets(self): '''update secrets will look for a dropbox token in the environment at SREGISTRY_DROPBOX_TOKEN and if found, create a client. If not, an error message is returned and the client exits. ''' # Retrieve the user token. Exit if not found token = self._get_and_update_setting('SREGISTRY_DROPBOX_TOKEN') if token is None: bot.error('You must export SREGISTRY_DROPBOX_TOKEN to use client.') sys.exit(1) # Create the dropbox client self.dbx = Dropbox(token) # Verify that the account is valid try: self.account = self.dbx.users_get_current_account() except AuthError as err: bot.error('Account invalid. Exiting.') sys.exit(1) def __str__(self): return type(self) def exists(self, path): '''determine if a path exists, return False if not.''' try: self.dbx.files_get_metadata(path) return True except: return False
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox folder. """ def __init__(self, token=ACCESS_TOKEN, location=ROOT_FOLDER): if not token: raise ImproperlyConfigured("You must configure an access token at " "'settings.DROPBOX_ACCESS_TOKEN'.") self.client = Dropbox(token) self.account_info = self.client.users_get_current_account() self.location = location or DEFAULT_ROOT_FOLDER self.base_url = 'https://dl.dropboxusercontent.com/' def _get_abs_path(self, name): return os.path.realpath(os.path.join(self.location, name)) def _open(self, name, mode='rb'): name = self._get_abs_path(name) remote_file = DropboxFile(name, self, mode=mode) return remote_file def _save(self, name, content): name = self._get_abs_path(name) directory = os.path.dirname(name) if not self.exists(directory) and directory: self.client.files_create_folder(directory) # response = self.client.files_get_metadata(directory) # if not response['is_dir']: # raise IOError("%s exists and is not a directory." % directory) abs_name = os.path.realpath(os.path.join(self.location, name)) self.client.files_upload(content.read(), abs_name) return name def delete(self, name): name = self._get_abs_path(name) try: self.client.files_delete(name) except ApiError as e: if isinstance(e.error, DeleteError)\ and e.error.is_path_lookup()\ and e.error.get_path_lookup().is_not_found(): # not found return False # error raise e # deleted return True def exists(self, name): name = self._get_abs_path(name) try: self.client.files_get_metadata(name) except ApiError as e: if hasattr(e.error, 'is_path')\ and e.error.is_path()\ and e.error.get_path().is_not_found(): # not found return False # error raise e # found return True def listdir(self, path): path = self._get_abs_path(path) response = self.client.files_list_folder(path) directories = [] files = [] for entry in response.entries: if isinstance(entry, FolderMetadata): directories.append(os.path.basename(entry.path_display)) elif isinstance(entry, FileMetadata): files.append(os.path.basename(entry.path_display)) return directories, files def size(self, name): name = self._get_abs_path(name) return self.client.files_get_metadata(name).size def url(self, name): name = self._get_abs_path(name) return self.client.files_get_temporary_link(name).link def modified_time(self, name): name = self._get_abs_path(name) return self.client.files_get_metadata(name).server_modified def accessed_time(self, name): name = self._get_abs_path(name) # Note to the unwary, this is actually an mtime return self.client.files_get_metadata(name).client_modified def get_available_name(self, name, max_length=None): """ Returns a filename that's free on the target storage system, and available for new content to be written to. """ name = self._get_abs_path(name) dir_name, file_name = os.path.split(name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a number (before # the file extension, if one exists) to the filename until the generated # filename doesn't exist. count = itertools.count(1) while self.exists(name): # file_ext includes the dot. _fn = "%s_%s%s" % (file_root, count.next(), file_ext) name = os.path.join(dir_name, _fn) return name
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None, timeout=None): oauth2_access_token = oauth2_access_token or setting( 'DROPBOX_OAUTH2_TOKEN') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') timeout = timeout or setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) if full_path == '/': full_path = '' metadata = self.client.files_list_folder(full_path) for entry in metadata.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.client_modified def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
class DropboxFS(FS): def __init__(self, accessToken): super().__init__() self.dropbox = Dropbox(accessToken) _meta = self._meta = { "case_insensitive": False, # I think? "invalid_path_chars": ":", # not sure what else "max_path_length": None, # don't know what the limit is "max_sys_path_length": None, # there's no syspath "network": True, "read_only": False, "supports_rename": False # since we don't have a syspath... } def __repr__(self): return "<DropboxDriveFS>" def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use rawInfo = { "basic": { "name": metadata.name, "is_dir": isinstance(metadata, FolderMetadata), } } if isinstance(metadata, FileMetadata): rawInfo.update({ "details": { "accessed": None, # not supported by Dropbox API "created": None, # not supported by Dropbox API?, "metadata_changed": None, # not supported by Dropbox "modified": datetime_to_epoch( metadata.server_modified ), # API documentation says that this is reliable "size": metadata.size, "type": 0 }, "dropbox": { "content_hash": metadata. content_hash, # see https://www.dropbox.com/developers/reference/content-hash "rev": metadata.rev, "client_modified": metadata. client_modified # unverified value coming from dropbox clients } }) if metadata.media_info is not None and metadata.media_info.is_metadata( ) is True: media_info_metadata = metadata.media_info.get_metadata() if media_info_metadata.time_taken is not None: rawInfo.update({ "media_info": { "taken_date_time": datetime_to_epoch(media_info_metadata.time_taken) } }) if media_info_metadata.location is not None: rawInfo.update({ "media_info": { "location_latitude": media_info_metadata.location.latitude, "location_longitude": media_info_metadata.location.longitude } }) # Dropbox doesn't parse some jpgs properly if media_info_metadata.dimensions is not None: rawInfo.update({ "media_info": { "dimensions_height": media_info_metadata.dimensions.height, "dimensions_width": media_info_metadata.dimensions.width } }) elif isinstance(metadata, FolderMetadata): rawInfo.update({ "details": { "accessed": None, # not supported by Dropbox API "created": None, # not supported by Dropbox API, "metadata_changed": None, # not supported by Dropbox "modified": None, # not supported for folders "size": None, # not supported for folders "type": 1 } }) else: assert False, f"{metadata.name}, {metadata}, {type(metadata)}" return Info(rawInfo) def getinfo(self, path, namespaces=None): if path == "/": return Info({"basic": {"name": "", "is_dir": True}}) try: if not path.startswith("/"): path = "/" + path metadata = self.dropbox.files_get_metadata(path, include_media_info=True) except ApiError as e: raise ResourceNotFound(path=path, exc=e) return self._infoFromMetadata(metadata) def setinfo(self, path, info): # pylint: disable=too-many-branches # dropbox doesn't support changing any of the metadata values pass def listdir(self, path): return [x.name for x in self.scandir(path)] def makedir(self, path, permissions=None, recreate=False): try: folderMetadata = self.dropbox.files_create_folder(path) except ApiError as e: assert isinstance(e.reason, CreateFolderError) # TODO - there are other possibilities raise DirectoryExpected(path=path) # don't need to close this filesystem so we return the non-closing version return SubFS(self, path) def openbin(self, path, mode="r", buffering=-1, **options): mode = Mode(mode) exists = True isDir = False try: isDir = self.getinfo(path).is_dir except ResourceNotFound: exists = False if mode.exclusive and exists: raise FileExists(path) elif mode.reading and not mode.create and not exists: raise ResourceNotFound(path) elif isDir: raise FileExpected(path) return DropboxFile(self.dropbox, path, mode) def remove(self, path): try: self.dropbox.files_delete(path) except ApiError as e: raise FileExpected(path=path, exc=e) def removedir(self, path): try: self.dropbox.files_delete(path) except ApiError as e: assert e.reason is DeleteError raise DirectoryExpected(path=path, exc=e) # non-essential method - for speeding up walk def scandir(self, path, namespaces=None, page=None): # if path == "/": path = "" # get all the avaliable metadata since it's cheap # TODO - this call has a recursive flag so we can either use that and cache OR override walk result = self.dropbox.files_list_folder(path, include_media_info=True) allEntries = result.entries while result.has_more: result = self.dropbox.files_list_folder_continue(result.cursor) allEntries += result.entries return [self._infoFromMetadata(x) for x in allEntries]
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting( 'DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, path): path = PurePosixPath(self.root_path) / path path = str(path) if path == '/': path = '' return path def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) result = self.client.files_list_folder(full_path) for entry in result.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) assert not result.has_more, "FIXME: Not implemented!" return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) # Note to the unwary, this is actually an mtime return metadata.client_modified def url(self, name): try: media = self.client.files_get_temporary_link(self._full_path(name)) return media.link except ApiError: raise ValueError("This file is not accessible via a URL.") def _open(self, name, mode='rb'): return DropBoxFile(self._full_path(name), self) def _save(self, name, content): try: content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) finally: content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
class DPBXBackend(duplicity.backend.Backend): """Connect to remote store using Dr*pB*x service""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) self.api_account = None self.api_client = None self.auth_flow = None self.login() def user_authenticated(self): try: account = self.api_client.users_get_current_account() log.Debug("User authenticated as ,%s" % account) return True except: log.Debug('User not authenticated') return False def load_access_token(self): return os.environ.get('DPBX_ACCESS_TOKEN', None) def save_access_token(self, access_token): raise BackendException( 'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token) def obtain_access_token(self): log.Info("dpbx: trying to obtain access token") for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']: if env_var not in os.environ: raise BackendException( 'dpbx: %s environment variable not set' % env_var) app_key = os.environ['DPBX_APP_KEY'] app_secret = os.environ['DPBX_APP_SECRET'] if not sys.stdout.isatty() or not sys.stdin.isatty(): log.FatalError( 'dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error) auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret) log.Debug('dpbx,auth_flow.start()') authorize_url = auth_flow.start() print print '-' * 72 print "1. Go to: " + authorize_url print "2. Click \"Allow\" (you might have to log in first)." print "3. Copy the authorization code." print '-' * 72 auth_code = raw_input("Enter the authorization code here: ").strip() try: log.Debug('dpbx,auth_flow.finish(%s)' % auth_code) authresult = auth_flow.finish(auth_code) except Exception as e: raise BackendException('dpbx: Unable to obtain access token: %s' % e) log.Info("dpbx: Authentication successfull") self.save_access_token(authresult.access_token) def login(self): if self.load_access_token() is None: self.obtain_access_token() self.api_client = Dropbox(self.load_access_token()) self.api_account = None try: log.Debug('dpbx,users_get_current_account([token])') self.api_account = self.api_client.users_get_current_account() log.Debug("dpbx,%s" % self.api_account) except (BadInputError, AuthError) as e: log.Debug('dpbx,exception: %s' % e) log.Info( "dpbx: Authentication failed. Trying to obtain new access token" ) self.obtain_access_token() # We're assuming obtain_access_token will throw exception. # So this line should not be reached raise BackendException( "dpbx: Please update DPBX_ACCESS_TOKEN and try again") log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name) def _error_code(self, operation, e): if isinstance(e, ApiError): err = e.error if isinstance(err, GetMetadataError) and err.is_path(): if err.get_path().is_not_found(): return log.ErrorCode.backend_not_found elif isinstance(err, DeleteError) and err.is_path_lookup(): lookup = e.error.get_path_lookup() if lookup.is_not_found(): return log.ErrorCode.backend_not_found @command() def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) progress.report_transfer(0, file_size) if file_size < DPBX_UPLOAD_CHUNK_SIZE: # Upload whole file at once to avoid extra server request res_metadata = self.put_file_small(source_path, remote_path) else: res_metadata = self.put_file_chunked(source_path, remote_path) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException( 'dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException( 'dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) def put_file_small(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: log.Debug('dpbx,files_upload(%s, [%d bytes])' % (remote_path, file_size)) res_metadata = self.api_client.files_upload( f.read(), remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) log.Debug('dpbx,files_upload(): %s' % res_metadata) progress.report_transfer(file_size, file_size) return res_metadata finally: f.close() def put_file_chunked(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug( 'dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries is_eof = False # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed # chunk while not is_eof or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) is_eof = f.tell() >= file_size if not is_eof and len(buf) == 0: continue # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if not is_eof: assert len(buf) != 0 log.Debug( 'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append( buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug( 'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish( buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError ) and error.is_incorrect_offset(): # Server reports that we should send another chunk. # Most likely this is caused by network error during # previous upload attempt. In such case we'll get # expected offset from server and it's enough to just # seek() and retry again new_offset = error.get_incorrect_offset( ).correct_offset log.Debug( 'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something # strange and no safe way to recover raise BackendException( "dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if not self.user_authenticated(): self.login() if retry_number == 0: raise # We don't know for sure, was partial upload successful or # not. So it's better to retry smaller amount to avoid extra # reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) return res_metadata finally: f.close() @command() def _get(self, remote_filename, local_path): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() log.Debug('dpbx,files_download(%s)' % remote_path) res_metadata, http_fd = self.api_client.files_download(remote_path) log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd)) file_size = res_metadata.size to_fd = None progress.report_transfer(0, file_size) try: to_fd = local_path.open('wb') for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE): to_fd.write(c) progress.report_transfer(to_fd.tell(), file_size) finally: if to_fd: to_fd.close() http_fd.close() # It's different from _query() check because we're not querying metadata # again. Since this check is free, it's better to have it here local_size = os.path.getsize(local_path.name) if local_size != file_size: raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size)) local_path.setdata() @command() def _list(self): # Do a long listing to avoid connection reset if not self.user_authenticated(): self.login() remote_dir = '/' + urllib.unquote( self.parsed_url.path.lstrip('/')).rstrip() log.Debug('dpbx.files_list_folder(%s)' % remote_dir) res = [] try: resp = self.api_client.files_list_folder(remote_dir) log.Debug('dpbx.list(%s): %s' % (remote_dir, resp)) while True: res.extend([entry.name for entry in resp.entries]) if not resp.has_more: break resp = self.api_client.files_list_folder_continue(resp.cursor) except ApiError as e: if (isinstance(e.error, ListFolderError) and e.error.is_path() and e.error.get_path().is_not_found()): log.Debug('dpbx.list(%s): ignore missing folder (%s)' % (remote_dir, e)) else: raise # Warn users of old version dpbx about automatically renamed files self.check_renamed_files(res) return res @command() def _delete(self, filename): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_delete(%s)' % remote_path) self.api_client.files_delete(remote_path) # files_permanently_delete seems to be better for backup purpose # but it's only available for Business accounts # self.api_client.files_permanently_delete(remote_path) @command() def _close(self): """close backend session? no! just "flush" the data""" log.Debug('dpbx.close():') @command() def _query(self, filename): if not self.user_authenticated(): self.login() remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, filename).rstrip() log.Debug('dpbx.files_get_metadata(%s)' % remote_path) info = self.api_client.files_get_metadata(remote_path) log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info)) return {'size': info.size} def check_renamed_files(self, file_list): if not self.user_authenticated(): self.login() bad_list = [ x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None ] if len(bad_list) == 0: return log.Warn('-' * 72) log.Warn( 'Warning! It looks like there are automatically renamed files on backend' ) log.Warn( 'They were probably created when using older version of duplicity.' ) log.Warn('') log.Warn( 'Please check your backup consistency. Most likely you will need to choose' ) log.Warn( 'largest file from duplicity-* (number).gpg and remove brackets from its name.' ) log.Warn('') log.Warn( 'These files are not managed by duplicity at all and will not be') log.Warn('removed/rotated automatically.') log.Warn('') log.Warn('Affected files:') for x in bad_list: log.Warn('\t%s' % x) log.Warn('') log.Warn('In any case it\'s better to create full backup.') log.Warn('-' * 72)
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox Public folder. """ def __init__(self, location='/Public'): self.client = Dropbox(ACCESS_TOKEN) self.account_info = self.client.users_get_current_account() self.location = location self.base_url = 'https://dl.dropboxusercontent.com/' def _get_abs_path(self, name): return os.path.realpath(os.path.join(self.location, name)) def _open(self, name, mode='rb'): name = self._get_abs_path(name) remote_file = DropboxFile(name, self, mode=mode) return remote_file def _save(self, name, content): name = self._get_abs_path(name) directory = os.path.dirname(name) if not self.exists(directory) and directory: self.client.files_create_folder(directory) # response = self.client.files_get_metadata(directory) # if not response['is_dir']: # raise IOError("%s exists and is not a directory." % directory) abs_name = os.path.realpath(os.path.join(self.location, name)) foo = self.client.files_upload(content.read(), abs_name) return name def delete(self, name): name = self._get_abs_path(name) self.client.files_delete(name) def exists(self, name): name = self._get_abs_path(name) try: self.client.files_get_metadata(name) except ApiError as e: if e.error.is_path() and e.error.get_path().is_not_found(): # not found return False raise e return True def listdir(self, path): path = self._get_abs_path(path) response = self.client.files_list_folder(path) directories = [] files = [] for entry in response.entries: if type(entry) == FolderMetadata: directories.append(os.path.basename(entry.path_display)) elif type(entry) == FileMetadata: files.append(os.path.basename(entry.path_display)) return directories, files def size(self, name): cache_key = 'django-dropbox-size:{}'.format(filepath_to_uri(name)) size = cache.get(cache_key) if not size: size = self.client.files_get_metadata(name).size cache.set(cache_key, size, CACHE_TIMEOUT) return size def url(self, name): if name.startswith(self.location): name = name[len(self.location) + 1:] name = os.path.basename(self.location) + "/" + name if self.base_url is None: raise ValueError("This file is not accessible via a URL.") myurl = urlparse.urljoin(self.base_url, filepath_to_uri(name)) if "static" not in self.location: # Use a dynamic URL for "non-static" files. try: new_name = os.path.dirname(self.location) + "/" + name fp = filepath_to_uri(new_name) cache_key = 'django-dropbox-size:{}'.format(fp) myurl = cache.get(cache_key) if not myurl: try: shared_link = self.client.sharing_create_shared_link(fp) myurl = shared_link.url + '&raw=1' logger.debug("shared link: {0}, myurl: {1}".format(shared_link, myurl)) except Exception,e: logger.exception(e) if myurl is None: temp_link = self.client.files_get_temporary_link(fp) myurl = temp_link.link logger.debug("temp link: {0}, myurl: {1}".format(temp_link, myurl)) cache.set(cache_key, myurl, SHARE_LINK_CACHE_TIMEOUT) except Exception,e: logger.exception(e) return myurl """