class DropBoxDataProvider(DataProviderBase): smoke_url = DROPBOX_SMOKE_URL def __init__(self, acs_token): self.dbx = Dropbox(acs_token) def api_smoke(self) -> int: return len(self.dbx.files_list_folder('').entries) def get_list_of_objects(self, dbx_folder='') -> list: result = namedtuple('Result', ['filename', 'filepatch']) return [ result(el.name, el.path_lower) for el in self.dbx.files_list_folder(dbx_folder).entries ] def file_delete(self, dbx_file) -> str: return self.dbx.files_delete_v2(dbx_file).metadata.path_lower def file_download(self, local_file, dbx_file) -> str: return self.dbx.files_download_to_file(local_file, dbx_file).path_lower def file_upload(self, local_file, dbx_file) -> str: if isinstance(local_file, str): if local_file.startswith("https://"): waiting_time = 0.1 waiting_attempt = 100 url_result = self.dbx.files_save_url(dbx_file, local_file) job_id = url_result.get_async_job_id() while waiting_attempt > 0: st = self.dbx.files_save_url_check_job_status(job_id) if st.is_complete(): return st.get_complete().path_lower sleep(waiting_time) waiting_attempt -= 1 else: with open(local_file, 'rb') as f: return self.dbx.files_upload( f.read(), dbx_file, autorename=True, strict_conflict=True).path_lower else: return self.dbx.files_upload(local_file.read(), dbx_file, autorename=True, strict_conflict=True).path_lower def file_move(self, dbx_file_from, dbx_file_to) -> str: return self.dbx.files_move_v2(dbx_file_from, dbx_file_to).metadata.path_lower def create_folder(self, dbx_folder) -> str: return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower def get_file_tmp_link(self, dbx_path) -> str: return self.dbx.files_get_temporary_link(dbx_path).link
class DropboxFS(FS): _meta = { "case_insensitive": False, "invalid_path_chars": "\0", "network": True, "read_only": False, "thread_safe": True, "unicode_paths": True, "virtual": False, } def __init__(self, accessToken, session=None): super(DropboxFS, self).__init__() self._lock = threading.RLock() self.dropbox = Dropbox(accessToken, session=session) def fix_path(self, path): if isinstance(path, bytes): try: path = path.decode("utf-8") except AttributeError: pass if not path.startswith("/"): path = "/" + path if path == "." or path == "./": path = "/" path = self.validatepath(path) return path def __repr__(self): return "<DropboxDriveFS>" def _infoFromMetadata(self, metadata): rawInfo = { "basic": { "name": metadata.name, "is_dir": isinstance(metadata, FolderMetadata), } } if isinstance(metadata, FileMetadata): rawInfo.update( {"details": {"size": metadata.size, "type": ResourceType.file}} ) else: rawInfo.update({"details": {"type": ResourceType.directory}}) return Info(rawInfo) def getinfo(self, path, namespaces=None): _path = self.fix_path(path) if _path == "/": info_dict = { "basic": {"name": "", "is_dir": True}, "details": {"type": ResourceType.directory}, } return Info(info_dict) try: metadata = self.dropbox.files_get_metadata( _path, include_media_info=True ) except ApiError as e: raise errors.ResourceNotFound(path=path, exc=e) return self._infoFromMetadata(metadata) def setinfo(self, path, info): if not self.exists(path): raise errors.ResourceNotFound(path) def listdir(self, path): _path = self.fix_path(path) if _path == "/": _path = "" if not self.exists(_path): raise errors.ResourceNotFound(path) meta = self.getinfo(_path) if meta.is_file: raise errors.DirectoryExpected(path) result = self.dropbox.files_list_folder(_path, include_media_info=True) allEntries = result.entries while result.has_more: result = self.dropbox.files_list_folder_continue(result.cursor) allEntries += result.entries return [x.name for x in allEntries] def makedir(self, path, permissions=None, recreate=False): path = self.fix_path(path) if self.exists(path) and not recreate: raise errors.DirectoryExists(path) if path == "/": return SubFS(self, path) if self.exists(path): meta = self.getinfo(path) if meta.is_dir: if recreate == False: raise errors.DirectoryExists(path) else: return SubFS(self, path) if meta.is_file: raise errors.DirectoryExpected(path) ppath = self.get_parent(path) if not self.exists(ppath): raise errors.ResourceNotFound(ppath) try: folderMetadata = self.dropbox.files_create_folder_v2(path) except ApiError as e: raise errors.DirectoryExpected(path=path) return SubFS(self, path) def openbin(self, path, mode="r", buffering=-1, **options): path = self.fix_path(path) _mode = Mode(mode) mode = _mode _mode.validate_bin() _path = self.validatepath(path) log.debug("openbin: %s, %s", path, mode) with self._lock: try: info = self.getinfo(_path) log.debug("Info: %s", info) except errors.ResourceNotFound: if not _mode.create: raise errors.ResourceNotFound(path) # Check the parent is an existing directory if not self.getinfo(self.get_parent(_path)).is_dir: raise errors.DirectoryExpected(path) else: if info.is_dir: raise errors.FileExpected(path) if _mode.exclusive: raise errors.FileExists(path) return DropboxFile(self.dropbox, path, mode) def remove(self, path): _path = self.fix_path(path) try: info = self.getinfo(path) if info.is_dir: raise errors.FileExpected(path=path) self.dropbox.files_delete_v2(_path) except ApiError as e: if isinstance(e.error._value, LookupError): raise errors.ResourceNotFound(path=path) log.debug(e) raise errors.FileExpected(path=path, exc=e) def removedir(self, path): _path = self.fix_path(path) if _path == "/": raise errors.RemoveRootError() try: info = self.getinfo(path) if not info.is_dir: raise errors.DirectoryExpected(path=path) if len(self.listdir(path)) > 0: raise errors.DirectoryNotEmpty(path=path) self.dropbox.files_delete_v2(_path) except ApiError as e: if isinstance(e.error._value, LookupError): raise errors.ResourceNotFound(path=path) raise errors.FileExpected(path=path, exc=e) def copy(self, src_path, dst_path, overwrite=False): src_path = self.fix_path(src_path) dst_path = self.fix_path(dst_path) try: src_meta = self.getinfo(src_path) if src_meta.is_dir: raise errors.FileExpected(src_path) except ApiError as e: raise errors.ResourceNotFound dst_meta = None try: dst_meta = self.getinfo(dst_path) except Exception as e: pass if dst_meta is not None: if overwrite == True: self.remove(dst_path) else: raise errors.DestinationExists(dst_path) parent_path = self.get_parent(dst_path) if not self.exists(parent_path): raise errors.ResourceNotFound(dst_path) self.dropbox.files_copy_v2(src_path, dst_path) def get_parent(self, dst_path): import os parent_path = os.path.abspath(os.path.join(dst_path, "..")) return parent_path def exists(self, path): path = self.fix_path(path) try: self.getinfo(path) return True except Exception as e: return False def move(self, src_path, dst_path, overwrite=False): _src_path = self.fix_path(src_path) _dst_path = self.fix_path(dst_path) if not self.getinfo(_src_path).is_file: raise errors.FileExpected(src_path) if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) if "/" in dst_path and not self.exists(self.get_parent(_dst_path)): raise errors.ResourceNotFound(src_path) with self._lock: try: if overwrite: try: # remove file anyways self.dropbox.files_delete_v2(_dst_path) except Exception as e: pass self.dropbox.files_move_v2(_src_path, _dst_path) except ApiError as e: raise errors.ResourceNotFound(src_path, exc=e) def apierror_map(self, error): log.debug(error) def geturl(self, path, purpose='download'): url = self.dropbox.sharing_create_shared_link(path).url url = url.replace('?dl=0', '?raw=1') return url