class DropBoxDataProvider(DataProviderBase):
smoke_url = DROPBOX_SMOKE_URL
def __init__(self, acs_token):
self.dbx = Dropbox(acs_token)
def api_smoke(self) -> int:
return len(self.dbx.files_list_folder('').entries)
def get_list_of_objects(self, dbx_folder='') -> list:
result = namedtuple('Result', ['filename', 'filepatch'])
return [
result(el.name, el.path_lower)
for el in self.dbx.files_list_folder(dbx_folder).entries
]
def file_delete(self, dbx_file) -> str:
return self.dbx.files_delete_v2(dbx_file).metadata.path_lower
def file_download(self, local_file, dbx_file) -> str:
return self.dbx.files_download_to_file(local_file, dbx_file).path_lower
def file_upload(self, local_file, dbx_file) -> str:
if isinstance(local_file, str):
if local_file.startswith("https://"):
waiting_time = 0.1
waiting_attempt = 100
url_result = self.dbx.files_save_url(dbx_file, local_file)
job_id = url_result.get_async_job_id()
while waiting_attempt > 0:
st = self.dbx.files_save_url_check_job_status(job_id)
if st.is_complete():
return st.get_complete().path_lower
sleep(waiting_time)
waiting_attempt -= 1
else:
with open(local_file, 'rb') as f:
return self.dbx.files_upload(
f.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
else:
return self.dbx.files_upload(local_file.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
def file_move(self, dbx_file_from, dbx_file_to) -> str:
return self.dbx.files_move_v2(dbx_file_from,
dbx_file_to).metadata.path_lower
def create_folder(self, dbx_folder) -> str:
return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower
def get_file_tmp_link(self, dbx_path) -> str:
return self.dbx.files_get_temporary_link(dbx_path).link
class DropboxFS(FS):
_meta = {
"case_insensitive": False,
"invalid_path_chars": "\0",
"network": True,
"read_only": False,
"thread_safe": True,
"unicode_paths": True,
"virtual": False,
}
def __init__(self, accessToken, session=None):
super(DropboxFS, self).__init__()
self._lock = threading.RLock()
self.dropbox = Dropbox(accessToken, session=session)
def fix_path(self, path):
if isinstance(path, bytes):
try:
path = path.decode("utf-8")
except AttributeError:
pass
if not path.startswith("/"):
path = "/" + path
if path == "." or path == "./":
path = "/"
path = self.validatepath(path)
return path
def __repr__(self):
return "<DropboxDriveFS>"
def _infoFromMetadata(self, metadata):
rawInfo = {
"basic": {
"name": metadata.name,
"is_dir": isinstance(metadata, FolderMetadata),
}
}
if isinstance(metadata, FileMetadata):
rawInfo.update(
{"details": {"size": metadata.size, "type": ResourceType.file}}
)
else:
rawInfo.update({"details": {"type": ResourceType.directory}})
return Info(rawInfo)
def getinfo(self, path, namespaces=None):
_path = self.fix_path(path)
if _path == "/":
info_dict = {
"basic": {"name": "", "is_dir": True},
"details": {"type": ResourceType.directory},
}
return Info(info_dict)
try:
metadata = self.dropbox.files_get_metadata(
_path, include_media_info=True
)
except ApiError as e:
raise errors.ResourceNotFound(path=path, exc=e)
return self._infoFromMetadata(metadata)
def setinfo(self, path, info):
if not self.exists(path):
raise errors.ResourceNotFound(path)
def listdir(self, path):
_path = self.fix_path(path)
if _path == "/":
_path = ""
if not self.exists(_path):
raise errors.ResourceNotFound(path)
meta = self.getinfo(_path)
if meta.is_file:
raise errors.DirectoryExpected(path)
result = self.dropbox.files_list_folder(_path, include_media_info=True)
allEntries = result.entries
while result.has_more:
result = self.dropbox.files_list_folder_continue(result.cursor)
allEntries += result.entries
return [x.name for x in allEntries]
def makedir(self, path, permissions=None, recreate=False):
path = self.fix_path(path)
if self.exists(path) and not recreate:
raise errors.DirectoryExists(path)
if path == "/":
return SubFS(self, path)
if self.exists(path):
meta = self.getinfo(path)
if meta.is_dir:
if recreate == False:
raise errors.DirectoryExists(path)
else:
return SubFS(self, path)
if meta.is_file:
raise errors.DirectoryExpected(path)
ppath = self.get_parent(path)
if not self.exists(ppath):
raise errors.ResourceNotFound(ppath)
try:
folderMetadata = self.dropbox.files_create_folder_v2(path)
except ApiError as e:
raise errors.DirectoryExpected(path=path)
return SubFS(self, path)
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if not self.getinfo(self.get_parent(_path)).is_dir:
raise errors.DirectoryExpected(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def remove(self, path):
_path = self.fix_path(path)
try:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
log.debug(e)
raise errors.FileExpected(path=path, exc=e)
def removedir(self, path):
_path = self.fix_path(path)
if _path == "/":
raise errors.RemoveRootError()
try:
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path=path)
if len(self.listdir(path)) > 0:
raise errors.DirectoryNotEmpty(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
raise errors.FileExpected(path=path, exc=e)
def copy(self, src_path, dst_path, overwrite=False):
src_path = self.fix_path(src_path)
dst_path = self.fix_path(dst_path)
try:
src_meta = self.getinfo(src_path)
if src_meta.is_dir:
raise errors.FileExpected(src_path)
except ApiError as e:
raise errors.ResourceNotFound
dst_meta = None
try:
dst_meta = self.getinfo(dst_path)
except Exception as e:
pass
if dst_meta is not None:
if overwrite == True:
self.remove(dst_path)
else:
raise errors.DestinationExists(dst_path)
parent_path = self.get_parent(dst_path)
if not self.exists(parent_path):
raise errors.ResourceNotFound(dst_path)
self.dropbox.files_copy_v2(src_path, dst_path)
def get_parent(self, dst_path):
import os
parent_path = os.path.abspath(os.path.join(dst_path, ".."))
return parent_path
def exists(self, path):
path = self.fix_path(path)
try:
self.getinfo(path)
return True
except Exception as e:
return False
def move(self, src_path, dst_path, overwrite=False):
_src_path = self.fix_path(src_path)
_dst_path = self.fix_path(dst_path)
if not self.getinfo(_src_path).is_file:
raise errors.FileExpected(src_path)
if not overwrite and self.exists(_dst_path):
raise errors.DestinationExists(dst_path)
if "/" in dst_path and not self.exists(self.get_parent(_dst_path)):
raise errors.ResourceNotFound(src_path)
with self._lock:
try:
if overwrite:
try:
# remove file anyways
self.dropbox.files_delete_v2(_dst_path)
except Exception as e:
pass
self.dropbox.files_move_v2(_src_path, _dst_path)
except ApiError as e:
raise errors.ResourceNotFound(src_path, exc=e)
def apierror_map(self, error):
log.debug(error)
def geturl(self, path, purpose='download'):
url = self.dropbox.sharing_create_shared_link(path).url
url = url.replace('?dl=0', '?raw=1')
return url
class ServiceDropbox(Service):
def __init__(self, access_token):
"""
starts a new connection to dropbox
:param str access_token: dbx access token
"""
self._access_token = access_token
self._session = session()
self._dbx = Dropbox(oauth2_access_token=self._access_token, session=self._session)
def __del__(self):
self._session.close()
def exists(self, path):
"""
check whether file exists or not
:param str path: file_path
:return bool : True, if file exists or False if not
:raise: ApiError on other dbx errors
"""
try:
self._dbx.files_get_metadata(path)
return True
except ApiError as e:
if e.error.get_path().is_not_found():
return False
else:
raise
def delete(self, path):
"""
delete some dir or file
:param str path: path to delete
"""
self._dbx.files_delete_v2(path)
def dirs(self, path, recursive=True):
"""
return files and dirs in current folder
:param str path: current folder
:param bool recursive: go deeper?
:return: list with files/dirs
"""
r = []
for entry in self._dbx.files_list_folder(path, recursive).entries:
file_name_len = len(entry.name)
if path.split("/")[-2] == entry.name:
continue
if isinstance(entry, FileMetadata):
r.append(File(entry.name, entry.path_lower[0:-file_name_len].lstrip("/"), entry.client_modified, entry.client_modified))
elif isinstance(entry, FolderMetadata):
r.append(Folder(entry.name, entry.path_lower[0:-file_name_len].lstrip("/")))
return r
def chunk(self, path, filename, size, offset=0):
"""
return one chunk of file
:param str path: path on server
:param str filename: name of file
:param int size: chunk-size
:param int offset: bits from the beginning
:return: tuple(File obj, content)
"""
p_session = session()
dbx_p = Dropbox(oauth2_access_token=self._access_token, headers={
"Range": "bytes=" + str(offset) + "-" + str(offset + size - 1)}, session=p_session) # fetch chunks from dropbox
meta, response = dbx_p.files_download(path+"/"+filename)
f = File(meta.name, meta.path_lower, meta.client_modified, meta.client_modified)
p_session.close()
return f, response.content
def file(self, path):
"""
imitates open-method of python by using context-manager,
so you can use "with" statement
:return: interateable object
"""
class _OpenStreamSession(ContextDecorator):
def __init__(self, dbx):
self._dbx = dbx
self._data_offset = 0
self._cur = None
def __enter__(self):
self._sess_id = self._dbx.files_upload_session_start(b'').session_id
return self
def write(self, content):
cur = UploadSessionCursor(self._sess_id, self._data_offset)
self._dbx.files_upload_session_append_v2(content, cur)
self._data_offset += len(content)
def __exit__(self, exc_type, exc_val, exc_tb):
commit = CommitInfo(path=path, mute=True)
cur = UploadSessionCursor(self._sess_id, self._data_offset)
self._dbx.files_upload_session_finish(b'', cur, commit)
return _OpenStreamSession(self._dbx)
def create_dir(self, path):
"""
create dir at specific location
:param path: location to create dir
"""
self._dbx.files_create_folder_v2(path)
def get_or_create_folder():
dbx = Dropbox(get_token())
try:
dbx.files_create_folder_v2('/Mainstay', autorename=False)
except ApiError:
pass
