def tearDownClass(cls):
# Clear sample files for tests
shutil.rmtree(cls.sample_dir)
# Clear Dropbox test files
if 'DROPBOX_TOKEN' in os.environ.keys():
dbx = Dropbox(os.environ['DROPBOX_TOKEN'])
try:
dbx.files_delete_v2('/' + cls.remote_test_dir)
except ApiError as e:
if not isinstance(e.error, DeleteError) \
or not e.error.is_path_lookup():
raise e
# Clear Google Drive test files
if os.path.isfile(cls.gdrive_creds_filepath):
gauth = cls.create_gdrive_auth()
gdrive = GoogleDrive(gauth)
query = "mimeType = 'application/vnd.google-apps.folder'" \
+ " and title = '{dir}' and trashed=false"
query = query.format(dir=cls.remote_test_dir)
folders = gdrive.ListFile({"q": query}).GetList()
if len(folders) == 1:
folder = folders[0]
query = "'{folder_id}' in parents and trashed=false"
files = gdrive.ListFile({
"q":
query.format(folder_id=folder['id'])
}).GetList()
for file_to_delete in files:
file_to_delete.Delete()
class DropBoxDataProvider(DataProviderBase):
smoke_url = DROPBOX_SMOKE_URL
def __init__(self, acs_token):
self.dbx = Dropbox(acs_token)
def api_smoke(self) -> int:
return len(self.dbx.files_list_folder('').entries)
def get_list_of_objects(self, dbx_folder='') -> list:
result = namedtuple('Result', ['filename', 'filepatch'])
return [
result(el.name, el.path_lower)
for el in self.dbx.files_list_folder(dbx_folder).entries
]
def file_delete(self, dbx_file) -> str:
return self.dbx.files_delete_v2(dbx_file).metadata.path_lower
def file_download(self, local_file, dbx_file) -> str:
return self.dbx.files_download_to_file(local_file, dbx_file).path_lower
def file_upload(self, local_file, dbx_file) -> str:
if isinstance(local_file, str):
if local_file.startswith("https://"):
waiting_time = 0.1
waiting_attempt = 100
url_result = self.dbx.files_save_url(dbx_file, local_file)
job_id = url_result.get_async_job_id()
while waiting_attempt > 0:
st = self.dbx.files_save_url_check_job_status(job_id)
if st.is_complete():
return st.get_complete().path_lower
sleep(waiting_time)
waiting_attempt -= 1
else:
with open(local_file, 'rb') as f:
return self.dbx.files_upload(
f.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
else:
return self.dbx.files_upload(local_file.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
def file_move(self, dbx_file_from, dbx_file_to) -> str:
return self.dbx.files_move_v2(dbx_file_from,
dbx_file_to).metadata.path_lower
def create_folder(self, dbx_folder) -> str:
return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower
def get_file_tmp_link(self, dbx_path) -> str:
return self.dbx.files_get_temporary_link(dbx_path).link
class DropBoxStorage(Storage):
"""
The default Storage base for all file storing of the Chat app
"""
def __init__(self, oauth2_access_token: str = None, root_path: str = None):
"""
The access token and root path may be provided here as well,
if they are not provided here, program will check settings for environment variables
:param oauth2_access_token: The OAUTH2 access token for the DropBox api
:param root_path: The root path for storing the files in the DropBox storage, defaults '/'
"""
oauth2_access_token = oauth2_access_token or settings.DROPBOX_OAUTH2_TOKEN
self.root_path = root_path or settings.DROPBOX_ROOT_PATH or '/'
if oauth2_access_token is None:
raise ImproperlyConfigured(
"You must configure an OATH2 access token ENV named "
"'DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def delete(self, name: str):
"""
Deletes the specified file from the storage system.
"""
self.client.files_delete_v2(join(self.root_path, basename(name)))
def exists(self, name: str):
"""
Returns True if a file referenced by the given name already exists in the
storage system, or False if the name is available for a new file.
"""
try:
return bool(
self.client.files_get_metadata(
join(self.root_path, basename(name))))
except ApiError:
return False
def url(self, name: str):
"""
Returns an absolute URL where the file's contents can be accessed
directly by a Web browser.
"""
media = self.client.files_get_temporary_link(
join(self.root_path, basename(name)))
return media.link
def _open(self, name: str, mode: str = 'rb'):
"""
Call DropBoxStorage.open(...) instead
"""
file = DropBoxFile(join(self.root_path, basename(name)), self.client)
return file
def _save(self, name: str, content: File):
"""
Call DropBoxStorage.save(...) instead
"""
self.client.files_upload(content.read(),
join(self.root_path, basename(name)))
return name
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {'mode': WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
'recursive': True,
'include_non_downloadable_files': False
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as err:
if _is_file_not_found_error(err):
raise KeyError(f"Key doesn't exist: {k}")
else:
raise ValueError(
"Some unknown error happened (sorry, the lazy dev didn't tell me more than that)."
)
if contents_response.status_code:
return contents_response.content
else:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
# def _entry_is_dir(entry):
# return not hasattr(entry, 'is_downloadable')
#
#
# def _entry_is_file(entry):
# return hasattr(entry, 'is_downloadable')
#
#
# def _extend_path(path, extension):
# extend_path = '/' + path + '/' + extension + '/'
# extend_path.replace('//', '/')
# return extend_path
#
#
# class DropboxLinkPersister(DropboxPersister):
# def __init__(self, url, oauth2_access_token):
# self._con = Dropbox(oauth2_access_token)
# self.url = url
# self.shared_link = SharedLink(url=url)
#
# def _yield_from_files_list_folder(self, path, path_gen):
# """
# yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
# in a depth search manner.
# """
# for x in path_gen.entries:
# try:
# if _entry_is_file(x):
# yield x.name
# else:
# folder_path = _extend_path(path, x.name)
# yield from self._get_path_gen_from_path(path=folder_path)
# except Exception as e:
# print(e)
# if path_gen.has_more:
# yield from self._get_path_gen_from_cursor(path_gen.cursor, path=path)
#
# def _get_path_gen_from_path(self, path):
# path_gen = self._con.files_list_folder(path=path, recursive=False, shared_link=self.shared_link)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def _get_path_gen_from_cursor(self, cursor, path):
# path_gen = self._con.files_list_folder_continue(cursor)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def __iter__(self):
# yield from self._get_path_gen_from_path(path='')
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(
self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None,
):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {"mode": WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
"recursive": True,
"include_non_downloadable_files": False,
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as e:
if _is_file_not_found_error(e):
raise KeyError(f"Key doesn't exist: {k}")
raise
if not contents_response.status_code:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
return contents_response.content
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
class DropboxFS(FS):
_meta = {
"case_insensitive": False,
"invalid_path_chars": "\0",
"network": True,
"read_only": False,
"thread_safe": True,
"unicode_paths": True,
"virtual": False,
}
def __init__(self, accessToken, session=None):
super(DropboxFS, self).__init__()
self._lock = threading.RLock()
self.dropbox = Dropbox(accessToken, session=session)
def fix_path(self, path):
if isinstance(path, bytes):
try:
path = path.decode("utf-8")
except AttributeError:
pass
if not path.startswith("/"):
path = "/" + path
if path == "." or path == "./":
path = "/"
path = self.validatepath(path)
return path
def __repr__(self):
return "<DropboxDriveFS>"
def _infoFromMetadata(self, metadata):
rawInfo = {
"basic": {
"name": metadata.name,
"is_dir": isinstance(metadata, FolderMetadata),
}
}
if isinstance(metadata, FileMetadata):
rawInfo.update(
{"details": {"size": metadata.size, "type": ResourceType.file}}
)
else:
rawInfo.update({"details": {"type": ResourceType.directory}})
return Info(rawInfo)
def getinfo(self, path, namespaces=None):
_path = self.fix_path(path)
if _path == "/":
info_dict = {
"basic": {"name": "", "is_dir": True},
"details": {"type": ResourceType.directory},
}
return Info(info_dict)
try:
metadata = self.dropbox.files_get_metadata(
_path, include_media_info=True
)
except ApiError as e:
raise errors.ResourceNotFound(path=path, exc=e)
return self._infoFromMetadata(metadata)
def setinfo(self, path, info):
if not self.exists(path):
raise errors.ResourceNotFound(path)
def listdir(self, path):
_path = self.fix_path(path)
if _path == "/":
_path = ""
if not self.exists(_path):
raise errors.ResourceNotFound(path)
meta = self.getinfo(_path)
if meta.is_file:
raise errors.DirectoryExpected(path)
result = self.dropbox.files_list_folder(_path, include_media_info=True)
allEntries = result.entries
while result.has_more:
result = self.dropbox.files_list_folder_continue(result.cursor)
allEntries += result.entries
return [x.name for x in allEntries]
def makedir(self, path, permissions=None, recreate=False):
path = self.fix_path(path)
if self.exists(path) and not recreate:
raise errors.DirectoryExists(path)
if path == "/":
return SubFS(self, path)
if self.exists(path):
meta = self.getinfo(path)
if meta.is_dir:
if recreate == False:
raise errors.DirectoryExists(path)
else:
return SubFS(self, path)
if meta.is_file:
raise errors.DirectoryExpected(path)
ppath = self.get_parent(path)
if not self.exists(ppath):
raise errors.ResourceNotFound(ppath)
try:
folderMetadata = self.dropbox.files_create_folder_v2(path)
except ApiError as e:
raise errors.DirectoryExpected(path=path)
return SubFS(self, path)
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if not self.getinfo(self.get_parent(_path)).is_dir:
raise errors.DirectoryExpected(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def remove(self, path):
_path = self.fix_path(path)
try:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
log.debug(e)
raise errors.FileExpected(path=path, exc=e)
def removedir(self, path):
_path = self.fix_path(path)
if _path == "/":
raise errors.RemoveRootError()
try:
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path=path)
if len(self.listdir(path)) > 0:
raise errors.DirectoryNotEmpty(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
raise errors.FileExpected(path=path, exc=e)
def copy(self, src_path, dst_path, overwrite=False):
src_path = self.fix_path(src_path)
dst_path = self.fix_path(dst_path)
try:
src_meta = self.getinfo(src_path)
if src_meta.is_dir:
raise errors.FileExpected(src_path)
except ApiError as e:
raise errors.ResourceNotFound
dst_meta = None
try:
dst_meta = self.getinfo(dst_path)
except Exception as e:
pass
if dst_meta is not None:
if overwrite == True:
self.remove(dst_path)
else:
raise errors.DestinationExists(dst_path)
parent_path = self.get_parent(dst_path)
if not self.exists(parent_path):
raise errors.ResourceNotFound(dst_path)
self.dropbox.files_copy_v2(src_path, dst_path)
def get_parent(self, dst_path):
import os
parent_path = os.path.abspath(os.path.join(dst_path, ".."))
return parent_path
def exists(self, path):
path = self.fix_path(path)
try:
self.getinfo(path)
return True
except Exception as e:
return False
def move(self, src_path, dst_path, overwrite=False):
_src_path = self.fix_path(src_path)
_dst_path = self.fix_path(dst_path)
if not self.getinfo(_src_path).is_file:
raise errors.FileExpected(src_path)
if not overwrite and self.exists(_dst_path):
raise errors.DestinationExists(dst_path)
if "/" in dst_path and not self.exists(self.get_parent(_dst_path)):
raise errors.ResourceNotFound(src_path)
with self._lock:
try:
if overwrite:
try:
# remove file anyways
self.dropbox.files_delete_v2(_dst_path)
except Exception as e:
pass
self.dropbox.files_move_v2(_src_path, _dst_path)
except ApiError as e:
raise errors.ResourceNotFound(src_path, exc=e)
def apierror_map(self, error):
log.debug(error)
def geturl(self, path, purpose='download'):
url = self.dropbox.sharing_create_shared_link(path).url
url = url.replace('?dl=0', '?raw=1')
return url
def delete_file(mapped_image_name):
dbx = Dropbox(token)
dbx.files_delete_v2('/' + mapped_image_name)
class ServiceDropbox(Service):
def __init__(self, access_token):
"""
starts a new connection to dropbox
:param str access_token: dbx access token
"""
self._access_token = access_token
self._session = session()
self._dbx = Dropbox(oauth2_access_token=self._access_token, session=self._session)
def __del__(self):
self._session.close()
def exists(self, path):
"""
check whether file exists or not
:param str path: file_path
:return bool : True, if file exists or False if not
:raise: ApiError on other dbx errors
"""
try:
self._dbx.files_get_metadata(path)
return True
except ApiError as e:
if e.error.get_path().is_not_found():
return False
else:
raise
def delete(self, path):
"""
delete some dir or file
:param str path: path to delete
"""
self._dbx.files_delete_v2(path)
def dirs(self, path, recursive=True):
"""
return files and dirs in current folder
:param str path: current folder
:param bool recursive: go deeper?
:return: list with files/dirs
"""
r = []
for entry in self._dbx.files_list_folder(path, recursive).entries:
file_name_len = len(entry.name)
if path.split("/")[-2] == entry.name:
continue
if isinstance(entry, FileMetadata):
r.append(File(entry.name, entry.path_lower[0:-file_name_len].lstrip("/"), entry.client_modified, entry.client_modified))
elif isinstance(entry, FolderMetadata):
r.append(Folder(entry.name, entry.path_lower[0:-file_name_len].lstrip("/")))
return r
def chunk(self, path, filename, size, offset=0):
"""
return one chunk of file
:param str path: path on server
:param str filename: name of file
:param int size: chunk-size
:param int offset: bits from the beginning
:return: tuple(File obj, content)
"""
p_session = session()
dbx_p = Dropbox(oauth2_access_token=self._access_token, headers={
"Range": "bytes=" + str(offset) + "-" + str(offset + size - 1)}, session=p_session) # fetch chunks from dropbox
meta, response = dbx_p.files_download(path+"/"+filename)
f = File(meta.name, meta.path_lower, meta.client_modified, meta.client_modified)
p_session.close()
return f, response.content
def file(self, path):
"""
imitates open-method of python by using context-manager,
so you can use "with" statement
:return: interateable object
"""
class _OpenStreamSession(ContextDecorator):
def __init__(self, dbx):
self._dbx = dbx
self._data_offset = 0
self._cur = None
def __enter__(self):
self._sess_id = self._dbx.files_upload_session_start(b'').session_id
return self
def write(self, content):
cur = UploadSessionCursor(self._sess_id, self._data_offset)
self._dbx.files_upload_session_append_v2(content, cur)
self._data_offset += len(content)
def __exit__(self, exc_type, exc_val, exc_tb):
commit = CommitInfo(path=path, mute=True)
cur = UploadSessionCursor(self._sess_id, self._data_offset)
self._dbx.files_upload_session_finish(b'', cur, commit)
return _OpenStreamSession(self._dbx)
def create_dir(self, path):
"""
create dir at specific location
:param path: location to create dir
"""
self._dbx.files_create_folder_v2(path)
class DropboxStorage:
def __init__(self):
"""Initialize the class object with attribute only for internal use."""
self._APP_KEY = 'orl3775x8jdgcg0'
self._APP_SECRET = 'cgz8tvz8k4uiubx'
self._dbx_user_account = None
self._access_token = None
@property
def access_token(self):
"""Return reference to the only internal attribute"""
return self._access_token
@property
def dbx_user_account(self):
"""Return user's Dropbox account information"""
return self._dbx_user_account.users_get_current_account()
def link_account(self):
"""Link user's Dropbox storage, in case of failure return False,
indicating that the error occurred."""
auth_flow = DropboxOAuth2FlowNoRedirect(self._APP_KEY,
self._APP_SECRET)
authorize_url = auth_flow.start()
print("You will be redirected to the authorization page.")
print("Copy the authorization code and paste it to the terminal window.")
time.sleep(3)
webbrowser.open(authorize_url)
auth_code = input("Enter the code: ").strip()
try:
oauth_result = auth_flow.finish(auth_code)
self._access_token = oauth_result.access_token
self._dbx_user_account = Dropbox(self._access_token)
return True
except:
print(
"An error occurred while connecting Dropbox storage.\nPlease, check you internet connection and try again.")
return False
def unlink_account(self):
"""Delete user's access tokens"""
self._access_token = None
self._dbx_user_account = None
return True
def upload_file(self, local_file, backup_path):
"""Upload file to the path, specified by backup_path argument,
in case of failure write the message to the stdout."""
try:
with open(local_file, mode='rb') as f:
try:
self._dbx_user_account.files_upload(f.read(), backup_path, mode=WriteMode('overwrite'))
return True
except ApiError as err:
print("Error occurred while uploading file to Dropbox.")
except:
print("Error occurred while opening local file.")
def delete_file(self, filename):
"""Delete a file in the user's Dropbox storage"""
try:
self._dbx_user_account.files_delete_v2(self.search(filename))
return True
except Exception:
print("File wasn't found in Dropbox.")
def sync(self, local_path):
"""Update the file in the user's Dropbox storage,
if file wasn't found, write the message to the stdout."""
try:
backup_path = self.search(local_path.split('/')[-1])
if '/' == local_path[0]:
local_path = local_path[1:]
self.upload_file(local_path, backup_path)
print("Synchronized with Dropbox.")
except:
print("Failed to synchronize file with Dropbox.")
def search(self, query_name):
"""Return the path to file if it was found in the user's Dropbox storage,
otherwise return False, indicating that file wasn't found."""
lst = [i.path_display for i in self._dbx_user_account.files_list_folder('', recursive=True).entries if
query_name in i.path_display]
if len(lst) > 0:
return lst[0]
else:
return False
def download_file(self, filename):
"""Open a link to download a file from the user's Dropbox storage,
in case of failure returns False."""
try:
temp_file = self._dbx_user_account.files_get_temporary_link(self.search(filename))
webbrowser.open(temp_file.link)
except:
return False
def list_files(self):
"""Write to the stdout the list of all files in the user's Dropbox storage."""
for i in self._dbx_user_account.files_list_folder('', recursive=True).entries:
print(i.path_display.split('/')[-1])
