def upload_folder(self,
local_file_folder: str,
remote_file_folder: str = None):
lfpp = FilePathParser(full_path_file_string=local_file_folder)
if is_blank(lfpp.source_path):
raise DropboxAPIException(
message="#upload_folder , local file folder is blank !")
if not os.path.exists(lfpp.source_path):
raise DropboxAPIException(
message="#upload_folder , local file folder is not exist !")
if not os.path.isdir(lfpp.source_path):
raise DropboxAPIException(
message="#upload_folder , local file folder is not folder !")
if is_not_blank(remote_file_folder):
remote_file_folder = DROPBOX_FILE_SEP + remote_file_folder if not remote_file_folder.startswith(
DROPBOX_FILE_SEP) else remote_file_folder
remote_file_folder = remote_file_folder + DROPBOX_FILE_SEP if not remote_file_folder.endswith(
DROPBOX_FILE_SEP) else remote_file_folder
def gen_local_file_path(folder: str):
global roots, dirs
for roots, dirs, files in os.walk(top=folder):
for _file in files:
local_file_path = os.path.join(roots, _file)
_f = FilePathParser(full_path_file_string=local_file_path)
remote_file_path = _f.translate_to_linux_file()
yield local_file_path, remote_file_path, _f.source_name
if len(dirs) > 0:
for _dir in dirs:
local_folder_path = os.path.join(roots, _dir)
gen_local_file_path(folder=local_folder_path)
for glfp in gen_local_file_path(folder=local_file_folder):
_local_file_path = glfp[0]
_remote_file_path = glfp[1]
_source_file_name = glfp[2]
if is_not_blank(remote_file_folder):
_remote_file_path = os.path.join(remote_file_folder,
_source_file_name)
if is_debug():
logger.debug("%s ==> %s " %
(_local_file_path, _remote_file_path))
# md = self.upload(local_file_path=_local_file_path, remote_file_path=_remote_file_path)
# print(md)
future = upload_file_pool.submit(
fn=self.upload,
local_file_path=_local_file_path,
remote_file_path=_remote_file_path)
pass
def download_as_bytes(self, remote_file_path: str) -> bytes:
"""
download as bytes
remote file path is not blank
:param remote_file_path: remote file path in dropbox
:return: byte array
"""
assert is_not_blank(remote_file_path)
return self.download_as_response(
remote_file_path=remote_file_path).content
def download_as_response(self, remote_file_path: str) -> Response:
"""
download as response
remote file path is not blank
:param remote_file_path: remote file path in dropbox
:return: requests.Response
"""
assert is_not_blank(remote_file_path)
_, response = self.async_download_from_dropbox(
remote_file_path=remote_file_path)
return response
def download_as_file(
self,
remote_file_path: str,
local_file_path: str,
excepted_name: str,
) -> SimpleFileMetadata:
"""
download as local file
# if `remote_file_path` not start with `/` , and combine `/` to head;
# sample as `foo/bar.jpg` to `/foo/bar.jpg`
# if source name not in remote file path , throw `DropboxAPIException`
# if source name not in local file path , first fetch excepted name , second fetch local source name
# sample as
remote_file_path = "/foo/bar.jpg" local_path_path="/local/ => "/local/bar.jpg"
# if excepted name exist , replace source name to excepted name.
# sample as
excepted_name = "cat.jpg" , local_path_file = "/local/bar.jpg" => "/local/cat.jpg"
excepted_name = "/animal/cat.jpg" , local_path_file = "/local/bar.jpg" => "/local/cat.jpg"
usage:
>>> self.download_as_file(remote_file_path="foo/bar.jpg",local_file_path="/local/bar.jpg",excepted_name="cat.jpg")
>>> self.download_as_file(remote_file_path="/foo/bar.jpg",local_file_path="/local/bar.jpg",excepted_name="cat.jpg")
>>> self.download_as_file(remote_file_path="/foo/bar.jpg",local_file_path="/local/bar.jpg",excepted_name="cat.jpg")
:param remote_file_path: remote file path and just support single file
:param local_file_path: local file path which download
:param excepted_name: excepted name just excepted name which want to rename
:return:
"""
if is_blank(remote_file_path):
raise DropboxAPIException(
message="#download_as_file, remote file is blank !")
if remote_file_path.endswith(DROPBOX_FILE_SEP):
raise DropboxAPIException(
message="#download_as_file, remote file path format error !")
# if `remote_file_path` not start with `/` , and combine `/` to head;
# transform `foo/bar.jpg` to `/foo/bar.jpg`
remote_file_path = remote_file_path if remote_file_path.startswith(
DROPBOX_FILE_SEP) else DROPBOX_FILE_SEP + remote_file_path
rfpp = FilePathParser(full_path_file_string=remote_file_path)
lfpp = FilePathParser(full_path_file_string=local_file_path)
# if source name not in remote file path , throw `DropboxAPIException`
if is_not_blank(rfpp.source_name):
raise DropboxAPIException(
message=
"#download_as_file , remote file path source name is blank !")
if lfpp.is_blank:
raise DropboxAPIException(
message="#download_as_file , local file path is blank !")
# fix if local folder is not exist , or create it
if not os.path.exists(lfpp.source_path):
os.mkdir(lfpp.source_path)
# if source name not in local file path , throw `DropboxAPIException`
# if is_blank(lfpp.source_name):
# raise DropboxAPIException(message="#download_as_file , local file path source name is blank !")
# rename local file path . if excepted name exist , replace source name to excepted name.
# excepted_name = "cat.jpg" , local_path_file = "/foo/bar.jgp" => "foo/cat.jpg"
if is_not_blank(excepted_name):
efpp = FilePathParser(full_path_file_string=excepted_name)
if is_not_blank(efpp.source_name):
local_file_path = lfpp.set_source_name(
excepted_source_name=excepted_name)
elif is_blank(lfpp.source_name):
local_file_path = lfpp.set_source_name(
excepted_source_name=rfpp.source_name)
simple_file_metadata, bytes_array = self.async_download_from_dropbox(
remote_file_path=remote_file_path)
buffer = io.StringIO()
buffer.write(bytes_array)
# local file write
with open_file(file_name=local_file_path, mode="wb") as lwf:
lwf.write(buffer.getvalue())
if not buffer.closed():
buffer.close()
return simple_file_metadata
def upload_from_external_url(self,
external_url: str,
remote_file_path: str = None,
remote_folder_path: str = None,
excepted_name: str = None,
**kwargs) -> SimpleFileMetadata:
"""
upload file which from external url
in case of all parameters is not none
`excepted_name` first priority
`remote_file_path` second priority
`remote_folder` third priority
:param external_url: url from external source
:param remote_file_path remote file path
:param remote_folder_path remote folder_path
:param excepted_name just only excepted name , not with dir path
:param kwargs: requests parameters
:return:
"""
if is_blank(remote_file_path):
if is_blank(remote_folder_path):
raise DropboxAPIException(
"#upload_from_external_url, remote_file_path is blank and remote_folder_path "
"is also blank !")
if not remote_folder_path.startswith(DROPBOX_FILE_SEP):
remote_folder_path = DROPBOX_FILE_SEP + remote_folder_path
if not remote_folder_path.endswith(DROPBOX_FILE_SEP):
remote_folder_path = remote_folder_path + DROPBOX_FILE_SEP
remote_file_path = remote_folder_path
if not remote_file_path.startswith(DROPBOX_FILE_SEP):
remote_file_path = DROPBOX_FILE_SEP + remote_file_path
euup = UrlPathParser(full_path_file_string=external_url)
if euup.is_blank:
raise DropboxAPIException(
"#upload_from_external , upload external url is blank!")
if euup.is_not_http:
raise DropboxAPIException(
"#upload_from_external , upload external url is unknown protocol!"
)
rfpp = FilePathParser(full_path_file_string=remote_file_path)
if is_not_blank(excepted_name):
if DROPBOX_FILE_SEP in excepted_name:
excepted_name = DROPBOX_FILE_SEP + excepted_name \
if not excepted_name.startswith(DROPBOX_FILE_SEP) else excepted_name
remote_file_path = excepted_name
else:
remote_file_path = rfpp.set_source_name(
excepted_name.split(DROPBOX_FILE_SEP)[-1])
else:
remote_file_source = rfpp.source_name
# case one:
# remote_file_path /DEFAULT/A/
# local_file_path /foo/bar.jpg
# auto set /DEFAULT/A/bar.jpg
if is_blank(remote_file_source):
remote_file_path = os.path.join(remote_file_path,
euup.source_name)
# case two:
# remote_file_path /DEFAULT/A/bar
# local_file_path /foo/bar.jpg
# auto set /DEFAULT/A/bar.jpg
elif not remote_file_source.__contains__(
DROPBOX_FILE_DOT) and is_not_blank(euup.source_suffix):
remote_file_path = remote_file_path + DROPBOX_FILE_DOT + euup.source_suffix
# case three:
# remote_file_path /DEFAULT/A/bar
# local_file_path /foo/bar
# auto set /DEFAULT/A/bar
if is_debug():
logger.debug(
"#upload_from_external_url , remote_file_path=%s, external_url=%s"
% (remote_file_path, external_url))
async def _arequest_external_url():
async def _inner_request():
return requests.get(url=external_url, **kwargs, stream=True)
_res = await _inner_request()
return _res
res = self.loop.run_until_complete(_arequest_external_url())
# Tips: Anti spider policy , 可以把status code 设置非200来迷惑爬虫
if res.status_code != 200:
error_msg = "#upload_from_external_url , fail to request , url=%s , msg=%s", (
external_url, res.txt)
logger.error(error_msg)
raise DropboxAPIException(message=error_msg)
# request success
content_type = res.headers.get("Content-Type")
if is_debug():
logger.info("#upload_from_external_url request <%s> success!" %
external_url)
if DROPBOX_FILE_DOT not in remote_file_path.split(
DROPBOX_FILE_SEP)[-1]:
# get file type (suffix)
file_suffix = get_suffix(mime=content_type)
if file_suffix is not None:
remote_file_path = remote_file_path + DROPBOX_FILE_DOT + file_suffix
buffer = io.BytesIO()
for chunk in res.iter_content(chunk_size=20 * 1024):
if chunk:
buffer.write(chunk)
else:
break
if is_debug():
logger.debug("#upload_from_external_url , remote_file_path=%s" %
remote_file_path)
md = self.async_upload_bytes(file_bytes=buffer.getvalue(),
remote_file_path=remote_file_path)
if not buffer.closed:
buffer.close()
return md
def upload_from_local(self,
local_file_path: str,
remote_file_path: str = None,
remote_folder_path: str = None,
excepted_name: str = None) -> SimpleFileMetadata:
"""
upload to dropbox with excepted name
`excepted_name` first priority
`remote_file_path` second priority
`remote_folder` third priority
:param local_file_path: file path in local
:param remote_file_path: file path in dropbox
:param remote_folder_path folder in dropbox
:param excepted_name: excepted name which want to rename
sample as
>>> self.download_as_file(local_file_path='/foo/bar.jpg',remote_file_path='/DEFAULT/cat.jpg')
# `/DEFAULT/cat.jpg`
>>> self.download_as_file(local_file_path='/foo/bar.jpg',remote_file_path='DEFAULT/cat.jpg')
# `/DEFAULT/cat.jpg`
>>> self.download_as_file(local_file_path='/foo/bar.jpg',remote_file_path='DEFAULT/cat.jpg',
>>> excepted_name='dog.jpg')
# `/DEFAULT/dog.jpg`
>>> self.download_as_file(local_file_path='/foo/bar.jpg',remote_file_path='DEFAULT/cat.jpg',
>>> excepted_name='EXCEPTED_DIR/dog.jpg')
# `/EXCEPTED_DIR/dog.jpg`
>>> self.download_as_file(local_file_path='/foo/bar.jpg',remote_folder_path='REMOTE_FOLDER',
>>> excepted_name='dog.jpg')
# `/REMOTE_FOLDER/dog.jpg`
:return:
"""
global metadata
if is_blank(remote_file_path):
if is_blank(remote_folder_path):
raise DropboxAPIException(
"#upload_from_local, remote_file_path is blank and remote_folder_path "
"is also blank !")
if not remote_folder_path.startswith(DROPBOX_FILE_SEP):
remote_folder_path = DROPBOX_FILE_SEP + remote_folder_path
if not remote_folder_path.endswith(DROPBOX_FILE_SEP):
remote_folder_path = remote_folder_path + DROPBOX_FILE_SEP
remote_file_path = remote_folder_path
if not remote_file_path.startswith(DROPBOX_FILE_SEP):
remote_file_path = DROPBOX_FILE_SEP + remote_file_path
lfpp = FilePathParser(full_path_file_string=local_file_path)
if lfpp.is_blank:
raise DropboxAPIException(
"#upload_from_local, local file path is blank !")
if lfpp.is_not_exist:
raise DropboxAPIException(
"#upload_from_local, local file path is not exist !")
if lfpp.is_not_file:
raise DropboxAPIException(
"#upload_from_local, local file path is not a file !")
rfpp = FilePathParser(full_path_file_string=remote_file_path)
if is_not_blank(excepted_name):
if DROPBOX_FILE_SEP in excepted_name:
remote_file_path = DROPBOX_FILE_SEP + excepted_name if not excepted_name.startswith(
DROPBOX_FILE_SEP) else excepted_name
else:
remote_file_path = rfpp.set_source_name(
excepted_name.split(DROPBOX_FILE_SEP)[-1])
else:
remote_file_source = rfpp.source_name
# case one:
# remote_file_path /DEFAULT/A/
# local_file_path /foo/bar.jpg
# auto set /DEFAULT/A/bar.jpg
if is_blank(remote_file_source):
remote_file_path = os.path.join(remote_file_path,
lfpp.source_name)
# case two:
# remote_file_path /DEFAULT/A/bar
# local_file_path /foo/bar.jpg
# auto set /DEFAULT/A/bar.jpg
elif not remote_file_source.__contains__(
DROPBOX_FILE_DOT) and is_not_blank(lfpp.source_suffix):
remote_file_path = remote_file_path + DROPBOX_FILE_DOT + lfpp.source_suffix
# case three:
# remote_file_path /DEFAULT/A/bar
# local_file_path /foo/bar
# auto set /DEFAULT/A/bar
file_size = os.path.getsize(local_file_path)
chunk_size = 10 * 1024 * 1024
if file_size <= chunk_size:
if is_debug():
logger.debug("> uploading tiny file")
with open_file(file_name=local_file_path, mode="rb") as fr:
metadata = self.async_upload_bytes(
file_bytes=fr.read(), remote_file_path=remote_file_path)
else:
if is_debug():
logger.debug("> uploading large file")
metadata = self.async_upload_with_large_file(
local_file_path=local_file_path,
remote_file_path=remote_file_path,
chunk_size=chunk_size)
if logger.level == logging.DEBUG:
logger.debug("upload_from_local metadata is %s" % metadata)
return metadata