def save_shared(
api: BaiduPCSApi,
shared_url: str,
remotedir: str,
password=Optional[str],
show_vcode: bool = True,
):
assert remotedir.startswith("/"), "`remotedir` must be an absolute path"
# Vertify with password
if password:
api.access_shared(shared_url, password, show_vcode=show_vcode)
shared_paths = deque(api.shared_paths(shared_url))
# Record the remotedir of each shared_path
_remotedirs: Dict[PcsSharedPath, str] = {}
for sp in shared_paths:
_remotedirs[sp] = remotedir
_dir_exists: Set[str] = set()
while shared_paths:
shared_path = shared_paths.popleft()
uk, share_id, bdstoken = (
shared_path.uk,
shared_path.share_id,
shared_path.bdstoken,
)
assert uk
assert share_id
assert bdstoken
rd = _remotedirs[shared_path]
if rd not in _dir_exists and not api.exists(rd):
api.makedir(rd)
_dir_exists.add(rd)
# rd = (Path(_remotedirs[shared_path]) / os.path.basename(shared_path.path)).as_posix()
try:
api.transfer_shared_paths(rd, [shared_path.fs_id], uk, share_id,
bdstoken, shared_url)
print(f"save: {shared_path.path} to {rd}")
continue
except BaiduPCSError as err:
if err.error_code not in (12, -33):
raise err
if err.error_code == 12: # -33: '一次支持操作999个，减点试试吧'
print(f"[yellow]WARNING[/]: {shared_path.path} has be in {rd}")
if err.error_code == -33: # -33: '一次支持操作999个，减点试试吧'
print(f"[yellow]WARNING[/]: {shared_path.path} "
"has more items and need to transfer one by one")
sub_paths = api.list_shared_paths(shared_path.path, uk, share_id,
bdstoken)
rd = (Path(rd) / os.path.basename(shared_path.path)).as_posix()
for sp in sub_paths:
_remotedirs[sp] = rd
shared_paths.extendleft(sub_paths[::-1])
def play(
api: BaiduPCSApi,
remotepaths: List[str],
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
shuffle: bool = False,
ignore_ext: bool = False,
out_cmd: bool = False,
local_server: str = "",
):
"""Play media file in `remotepaths`
Args:
`from_index` (int): The start index of playing entries from EACH remote directory
"""
if shuffle:
rg = random.Random(time.time())
rg.shuffle(remotepaths)
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
play_file(
api,
rp,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
else:
play_dir(
api,
rp,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
shuffle=shuffle,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
def download(
api: BaiduPCSApi,
remotepaths: List[str],
localdir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
):
"""Download `remotepaths` to the `localdir`
Args:
`from_index` (int): The start index of downloading entries from EACH remote directory
"""
remotepaths = sift(remotepaths, sifters)
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
download_file(
api,
rp,
localdir,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
)
else:
_localdir = str(Path(localdir) / os.path.basename(rp))
download_dir(
api,
rp,
_localdir,
sifters=sifters,
recursive=recursive,
from_index=from_index,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
)
if downloader == Downloader.me:
MeDownloader._exit_executor()
_progress.stop()
def play(
api: BaiduPCSApi,
remotepaths: List[str],
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
out_cmd: bool = False,
):
"""Play media file in `remotepaths`
Args:
`from_index` (int): The start index of playing entries from EACH remote directory
"""
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
play_file(
api,
rp,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
else:
play_dir(
api,
rp,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
def sync(
api: BaiduPCSApi,
localdir: str,
remotedir: str,
encrypt_key: Any = None,
salt: Any = None,
encrypt_type: EncryptType = EncryptType.No,
max_workers: int = CPU_NUM,
slice_size: int = DEFAULT_SLICE_SIZE,
show_progress: bool = True,
):
localdir = Path(localdir).as_posix()
remotedir = Path(remotedir).as_posix()
is_file = api.is_file(remotedir)
assert not is_file, "remotedir must be a directory"
if not api.exists(remotedir):
all_pcs_files = {}
else:
all_pcs_files = {
pcs_file.path[len(remotedir) + 1:]: pcs_file
for pcs_file in recursive_list(api, remotedir)
}
fts: List[FromTo] = []
check_list: List[Tuple[str, PcsFile]] = []
all_localpaths = set()
for localpath in walk(localdir):
path = localpath[len(localdir) + 1:]
all_localpaths.add(path)
if path not in all_pcs_files:
fts.append(FromTo(localpath, join_path(remotedir, path)))
else:
check_list.append((localpath, all_pcs_files[path]))
semaphore = Semaphore(max_workers)
with ThreadPoolExecutor(max_workers=CPU_NUM) as executor:
tasks = {}
for lp, pf in check_list:
semaphore.acquire()
fut = executor.submit(sure_release, semaphore, check_file_md5, lp,
pf)
tasks[fut] = (lp, pf)
for fut in as_completed(tasks):
is_equal = fut.result()
lp, pf = tasks[fut]
if not is_equal:
fts.append(FromTo(lp, pf.path))
_upload(
api,
fts,
encrypt_key=encrypt_key,
salt=salt,
encrypt_type=encrypt_type,
max_workers=max_workers,
slice_size=slice_size,
ignore_existing=False,
show_progress=show_progress,
)
to_deletes = []
for rp in all_pcs_files.keys():
if rp not in all_localpaths:
to_deletes.append(all_pcs_files[rp].path)
if to_deletes:
api.remove(*to_deletes)
print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")
def sync(
api: BaiduPCSApi,
localdir: str,
remotedir: str,
encrypt_password: bytes = b"",
encrypt_type: EncryptType = EncryptType.No,
max_workers: int = CPU_NUM,
slice_size: int = DEFAULT_SLICE_SIZE,
show_progress: bool = True,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
):
localdir = Path(localdir).as_posix()
remotedir = Path(remotedir).as_posix()
is_file = api.is_file(remotedir)
assert not is_file, "remotedir must be a directory"
if not api.exists(remotedir):
all_pcs_files = {}
else:
all_pcs_files = {
pcs_file.path[len(remotedir) + 1 :]: pcs_file
for pcs_file in recursive_list(api, remotedir)
}
fts: List[FromTo] = []
check_list: List[Tuple[str, PcsFile]] = []
all_localpaths = set()
for localpath in walk(localdir):
path = localpath[len(localdir) + 1 :]
all_localpaths.add(path)
if path not in all_pcs_files:
fts.append(FromTo(localpath, join_path(remotedir, path)))
else:
check_list.append((localpath, all_pcs_files[path]))
for lp, pf in check_list:
lstat = Path(lp).stat()
if int(lstat.st_mtime) != pf.local_mtime or lstat.st_size != pf.size:
fts.append(FromTo(lp, pf.path))
to_deletes = []
for rp in all_pcs_files.keys():
if rp not in all_localpaths:
to_deletes.append(all_pcs_files[rp].path)
logger.debug(
"`sync`: all localpaths: %s, "
"localpaths needed to upload: %s, "
"remotepaths needed to delete: %s",
len(all_localpaths),
len(fts),
len(to_deletes),
)
# The md5 of remote file is incorrect at most time, so we don't compare md5
#
# # Compare localpath content md5 with remotepath content md5
# semaphore = Semaphore(max_workers)
# with ThreadPoolExecutor(max_workers=CPU_NUM) as executor:
# tasks = {}
# for lp, pf in check_list:
# semaphore.acquire()
# fut = executor.submit(sure_release, semaphore, check_file_md5, lp, pf)
# tasks[fut] = (lp, pf)
#
# for fut in as_completed(tasks):
# is_equal = fut.result()
# lp, pf = tasks[fut]
# if not is_equal:
# fts.append(FromTo(lp, pf.path))
_upload(
api,
fts,
encrypt_password=encrypt_password,
encrypt_type=encrypt_type,
max_workers=max_workers,
slice_size=slice_size,
ignore_existing=False,
show_progress=show_progress,
rapiduploadinfo_file=rapiduploadinfo_file,
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
)
if to_deletes:
api.remove(*to_deletes)
print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")
def upload_file(
api: BaiduPCSApi,
from_to: FromTo,
ondup: str,
encrypt_password: bytes = b"",
encrypt_type: EncryptType = EncryptType.No,
slice_size: int = DEFAULT_SLICE_SIZE,
ignore_existing: bool = True,
task_id: Optional[TaskID] = None,
):
_wait_start()
localpath, remotepath = from_to
assert exists(localpath), f"`{localpath}` does not exist"
if ignore_existing:
try:
if api.exists(remotepath):
print(f"`{remotepath}` already exists.")
logger.debug("`upload`: remote file already exists")
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
return
except Exception as err:
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
raise err
logger.debug("`upload`: encrypt_type: %s", encrypt_type)
encrypt_io = encrypt_type.encrypt_io(open(localpath, "rb"),
encrypt_password)
# IO Length
encrypt_io_len = total_len(encrypt_io)
logger.debug("`upload`: encrypt_io_len: %s", encrypt_io_len)
# Progress bar
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, total=encrypt_io_len)
_progress.start_task(task_id)
def callback(monitor: MultipartEncoderMonitor):
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, completed=monitor.bytes_read)
slice_completed = 0
def callback_for_slice(monitor: MultipartEncoderMonitor):
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id,
completed=slice_completed + monitor.bytes_read)
if encrypt_io_len > 256 * constant.OneK:
# Rapid Upload
logger.debug("`upload`: rapid_upload starts")
try:
slice_md5, content_md5, content_crc32, encrypt_io_len = rapid_upload_params(
encrypt_io)
api.rapid_upload_file(
slice_md5,
content_md5,
content_crc32,
encrypt_io_len,
remotepath,
ondup=ondup,
)
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, completed=encrypt_io_len)
_progress.remove_task(task_id)
logger.debug("`upload`: rapid_upload success")
return
except BaiduPCSError as err:
logger.debug("`upload`: rapid_upload fails")
if err.error_code != 31079: # 31079: '未找到文件MD5，请使用上传API上传整个文件。'
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
logger.warning("`rapid_upload`: unknown error: %s", err)
raise err
else:
logger.info("`rapid_upload`: %s, no exist in remote",
localpath)
if task_id is not None and progress_task_exists(task_id):
_progress.reset(task_id)
try:
if encrypt_io_len < slice_size:
# Upload file
logger.debug("`upload`: upload_file starts")
reset_encrypt_io(encrypt_io)
retry(
30,
except_callback=lambda err, fail_count: (
logger.warning(
"`upload`: `upload_file`: error: %s, fail_count: %s",
err,
fail_count,
),
_wait_start(),
),
)(api.upload_file)(encrypt_io,
remotepath,
ondup=ondup,
callback=callback)
logger.debug("`upload`: upload_file success")
else:
# Upload file slice
logger.debug("`upload`: upload_slice starts")
slice_md5s = []
reset_encrypt_io(encrypt_io)
while True:
_wait_start()
logger.debug("`upload`: upload_slice: slice_completed: %s",
slice_completed)
size = min(slice_size, encrypt_io_len - slice_completed)
if size == 0:
break
data = encrypt_io.read(size) or b""
io = BytesIO(data)
logger.debug("`upload`: upload_slice: size should be %s == %s",
size, len(data))
# Retry upload until success
slice_md5 = retry(
-1,
except_callback=lambda err, fail_count: (
io.seek(0, 0),
logger.warning(
"`upload`: `upload_slice`: error: %s, fail_count: %s",
err,
fail_count,
),
_wait_start(),
),
)(api.upload_slice)(io, callback=callback_for_slice)
slice_md5s.append(slice_md5)
slice_completed += size
# Combine slices
retry(
30,
except_callback=lambda err, fail_count: logger.warning(
"`upload`: `combine_slices`: error: %s, fail_count: %s",
err,
fail_count,
),
)(api.combine_slices)(slice_md5s, remotepath, ondup=ondup)
logger.debug("`upload`: upload_slice and combine_slices success")
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
except Exception as err:
logger.warning("`upload`: error: %s", err)
raise err
finally:
encrypt_io.close()
if task_id is not None and progress_task_exists(task_id):
_progress.reset(task_id)
def upload_file(
api: BaiduPCSApi,
from_to: FromTo,
ondup: str,
slice_size: int = DEFAULT_SLICE_SIZE,
ignore_existing: bool = True,
task_id: Optional[TaskID] = None,
):
localpath, remotepath = from_to
assert exists(localpath), f"`{localpath}` does not exist"
if ignore_existing:
try:
if api.exists(remotepath):
print(f"`{remotepath}` already exists.")
if task_id is not None:
_progress.remove_task(task_id)
return
except Exception as err:
if task_id is not None:
_progress.remove_task(task_id)
raise err
local_size = Path(localpath).stat().st_size
if task_id is not None:
_progress.update(task_id, total=local_size)
_progress.start_task(task_id)
def callback(monitor: MultipartEncoderMonitor):
if task_id is not None:
_progress.update(task_id, completed=monitor.bytes_read)
slice_completed = 0
def callback_for_slice(monitor: MultipartEncoderMonitor):
if task_id is not None:
_progress.update(task_id, completed=slice_completed + monitor.bytes_read)
if local_size > 256 * constant.OneK:
try:
api.rapid_upload_file(localpath, remotepath, ondup=ondup)
if task_id is not None:
_progress.update(task_id, completed=local_size)
_progress.remove_task(task_id)
return
except BaiduPCSError as err:
if err.error_code != 31079: # 31079: '未找到文件MD5，请使用上传API上传整个文件。'
if task_id is not None:
_progress.remove_task(task_id)
raise err
else:
if task_id is not None:
_progress.reset(task_id)
try:
if local_size < slice_size:
api.upload_file(localpath, remotepath, ondup=ondup, callback=callback)
else:
slice_md5s = []
fd = open(localpath, "rb")
while True:
buf = fd.read(slice_size)
if not buf:
break
slice_md5 = api.upload_slice(buf, callback=callback_for_slice)
slice_md5s.append(slice_md5)
slice_completed += len(buf)
api.combine_slices(slice_md5s, remotepath, ondup=ondup)
finally:
if task_id is not None:
_progress.remove_task(task_id)
def upload_file(
api: BaiduPCSApi,
from_to: FromTo,
ondup: str,
encrypt_password: bytes = b"",
encrypt_type: EncryptType = EncryptType.No,
slice_size: int = DEFAULT_SLICE_SIZE,
ignore_existing: bool = True,
task_id: Optional[TaskID] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
):
_wait_start()
localpath, remotepath = from_to
assert exists(localpath), f"`{localpath}` does not exist"
if ignore_existing:
try:
if api.exists(remotepath):
print(f"`{remotepath}` already exists.")
logger.debug("`upload`: remote file already exists")
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
return
except Exception as err:
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
raise err
logger.debug(
"`upload`: encrypt_type: %s, localpath: %s, remotepath, %s",
encrypt_type,
localpath,
remotepath,
)
stat = Path(localpath).stat()
local_ctime, local_mtime = int(stat.st_ctime), int(stat.st_mtime)
encrypt_io = encrypt_type.encrypt_io(open(localpath, "rb"), encrypt_password)
# IO Length
encrypt_io_len = total_len(encrypt_io)
logger.debug("`upload`: encrypt_io_len: %s", encrypt_io_len)
# Progress bar
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, total=encrypt_io_len)
_progress.start_task(task_id)
def callback(monitor: MultipartEncoderMonitor):
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, completed=monitor.bytes_read)
slice_completed = 0
def callback_for_slice(monitor: MultipartEncoderMonitor):
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, completed=slice_completed + monitor.bytes_read)
slice256k_md5 = ""
content_md5 = ""
content_crc32 = 0
io_len = 0
if encrypt_io_len > 256 * constant.OneK:
# Rapid Upload
logger.debug("`upload`: rapid_upload starts")
try:
slice256k_md5, content_md5, content_crc32, io_len = rapid_upload_params(
encrypt_io
)
api.rapid_upload_file(
slice256k_md5,
content_md5,
0, # not needed
encrypt_io_len,
remotepath,
local_ctime=local_ctime,
local_mtime=local_mtime,
ondup=ondup,
)
if _rapiduploadinfo_file:
save_rapid_upload_info(
_rapiduploadinfo_file,
slice256k_md5,
content_md5,
content_crc32,
io_len,
localpath=localpath,
remotepath=remotepath,
encrypt_password=encrypt_password,
encrypt_type=encrypt_type.value,
user_id=user_id,
user_name=user_name,
)
if task_id is not None and progress_task_exists(task_id):
_progress.update(task_id, completed=encrypt_io_len)
_progress.remove_task(task_id)
logger.debug("`upload`: rapid_upload success, task_id: %s", task_id)
return
except BaiduPCSError as err:
logger.warning("`upload`: rapid_upload fails")
if err.error_code != 31079: # 31079: '未找到文件MD5，请使用上传API上传整个文件。'
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
logger.warning("`rapid_upload`: unknown error: %s", err)
raise err
else:
logger.debug("`rapid_upload`: %s, no exist in remote", localpath)
if task_id is not None and progress_task_exists(task_id):
_progress.reset(task_id)
try:
# Upload file slice
logger.debug("`upload`: upload_slice starts")
slice_md5s = []
reset_encrypt_io(encrypt_io)
while True:
_wait_start()
logger.debug("`upload`: upload_slice: slice_completed: %s", slice_completed)
size = min(slice_size, encrypt_io_len - slice_completed)
if size == 0:
break
data = encrypt_io.read(size) or b""
io = BytesIO(data)
logger.debug(
"`upload`: upload_slice: size should be %s == %s", size, len(data)
)
# Retry upload until success
slice_md5 = retry(
-1,
except_callback=lambda err, fail_count: (
io.seek(0, 0),
logger.warning(
"`upload`: `upload_slice`: error: %s, fail_count: %s",
err,
fail_count,
exc_info=err,
),
_wait_start(),
),
)(api.upload_slice)(io, callback=callback_for_slice)
slice_md5s.append(slice_md5)
slice_completed += size
# Combine slices
def _handle_combin_slices_error(err, fail_count):
logger.warning(
"`upload`: `combine_slices`: error: %s, fail_count: %s",
err,
fail_count,
exc_info=err,
)
# If following errors occur, we need to re-upload
if (
isinstance(err, BaiduPCSError)
and err.error_code == 31352 # commit superfile2 failed
or err.error_code == 31363 # block miss in superfile2
):
raise err
retry(20, except_callback=_handle_combin_slices_error)(api.combine_slices)(
slice_md5s,
remotepath,
local_ctime=local_ctime,
local_mtime=local_mtime,
ondup=ondup,
)
logger.debug(
"`upload`: upload_slice and combine_slices success, task_id: %s", task_id
)
# `combine_slices` can not get right content md5.
# We need to check whether server updates by hand.
if check_md5:
_check_md5(
api,
localpath,
remotepath,
slice256k_md5,
content_md5,
content_crc32,
io_len,
encrypt_password=encrypt_password,
encrypt_type=encrypt_type.value,
user_id=user_id,
user_name=user_name,
)
if task_id is not None and progress_task_exists(task_id):
_progress.remove_task(task_id)
except Exception as err:
logger.warning("`upload`: error: %s", err)
raise err
finally:
encrypt_io.close()
if task_id is not None and progress_task_exists(task_id):
_progress.reset(task_id)
def rapid_upload(
api: BaiduPCSApi,
remotedir: str,
link: str = "",
slice_md5: str = "",
content_md5: str = "",
content_crc32: int = 0,
content_length: int = 0,
filename: str = "",
no_ignore_existing: bool = False,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
):
"""Rapid upload with params
If given `link` and `filename`, then filename of link will be replace by `filename`
"""
if link:
slice_md5, content_md5, content_crc32, content_length, _filename = _parse_link(
link)
content_crc32 = content_crc32 or 0
filename = filename or _filename
remotepath = join_path(remotedir, filename)
assert all([slice_md5, content_md5, content_length
]), f"`rapid_upload`: parsing rapid upload link fails: {link}"
if not no_ignore_existing:
if api.exists(remotepath):
return
try:
pcs_file = api.rapid_upload_file(
slice_md5,
content_md5,
content_crc32,
content_length,
remotepath,
ondup="overwrite",
)
if rapiduploadinfo_file:
save_rapid_upload_info(
rapiduploadinfo_file,
slice_md5,
content_md5,
content_crc32,
content_length,
remotepath=remotepath,
user_id=user_id,
user_name=user_name,
)
print(f"[i blue]Save to[/i blue] {pcs_file.path}")
except Exception as err:
link = PcsRapidUploadInfo(
slice_md5=slice_md5,
content_md5=content_md5,
content_crc32=content_crc32,
content_length=content_length,
remotepath=remotepath,
).cs3l()
print(
f"[i yellow]Rapid Upload fails[/i yellow]: error: {err} link: {link}",
)
def download(
api: BaiduPCSApi,
remotepaths: List[str],
localdir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
encrypt_password: bytes = b"",
):
"""Download `remotepaths` to the `localdir`
Args:
`from_index` (int): The start index of downloading entries from EACH remote directory
"""
logger.debug(
"`download`: sifters: %s, recursive: %s, from_index: %s, "
"downloader: %s, downloadparams: %s, out_cmd: %s, has encrypt_password: %s",
sifters,
recursive,
from_index,
downloader,
downloadparams,
out_cmd,
bool(encrypt_password),
)
logger.debug(
"`download`: remotepaths should be uniq %s == %s",
len(remotepaths),
len(set(remotepaths)),
)
assert (
human_size_to_int(downloadparams.chunk_size) <= MAX_CHUNK_SIZE
), f"`chunk_size` must be less or equal then {human_size(MAX_CHUNK_SIZE)}"
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
download_file(
api,
rp,
localdir,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_password=encrypt_password,
)
else:
_localdir = str(Path(localdir) / os.path.basename(rp))
download_dir(
api,
rp,
_localdir,
sifters=sifters,
recursive=recursive,
from_index=from_index,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_password=encrypt_password,
)
if downloader == Downloader.me:
MeDownloader._exit_executor()
_progress.stop()
def download(
api: BaiduPCSApi,
remotepaths: List[str],
localdir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
encrypt_key: Optional[str] = None,
):
"""Download `remotepaths` to the `localdir`
Args:
`from_index` (int): The start index of downloading entries from EACH remote directory
"""
logger.debug(
"`download`: sifters: %s, recursive: %s, from_index: %s, "
"downloader: %s, downloadparams: %s, out_cmd: %s, has encrypt_key: %s",
sifters,
recursive,
from_index,
downloader,
downloadparams,
out_cmd,
bool(encrypt_key),
)
logger.debug(
"`download`: remotepaths should be uniq %s == %s",
len(remotepaths),
len(set(remotepaths)),
)
remotepaths = sift(remotepaths, sifters)
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
download_file(
api,
rp,
localdir,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_key=encrypt_key,
)
else:
_localdir = str(Path(localdir) / os.path.basename(rp))
download_dir(
api,
rp,
_localdir,
sifters=sifters,
recursive=recursive,
from_index=from_index,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_key=encrypt_key,
)
if downloader == Downloader.me:
MeDownloader._exit_executor()
_progress.stop()
