def add_task(
api: BaiduPCSApi,
task_url: str,
remotedir: str,
file_types: List[FileType] = [FileType.Media],
):
if is_magnet(task_url):
logger.warning("Add cloud task: magnet: %s", task_url)
pmfs = api.magnet_info(task_url)
selected_idx = []
for idx, pmf in enumerate(pmfs, 1):
ext = os.path.splitext(pmf.path)[-1]
for file_type in file_types:
if file_type.sift(ext):
selected_idx.append(idx)
break
if not selected_idx:
logger.warning("`add_task`: No selected idx for %s", task_url)
return
task_id = api.add_magnet_task(task_url, remotedir, selected_idx)
tasks(api, task_id)
else:
logger.warning("Add cloud task: %s", task_url)
task_id = api.add_task(task_url, remotedir)
def list_shared_paths(
api: BaiduPCSApi,
shared_url: str,
password: Optional[str] = None,
show_vcode: bool = True,
):
# Vertify with password
if password:
api.access_shared(shared_url, password, show_vcode=show_vcode)
all_shared_paths: List[PcsSharedPath] = []
shared_paths = deque(api.shared_paths(shared_url))
all_shared_paths += shared_paths
while shared_paths:
shared_path = shared_paths.popleft()
uk, share_id, bdstoken = (
shared_path.uk,
shared_path.share_id,
shared_path.bdstoken,
)
assert uk
assert share_id
assert bdstoken
if shared_path.is_dir:
# Take all sub paths
sub_paths = list_all_sub_paths(api, shared_path.path, uk, share_id,
bdstoken)
all_shared_paths += sub_paths
shared_paths.extendleft(sub_paths[::-1])
display_shared_paths(*all_shared_paths)
def play_file(
api: BaiduPCSApi,
remotepath: str,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
out_cmd: bool = False,
):
if not _with_media_ext(remotepath):
return
print(f"[italic blue]Play[/italic blue]: {remotepath} {'(m3u8)' if m3u8 else ''}")
if m3u8:
m3u8_cn = api.m3u8_stream(remotepath)
with open(DEFAULT_TEMP_M3U8, "w") as fd:
fd.write(m3u8_cn)
url = DEFAULT_TEMP_M3U8
else:
url = api.download_link(remotepath)
player.play(
url,
api.cookies,
m3u8=m3u8,
quiet=quiet,
player_params=player_params,
out_cmd=out_cmd,
)
def play(
api: BaiduPCSApi,
remotepaths: List[str],
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
shuffle: bool = False,
ignore_ext: bool = False,
out_cmd: bool = False,
local_server: str = "",
):
"""Play media file in `remotepaths`
Args:
`from_index` (int): The start index of playing entries from EACH remote directory
"""
if shuffle:
rg = random.Random(time.time())
rg.shuffle(remotepaths)
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
play_file(
api,
rp,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
else:
play_dir(
api,
rp,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
shuffle=shuffle,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
def save_shared(
api: BaiduPCSApi,
shared_url: str,
remotedir: str,
password=Optional[str],
show_vcode: bool = True,
):
assert remotedir.startswith("/"), "`remotedir` must be an absolute path"
# Vertify with password
if password:
api.access_shared(shared_url, password, show_vcode=show_vcode)
shared_paths = deque(api.shared_paths(shared_url))
# Record the remotedir of each shared_path
_remotedirs: Dict[PcsSharedPath, str] = {}
for sp in shared_paths:
_remotedirs[sp] = remotedir
_dir_exists: Set[str] = set()
while shared_paths:
shared_path = shared_paths.popleft()
uk, share_id, bdstoken = (
shared_path.uk,
shared_path.share_id,
shared_path.bdstoken,
)
assert uk
assert share_id
assert bdstoken
rd = _remotedirs[shared_path]
if rd not in _dir_exists and not api.exists(rd):
api.makedir(rd)
_dir_exists.add(rd)
# rd = (Path(_remotedirs[shared_path]) / os.path.basename(shared_path.path)).as_posix()
try:
api.transfer_shared_paths(rd, [shared_path.fs_id], uk, share_id,
bdstoken, shared_url)
print(f"save: {shared_path.path} to {rd}")
continue
except BaiduPCSError as err:
if err.error_code not in (12, -33):
raise err
if err.error_code == 12: # -33: '一次支持操作999个，减点试试吧'
print(f"[yellow]WARNING[/]: {shared_path.path} has be in {rd}")
if err.error_code == -33: # -33: '一次支持操作999个，减点试试吧'
print(f"[yellow]WARNING[/]: {shared_path.path} "
"has more items and need to transfer one by one")
sub_paths = api.list_shared_paths(shared_path.path, uk, share_id,
bdstoken)
rd = (Path(rd) / os.path.basename(shared_path.path)).as_posix()
for sp in sub_paths:
_remotedirs[sp] = rd
shared_paths.extendleft(sub_paths[::-1])
def list_file(
api: BaiduPCSApi,
remotepath: str,
desc: bool = False,
name: bool = False,
time: bool = False,
size: bool = False,
recursive: bool = False,
sifters: List[Sifter] = [],
highlight: bool = False,
show_size: bool = False,
show_date: bool = False,
show_md5: bool = False,
show_absolute_path: bool = False,
):
is_dir = api.is_dir(remotepath)
if is_dir:
pcs_files = api.list(remotepath, desc=desc, name=name, time=time, size=size)
else:
pcs_files = api.meta(remotepath)
display_files(
pcs_files,
remotepath,
sifters=sifters,
highlight=highlight,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=show_absolute_path,
)
if is_dir and recursive:
for pcs_file in pcs_files:
if pcs_file.is_dir:
list_file(
api,
pcs_file.path,
desc=desc,
name=name,
time=time,
size=size,
recursive=recursive,
sifters=sifters,
highlight=highlight,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=show_absolute_path,
)
def download(
api: BaiduPCSApi,
remotepaths: List[str],
localdir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
):
"""Download `remotepaths` to the `localdir`
Args:
`from_index` (int): The start index of downloading entries from EACH remote directory
"""
remotepaths = sift(remotepaths, sifters)
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
download_file(
api,
rp,
localdir,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
)
else:
_localdir = str(Path(localdir) / os.path.basename(rp))
download_dir(
api,
rp,
_localdir,
sifters=sifters,
recursive=recursive,
from_index=from_index,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
)
if downloader == Downloader.me:
MeDownloader._exit_executor()
_progress.stop()
def search(
api: BaiduPCSApi,
keyword: str,
remotedir: str = "/",
recursive: bool = False,
sifters: Optional[List[Sifter]] = None,
highlight: bool = False,
show_size: bool = False,
show_date: bool = False,
show_md5: bool = False,
csv: bool = False,
):
pcs_files = api.search(keyword, remotedir, recursive=recursive)
sifters = [*(sifters or []), IncludeSifter(keyword)]
display_files(
pcs_files,
remotedir,
sifters=sifters,
highlight=highlight,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=True,
csv=csv,
)
def download_file(
api: BaiduPCSApi,
remotepath: str,
localdir: str,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
):
localpath = Path(localdir) / os.path.basename(remotepath)
# Make sure parent directory existed
if not localpath.parent.exists():
localpath.parent.mkdir(parents=True)
if not out_cmd and localpath.exists():
print(f"[yellow]{localpath}[/yellow] is ready existed.")
return
dlink = api.download_link(remotepath)
if downloader != Downloader.me:
print(f"[italic blue]Download[/italic blue]: {remotepath} to {localpath}")
downloader.download(
dlink,
str(localpath),
api.cookies,
downloadparams=downloadparams,
out_cmd=out_cmd,
)
def _get_download_link_and_rapid_upload_info(
api: BaiduPCSApi,
pcs_file: PcsFile,
show_dl_link: bool = False,
show_hash_link: bool = False,
check_md5: bool = True,
) -> Tuple[Optional[str], Optional[PcsRapidUploadInfo]]:
dl_link = None
if show_dl_link:
dl_link = api.download_link(pcs_file.path)
rpinfo = None
if show_hash_link:
rpinfo = api.rapid_upload_info(pcs_file.path, check=check_md5)
return dl_link, rpinfo
def play_file(
api: BaiduPCSApi,
remotepath: str,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
ignore_ext: bool = False,
out_cmd: bool = False,
local_server: str = "",
):
if not ignore_ext and not _with_media_ext(remotepath):
return
print(
f"[italic blue]Play[/italic blue]: {remotepath} {'(m3u8)' if m3u8 else ''}"
)
# For typing
url: Optional[str] = None
if m3u8:
m3u8_cn = api.m3u8_stream(remotepath)
with open(DEFAULT_TEMP_M3U8, "w") as fd:
fd.write(m3u8_cn)
url = DEFAULT_TEMP_M3U8
use_local_server = bool(local_server)
if use_local_server:
url = f"{local_server}{quote(remotepath)}"
print("url:", url)
else:
url = api.download_link(remotepath)
if not url:
display_blocked_remotepath(remotepath)
return
player.play(
url,
api.cookies,
m3u8=m3u8,
quiet=quiet,
player_params=player_params,
out_cmd=out_cmd,
use_local_server=use_local_server,
)
def play(
api: BaiduPCSApi,
remotepaths: List[str],
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
out_cmd: bool = False,
):
"""Play media file in `remotepaths`
Args:
`from_index` (int): The start index of playing entries from EACH remote directory
"""
for rp in remotepaths:
if not api.exists(rp):
print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.")
continue
if api.is_file(rp):
play_file(
api,
rp,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
else:
play_dir(
api,
rp,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
def makedir(api: BaiduPCSApi, *remotedirs: str, show: bool = False):
pcs_files = []
for d in remotedirs:
pcs_file = api.makedir(d)
pcs_files.append(pcs_file)
if show:
display_files(pcs_files, "/", show_absolute_path=True)
def list_shared(api: BaiduPCSApi, show_all=True):
page = 1
while True:
shared_links = api.list_shared(page=page)
if not shared_links:
break
page += 1
for sl in shared_links:
if sl.has_password():
assert sl.share_id
pwd = api.shared_password(sl.share_id)
sl = sl._replace(password=pwd)
if show_all or sl.available():
display_shared_links(sl)
def remotepath_exists(api: BaiduPCSApi,
name: str,
rd: str,
_cache: Dict[str, Set[str]] = {}) -> bool:
names = _cache.get(rd)
if not names:
names = set([PurePosixPath(sp.path).name for sp in api.list(rd)])
_cache[rd] = names
return name in names
def pcsapi(self) -> BaiduPCSApi:
auth = self.user.auth
assert auth, f"{self}.user.auth is None"
return BaiduPCSApi(
bduss=auth.bduss,
stoken=auth.stoken,
ptoken=auth.ptoken,
cookies=auth.cookies,
user_id=self.user.user_id,
)
def list_shared(api: BaiduPCSApi, page: int = 1, show_all=True):
_shared_links = api.list_shared(page=page)
if not _shared_links:
return
shared_links = []
for sl in _shared_links:
if sl.has_password():
assert sl.share_id
pwd = api.shared_password(sl.share_id)
sl = sl._replace(password=pwd)
if show_all:
shared_links.append(sl)
continue
if sl.available():
shared_links.append(sl)
display_shared_links(*shared_links)
def recursive_list(api: BaiduPCSApi, remotedir: Union[str, PcsFile]) -> List[PcsFile]:
if isinstance(remotedir, PcsFile):
remotedir = remotedir.path
pcs_files = []
for pcs_file in api.list(remotedir):
if pcs_file.is_file:
pcs_files.append(pcs_file)
else:
pcs_files.extend(recursive_list(api, pcs_file.path))
return pcs_files
def play_dir(
api: BaiduPCSApi,
remotedir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
shuffle: bool = False,
ignore_ext: bool = False,
out_cmd: bool = False,
local_server: str = "",
):
remotepaths = api.list(remotedir)
remotepaths = sift(remotepaths, sifters, recursive=recursive)
if shuffle:
rg = random.Random(time.time())
rg.shuffle(remotepaths)
for rp in remotepaths[from_index:]:
if rp.is_file:
play_file(
api,
rp.path,
player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
else: # is_dir
if recursive:
play_dir(
api,
rp.path,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
shuffle=shuffle,
ignore_ext=ignore_ext,
out_cmd=out_cmd,
local_server=local_server,
)
def cat(
api: BaiduPCSApi,
remotepath: str,
max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE,
encoding: Optional[str] = None,
):
rangeRequestIO = api.file_stream(remotepath)
cn = rangeRequestIO.read()
if cn:
if encoding:
print(cn.decode(encoding))
else:
r = chardet.detect(cn)
if r["confidence"] > 0.5:
print(cn.decode(r["encoding"]))
else:
print(cn)
def list_all_sub_paths(
api: BaiduPCSApi,
sharedpath: str,
uk: int,
share_id: int,
bdstoken: str,
) -> List[PcsSharedPath]:
sub_paths = []
page = 1
size = 100
while True:
sps = api.list_shared_paths(
sharedpath, uk, share_id, bdstoken, page=page, size=size
)
sub_paths += sps
if len(sps) < 100:
break
page += 1
return sub_paths
def download_dir(
api: BaiduPCSApi,
remotedir: str,
localdir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
downloader: Downloader = DEFAULT_DOWNLOADER,
downloadparams=DEFAULT_DOWNLOADPARAMS,
out_cmd: bool = False,
encrypt_password: bytes = b"",
):
remotepaths = api.list(remotedir)
remotepaths = sift(remotepaths, sifters, recursive=recursive)
for rp in remotepaths[from_index:]:
if rp.is_file:
download_file(
api,
rp.path,
localdir,
downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_password=encrypt_password,
)
else: # is_dir
if recursive:
_localdir = Path(localdir) / os.path.basename(rp.path)
download_dir(
api,
rp.path,
str(_localdir),
sifters=sifters,
recursive=recursive,
from_index=from_index,
downloader=downloader,
downloadparams=downloadparams,
out_cmd=out_cmd,
encrypt_password=encrypt_password,
)
def cat(
api: BaiduPCSApi,
remotepath: str,
max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE,
encoding: Optional[str] = None,
encrypt_password: bytes = b"",
):
fs = api.file_stream(remotepath, encrypt_password=encrypt_password)
if not fs:
display_blocked_remotepath(remotepath)
return
cn = fs.read()
if cn:
if encoding:
print(cn.decode(encoding))
else:
r = chardet.detect(cn)
if r["confidence"] > 0.5:
print(cn.decode(r["encoding"]))
else:
print(cn)
def play_dir(
api: BaiduPCSApi,
remotedir: str,
sifters: List[Sifter] = [],
recursive: bool = False,
from_index: int = 0,
player: Player = DEFAULT_PLAYER,
player_params: List[str] = [],
m3u8: bool = False,
quiet: bool = False,
out_cmd: bool = False,
):
remotepaths = api.list(remotedir)
remotepaths = sift(remotepaths, sifters)
for rp in remotepaths[from_index:]:
if rp.is_file:
play_file(
api,
rp.path,
player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
else: # is_dir
play_dir(
api,
rp.path,
sifters=sifters,
recursive=recursive,
from_index=from_index,
player=player,
player_params=player_params,
m3u8=m3u8,
quiet=quiet,
out_cmd=out_cmd,
)
def sync(
api: BaiduPCSApi,
localdir: str,
remotedir: str,
encrypt_password: bytes = b"",
encrypt_type: EncryptType = EncryptType.No,
max_workers: int = CPU_NUM,
slice_size: int = DEFAULT_SLICE_SIZE,
show_progress: bool = True,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
check_md5: bool = False,
):
localdir = Path(localdir).as_posix()
remotedir = Path(remotedir).as_posix()
is_file = api.is_file(remotedir)
assert not is_file, "remotedir must be a directory"
if not api.exists(remotedir):
all_pcs_files = {}
else:
all_pcs_files = {
pcs_file.path[len(remotedir) + 1 :]: pcs_file
for pcs_file in recursive_list(api, remotedir)
}
fts: List[FromTo] = []
check_list: List[Tuple[str, PcsFile]] = []
all_localpaths = set()
for localpath in walk(localdir):
path = localpath[len(localdir) + 1 :]
all_localpaths.add(path)
if path not in all_pcs_files:
fts.append(FromTo(localpath, join_path(remotedir, path)))
else:
check_list.append((localpath, all_pcs_files[path]))
for lp, pf in check_list:
lstat = Path(lp).stat()
if int(lstat.st_mtime) != pf.local_mtime or lstat.st_size != pf.size:
fts.append(FromTo(lp, pf.path))
to_deletes = []
for rp in all_pcs_files.keys():
if rp not in all_localpaths:
to_deletes.append(all_pcs_files[rp].path)
logger.debug(
"`sync`: all localpaths: %s, "
"localpaths needed to upload: %s, "
"remotepaths needed to delete: %s",
len(all_localpaths),
len(fts),
len(to_deletes),
)
# The md5 of remote file is incorrect at most time, so we don't compare md5
#
# # Compare localpath content md5 with remotepath content md5
# semaphore = Semaphore(max_workers)
# with ThreadPoolExecutor(max_workers=CPU_NUM) as executor:
# tasks = {}
# for lp, pf in check_list:
# semaphore.acquire()
# fut = executor.submit(sure_release, semaphore, check_file_md5, lp, pf)
# tasks[fut] = (lp, pf)
#
# for fut in as_completed(tasks):
# is_equal = fut.result()
# lp, pf = tasks[fut]
# if not is_equal:
# fts.append(FromTo(lp, pf.path))
_upload(
api,
fts,
encrypt_password=encrypt_password,
encrypt_type=encrypt_type,
max_workers=max_workers,
slice_size=slice_size,
ignore_existing=False,
show_progress=show_progress,
rapiduploadinfo_file=rapiduploadinfo_file,
user_id=user_id,
user_name=user_name,
check_md5=check_md5,
)
if to_deletes:
api.remove(*to_deletes)
print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")
def cancel_shared(api: BaiduPCSApi, *share_ids: int):
api.cancel_shared(*share_ids)
def copy(api: BaiduPCSApi, *remotepaths: str, show: bool = False):
from_to_list = api.copy(*remotepaths)
if show:
display_from_to(*from_to_list)
def share_files(api: BaiduPCSApi, *remotepaths: str, password: Optional[str] = None):
shared_link = api.share(*remotepaths, password=password)
display_shared_links(shared_link)
def remove(api: BaiduPCSApi, *remotepaths: str):
api.remove(*remotepaths)
def sync(
api: BaiduPCSApi,
localdir: str,
remotedir: str,
encrypt_key: Any = None,
salt: Any = None,
encrypt_type: EncryptType = EncryptType.No,
max_workers: int = CPU_NUM,
slice_size: int = DEFAULT_SLICE_SIZE,
show_progress: bool = True,
):
localdir = Path(localdir).as_posix()
remotedir = Path(remotedir).as_posix()
is_file = api.is_file(remotedir)
assert not is_file, "remotedir must be a directory"
if not api.exists(remotedir):
all_pcs_files = {}
else:
all_pcs_files = {
pcs_file.path[len(remotedir) + 1:]: pcs_file
for pcs_file in recursive_list(api, remotedir)
}
fts: List[FromTo] = []
check_list: List[Tuple[str, PcsFile]] = []
all_localpaths = set()
for localpath in walk(localdir):
path = localpath[len(localdir) + 1:]
all_localpaths.add(path)
if path not in all_pcs_files:
fts.append(FromTo(localpath, join_path(remotedir, path)))
else:
check_list.append((localpath, all_pcs_files[path]))
semaphore = Semaphore(max_workers)
with ThreadPoolExecutor(max_workers=CPU_NUM) as executor:
tasks = {}
for lp, pf in check_list:
semaphore.acquire()
fut = executor.submit(sure_release, semaphore, check_file_md5, lp,
pf)
tasks[fut] = (lp, pf)
for fut in as_completed(tasks):
is_equal = fut.result()
lp, pf = tasks[fut]
if not is_equal:
fts.append(FromTo(lp, pf.path))
_upload(
api,
fts,
encrypt_key=encrypt_key,
salt=salt,
encrypt_type=encrypt_type,
max_workers=max_workers,
slice_size=slice_size,
ignore_existing=False,
show_progress=show_progress,
)
to_deletes = []
for rp in all_pcs_files.keys():
if rp not in all_localpaths:
to_deletes.append(all_pcs_files[rp].path)
if to_deletes:
api.remove(*to_deletes)
print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")
