def add_task( api: BaiduPCSApi, task_url: str, remotedir: str, file_types: List[FileType] = [FileType.Media], ): if is_magnet(task_url): logger.warning("Add cloud task: magnet: %s", task_url) pmfs = api.magnet_info(task_url) selected_idx = [] for idx, pmf in enumerate(pmfs, 1): ext = os.path.splitext(pmf.path)[-1] for file_type in file_types: if file_type.sift(ext): selected_idx.append(idx) break if not selected_idx: logger.warning("`add_task`: No selected idx for %s", task_url) return task_id = api.add_magnet_task(task_url, remotedir, selected_idx) tasks(api, task_id) else: logger.warning("Add cloud task: %s", task_url) task_id = api.add_task(task_url, remotedir)
def list_shared_paths( api: BaiduPCSApi, shared_url: str, password: Optional[str] = None, show_vcode: bool = True, ): # Vertify with password if password: api.access_shared(shared_url, password, show_vcode=show_vcode) all_shared_paths: List[PcsSharedPath] = [] shared_paths = deque(api.shared_paths(shared_url)) all_shared_paths += shared_paths while shared_paths: shared_path = shared_paths.popleft() uk, share_id, bdstoken = ( shared_path.uk, shared_path.share_id, shared_path.bdstoken, ) assert uk assert share_id assert bdstoken if shared_path.is_dir: # Take all sub paths sub_paths = list_all_sub_paths(api, shared_path.path, uk, share_id, bdstoken) all_shared_paths += sub_paths shared_paths.extendleft(sub_paths[::-1]) display_shared_paths(*all_shared_paths)
def play_file( api: BaiduPCSApi, remotepath: str, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, out_cmd: bool = False, ): if not _with_media_ext(remotepath): return print(f"[italic blue]Play[/italic blue]: {remotepath} {'(m3u8)' if m3u8 else ''}") if m3u8: m3u8_cn = api.m3u8_stream(remotepath) with open(DEFAULT_TEMP_M3U8, "w") as fd: fd.write(m3u8_cn) url = DEFAULT_TEMP_M3U8 else: url = api.download_link(remotepath) player.play( url, api.cookies, m3u8=m3u8, quiet=quiet, player_params=player_params, out_cmd=out_cmd, )
def play( api: BaiduPCSApi, remotepaths: List[str], sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, shuffle: bool = False, ignore_ext: bool = False, out_cmd: bool = False, local_server: str = "", ): """Play media file in `remotepaths` Args: `from_index` (int): The start index of playing entries from EACH remote directory """ if shuffle: rg = random.Random(time.time()) rg.shuffle(remotepaths) for rp in remotepaths: if not api.exists(rp): print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.") continue if api.is_file(rp): play_file( api, rp, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, ignore_ext=ignore_ext, out_cmd=out_cmd, local_server=local_server, ) else: play_dir( api, rp, sifters=sifters, recursive=recursive, from_index=from_index, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, shuffle=shuffle, ignore_ext=ignore_ext, out_cmd=out_cmd, local_server=local_server, )
def save_shared( api: BaiduPCSApi, shared_url: str, remotedir: str, password=Optional[str], show_vcode: bool = True, ): assert remotedir.startswith("/"), "`remotedir` must be an absolute path" # Vertify with password if password: api.access_shared(shared_url, password, show_vcode=show_vcode) shared_paths = deque(api.shared_paths(shared_url)) # Record the remotedir of each shared_path _remotedirs: Dict[PcsSharedPath, str] = {} for sp in shared_paths: _remotedirs[sp] = remotedir _dir_exists: Set[str] = set() while shared_paths: shared_path = shared_paths.popleft() uk, share_id, bdstoken = ( shared_path.uk, shared_path.share_id, shared_path.bdstoken, ) assert uk assert share_id assert bdstoken rd = _remotedirs[shared_path] if rd not in _dir_exists and not api.exists(rd): api.makedir(rd) _dir_exists.add(rd) # rd = (Path(_remotedirs[shared_path]) / os.path.basename(shared_path.path)).as_posix() try: api.transfer_shared_paths(rd, [shared_path.fs_id], uk, share_id, bdstoken, shared_url) print(f"save: {shared_path.path} to {rd}") continue except BaiduPCSError as err: if err.error_code not in (12, -33): raise err if err.error_code == 12: # -33: '一次支持操作999个,减点试试吧' print(f"[yellow]WARNING[/]: {shared_path.path} has be in {rd}") if err.error_code == -33: # -33: '一次支持操作999个,减点试试吧' print(f"[yellow]WARNING[/]: {shared_path.path} " "has more items and need to transfer one by one") sub_paths = api.list_shared_paths(shared_path.path, uk, share_id, bdstoken) rd = (Path(rd) / os.path.basename(shared_path.path)).as_posix() for sp in sub_paths: _remotedirs[sp] = rd shared_paths.extendleft(sub_paths[::-1])
def list_file( api: BaiduPCSApi, remotepath: str, desc: bool = False, name: bool = False, time: bool = False, size: bool = False, recursive: bool = False, sifters: List[Sifter] = [], highlight: bool = False, show_size: bool = False, show_date: bool = False, show_md5: bool = False, show_absolute_path: bool = False, ): is_dir = api.is_dir(remotepath) if is_dir: pcs_files = api.list(remotepath, desc=desc, name=name, time=time, size=size) else: pcs_files = api.meta(remotepath) display_files( pcs_files, remotepath, sifters=sifters, highlight=highlight, show_size=show_size, show_date=show_date, show_md5=show_md5, show_absolute_path=show_absolute_path, ) if is_dir and recursive: for pcs_file in pcs_files: if pcs_file.is_dir: list_file( api, pcs_file.path, desc=desc, name=name, time=time, size=size, recursive=recursive, sifters=sifters, highlight=highlight, show_size=show_size, show_date=show_date, show_md5=show_md5, show_absolute_path=show_absolute_path, )
def download( api: BaiduPCSApi, remotepaths: List[str], localdir: str, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, downloader: Downloader = DEFAULT_DOWNLOADER, downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, out_cmd: bool = False, ): """Download `remotepaths` to the `localdir` Args: `from_index` (int): The start index of downloading entries from EACH remote directory """ remotepaths = sift(remotepaths, sifters) for rp in remotepaths: if not api.exists(rp): print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.") continue if api.is_file(rp): download_file( api, rp, localdir, downloader=downloader, downloadparams=downloadparams, out_cmd=out_cmd, ) else: _localdir = str(Path(localdir) / os.path.basename(rp)) download_dir( api, rp, _localdir, sifters=sifters, recursive=recursive, from_index=from_index, downloader=downloader, downloadparams=downloadparams, out_cmd=out_cmd, ) if downloader == Downloader.me: MeDownloader._exit_executor() _progress.stop()
def search( api: BaiduPCSApi, keyword: str, remotedir: str = "/", recursive: bool = False, sifters: Optional[List[Sifter]] = None, highlight: bool = False, show_size: bool = False, show_date: bool = False, show_md5: bool = False, csv: bool = False, ): pcs_files = api.search(keyword, remotedir, recursive=recursive) sifters = [*(sifters or []), IncludeSifter(keyword)] display_files( pcs_files, remotedir, sifters=sifters, highlight=highlight, show_size=show_size, show_date=show_date, show_md5=show_md5, show_absolute_path=True, csv=csv, )
def download_file( api: BaiduPCSApi, remotepath: str, localdir: str, downloader: Downloader = DEFAULT_DOWNLOADER, downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, out_cmd: bool = False, ): localpath = Path(localdir) / os.path.basename(remotepath) # Make sure parent directory existed if not localpath.parent.exists(): localpath.parent.mkdir(parents=True) if not out_cmd and localpath.exists(): print(f"[yellow]{localpath}[/yellow] is ready existed.") return dlink = api.download_link(remotepath) if downloader != Downloader.me: print(f"[italic blue]Download[/italic blue]: {remotepath} to {localpath}") downloader.download( dlink, str(localpath), api.cookies, downloadparams=downloadparams, out_cmd=out_cmd, )
def _get_download_link_and_rapid_upload_info( api: BaiduPCSApi, pcs_file: PcsFile, show_dl_link: bool = False, show_hash_link: bool = False, check_md5: bool = True, ) -> Tuple[Optional[str], Optional[PcsRapidUploadInfo]]: dl_link = None if show_dl_link: dl_link = api.download_link(pcs_file.path) rpinfo = None if show_hash_link: rpinfo = api.rapid_upload_info(pcs_file.path, check=check_md5) return dl_link, rpinfo
def play_file( api: BaiduPCSApi, remotepath: str, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, ignore_ext: bool = False, out_cmd: bool = False, local_server: str = "", ): if not ignore_ext and not _with_media_ext(remotepath): return print( f"[italic blue]Play[/italic blue]: {remotepath} {'(m3u8)' if m3u8 else ''}" ) # For typing url: Optional[str] = None if m3u8: m3u8_cn = api.m3u8_stream(remotepath) with open(DEFAULT_TEMP_M3U8, "w") as fd: fd.write(m3u8_cn) url = DEFAULT_TEMP_M3U8 use_local_server = bool(local_server) if use_local_server: url = f"{local_server}{quote(remotepath)}" print("url:", url) else: url = api.download_link(remotepath) if not url: display_blocked_remotepath(remotepath) return player.play( url, api.cookies, m3u8=m3u8, quiet=quiet, player_params=player_params, out_cmd=out_cmd, use_local_server=use_local_server, )
def play( api: BaiduPCSApi, remotepaths: List[str], sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, out_cmd: bool = False, ): """Play media file in `remotepaths` Args: `from_index` (int): The start index of playing entries from EACH remote directory """ for rp in remotepaths: if not api.exists(rp): print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.") continue if api.is_file(rp): play_file( api, rp, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, out_cmd=out_cmd, ) else: play_dir( api, rp, sifters=sifters, recursive=recursive, from_index=from_index, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, out_cmd=out_cmd, )
def makedir(api: BaiduPCSApi, *remotedirs: str, show: bool = False): pcs_files = [] for d in remotedirs: pcs_file = api.makedir(d) pcs_files.append(pcs_file) if show: display_files(pcs_files, "/", show_absolute_path=True)
def list_shared(api: BaiduPCSApi, show_all=True): page = 1 while True: shared_links = api.list_shared(page=page) if not shared_links: break page += 1 for sl in shared_links: if sl.has_password(): assert sl.share_id pwd = api.shared_password(sl.share_id) sl = sl._replace(password=pwd) if show_all or sl.available(): display_shared_links(sl)
def remotepath_exists(api: BaiduPCSApi, name: str, rd: str, _cache: Dict[str, Set[str]] = {}) -> bool: names = _cache.get(rd) if not names: names = set([PurePosixPath(sp.path).name for sp in api.list(rd)]) _cache[rd] = names return name in names
def pcsapi(self) -> BaiduPCSApi: auth = self.user.auth assert auth, f"{self}.user.auth is None" return BaiduPCSApi( bduss=auth.bduss, stoken=auth.stoken, ptoken=auth.ptoken, cookies=auth.cookies, user_id=self.user.user_id, )
def list_shared(api: BaiduPCSApi, page: int = 1, show_all=True): _shared_links = api.list_shared(page=page) if not _shared_links: return shared_links = [] for sl in _shared_links: if sl.has_password(): assert sl.share_id pwd = api.shared_password(sl.share_id) sl = sl._replace(password=pwd) if show_all: shared_links.append(sl) continue if sl.available(): shared_links.append(sl) display_shared_links(*shared_links)
def recursive_list(api: BaiduPCSApi, remotedir: Union[str, PcsFile]) -> List[PcsFile]: if isinstance(remotedir, PcsFile): remotedir = remotedir.path pcs_files = [] for pcs_file in api.list(remotedir): if pcs_file.is_file: pcs_files.append(pcs_file) else: pcs_files.extend(recursive_list(api, pcs_file.path)) return pcs_files
def play_dir( api: BaiduPCSApi, remotedir: str, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, shuffle: bool = False, ignore_ext: bool = False, out_cmd: bool = False, local_server: str = "", ): remotepaths = api.list(remotedir) remotepaths = sift(remotepaths, sifters, recursive=recursive) if shuffle: rg = random.Random(time.time()) rg.shuffle(remotepaths) for rp in remotepaths[from_index:]: if rp.is_file: play_file( api, rp.path, player, player_params=player_params, m3u8=m3u8, quiet=quiet, ignore_ext=ignore_ext, out_cmd=out_cmd, local_server=local_server, ) else: # is_dir if recursive: play_dir( api, rp.path, sifters=sifters, recursive=recursive, from_index=from_index, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, shuffle=shuffle, ignore_ext=ignore_ext, out_cmd=out_cmd, local_server=local_server, )
def cat( api: BaiduPCSApi, remotepath: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, encoding: Optional[str] = None, ): rangeRequestIO = api.file_stream(remotepath) cn = rangeRequestIO.read() if cn: if encoding: print(cn.decode(encoding)) else: r = chardet.detect(cn) if r["confidence"] > 0.5: print(cn.decode(r["encoding"])) else: print(cn)
def list_all_sub_paths( api: BaiduPCSApi, sharedpath: str, uk: int, share_id: int, bdstoken: str, ) -> List[PcsSharedPath]: sub_paths = [] page = 1 size = 100 while True: sps = api.list_shared_paths( sharedpath, uk, share_id, bdstoken, page=page, size=size ) sub_paths += sps if len(sps) < 100: break page += 1 return sub_paths
def download_dir( api: BaiduPCSApi, remotedir: str, localdir: str, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, downloader: Downloader = DEFAULT_DOWNLOADER, downloadparams=DEFAULT_DOWNLOADPARAMS, out_cmd: bool = False, encrypt_password: bytes = b"", ): remotepaths = api.list(remotedir) remotepaths = sift(remotepaths, sifters, recursive=recursive) for rp in remotepaths[from_index:]: if rp.is_file: download_file( api, rp.path, localdir, downloader, downloadparams=downloadparams, out_cmd=out_cmd, encrypt_password=encrypt_password, ) else: # is_dir if recursive: _localdir = Path(localdir) / os.path.basename(rp.path) download_dir( api, rp.path, str(_localdir), sifters=sifters, recursive=recursive, from_index=from_index, downloader=downloader, downloadparams=downloadparams, out_cmd=out_cmd, encrypt_password=encrypt_password, )
def cat( api: BaiduPCSApi, remotepath: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, encoding: Optional[str] = None, encrypt_password: bytes = b"", ): fs = api.file_stream(remotepath, encrypt_password=encrypt_password) if not fs: display_blocked_remotepath(remotepath) return cn = fs.read() if cn: if encoding: print(cn.decode(encoding)) else: r = chardet.detect(cn) if r["confidence"] > 0.5: print(cn.decode(r["encoding"])) else: print(cn)
def play_dir( api: BaiduPCSApi, remotedir: str, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, player: Player = DEFAULT_PLAYER, player_params: List[str] = [], m3u8: bool = False, quiet: bool = False, out_cmd: bool = False, ): remotepaths = api.list(remotedir) remotepaths = sift(remotepaths, sifters) for rp in remotepaths[from_index:]: if rp.is_file: play_file( api, rp.path, player, player_params=player_params, m3u8=m3u8, quiet=quiet, out_cmd=out_cmd, ) else: # is_dir play_dir( api, rp.path, sifters=sifters, recursive=recursive, from_index=from_index, player=player, player_params=player_params, m3u8=m3u8, quiet=quiet, out_cmd=out_cmd, )
def sync( api: BaiduPCSApi, localdir: str, remotedir: str, encrypt_password: bytes = b"", encrypt_type: EncryptType = EncryptType.No, max_workers: int = CPU_NUM, slice_size: int = DEFAULT_SLICE_SIZE, show_progress: bool = True, rapiduploadinfo_file: Optional[str] = None, user_id: Optional[int] = None, user_name: Optional[str] = None, check_md5: bool = False, ): localdir = Path(localdir).as_posix() remotedir = Path(remotedir).as_posix() is_file = api.is_file(remotedir) assert not is_file, "remotedir must be a directory" if not api.exists(remotedir): all_pcs_files = {} else: all_pcs_files = { pcs_file.path[len(remotedir) + 1 :]: pcs_file for pcs_file in recursive_list(api, remotedir) } fts: List[FromTo] = [] check_list: List[Tuple[str, PcsFile]] = [] all_localpaths = set() for localpath in walk(localdir): path = localpath[len(localdir) + 1 :] all_localpaths.add(path) if path not in all_pcs_files: fts.append(FromTo(localpath, join_path(remotedir, path))) else: check_list.append((localpath, all_pcs_files[path])) for lp, pf in check_list: lstat = Path(lp).stat() if int(lstat.st_mtime) != pf.local_mtime or lstat.st_size != pf.size: fts.append(FromTo(lp, pf.path)) to_deletes = [] for rp in all_pcs_files.keys(): if rp not in all_localpaths: to_deletes.append(all_pcs_files[rp].path) logger.debug( "`sync`: all localpaths: %s, " "localpaths needed to upload: %s, " "remotepaths needed to delete: %s", len(all_localpaths), len(fts), len(to_deletes), ) # The md5 of remote file is incorrect at most time, so we don't compare md5 # # # Compare localpath content md5 with remotepath content md5 # semaphore = Semaphore(max_workers) # with ThreadPoolExecutor(max_workers=CPU_NUM) as executor: # tasks = {} # for lp, pf in check_list: # semaphore.acquire() # fut = executor.submit(sure_release, semaphore, check_file_md5, lp, pf) # tasks[fut] = (lp, pf) # # for fut in as_completed(tasks): # is_equal = fut.result() # lp, pf = tasks[fut] # if not is_equal: # fts.append(FromTo(lp, pf.path)) _upload( api, fts, encrypt_password=encrypt_password, encrypt_type=encrypt_type, max_workers=max_workers, slice_size=slice_size, ignore_existing=False, show_progress=show_progress, rapiduploadinfo_file=rapiduploadinfo_file, user_id=user_id, user_name=user_name, check_md5=check_md5, ) if to_deletes: api.remove(*to_deletes) print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")
def cancel_shared(api: BaiduPCSApi, *share_ids: int): api.cancel_shared(*share_ids)
def copy(api: BaiduPCSApi, *remotepaths: str, show: bool = False): from_to_list = api.copy(*remotepaths) if show: display_from_to(*from_to_list)
def share_files(api: BaiduPCSApi, *remotepaths: str, password: Optional[str] = None): shared_link = api.share(*remotepaths, password=password) display_shared_links(shared_link)
def remove(api: BaiduPCSApi, *remotepaths: str): api.remove(*remotepaths)
def sync( api: BaiduPCSApi, localdir: str, remotedir: str, encrypt_key: Any = None, salt: Any = None, encrypt_type: EncryptType = EncryptType.No, max_workers: int = CPU_NUM, slice_size: int = DEFAULT_SLICE_SIZE, show_progress: bool = True, ): localdir = Path(localdir).as_posix() remotedir = Path(remotedir).as_posix() is_file = api.is_file(remotedir) assert not is_file, "remotedir must be a directory" if not api.exists(remotedir): all_pcs_files = {} else: all_pcs_files = { pcs_file.path[len(remotedir) + 1:]: pcs_file for pcs_file in recursive_list(api, remotedir) } fts: List[FromTo] = [] check_list: List[Tuple[str, PcsFile]] = [] all_localpaths = set() for localpath in walk(localdir): path = localpath[len(localdir) + 1:] all_localpaths.add(path) if path not in all_pcs_files: fts.append(FromTo(localpath, join_path(remotedir, path))) else: check_list.append((localpath, all_pcs_files[path])) semaphore = Semaphore(max_workers) with ThreadPoolExecutor(max_workers=CPU_NUM) as executor: tasks = {} for lp, pf in check_list: semaphore.acquire() fut = executor.submit(sure_release, semaphore, check_file_md5, lp, pf) tasks[fut] = (lp, pf) for fut in as_completed(tasks): is_equal = fut.result() lp, pf = tasks[fut] if not is_equal: fts.append(FromTo(lp, pf.path)) _upload( api, fts, encrypt_key=encrypt_key, salt=salt, encrypt_type=encrypt_type, max_workers=max_workers, slice_size=slice_size, ignore_existing=False, show_progress=show_progress, ) to_deletes = [] for rp in all_pcs_files.keys(): if rp not in all_localpaths: to_deletes.append(all_pcs_files[rp].path) if to_deletes: api.remove(*to_deletes) print(f"Delete: [i]{len(to_deletes)}[/i] remote paths")