def rapid_upload_search(
rapiduploadinfo_file: str,
keyword: str,
in_filename: bool = False,
in_localpath: bool = False,
in_remotepath: bool = False,
in_user_name: bool = False,
in_md5: bool = False,
hash_link_protocol: str = PcsRapidUploadInfo.default_hash_link_protocol(),
show_all: bool = False,
only_hash_link: bool = False,
):
rapiduploadinfo = RapidUploadInfo(rapiduploadinfo_file)
rows = rapiduploadinfo.search(
keyword,
in_filename=in_filename,
in_localpath=in_localpath,
in_remotepath=in_remotepath,
in_user_name=in_user_name,
in_md5=in_md5,
)
_display(
rows,
hash_link_protocol=hash_link_protocol,
show_all=show_all,
only_hash_link=only_hash_link,
)
def _display(
rows: List[Dict[str, Any]],
hash_link_protocol: str = PcsRapidUploadInfo.default_hash_link_protocol(),
show_all: bool = False,
only_hash_link: bool = False,
):
if show_all and not only_hash_link:
display_rapid_upload_infos(rows)
else:
display_rapid_upload_links(rows,
hash_link_protocol=hash_link_protocol,
only_hash_link=only_hash_link)
def list_files(
api: BaiduPCSApi,
*remotepaths: str,
desc: bool = False,
name: bool = False,
time: bool = False,
size: bool = False,
recursive: bool = False,
sifters: List[Sifter] = [],
highlight: bool = False,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
show_size: bool = False,
show_date: bool = False,
show_md5: bool = False,
show_absolute_path: bool = False,
show_dl_link: bool = False,
show_hash_link: bool = False,
hash_link_protocol: str = PcsRapidUploadInfo.default_hash_link_protocol(),
check_md5: bool = True,
csv: bool = False,
only_dl_link: bool = False,
only_hash_link: bool = False,
):
for rp in remotepaths:
list_file(
api,
rp,
desc=desc,
name=name,
time=time,
size=size,
recursive=recursive,
sifters=sifters,
highlight=highlight,
rapiduploadinfo_file=rapiduploadinfo_file,
user_id=user_id,
user_name=user_name,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=show_absolute_path,
show_dl_link=show_dl_link,
show_hash_link=show_hash_link,
hash_link_protocol=hash_link_protocol,
check_md5=check_md5,
csv=csv,
only_dl_link=only_dl_link,
only_hash_link=only_hash_link,
)
def rapid_upload_list(
rapiduploadinfo_file: str,
ids: List[int] = [],
filename: bool = False,
time: bool = False,
size: bool = False,
localpath: bool = False,
remotepath: bool = False,
user_id: bool = False,
user_name: bool = False,
desc: bool = False,
limit: int = 0,
offset: int = -1,
hash_link_protocol: str = PcsRapidUploadInfo.default_hash_link_protocol(),
show_all: bool = False,
only_hash_link: bool = False,
):
rapiduploadinfo = RapidUploadInfo(rapiduploadinfo_file)
rows = rapiduploadinfo.list(
ids=ids,
by_filename=filename,
by_time=time,
by_size=size,
by_localpath=localpath,
by_remotepath=remotepath,
by_user_id=user_id,
by_user_name=user_name,
desc=desc,
limit=limit,
offset=offset,
)
_display(
rows,
hash_link_protocol=hash_link_protocol,
show_all=show_all,
only_hash_link=only_hash_link,
)
@click.option("--include-regex", "--IR", type=str, help="筛选包含这个正则表达式的文件")
@click.option("--exclude", "-E", type=str, help="筛选 不 包含这个字符串的文件")
@click.option("--exclude-regex", "--ER", type=str, help="筛选 不 包含这个正则表达式的文件")
@click.option("--is-file", "-f", is_flag=True, help="筛选 非 目录文件")
@click.option("--is-dir", "-d", is_flag=True, help="筛选目录文件")
@click.option("--no-highlight", "--NH", is_flag=True, help="取消匹配高亮")
@click.option("--show-size", "-S", is_flag=True, help="显示文件大小")
@click.option("--show-date", "-D", is_flag=True, help="显示文件创建时间")
@click.option("--show-md5", "-M", is_flag=True, help="显示文件md5")
@click.option("--show-absolute-path", "-A", is_flag=True, help="显示文件绝对路径")
@click.option("--show-dl-link", "--DL", is_flag=True, help="显示文件下载连接")
@click.option("--show-hash-link", "--HL", is_flag=True, help="显示文件秒传连接")
@click.option(
"--hash-link-protocol",
"--HLP",
type=click.Choice(PcsRapidUploadInfo.hash_link_protocols()),
default=PcsRapidUploadInfo.default_hash_link_protocol(),
help="显示文件hash链接，并指定协议",
)
@click.option(
"--no-check-md5",
"--NC",
is_flag=True,
help="显示文件 cs3l:// 连接时不检查md5",
)
@click.option("--csv",
is_flag=True,
help="用 csv 格式显示，单行显示，推荐和 --DL 或 --HL 一起用")
@click.option("--only-dl-link", "--ODL", is_flag=True, help="只显示文件下载连接")
@click.option("--only-hash-link", "--OHL", is_flag=True, help="只显示文件秒传连接")
@click.pass_context
def rapid_upload_info(self,
remotepath: str,
check: bool = True) -> Optional[PcsRapidUploadInfo]:
pcs_file = self.meta(remotepath)[0]
content_length = pcs_file.size
if content_length < 256 * constant.OneK:
return None
fs = self.file_stream(remotepath, pcs=False)
if not fs:
return None
data = fs.read(256 * constant.OneK)
assert data and len(data) == 256 * constant.OneK
slice_md5 = calu_md5(data)
assert (content_length
and content_length == fs._auto_decrypt_request.content_length)
content_md5 = fs._auto_decrypt_request.content_md5
content_crc32 = fs._auto_decrypt_request.content_crc32 or 0
if not content_md5:
return None
block_list = pcs_file.block_list
if block_list and len(
block_list) == 1 and block_list[0] == pcs_file.md5:
return PcsRapidUploadInfo(
slice_md5=slice_md5,
content_md5=content_md5,
content_crc32=content_crc32,
content_length=content_length,
remotepath=pcs_file.path,
)
if check:
try:
# Try rapid_upload_file
self.rapid_upload_file(
slice_md5,
content_md5,
content_crc32,
content_length,
pcs_file.path,
local_ctime=pcs_file.local_ctime,
local_mtime=pcs_file.local_mtime,
ondup="overwrite",
)
except BaiduPCSError as err:
# 31079: "未找到文件MD5"
if err.error_code != 31079:
raise err
return None
return PcsRapidUploadInfo(
slice_md5=slice_md5,
content_md5=content_md5,
content_crc32=content_crc32,
content_length=content_length,
remotepath=pcs_file.path,
)
def list_file(
api: BaiduPCSApi,
remotepath: str,
desc: bool = False,
name: bool = False,
time: bool = False,
size: bool = False,
recursive: bool = False,
sifters: List[Sifter] = [],
highlight: bool = False,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
show_size: bool = False,
show_date: bool = False,
show_md5: bool = False,
show_absolute_path: bool = False,
show_dl_link: bool = False,
show_hash_link: bool = False,
hash_link_protocol: str = PcsRapidUploadInfo.default_hash_link_protocol(),
check_md5: bool = True,
csv: bool = False,
only_dl_link: bool = False,
only_hash_link: bool = False,
):
is_dir = api.is_dir(remotepath)
if is_dir:
pcs_files = api.list(remotepath,
desc=desc,
name=name,
time=time,
size=size)
else:
pcs_files = api.meta(remotepath)
pcs_files = sift(pcs_files, sifters, recursive=recursive)
if not pcs_files:
return
if show_dl_link or show_hash_link:
# Concurrently request rapiduploadinfo
max_workers = DEFAULT_MAX_WORKERS
semaphore = Semaphore(max_workers)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futs = {}
for i in range(len(pcs_files)):
if pcs_files[i].is_dir:
continue
semaphore.acquire()
fut = executor.submit(
sure_release,
semaphore,
_get_download_link_and_rapid_upload_info,
api,
pcs_files[i],
show_dl_link=show_dl_link,
show_hash_link=show_hash_link,
check_md5=check_md5,
)
futs[fut] = i
for fut in as_completed(futs):
i = futs[fut]
e = fut.exception()
if e is None:
dl_link, rpinfo = fut.result()
if rapiduploadinfo_file and rpinfo:
save_rapid_upload_info(
rapiduploadinfo_file,
rpinfo.slice_md5,
rpinfo.content_md5,
rpinfo.content_crc32,
rpinfo.content_length,
remotepath=pcs_files[i].path,
user_id=user_id,
user_name=user_name,
)
pcs_files[i] = pcs_files[i]._replace(
dl_link=dl_link, rapid_upload_info=rpinfo)
if only_dl_link and dl_link:
print(dl_link)
if only_hash_link and rpinfo:
hash_link = getattr(rpinfo, hash_link_protocol)()
print(hash_link)
else:
logger.error(
"`list_file`: `_get_download_link_and_rapid_upload_info` error: %s",
e,
)
if not only_dl_link and not only_hash_link:
display_files(
pcs_files,
remotepath,
sifters=sifters,
highlight=highlight,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=show_absolute_path,
show_dl_link=show_dl_link,
show_hash_link=show_hash_link,
hash_link_protocol=hash_link_protocol,
csv=csv,
)
if is_dir and recursive:
for pcs_file in pcs_files:
if pcs_file.is_dir:
list_file(
api,
pcs_file.path,
desc=desc,
name=name,
time=time,
size=size,
recursive=recursive,
sifters=sifters,
highlight=highlight,
rapiduploadinfo_file=rapiduploadinfo_file,
user_id=user_id,
user_name=user_name,
show_size=show_size,
show_date=show_date,
show_md5=show_md5,
show_absolute_path=show_absolute_path,
show_dl_link=show_dl_link,
show_hash_link=show_hash_link,
hash_link_protocol=hash_link_protocol,
check_md5=check_md5,
csv=csv,
only_dl_link=only_dl_link,
only_hash_link=only_hash_link,
)
def rapid_upload(
api: BaiduPCSApi,
remotedir: str,
link: str = "",
slice_md5: str = "",
content_md5: str = "",
content_crc32: int = 0,
content_length: int = 0,
filename: str = "",
no_ignore_existing: bool = False,
rapiduploadinfo_file: Optional[str] = None,
user_id: Optional[int] = None,
user_name: Optional[str] = None,
):
"""Rapid upload with params
If given `link` and `filename`, then filename of link will be replace by `filename`
"""
if link:
slice_md5, content_md5, content_crc32, content_length, _filename = _parse_link(
link)
content_crc32 = content_crc32 or 0
filename = filename or _filename
remotepath = join_path(remotedir, filename)
assert all([slice_md5, content_md5, content_length
]), f"`rapid_upload`: parsing rapid upload link fails: {link}"
if not no_ignore_existing:
if api.exists(remotepath):
return
try:
pcs_file = api.rapid_upload_file(
slice_md5,
content_md5,
content_crc32,
content_length,
remotepath,
ondup="overwrite",
)
if rapiduploadinfo_file:
save_rapid_upload_info(
rapiduploadinfo_file,
slice_md5,
content_md5,
content_crc32,
content_length,
remotepath=remotepath,
user_id=user_id,
user_name=user_name,
)
print(f"[i blue]Save to[/i blue] {pcs_file.path}")
except Exception as err:
link = PcsRapidUploadInfo(
slice_md5=slice_md5,
content_md5=content_md5,
content_crc32=content_crc32,
content_length=content_length,
remotepath=remotepath,
).cs3l()
print(
f"[i yellow]Rapid Upload fails[/i yellow]: error: {err} link: {link}",
)