def TiebaRequest(url, data, first=False):
JsonRetry = 0
while (JsonRetry <= 5):
try:
if (first):
req = Retry(requests.post,
args=(url, ),
kwargs={"data": SignRequest(data)},
cfunc=(lambda x: x.status_code == 200),
ffunc=print,
fargs=("连接失败,正在重试...\n", ),
times=10)
else:
req = Retry(requests.post,
args=(url, ),
kwargs={"data": SignRequest(data)},
cfunc=(lambda x: x.status_code == 200),
ffunc=Progress.set_description,
fargs=("连接失败,正在重试...", ),
times=10)
req.encoding = 'utf-8'
ret = req.json() # JSON错误的源头
break
except:
Avalon.warning("遇到一次JSON错误,正在重试")
JsonRetry += 1
continue
#print(ret)
if (int(ret["error_code"]) != 0):
raise RequestError({
"code": int(ret["error_code"]),
"msg": str(ret["error_msg"])
})
return ret
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
if (str(s["type"]) == "0"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "1"):
content += ProcessUrl(s["link"], s["text"])
elif (str(s["type"]) == "2"):
content += ProcessEmotion(floor, s["text"], s["c"])
elif (str(s["type"]) == "3"):
content += ProcessImg(s["origin_src"])
elif (str(s["type"]) == "4"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "5"):
content += ProcessVideo(s["link"], s["src"])
elif (str(s["type"]) == "9"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "10"):
content += ProcessAudio(s["voice_md5"])
elif (str(s["type"]) == "11"):
content += ProcessImg(s["static"])
elif (str(s["type"]) == "20"):
content += ProcessImg(s["src"])
else:
Avalon.warning("floor %s: content data wrong: \n%s\n" %
(floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
return content
def delete_old_files():
Avalon.info("删除3天以前的备份...")
Dirname_backuped = DirName + "-" + time.strftime(
"%Y%m%d-%Hh", time.localtime(int(time.time()) - 86400 * 3))
if os.path.exists(Dirname_backuped):
Avalon.warning("检测到3天前的备份，删除ing...")
shutil.rmtree(Dirname_backuped)
Avalon.info("删除完毕")
else:
Avalon.warning("未发现3天前的备份，跳过")
def Init(pid, overwrite):
global FileHandle,Progress,AudioCount,VideoCount,ImageCount,\
Pool,IsDownload,DirName,IsCreate,OutputHTML,FFmpeg
IsDownload = set()
IsCreate = set()
AudioCount = VideoCount = ImageCount = 0
if (os.path.isdir(DirName)):
Avalon.warning("\"%s\"已存在" % DirName)
if (overwrite == 1):
Avalon.warning("跳过%d" % pid)
elif (overwrite == 2):
Avalon.warning("默认覆盖\"%s\"" % DirName)
elif (not Avalon.ask("是否覆盖?", False)):
raise UserCancelled("...")
elif (os.path.exists(DirName)):
raise OSError("存在同名文件")
else:
os.makedirs(DirName)
if (OutputHTML):
FileHandle = open("%s/%d.html" % (DirName, pid), "w", encoding="utf-8")
Write('<!doctype html><html lang="zh-cn"><head><link rel="stylesheet"'\
' type="text/css" href="main.css"></head><body><div id="write">')
shutil.copy("main.css", DirName + "/")
else:
FileHandle = open("%s/%d.md" % (DirName, pid), "w", encoding="utf-8")
try:
subprocess.Popen("ffmpeg",stdout=subprocess.DEVNULL,\
stderr=subprocess.DEVNULL).wait()
FFmpeg = 1
except FileNotFoundError:
Avalon.warning("未找到ffmpeg,语音将不会被转为mp3")
FFmpeg = 0
Pool = DownloadPool(DirName + "/", "file")
Progress = tqdm(unit="floor")
def send_wxmsg(_sckey, _title="标题", _context="正文"):
url = "https://sc.ftqq.com/%s.send" % _sckey
_context = _context + " \n\n" + time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime())
data = {"text": "%s" % _title, "desp": "%s" % _context}
try:
res = requests.post(url=url, data=data)
msg_back = json.loads(res.text)
if msg_back["errmsg"] == "success":
Avalon.info("返回值：%s" % (msg_back["errmsg"]))
else:
Avalon.warning("返回值：%s" % (msg_back["errmsg"]))
except Exception:
Avalon.error("消息发送错误")
def copydir_overwrite(_from_path, _to_path):
if os.path.exists(_from_path):
if os.path.exists(_to_path):
Avalon.warning("目标目录已存在，删除ing...")
try:
shutil.rmtree(_to_path)
except Exception as err1:
Avalon.error("删除目标文件失败，此次复制取消\n" + str(err1))
return 1
try:
shutil.copytree(_from_path, _to_path)
except Exception as err2:
Avalon.error("复制失败！\n" + str(err2))
else:
Avalon.info("复制成功")
else:
Avalon.warning("源目录不存在，跳过")
def customized_tools():
# 以下--删除3天以前的备份
Tools.delete_old_files()
# 以下--复制文件到网站目录
if copy_to_website == 1:
Avalon.info("准备复制文件到网站目录----")
Tools.copydir_overwrite(_from_path="./%s" % DirName,
_to_path=website_dir + DirName)
elif copy_to_website == 0:
Avalon.info("跳过 复制文件到网站目录")
# 以下--向server酱推送消息
if not sckey == "":
Avalon.info("尝试向Server酱推送消息……")
Tools.send_wxmsg(_sckey=sckey,
_title="贴吧备份（全部楼层）",
_context="今日份的备份已完成...\n\n一切顺利..Maybe..\n\n帖子id：%d" %
pid)
else:
Avalon.warning("SCKEY为空，跳过微信推送")
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
try:
if str(s["type"]) == "0":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "1":
content += ProcessUrl(s["link"], s["text"])
elif str(s["type"]) == "2":
content += ProcessEmotion(floor, s["text"], s["c"])
elif str(s["type"]) == "3":
content += ProcessImg(s["origin_src"])
elif str(s["type"]) == "4":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "5":
try:
content += ProcessVideo(s["link"], s["src"])
except KeyError:
content += ProcessQuoteVideo(s["text"])
elif str(s["type"]) == "9":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "10":
content += ProcessAudio(s["voice_md5"])
elif str(s["type"]) == "11":
content += ProcessImg(s["static"])
elif str(s["type"]) == "20":
content += ProcessImg(s["src"])
else:
Avalon.warning("floor %s: content data wrong: \n%s\n" %
(floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
except KeyError:
write_err_info()
Avalon.error("KeyError! 建议修改源码中字典的Key\n" + traceback.format_exc(),
front="\n")
content += '\n<a>[Error] 这里似乎出错了...类型 KeyError</a>\n'
except Exception:
write_err_info()
Avalon.error("发生异常:\n" + traceback.format_exc(), front="\n")
content += '\n<a>[Error] 这里似乎出错了...</a>\n'
else:
return content
def ProcessEmotion(floor, name, text):
global DirName, IsDownload
MakeDir(DirName + "/images")
lname = len(name)
if (name == "image_emoticon"):
name += "1"
lname += 1
url = ""
if (lname >= 3 and name[0:3] == "ali"):
url = "%s%s.gif" % (const.AliUrl, name)
name += ".gif"
elif (lname >= 14 and name[0:14] == "image_emoticon"):
url = "%s%s.png" % (const.EmotionUrl, name)
name += ".png"
else:
Avalon.warning("第%s楼出现未知表情:%s\n" % (floor, name), front="\n")
return ''
if (not name in IsDownload):
IsDownload.add(name)
Pool.Download(url, "images/%s" % name)
return '<img src="images/%s" alt="%s" title="%s" />' % (name, text, text)
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
#TODO:断点续传
if (str(s["type"]) == "0"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "1"):
content += ProcessUrl(s["link"], s["text"])
elif (str(s["type"]) == "2"):
content += ProcessEmotion(floor, s["text"], s["c"])
elif (str(s["type"]) == "3"):
try:
content += ProcessImg(s["origin_src"])
except:
content += "无法获取该图片,已自动跳过!" + str(s)
Avalon.warning("第 %s 楼: 出现不支持的图片: \n%s\n" % (floor, str(s)),
front="\n")
elif (str(s["type"]) == "4"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "5"):
try:
content += ProcessVideo(s["link"], s["src"])
except:
content += "无法获取该视频,已自动跳过!" + str(s)
Avalon.warning("第 %s 楼: 出现不支持的视频: \n%s\n" % (floor, str(s)),
front="\n")
elif (str(s["type"]) == "9"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "10"):
content += ProcessAudio(s["voice_md5"])
elif (str(s["type"]) == "11"):
content += ProcessImg(s["static"])
elif (str(s["type"]) == "20"):
content += ProcessImg(s["src"])
else:
Avalon.warning("第 %s 楼: 请求数据错误: \n%s\n" % (floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
return content
# print(fid,lastfid)
lastfid = fid
while (1):
try:
if (Avalon.ask("批量模式?", False)):
PreSet = True
lz = Avalon.ask("只看楼主?", False)
comment = (0 if lz else Avalon.ask("包括评论?", True))
OutputHTML = Avalon.ask("输出HTML(否则表示输出Makrdown)?:", True)
overwrite = Avalon.ask("默认覆盖?", False)
Avalon.info("选定:%s && %s评论 , 目录:\"吧名\\标题\"" %
(("楼主" if lz else "全部"), ("全" if comment else "无")))
if (not Avalon.ask("确认无误?", True)):
Avalon.warning("请重新输入")
else:
break
else:
PreSet = False
break
except KeyboardInterrupt:
ForceStop()
Avalon.error("Control-C,exiting", front="\n")
exit(0)
while (1):
try:
try:
pid = int((Avalon.gets("请输入帖子链接或id(输入0退出):").split('/')
)[-1].split('?')[0])
def Init(pid, overwrite):
global FileHandle, Progress, AudioCount, VideoCount, ImageCount, Pool, IsDownload, DirName, IsCreate, OutputHTML, FFmpeg, OutPutFileName, DirName_New
IsDownload = set()
IsCreate = set()
AudioCount = VideoCount = ImageCount = 0
if (os.path.isdir(DirName)):
Avalon.warning("\"%s\"已存在" % DirName)
if (overwrite == 1):
Avalon.warning("跳过%d" % pid)
elif (overwrite == 2):
Avalon.warning("默认覆盖\"%s\"" % DirName)
elif (overwrite == 3):
if (len(DirName) > 30):
DirName_New = DirName[:-11]
else:
DirName_New = DirName
if (Server):
Avalon.info("已将已存在文件夹备份至" +
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
(datetime.datetime.now() -
datetime.timedelta(hours=12)).strftime('%H-%M'))
shutil.move(DirName,
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
(datetime.datetime.now() -
datetime.timedelta(hours=12)).strftime('%H-%M'))
if not (Server):
Avalon.info("已将已存在文件夹备份至" +
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
datetime.datetime.now().strftime('%H-%M'))
shutil.move(DirName,
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
datetime.datetime.now().strftime('%H-%M'))
os.makedirs(DirName)
elif (not Avalon.ask("是否覆盖?", False)):
raise UserCancelled("...")
elif (os.path.exists(DirName)):
raise OSError("存在同名文件")
else:
os.makedirs(DirName)
if (OutputHTML):
FileHandle = open("%s/%s.html" % (DirName, OutPutFileName),
"w+",
encoding="utf-8")
Write(Html_Header)
Write("<title>%s</title>\n" % (title_name))
if (exe_mode):
if (not_work_path):
shutil.copy("main.css", DirName + "/")
elif not (not_work_path):
shutil.copy(exe_path + "/main.css", DirName + "/")
else:
shutil.copy(sys.path[0] + "/main.css", DirName + "/")
else:
FileHandle = open("%s/%s.md" % (DirName, OutPutFileName),
"w+",
encoding="utf-8")
try:
subprocess.Popen("ffmpeg",stdout=subprocess.DEVNULL,\
stderr=subprocess.DEVNULL).wait()
FFmpeg = 1
except FileNotFoundError:
Avalon.warning("未找到ffmpeg,语音将不会被转为mp3")
FFmpeg = 0
Pool = DownloadPool(DirName + "/", "file")
Progress = tqdm(unit="floor")
if (Auto_Clean):
Avalon.info("自动清理已开启,将会自动删除" + str(Auto_Clean_time) + "天前的备份文件")
not_work_path = False
exe_path = os.getcwd()
if (os.path.exists(Work_path)):
try:
os.chdir(Work_path)
Avalon.info("当前工作目录为" + os.getcwd())
except KeyboardInterrupt:
ForceStop()
Avalon.error("Control-C,exiting", front="\n")
sys.exit()
except UserCancelled:
Avalon.warning("用户取消")
except Exception as err:
ForceStop()
Avalon.error("发生异常:\n" + traceback.format_exc() +
str(datetime.datetime.now()),
front="\n")
sys.exit()
else:
Avalon.warning("指定工作目录不存在!请检查!当前工作目录为" + os.getcwd())
not_work_path = True
#TODO:将其更换为第三方库实现
'''if (len(sys.argv) > 1):
if (sys.argv[1] == "--lz-T"):
lz = True
Avalon.info("从传入参数获取新的只看楼主模式:True")
GetPost(pid, lz, comment)
Done()
ConvertAudio(DirName)
except KeyboardInterrupt:
ForceStop()
Avalon.error("Raised Control-C", front="\n")
write_err_info()
t_in = Avalon.gets("请选择：1.退出程序 2.退出当前帖子\n", front="\n")
if "1" in t_in:
exit(0)
elif "2" in t_in:
continue
else:
continue
except UserCancelled:
Avalon.warning("用户取消")
except RequestError as err:
err = err.data
Avalon.error("百度贴吧API返回错误,代码:%d\n描述:%s" %
(err["code"], err["msg"]),
front="\n")
except Exception:
ForceStop()
Avalon.error("发生异常:\n" + traceback.format_exc(), front="\n")
exit(0)
else:
customized_tools() # 运行自定义工具
Avalon.info("完成 %d" % pid)
try:
if pids.index(pid) < len(pids) - 1:
Avalon.info("10s后进行下一个帖子...\n", front="\n\n")