def TiebaRequest(url, data, first=False):
JsonRetry = 0
while (JsonRetry <= 5):
try:
if (first):
req = Retry(requests.post,
args=(url, ),
kwargs={"data": SignRequest(data)},
cfunc=(lambda x: x.status_code == 200),
ffunc=print,
fargs=("连接失败,正在重试...\n", ),
times=10)
else:
req = Retry(requests.post,
args=(url, ),
kwargs={"data": SignRequest(data)},
cfunc=(lambda x: x.status_code == 200),
ffunc=Progress.set_description,
fargs=("连接失败,正在重试...", ),
times=10)
req.encoding = 'utf-8'
ret = req.json() # JSON错误的源头
break
except:
Avalon.warning("遇到一次JSON错误,正在重试")
JsonRetry += 1
continue
#print(ret)
if (int(ret["error_code"]) != 0):
raise RequestError({
"code": int(ret["error_code"]),
"msg": str(ret["error_msg"])
})
return ret
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
if (str(s["type"]) == "0"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "1"):
content += ProcessUrl(s["link"], s["text"])
elif (str(s["type"]) == "2"):
content += ProcessEmotion(floor, s["text"], s["c"])
elif (str(s["type"]) == "3"):
content += ProcessImg(s["origin_src"])
elif (str(s["type"]) == "4"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "5"):
content += ProcessVideo(s["link"], s["src"])
elif (str(s["type"]) == "9"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "10"):
content += ProcessAudio(s["voice_md5"])
elif (str(s["type"]) == "11"):
content += ProcessImg(s["static"])
elif (str(s["type"]) == "20"):
content += ProcessImg(s["src"])
else:
Avalon.warning("floor %s: content data wrong: \n%s\n" %
(floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
return content
def backup_existed_file():
if os.path.exists(DirName): # 备份一下本地已经爬到的文件
Avalon.info("备份本地已经爬到的文件")
try:
os.rename(
DirName, DirName + "-" + time.strftime(
"%Y%m%d-%Hh", time.localtime(int(time.time()))))
except OSError:
Avalon.info("备份文件已经存在")
def Init(pid, overwrite):
global FileHandle,Progress,AudioCount,VideoCount,ImageCount,\
Pool,IsDownload,DirName,IsCreate,OutputHTML,FFmpeg
IsDownload = set()
IsCreate = set()
AudioCount = VideoCount = ImageCount = 0
if (os.path.isdir(DirName)):
Avalon.warning("\"%s\"已存在" % DirName)
if (overwrite == 1):
Avalon.warning("跳过%d" % pid)
elif (overwrite == 2):
Avalon.warning("默认覆盖\"%s\"" % DirName)
elif (not Avalon.ask("是否覆盖?", False)):
raise UserCancelled("...")
elif (os.path.exists(DirName)):
raise OSError("存在同名文件")
else:
os.makedirs(DirName)
if (OutputHTML):
FileHandle = open("%s/%d.html" % (DirName, pid), "w", encoding="utf-8")
Write('<!doctype html><html lang="zh-cn"><head><link rel="stylesheet"'\
' type="text/css" href="main.css"></head><body><div id="write">')
shutil.copy("main.css", DirName + "/")
else:
FileHandle = open("%s/%d.md" % (DirName, pid), "w", encoding="utf-8")
try:
subprocess.Popen("ffmpeg",stdout=subprocess.DEVNULL,\
stderr=subprocess.DEVNULL).wait()
FFmpeg = 1
except FileNotFoundError:
Avalon.warning("未找到ffmpeg,语音将不会被转为mp3")
FFmpeg = 0
Pool = DownloadPool(DirName + "/", "file")
Progress = tqdm(unit="floor")
async def _init_command(self, message):
'''-init Initialize a new game'''
if self.avalon:
await message.channel.send(f'Game already started.')
else:
self.avalon = Avalon(players=[])
self.avalon.add_player(MockMember(name='Bob'))
self.avalon.add_player(MockMember(name='joe'))
self.avalon.add_player(MockMember(name='Mike'))
self.avalon.add_player(MockMember(name='Sally'))
await message.channel.send(f'Started game {self.avalon.id}')
def copydir_overwrite(_from_path, _to_path):
if os.path.exists(_from_path):
if os.path.exists(_to_path):
Avalon.warning("目标目录已存在，删除ing...")
try:
shutil.rmtree(_to_path)
except Exception as err1:
Avalon.error("删除目标文件失败，此次复制取消\n" + str(err1))
return 1
try:
shutil.copytree(_from_path, _to_path)
except Exception as err2:
Avalon.error("复制失败！\n" + str(err2))
else:
Avalon.info("复制成功")
else:
Avalon.warning("源目录不存在，跳过")
def do_PUT(self):
self.requestPrepHelper()
if not self.routePath:
self.send_res(400,{"response": "failure", "message": "bad path"})
elif self.routePath[0] == "games" and self.routePath[2] == "players" and len(self.routePath) == 3:
# print(self.routePath)
ava = Avalon()
gameCharacters = dict(ava.characters)
players = db.get(self.routePath[1], True) # get players in game
count = len(players)
character_list = ava.getCharacters(count)
random.shuffle(character_list)
for i in range(count):
players[i]['character_title'] = gameCharacters[character_list[i]]['name']
players[i]['knownby'] = gameCharacters[character_list[i]]['knownby']
for player in players:
player['character_data'] = ""
for player in players:
if player['knownby'] == "wizards":
merlin = [element for element in players if element['character_title'] == 'merlin']
i = players.index(merlin[0])
players[i]['character_data'] += str(player['player_name']) + " is a minion \n"
mordred = [element for element in players if element['character_title'] == 'mordred']
i = players.index(mordred[0])
players[i]['character_data'] += str(player['player_name']) + " is a minion \n"
if player['knownby'] == "percival" and count > 7:
p = [element for element in players if element['character_title'] == 'percival']
i = players.index(p[0])
players[i]['character_data'] += str(player['player_name']) +" is merlin \n"
for player in players:
player['is_active'] = 0
db.update(player)
self.send_res(200,players)
else:
self.send_res(404,{"response":"bad path"})
def delete_old_files():
Avalon.info("删除3天以前的备份...")
Dirname_backuped = DirName + "-" + time.strftime(
"%Y%m%d-%Hh", time.localtime(int(time.time()) - 86400 * 3))
if os.path.exists(Dirname_backuped):
Avalon.warning("检测到3天前的备份，删除ing...")
shutil.rmtree(Dirname_backuped)
Avalon.info("删除完毕")
else:
Avalon.warning("未发现3天前的备份，跳过")
def test_script(self):
from avalon import story, Avalon, Script
from datetime import timedelta
av = Avalon()
@story
def bad_story():
raise ValueError("Im bad")
script = av.register(bad_story, timeout=0.5)
self.assertTrue(script.avalon)
self.assertEqual(script.timeout, 0.5)
av.start()
script = Script(bad_story, timeout=timedelta(hours=3))
self.assertEqual(script.timeout, 10800)
def ProcessEmotion(floor, name, text):
global DirName, IsDownload
MakeDir(DirName + "/images")
lname = len(name)
if (name == "image_emoticon"):
name += "1"
lname += 1
url = ""
if (lname >= 3 and name[0:3] == "ali"):
url = "%s%s.gif" % (const.AliUrl, name)
name += ".gif"
elif (lname >= 14 and name[0:14] == "image_emoticon"):
url = "%s%s.png" % (const.EmotionUrl, name)
name += ".png"
else:
Avalon.warning("第%s楼出现未知表情:%s\n" % (floor, name), front="\n")
return ''
if (not name in IsDownload):
IsDownload.add(name)
Pool.Download(url, "images/%s" % name)
return '<img src="images/%s" alt="%s" title="%s" />' % (name, text, text)
def customized_tools():
# 以下--删除3天以前的备份
Tools.delete_old_files()
# 以下--复制文件到网站目录
if copy_to_website == 1:
Avalon.info("准备复制文件到网站目录----")
Tools.copydir_overwrite(_from_path="./%s" % DirName,
_to_path=website_dir + DirName)
elif copy_to_website == 0:
Avalon.info("跳过 复制文件到网站目录")
# 以下--向server酱推送消息
if not sckey == "":
Avalon.info("尝试向Server酱推送消息……")
Tools.send_wxmsg(_sckey=sckey,
_title="贴吧备份（全部楼层）",
_context="今日份的备份已完成...\n\n一切顺利..Maybe..\n\n帖子id：%d" %
pid)
else:
Avalon.warning("SCKEY为空，跳过微信推送")
def send_wxmsg(_sckey, _title="标题", _context="正文"):
url = "https://sc.ftqq.com/%s.send" % _sckey
_context = _context + " \n\n" + time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime())
data = {"text": "%s" % _title, "desp": "%s" % _context}
try:
res = requests.post(url=url, data=data)
msg_back = json.loads(res.text)
if msg_back["errmsg"] == "success":
Avalon.info("返回值：%s" % (msg_back["errmsg"]))
else:
Avalon.warning("返回值：%s" % (msg_back["errmsg"]))
except Exception:
Avalon.error("消息发送错误")
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
try:
if str(s["type"]) == "0":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "1":
content += ProcessUrl(s["link"], s["text"])
elif str(s["type"]) == "2":
content += ProcessEmotion(floor, s["text"], s["c"])
elif str(s["type"]) == "3":
content += ProcessImg(s["origin_src"])
elif str(s["type"]) == "4":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "5":
try:
content += ProcessVideo(s["link"], s["src"])
except KeyError:
content += ProcessQuoteVideo(s["text"])
elif str(s["type"]) == "9":
content += ProcessText(s["text"], in_comment)
elif str(s["type"]) == "10":
content += ProcessAudio(s["voice_md5"])
elif str(s["type"]) == "11":
content += ProcessImg(s["static"])
elif str(s["type"]) == "20":
content += ProcessImg(s["src"])
else:
Avalon.warning("floor %s: content data wrong: \n%s\n" %
(floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
except KeyError:
write_err_info()
Avalon.error("KeyError! 建议修改源码中字典的Key\n" + traceback.format_exc(),
front="\n")
content += '\n<a>[Error] 这里似乎出错了...类型 KeyError</a>\n'
except Exception:
write_err_info()
Avalon.error("发生异常:\n" + traceback.format_exc(), front="\n")
content += '\n<a>[Error] 这里似乎出错了...</a>\n'
else:
return content
def ProcessContent(floor, data, in_comment):
content = ""
for s in data:
#TODO:断点续传
if (str(s["type"]) == "0"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "1"):
content += ProcessUrl(s["link"], s["text"])
elif (str(s["type"]) == "2"):
content += ProcessEmotion(floor, s["text"], s["c"])
elif (str(s["type"]) == "3"):
try:
content += ProcessImg(s["origin_src"])
except:
content += "无法获取该图片,已自动跳过!" + str(s)
Avalon.warning("第 %s 楼: 出现不支持的图片: \n%s\n" % (floor, str(s)),
front="\n")
elif (str(s["type"]) == "4"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "5"):
try:
content += ProcessVideo(s["link"], s["src"])
except:
content += "无法获取该视频,已自动跳过!" + str(s)
Avalon.warning("第 %s 楼: 出现不支持的视频: \n%s\n" % (floor, str(s)),
front="\n")
elif (str(s["type"]) == "9"):
content += ProcessText(s["text"], in_comment)
elif (str(s["type"]) == "10"):
content += ProcessAudio(s["voice_md5"])
elif (str(s["type"]) == "11"):
content += ProcessImg(s["static"])
elif (str(s["type"]) == "20"):
content += ProcessImg(s["src"])
else:
Avalon.warning("第 %s 楼: 请求数据错误: \n%s\n" % (floor, str(s)),
front="\n")
# raise UndifiedMsgType("content data wrong: \n%s\n"%str(s))
return content
if not sckey == "":
Avalon.info("尝试向Server酱推送消息……")
Tools.send_wxmsg(_sckey=sckey,
_title="贴吧备份（全部楼层）",
_context="今日份的备份已完成...\n\n一切顺利..Maybe..\n\n帖子id：%d" %
pid)
else:
Avalon.warning("SCKEY为空，跳过微信推送")
if __name__ == '__main__':
os.chdir(sys.path[0]) # 切换至脚本文件所在目录
DirName_flag = 0
for pid in pids:
try:
Avalon.info("当前时间: " + time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))))
Avalon.info("帖子id: %d" % pid)
title = GetTitle(pid)
title["forum"] = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "_",
title["forum"])
title["post"] = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "_", title["post"])
# lz = Avalon.ask("只看楼主?", False)
Avalon.info("只看楼主: " + str(lz))
# comment = (0 if lz else Avalon.ask("包括评论?", True))
Avalon.info("包括评论: " + str(comment))
# DirName = Avalon.gets("文件夹名(空则表示使用\"吧名-标题\"):")
DirName = DirNames[DirName_flag]
Avalon.info("目录名: " + DirName)
Tools.backup_existed_file() # 备份已爬取的文件
# OutputHTML = Avalon.ask("输出HTML(否则表示输出Makrdown)?:", True)
Avalon.info("输出为Html: " + str(OutputHTML))
def add_css():
with open("main.css", "wb+") as code:
code.write(css.encode())
Avalon.info("CSS文件已释放")
def write_err_info():
with open(save_path + "errors.txt", "a", encoding="utf-8") as f_err_info:
f_err_info.writelines("https://tieba.baidu.com/p/" + str(pid) + "\n")
Avalon.error(f"帖子 {pid} 可能出错, 帖子id已保存至 {save_path}errors.txt")
# coding=UTF-8
import discord
from discord.ext.commands import Bot
from discord.ext import commands
from avalon import Avalon
bot = Bot(command_prefix="!")
bot.add_cog(Avalon(bot))
bot.run("----------bot-token----------")
DirName_New = ""
OverWriteMode = 0
PreList = list()
PreDirList = list()
CopyDir = ""
Copy = False
Delay = False
Delay_time = ""
Delay_times = ""
exe_mode = False
if ('.exe' in sys.argv[0]):
add_css()
exe_mode = True
Avalon.info("正在以exe模式运行")
if (Server):
pid = 6684644957 #6477171863#4016579360
#6477171863
#帖子链接/id
lz = False
#是否只看楼主
comment = True
#是否包含评论
DirName = "ppp"
#输出文件夹路径,为空则默认为(\"吧名\\标题\)
OutPutFileName = "index"
#输出文件名,方便网站部署,为空则默认为帖子pid
OutputHTML = True
#是否输出html,否则输出markdown
class MyClient(discord.Client):
async def on_ready(self):
print('Logged on as {0}!'.format(self.user))
self.avalon = None
async def on_message(self, message):
# The bot should ignore private messages
if message.channel.type == discord.enums.ChannelType.private:
return
# the bot should ignore any of it's own messages
if message.author == self.user:
return
match = re.match('-([A-Za-z])+', message.content)
if match:
try:
command = getattr(self, '_'+match.group()[1:]+'_command')
await command(message)
except AttributeError as e:
print(f'{message.content} not found')
async def _init_command(self, message):
'''-init Initialize a new game'''
if self.avalon:
await message.channel.send(f'Game already started.')
else:
self.avalon = Avalon(players=[])
self.avalon.add_player(MockMember(name='Bob'))
self.avalon.add_player(MockMember(name='joe'))
self.avalon.add_player(MockMember(name='Mike'))
self.avalon.add_player(MockMember(name='Sally'))
await message.channel.send(f'Started game {self.avalon.id}')
async def _join_command(self, message):
'''-join Join a game before it starts'''
if self.avalon is None:
await message.channel.send('No active game to join!')
elif len(self.avalon.players) == 10:
await message.channel.send('Sorry! the max number of players for Avalon is 10')
else:
try:
self.avalon.add_player(message.author)
await message.channel.send(f'Player {message.author} added')
await message.author.send(f'You joined game {self.avalon.id}!')
except DuplicatePlayerException as e:
await message.channel.send(f'Player {message.author} already added!')
async def _start_command(self, message):
'''-start Start a game'''
if not self.avalon:
await message.channel.send('No active game!')
try:
self.avalon.start()
except PlayerCountException:
await message.channel.send(f'Not enough players!')
return
for player in self.avalon.players:
await player.author.send(f'\nYou are {player.character.name}. You are on the {player.character.get_alignment()} team.\n{player.character.description}\n')
await message.channel.send(f'Night is about to start, Here are the characters for this game: {[player.character.name for player in self.avalon.players]}')
for vision in self.avalon.night_phase():
player = vision[0]
other_player = vision[1]
await player.author.send(f'You saw {other_player.author.name} during the night phase')
print(
f'{player.author.name} ({player.character.name}) saw {other_player.author.name} ({other_player.character.name})')
await message.channel.send(f'Night phase complete! Check your DM\'s for details!')
async def _players_command(self, message):
'''-players Get a list of players in the active game'''
if not self.avalon:
await message.channel.send('No active game!')
return
await message.channel.send(f'Active Players for game id: {self.avalon.id}: {[player.author.name for player in self.avalon.players]}')
async def _reset_command(self, message):
'''-reset Clear the active game'''
self.avalon = None
async def _help_command(self, message):
'''-help Display the help menu'''
help_message='Available commands:\n'
for method in dir(self):
if method[-8:] == '_command':
help_message+=getattr(self, method).__doc__+'\n'
await message.channel.send(help_message)
def Init(pid, overwrite):
global FileHandle, Progress, AudioCount, VideoCount, ImageCount, Pool, IsDownload, DirName, IsCreate, OutputHTML, FFmpeg, OutPutFileName, DirName_New
IsDownload = set()
IsCreate = set()
AudioCount = VideoCount = ImageCount = 0
if (os.path.isdir(DirName)):
Avalon.warning("\"%s\"已存在" % DirName)
if (overwrite == 1):
Avalon.warning("跳过%d" % pid)
elif (overwrite == 2):
Avalon.warning("默认覆盖\"%s\"" % DirName)
elif (overwrite == 3):
if (len(DirName) > 30):
DirName_New = DirName[:-11]
else:
DirName_New = DirName
if (Server):
Avalon.info("已将已存在文件夹备份至" +
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
(datetime.datetime.now() -
datetime.timedelta(hours=12)).strftime('%H-%M'))
shutil.move(DirName,
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
(datetime.datetime.now() -
datetime.timedelta(hours=12)).strftime('%H-%M'))
if not (Server):
Avalon.info("已将已存在文件夹备份至" +
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
datetime.datetime.now().strftime('%H-%M'))
shutil.move(DirName,
(datetime.datetime.now().strftime('%m-%d')) + "/" +
DirName_New + " " +
datetime.datetime.now().strftime('%H-%M'))
os.makedirs(DirName)
elif (not Avalon.ask("是否覆盖?", False)):
raise UserCancelled("...")
elif (os.path.exists(DirName)):
raise OSError("存在同名文件")
else:
os.makedirs(DirName)
if (OutputHTML):
FileHandle = open("%s/%s.html" % (DirName, OutPutFileName),
"w+",
encoding="utf-8")
Write(Html_Header)
Write("<title>%s</title>\n" % (title_name))
if (exe_mode):
if (not_work_path):
shutil.copy("main.css", DirName + "/")
elif not (not_work_path):
shutil.copy(exe_path + "/main.css", DirName + "/")
else:
shutil.copy(sys.path[0] + "/main.css", DirName + "/")
else:
FileHandle = open("%s/%s.md" % (DirName, OutPutFileName),
"w+",
encoding="utf-8")
try:
subprocess.Popen("ffmpeg",stdout=subprocess.DEVNULL,\
stderr=subprocess.DEVNULL).wait()
FFmpeg = 1
except FileNotFoundError:
Avalon.warning("未找到ffmpeg,语音将不会被转为mp3")
FFmpeg = 0
Pool = DownloadPool(DirName + "/", "file")
Progress = tqdm(unit="floor")
ProcessFloor(fnum, userlist[floor["author_id"]]["name"],
floor["time"],
ProcessContent(fnum, floor["content"], 0)))
if (int(floor["sub_post_number"]) == 0):
continue
if (comment):
GetComment(fnum, pid, floor["id"])
if (lastfid == fid):
break
# print(fid,lastfid)
lastfid = fid
while (1):
try:
if (Avalon.ask("批量模式?", False)):
PreSet = True
lz = Avalon.ask("只看楼主?", False)
comment = (0 if lz else Avalon.ask("包括评论?", True))
OutputHTML = Avalon.ask("输出HTML(否则表示输出Makrdown)?:", True)
overwrite = Avalon.ask("默认覆盖?", False)
Avalon.info("选定:%s && %s评论 , 目录:\"吧名\\标题\"" %
(("楼主" if lz else "全部"), ("全" if comment else "无")))
if (not Avalon.ask("确认无误?", True)):
Avalon.warning("请重新输入")
else:
break
else:
PreSet = False
break
except KeyboardInterrupt:
