async def cog_before_invoke(self, ctx):
if Sudo.is_authorized_command(self.bot, ctx):
old_userSettings = deepcopy(self.bot.userSettings)
self.bot.userSettings = Sudo.user_settings_check(
self.bot.userSettings, ctx.author.id)
if old_userSettings != self.bot.userSettings:
Sudo.save_configs(self.bot)
#Leveling system
self.bot.userSettings[ctx.author.id]['level']['xp'] += 1
if self.bot.userSettings[
ctx.author.id]['level']['xp'] >= self.bot.userSettings[
ctx.author.id]['level']['rank'] * 10:
self.bot.userSettings[ctx.author.id]['level']['xp'] = 0
self.bot.userSettings[ctx.author.id]['level']['rank'] += 1
Sudo.save_configs(self.bot)
await ctx.send(embed=Embed(
description=
f"Congratulations {ctx.author}, you are now level {self.bot.userSettings[ctx.author.id]['level']['rank']}"
))
# if ctx.command.name == 's' and self.bot.userSettings[ctx.author.id]['searchAlias'] is not None:
# Log.append_to_log(ctx, self.bot.userSettings[ctx.author.id]['searchAlias'])
# elif ctx.command.name == 's':
# Log.append_to_log(ctx, 's', 'Not set')
# else:
Log.append_to_log(ctx)
return
def sectionInfo(self, bookInfo, muluUrl, bookMuluSoup):
Log.I("[I] on get sectionInfo");
model = SectionInfoModel();
model.bookInfo = bookInfo;
muluList = bookMuluSoup.find(lambda tag: tag.name == "ul" and tag.has_attr("class") and tag["class"][0] == "mulu_list");
if muluList == None or muluList.contents == None or len(muluList.contents) <= 0:
return None;
setted = False;
for c in muluList.contents:
atag = c.find("a");
if atag != -1:
href = Utils.absoluteUrl(atag["href"], muluUrl, None) ;
title = atag.string;
if href != None and title != None:
model.addChapter(str(href), str(title));
setted = True;
else:
Log.W(" on getSection found invalid tag " + str(atag) + ", href=" + str(href) + ",title=" + str(title));
if not setted:
return None;
return model;
def analytics(**kwargs):
# python cf.py --analytics --readfile output12.txt --stat t
# python cf.py --analytics --readfile output12.txt --stat win_player --pct_analytics 1
# python cf.py --runs 300 --strat_player (1,) --strat_me --strat_you
# python cf.py --runs 300 --strat_fork (1,)
# python cf.py --analytics --readfile output91.txt --stat win_player --pct_analytics 1
log = Log(noisy=False)
data = log.load_from_file(filename=args['readfile'])
stat_str = kwargs.get('stat', args['stat'])
stat = log.get_stat(data, stat=stat_str)
#filter the Nones
stat_v = [val for val in stat if val is not None]
#analytics
avg = float(sum(stat_v)) / float(len(stat_v))
_min, _max = min(stat_v), max(stat_v)
print 'Var: ', str(stat_str.upper())
print 'N: ', str(len(stat_v))
print 'Avg: ', str(avg)[:5]
print 'Min: ', str(_min), ' Max: ', str(_max)
if len(args['pct_analytics']) > 0:
match = str(args['pct_analytics'])
num_match = [str(x) for x in stat_v].count(match)
pct_match = 100.0 * float(num_match) / float(len(stat_v))
print 'PCT of ', stat_str, ' = ', match, ' : ', str(pct_match)[:5]
def test(cls):
aescode = "123";
uniqueKey = "@$sswq112$FF";
s = "helloworld~~~";
encoded = cls.encode(s, aescode, uniqueKey);
Log.D("encode result = " + str(encoded));
decoded = cls.decode(encoded, aescode, uniqueKey);
Log.D("decode result = " + str(decoded));
#图片测试
from Config import Config
import os
jpgFile = Config.shared.outPath + os.sep + "nocover.jpg";
jpgFd = open(jpgFile, "r");
jpgData = jpgFd.read();
jpgFd.close();
encodeJpgData = cls.encode(jpgData, aescode, uniqueKey, False);
jpgFd = open(Config.shared.outPath + os.sep + "nocover-encode.jpg", "w");
jpgFd.write(encodeJpgData);
jpgFd.flush();
jpgFd.close();
decodeJpgData = cls.decode(encodeJpgData, aescode, uniqueKey, False);
jpgFd = open(Config.shared.outPath + os.sep + "nocover-decode.jpg", "w");
jpgFd.write(decodeJpgData);
jpgFd.flush();
jpgFd.close();
async def wikilang(self, ctx):
global serverSettings
if ctx.author.id not in serverSettings[ctx.guild.id]['blacklist']:
Log.appendToLog(ctx, 'wikilang')
await WikipediaSearch(bot, ctx, "en").lang()
return
def dropAllTables():
from src.db import Db
Db.instance.executeSql('''show tables;''')
ret = Db.instance.fetchAll()
for item in ret:
for k, v in item.items():
Db.instance.executeSql('''drop table if exists {};'''.format(v))
Log.D("droping table " + str(v))
Log.D("droped all tables finished")
def test(self):
self.createTable()
self.set("testkey", None)
self.set("testkey", "testValue")
self.set("testkey", "testValue")
self.set("testkey", None)
self.set("testkey", "testValue")
Log.D(str(self.get("testkey")))
Log.D(str(self.get("testkey1")))
def bookMuluUrl(self, bookPageSoup):
Log.D("[D] bookMuluSoup input = None " + str(bookPageSoup == None));
spans = Utils.findAllClassTag(bookPageSoup, "span", "btopt");
if len(spans) <= 0 or len(spans[0].contents) <= 0:
return None;
span = spans[0];
a = span.contents[0];
url = a["href"];
Log.D("[D] muluurl = " + str(url));
return url;
def executeSql(self, sql):
try:
self.cursor.execute(sql);
Log.I("[I] 执行 " + sql.strip() + " 成功");
return True;
except Exception as e:
if not isinstance(e, pymysql.err.IntegrityError) or len(e.args) <= 0 or e.args[0] != 1062:
Log.E("[I] 执行 " + sql.strip() + " 失败");
Log.Exc(e);
return False;
def check(self):
#url
urlParseResult = urlparse.urlsplit(self.url)
if urlParseResult == None:
self.error("Config.url配置错误")
self.scheme = urlParseResult.scheme
self.host = urlParseResult.netloc
if not Utils.isValidStr(self.scheme) or not Utils.isValidStr(
self.host):
self.error("Config.url配置错误")
#outPath
if not Utils.isValidStr(self.outPath):
self.error("Config.outPath配置错误")
if not os.path.isdir(self.outPath):
if not Utils.createDir(self.outPath):
self.error("Config.outPath配置错误")
from src.utils import Log
Log.setOutPath(self.outPath)
self.tmpPath = self.outPath + os.sep + ".tmp"
if not os.path.isdir(self.tmpPath):
if not Utils.createDir(self.tmpPath):
self.error("Config.outPath配置错误")
#parser
if self.parserName != "YbduParser":
self.error("Config.parserName配置错误")
#storge
if self.storgeName != "FileStorge" and self.storgeName != "QiniuStorge" and self.storgeName != "AWSStorge":
self.error("Config.storgeName配置错误")
if self.storgeName == "QiniuStorge":
if not Utils.isValidStr(self.qnAK) or not Utils.isValidStr(
self.qnSK) or not Utils.isValidStr(self.qnBK):
self.error("未配置七牛云的 access key，secret key，bucketname")
elif self.storgeName == "AWSStorge":
if not Utils.isValidStr(self.awsBK):
self.error("未配置AWS的bucketname")
#aescode
if self.aesCode != None and not Utils.isValidStr(self.aesCode):
self.error("Config.aesCode配置错误")
#parserType
if self.parseType != "all" and self.parseType != "single":
self.error("Config.parsetType配置错误")
#数据库
if not Utils.isValidStr(self.dbUser) or not Utils.isValidStr(
self.dbPwd) or not Utils.isValidStr(self.dbName):
self.error("未配置mysql的用户名，密码或数据库名称")
def downloadSection(self, sectionInfo):
sectionCount = len(sectionInfo.chapters)
Log.I("[I] on downloadSection() enter will download section count %s" %
(str(sectionCount)))
self.sectionDownloadSuccCount = 0
self.sectionDownloadFailedCount = 0
toDir = sectionInfo.bookInfo.uniqueKey
for i in range(0, sectionCount):
self.downloadOneSection(i, sectionInfo.chapters[i], toDir)
Log.I(
"[I] on downloadSection() exit download all section(%s) succ section(%s) failed section(%s)"
% (str(sectionCount), str(self.sectionDownloadSuccCount),
str(self.sectionDownloadFailedCount)))
async def sudo(self, ctx, *args):
global serverSettings
args = list(args)
if Sudo.isSudoer(bot, ctx, serverSettings) == False:
await ctx.send(f"{ctx.author} is not in the sudoers file. This incident will be reported.")
Log.appendToLog(ctx, 'sudo', 'unauthorised')
else:
Log.appendToLog(ctx, "sudo", ' '.join(args).strip())
command = Sudo(bot, ctx, serverSettings)
await command.sudo(args)
with open('serverSettings.json', 'r') as data:
serverSettings = json.load(data, object_hook=lambda d: {int(k) if k.isdigit() else k: v for k, v in d.items()})
return
async def sudo(self, ctx, *args):
if Sudo.is_sudoer(self.bot, ctx):
Log.append_to_log(ctx, None, args)
command = Sudo(self.bot, ctx)
self.bot.serverSettings, self.bot.userSettings = await command.sudo(
list(args))
Sudo.save_configs(self.bot)
else:
await ctx.send(
f"`{ctx.author}` is not in the sudoers file. This incident will be reported."
)
Log.append_to_log(ctx, None, 'unauthorised')
return
def downloadBookImg(self, url, toDir):
Log.I("[I] dowlonad bookImg " + url)
uniqueKey = Utils.md5str(url)
self.storge.checkFileExists(
uniqueKey, toDir,
lambda exists: not exists and self._downloadBookImg(
url, uniqueKey, toDir))
async def invite(self, ctx):
try:
Log.append_to_log(ctx)
dm = await ctx.author.create_dm()
await dm.send(
discord.utils.oauth_url(
client_id=self.bot.botuser.id,
permissions=discord.Permissions(4228381776),
scopes=['bot', 'applications.commands']))
except discord.errors.Forbidden:
await ctx.send(
'Sorry, I cannot open a DM at this time. Please check your privacy settings'
)
except Exception as e:
await error_handler(self.bot, ctx, e)
finally:
return
def onDownloadSectionCompleted(self, idx, uniqueKey, succ):
Log.I("[I] download section(%s) completed succ(%s)" %
(str(idx), str(succ)))
if succ:
self.sectionDownloadSuccCount += 1
self.chapterDb.setDownloaded(uniqueKey, 1)
else:
self.sectionDownloadFailedCount += 1
self.chapterDb.setDownloaded(uniqueKey, 2)
async def config(self, ctx, *args):
args = list(args)
command = Sudo(self.bot, ctx)
if len(args) > 0:
localSetting = args[0] in ['locale', 'alias']
else:
localSetting = False
if Sudo.is_sudoer(self.bot, ctx) or localSetting:
self.bot.serverSettings, self.bot.userSettings = await command.config(
args)
else:
self.bot.serverSettings, self.bot.userSettings = await command.config(
[])
Sudo.save_configs(self.bot)
Log.append_to_log(ctx)
def pushContent(self, content, fileuniquekey, toDir, complete):
outDir = self.filepath
if toDir != None:
outDir = outDir + os.sep + toDir
file = outDir + os.sep + fileuniquekey
pushRet = None
try:
pushRet = self.bucket.put_object(Key=file, Body=content)
except Exception, e:
Log.Exc(e)
def downloadOneSection(self, idx, oneSectionModel, toDir):
Log.I("[I] downloading section(%s) (%s) (%s)" %
(str(idx), str(
oneSectionModel.title), str(oneSectionModel.downUrl)))
uniqueKey = oneSectionModel.uniqueKey
url = oneSectionModel.downUrl
self.storge.checkFileExists(
oneSectionModel.uniqueKey, toDir, lambda exists: not exists and
(self._downloadOneSection(idx, url, uniqueKey, toDir) or True
) or self.onDownloadSectionCompleted(idx, uniqueKey, True))
def checkFileExists(self, fileuniquekey, toDir, complete):
outDir = self.filepath
if toDir != None:
outDir = outDir + os.sep + toDir
file = outDir + os.sep + fileuniquekey
ret = None
try:
ret, resp = self.bkManager.stat(self.bk, file)
except Exception, e:
Log.Exc(e)
def test(self):
self.createTable();
visitingModel = VisitUrlModel("http://www.baidu.com", 0);
self.insert(visitingModel);
visitingModel.visiting = 1;
self.update(visitingModel);
Log.D(" -- test visiting url = " + str(self.getVisitingUrl()));
def checkFileExists(self, fileuniquekey, toDir, complete):
outDir = self.filepath
if toDir != None:
outDir = outDir + os.sep + toDir
file = outDir + os.sep + fileuniquekey
objSummary = None
try:
objSummary = self.s3.ObjectSummary(self.bk, file).get()
except Exception, e:
Log.Exc(e)
class History(Callback):
"""
Callback that is used to store information about
the training during the training, and writes it to
a file to the end to be able to read it later.
"""
def __init__(self):
super(History, self).__init__()
self.logs = Log()
def set_trainer(self, trainer):
self.trainer = trainer
def on_epoch_begin(self, info=None):
pass
def on_epoch_end(self, info=None):
self.logs.write_to_log('Train', info['train_losses'],
info['train_metrics'])
def on_batch_begin(self, batch, info=None):
pass
def on_batch_end(self, batch, info=None):
if info['print']:
self.logs.update_step(info['step'])
for m_data in self.trainer.monitor_data:
self.logs.write_to_log(m_data, info['monitor_losses'][m_data],
info['monitor_metrics'][m_data])
def on_train_begin(self, info=None):
pass
def on_train_end(self, info=None):
pass
def __init__(self):
Log.D("[I] init class Db");
connConfig = {
'host': '127.0.0.1',
'user': Config.shared.dbUser,
'password': Config.shared.dbPwd,
'db': Config.shared.dbName,
'charset': 'utf8',
'cursorclass': pymysql.cursors.DictCursor,
'autocommit': True
};
self.conn = pymysql.connect(**connConfig);
self.cursor = self.conn.cursor();
def visit(self, url):
Log.I("[I] on visit() " + str(url))
self.setVisitingUrl(url)
soup = Utils.soupUrl(url)
if not soup:
Log.W("[W] on visit() soup is None " + str(url))
return
Log.I("[I] on visit() did get soup")
#将本页所有url放入数据库中
urls = self.addUrlsFromSoup(soup, url)
if not Utils.isValidArr(urls):
Log.W("[W] on visit() urls not found")
return
#获取匹配的书页
bookUrls = self.addBookPageUrls(urls)
#遍历书页
if len(bookUrls) > 0:
for bookUrl in bookUrls:
self.downloadBook(bookUrl)
self.removeVisitUrl(url)
Log.I("[I] on visit() finished " + str(url))
def execute(self):
# self.test();
#如果在数据库中能找到visitUrl，说明程序已经运行过了，可以从数据库中恢复现场
Log.V("[I] onExecute()")
#检查是否有visiting的book
visitingBookUrl = self.visitingBookUrl()
if visitingBookUrl != None:
self.downloadBook(visitingBookUrl)
#检查是否有visiting的url
visitingUrl = self.visitingUrl()
if visitingUrl != None:
self.visit(visitUrl)
#否则从root开始搜索
visitUrl = self.nextVisitUrl()
self.visit(visitUrl)
Log.I("[I] willVisitNext")
#处理下一个
while True:
visitUrl = self.nextVisitUrl()
if visitUrl != None:
self.visit(visitUrl)
else:
break
Log.I("[I] willVisitNextPageUrl")
#网页遍历完成
while True:
bookUrl = self.nextPageUrl()
if bookUrl != None:
self.downloadBook(bookUrl)
else:
break
Log.V("------parse finished------")
def chapterContent(self, chapterSoup):
tag = chapterSoup.find(id = "htmlContent");
if tag == None:
return None;
content = "";
for c in tag.contents:
if Utils.isSoupStr(c):
part = c;
part = part.strip();
if "全本小说" in part:
Log.I("[W] ignore line " + str(part));
continue;
if len(part) > 0:
content += part + "\n";
return content;
def test(self):
self.createTable()
bookInfoModel = BookInfoModel()
bookInfoModel.bookId = "123456"
bookInfoModel.title = "斗破苍穹"
bookInfoModel.author = "天蚕土豆"
bookInfoModel.category = "玄幻"
bookInfoModel.des = "斗气大法"
bookInfoModel.setUniqueKey()
Log.D(" --- toDict = " + str(bookInfoModel.toDict(True)))
# self.bookInfoDelete(bookInfoModel);
self.insert(bookInfoModel)
bookInfoModel.title = "武动乾坤"
bookInfoModel.setUniqueKey()
self.update(bookInfoModel)
def pushContent(self, content, fileuniquekey, toDir, complete):
outDir = self.filepath
if toDir != None:
outDir = outDir + os.sep + toDir
file = outDir + os.sep + fileuniquekey
ret = None
tmpFile = None
try:
tmpFile = Options.shared.tmpPath + os.sep + (
toDir != None and str(toDir) + "-" or "") + fileuniquekey
tmpFileFd = open(tmpFile, "w")
tmpFileFd.write(content)
tmpFileFd.flush()
tmpFileFd.close()
token = self.qn.upload_token(self.bk, file, 3600)
ret, info = put_file(token, file, tmpFile)
except Exception, e:
Log.Exc(e)
def bookInfo(self, bookPageSoup, bookMuluSoup):
Log.I("[I] on get bookInfo ");
model = BookInfoModel();
#检查书页中的tlj标签
tLJTags = Utils.findAllClassTag(bookPageSoup, "div", "tLJ");
for tag in tLJTags:
self._checkTLJTag(tag, model);
#字数
find = Utils.findAll(r"已写了(\d+)字", str(bookPageSoup));
if find and len(find) > 0:
model.wordsCount = find[0];
#检查目录页中的数据
metaTags = bookMuluSoup.find_all(lambda t: t.name == "meta" and t.has_attr("property") and Utils.isMatch("og:.+?\"", t["property"]) != None);
for tag in metaTags:
self._checkMetaTag(tag, model);
return model;
