async def cog_before_invoke(self, ctx): if Sudo.is_authorized_command(self.bot, ctx): old_userSettings = deepcopy(self.bot.userSettings) self.bot.userSettings = Sudo.user_settings_check( self.bot.userSettings, ctx.author.id) if old_userSettings != self.bot.userSettings: Sudo.save_configs(self.bot) #Leveling system self.bot.userSettings[ctx.author.id]['level']['xp'] += 1 if self.bot.userSettings[ ctx.author.id]['level']['xp'] >= self.bot.userSettings[ ctx.author.id]['level']['rank'] * 10: self.bot.userSettings[ctx.author.id]['level']['xp'] = 0 self.bot.userSettings[ctx.author.id]['level']['rank'] += 1 Sudo.save_configs(self.bot) await ctx.send(embed=Embed( description= f"Congratulations {ctx.author}, you are now level {self.bot.userSettings[ctx.author.id]['level']['rank']}" )) # if ctx.command.name == 's' and self.bot.userSettings[ctx.author.id]['searchAlias'] is not None: # Log.append_to_log(ctx, self.bot.userSettings[ctx.author.id]['searchAlias']) # elif ctx.command.name == 's': # Log.append_to_log(ctx, 's', 'Not set') # else: Log.append_to_log(ctx) return
def sectionInfo(self, bookInfo, muluUrl, bookMuluSoup): Log.I("[I] on get sectionInfo"); model = SectionInfoModel(); model.bookInfo = bookInfo; muluList = bookMuluSoup.find(lambda tag: tag.name == "ul" and tag.has_attr("class") and tag["class"][0] == "mulu_list"); if muluList == None or muluList.contents == None or len(muluList.contents) <= 0: return None; setted = False; for c in muluList.contents: atag = c.find("a"); if atag != -1: href = Utils.absoluteUrl(atag["href"], muluUrl, None) ; title = atag.string; if href != None and title != None: model.addChapter(str(href), str(title)); setted = True; else: Log.W(" on getSection found invalid tag " + str(atag) + ", href=" + str(href) + ",title=" + str(title)); if not setted: return None; return model;
def analytics(**kwargs): # python cf.py --analytics --readfile output12.txt --stat t # python cf.py --analytics --readfile output12.txt --stat win_player --pct_analytics 1 # python cf.py --runs 300 --strat_player (1,) --strat_me --strat_you # python cf.py --runs 300 --strat_fork (1,) # python cf.py --analytics --readfile output91.txt --stat win_player --pct_analytics 1 log = Log(noisy=False) data = log.load_from_file(filename=args['readfile']) stat_str = kwargs.get('stat', args['stat']) stat = log.get_stat(data, stat=stat_str) #filter the Nones stat_v = [val for val in stat if val is not None] #analytics avg = float(sum(stat_v)) / float(len(stat_v)) _min, _max = min(stat_v), max(stat_v) print 'Var: ', str(stat_str.upper()) print 'N: ', str(len(stat_v)) print 'Avg: ', str(avg)[:5] print 'Min: ', str(_min), ' Max: ', str(_max) if len(args['pct_analytics']) > 0: match = str(args['pct_analytics']) num_match = [str(x) for x in stat_v].count(match) pct_match = 100.0 * float(num_match) / float(len(stat_v)) print 'PCT of ', stat_str, ' = ', match, ' : ', str(pct_match)[:5]
def test(cls): aescode = "123"; uniqueKey = "@$sswq112$FF"; s = "helloworld~~~"; encoded = cls.encode(s, aescode, uniqueKey); Log.D("encode result = " + str(encoded)); decoded = cls.decode(encoded, aescode, uniqueKey); Log.D("decode result = " + str(decoded)); #图片测试 from Config import Config import os jpgFile = Config.shared.outPath + os.sep + "nocover.jpg"; jpgFd = open(jpgFile, "r"); jpgData = jpgFd.read(); jpgFd.close(); encodeJpgData = cls.encode(jpgData, aescode, uniqueKey, False); jpgFd = open(Config.shared.outPath + os.sep + "nocover-encode.jpg", "w"); jpgFd.write(encodeJpgData); jpgFd.flush(); jpgFd.close(); decodeJpgData = cls.decode(encodeJpgData, aescode, uniqueKey, False); jpgFd = open(Config.shared.outPath + os.sep + "nocover-decode.jpg", "w"); jpgFd.write(decodeJpgData); jpgFd.flush(); jpgFd.close();
async def wikilang(self, ctx): global serverSettings if ctx.author.id not in serverSettings[ctx.guild.id]['blacklist']: Log.appendToLog(ctx, 'wikilang') await WikipediaSearch(bot, ctx, "en").lang() return
def dropAllTables(): from src.db import Db Db.instance.executeSql('''show tables;''') ret = Db.instance.fetchAll() for item in ret: for k, v in item.items(): Db.instance.executeSql('''drop table if exists {};'''.format(v)) Log.D("droping table " + str(v)) Log.D("droped all tables finished")
def test(self): self.createTable() self.set("testkey", None) self.set("testkey", "testValue") self.set("testkey", "testValue") self.set("testkey", None) self.set("testkey", "testValue") Log.D(str(self.get("testkey"))) Log.D(str(self.get("testkey1")))
def bookMuluUrl(self, bookPageSoup): Log.D("[D] bookMuluSoup input = None " + str(bookPageSoup == None)); spans = Utils.findAllClassTag(bookPageSoup, "span", "btopt"); if len(spans) <= 0 or len(spans[0].contents) <= 0: return None; span = spans[0]; a = span.contents[0]; url = a["href"]; Log.D("[D] muluurl = " + str(url)); return url;
def executeSql(self, sql): try: self.cursor.execute(sql); Log.I("[I] 执行 " + sql.strip() + " 成功"); return True; except Exception as e: if not isinstance(e, pymysql.err.IntegrityError) or len(e.args) <= 0 or e.args[0] != 1062: Log.E("[I] 执行 " + sql.strip() + " 失败"); Log.Exc(e); return False;
def check(self): #url urlParseResult = urlparse.urlsplit(self.url) if urlParseResult == None: self.error("Config.url配置错误") self.scheme = urlParseResult.scheme self.host = urlParseResult.netloc if not Utils.isValidStr(self.scheme) or not Utils.isValidStr( self.host): self.error("Config.url配置错误") #outPath if not Utils.isValidStr(self.outPath): self.error("Config.outPath配置错误") if not os.path.isdir(self.outPath): if not Utils.createDir(self.outPath): self.error("Config.outPath配置错误") from src.utils import Log Log.setOutPath(self.outPath) self.tmpPath = self.outPath + os.sep + ".tmp" if not os.path.isdir(self.tmpPath): if not Utils.createDir(self.tmpPath): self.error("Config.outPath配置错误") #parser if self.parserName != "YbduParser": self.error("Config.parserName配置错误") #storge if self.storgeName != "FileStorge" and self.storgeName != "QiniuStorge" and self.storgeName != "AWSStorge": self.error("Config.storgeName配置错误") if self.storgeName == "QiniuStorge": if not Utils.isValidStr(self.qnAK) or not Utils.isValidStr( self.qnSK) or not Utils.isValidStr(self.qnBK): self.error("未配置七牛云的 access key,secret key,bucketname") elif self.storgeName == "AWSStorge": if not Utils.isValidStr(self.awsBK): self.error("未配置AWS的bucketname") #aescode if self.aesCode != None and not Utils.isValidStr(self.aesCode): self.error("Config.aesCode配置错误") #parserType if self.parseType != "all" and self.parseType != "single": self.error("Config.parsetType配置错误") #数据库 if not Utils.isValidStr(self.dbUser) or not Utils.isValidStr( self.dbPwd) or not Utils.isValidStr(self.dbName): self.error("未配置mysql的用户名,密码或数据库名称")
def downloadSection(self, sectionInfo): sectionCount = len(sectionInfo.chapters) Log.I("[I] on downloadSection() enter will download section count %s" % (str(sectionCount))) self.sectionDownloadSuccCount = 0 self.sectionDownloadFailedCount = 0 toDir = sectionInfo.bookInfo.uniqueKey for i in range(0, sectionCount): self.downloadOneSection(i, sectionInfo.chapters[i], toDir) Log.I( "[I] on downloadSection() exit download all section(%s) succ section(%s) failed section(%s)" % (str(sectionCount), str(self.sectionDownloadSuccCount), str(self.sectionDownloadFailedCount)))
async def sudo(self, ctx, *args): global serverSettings args = list(args) if Sudo.isSudoer(bot, ctx, serverSettings) == False: await ctx.send(f"{ctx.author} is not in the sudoers file. This incident will be reported.") Log.appendToLog(ctx, 'sudo', 'unauthorised') else: Log.appendToLog(ctx, "sudo", ' '.join(args).strip()) command = Sudo(bot, ctx, serverSettings) await command.sudo(args) with open('serverSettings.json', 'r') as data: serverSettings = json.load(data, object_hook=lambda d: {int(k) if k.isdigit() else k: v for k, v in d.items()}) return
async def sudo(self, ctx, *args): if Sudo.is_sudoer(self.bot, ctx): Log.append_to_log(ctx, None, args) command = Sudo(self.bot, ctx) self.bot.serverSettings, self.bot.userSettings = await command.sudo( list(args)) Sudo.save_configs(self.bot) else: await ctx.send( f"`{ctx.author}` is not in the sudoers file. This incident will be reported." ) Log.append_to_log(ctx, None, 'unauthorised') return
def downloadBookImg(self, url, toDir): Log.I("[I] dowlonad bookImg " + url) uniqueKey = Utils.md5str(url) self.storge.checkFileExists( uniqueKey, toDir, lambda exists: not exists and self._downloadBookImg( url, uniqueKey, toDir))
async def invite(self, ctx): try: Log.append_to_log(ctx) dm = await ctx.author.create_dm() await dm.send( discord.utils.oauth_url( client_id=self.bot.botuser.id, permissions=discord.Permissions(4228381776), scopes=['bot', 'applications.commands'])) except discord.errors.Forbidden: await ctx.send( 'Sorry, I cannot open a DM at this time. Please check your privacy settings' ) except Exception as e: await error_handler(self.bot, ctx, e) finally: return
def onDownloadSectionCompleted(self, idx, uniqueKey, succ): Log.I("[I] download section(%s) completed succ(%s)" % (str(idx), str(succ))) if succ: self.sectionDownloadSuccCount += 1 self.chapterDb.setDownloaded(uniqueKey, 1) else: self.sectionDownloadFailedCount += 1 self.chapterDb.setDownloaded(uniqueKey, 2)
async def config(self, ctx, *args): args = list(args) command = Sudo(self.bot, ctx) if len(args) > 0: localSetting = args[0] in ['locale', 'alias'] else: localSetting = False if Sudo.is_sudoer(self.bot, ctx) or localSetting: self.bot.serverSettings, self.bot.userSettings = await command.config( args) else: self.bot.serverSettings, self.bot.userSettings = await command.config( []) Sudo.save_configs(self.bot) Log.append_to_log(ctx)
def pushContent(self, content, fileuniquekey, toDir, complete): outDir = self.filepath if toDir != None: outDir = outDir + os.sep + toDir file = outDir + os.sep + fileuniquekey pushRet = None try: pushRet = self.bucket.put_object(Key=file, Body=content) except Exception, e: Log.Exc(e)
def downloadOneSection(self, idx, oneSectionModel, toDir): Log.I("[I] downloading section(%s) (%s) (%s)" % (str(idx), str( oneSectionModel.title), str(oneSectionModel.downUrl))) uniqueKey = oneSectionModel.uniqueKey url = oneSectionModel.downUrl self.storge.checkFileExists( oneSectionModel.uniqueKey, toDir, lambda exists: not exists and (self._downloadOneSection(idx, url, uniqueKey, toDir) or True ) or self.onDownloadSectionCompleted(idx, uniqueKey, True))
def checkFileExists(self, fileuniquekey, toDir, complete): outDir = self.filepath if toDir != None: outDir = outDir + os.sep + toDir file = outDir + os.sep + fileuniquekey ret = None try: ret, resp = self.bkManager.stat(self.bk, file) except Exception, e: Log.Exc(e)
def test(self): self.createTable(); visitingModel = VisitUrlModel("http://www.baidu.com", 0); self.insert(visitingModel); visitingModel.visiting = 1; self.update(visitingModel); Log.D(" -- test visiting url = " + str(self.getVisitingUrl()));
def checkFileExists(self, fileuniquekey, toDir, complete): outDir = self.filepath if toDir != None: outDir = outDir + os.sep + toDir file = outDir + os.sep + fileuniquekey objSummary = None try: objSummary = self.s3.ObjectSummary(self.bk, file).get() except Exception, e: Log.Exc(e)
class History(Callback): """ Callback that is used to store information about the training during the training, and writes it to a file to the end to be able to read it later. """ def __init__(self): super(History, self).__init__() self.logs = Log() def set_trainer(self, trainer): self.trainer = trainer def on_epoch_begin(self, info=None): pass def on_epoch_end(self, info=None): self.logs.write_to_log('Train', info['train_losses'], info['train_metrics']) def on_batch_begin(self, batch, info=None): pass def on_batch_end(self, batch, info=None): if info['print']: self.logs.update_step(info['step']) for m_data in self.trainer.monitor_data: self.logs.write_to_log(m_data, info['monitor_losses'][m_data], info['monitor_metrics'][m_data]) def on_train_begin(self, info=None): pass def on_train_end(self, info=None): pass
def __init__(self): Log.D("[I] init class Db"); connConfig = { 'host': '127.0.0.1', 'user': Config.shared.dbUser, 'password': Config.shared.dbPwd, 'db': Config.shared.dbName, 'charset': 'utf8', 'cursorclass': pymysql.cursors.DictCursor, 'autocommit': True }; self.conn = pymysql.connect(**connConfig); self.cursor = self.conn.cursor();
def visit(self, url): Log.I("[I] on visit() " + str(url)) self.setVisitingUrl(url) soup = Utils.soupUrl(url) if not soup: Log.W("[W] on visit() soup is None " + str(url)) return Log.I("[I] on visit() did get soup") #将本页所有url放入数据库中 urls = self.addUrlsFromSoup(soup, url) if not Utils.isValidArr(urls): Log.W("[W] on visit() urls not found") return #获取匹配的书页 bookUrls = self.addBookPageUrls(urls) #遍历书页 if len(bookUrls) > 0: for bookUrl in bookUrls: self.downloadBook(bookUrl) self.removeVisitUrl(url) Log.I("[I] on visit() finished " + str(url))
def execute(self): # self.test(); #如果在数据库中能找到visitUrl,说明程序已经运行过了,可以从数据库中恢复现场 Log.V("[I] onExecute()") #检查是否有visiting的book visitingBookUrl = self.visitingBookUrl() if visitingBookUrl != None: self.downloadBook(visitingBookUrl) #检查是否有visiting的url visitingUrl = self.visitingUrl() if visitingUrl != None: self.visit(visitUrl) #否则从root开始搜索 visitUrl = self.nextVisitUrl() self.visit(visitUrl) Log.I("[I] willVisitNext") #处理下一个 while True: visitUrl = self.nextVisitUrl() if visitUrl != None: self.visit(visitUrl) else: break Log.I("[I] willVisitNextPageUrl") #网页遍历完成 while True: bookUrl = self.nextPageUrl() if bookUrl != None: self.downloadBook(bookUrl) else: break Log.V("------parse finished------")
def chapterContent(self, chapterSoup): tag = chapterSoup.find(id = "htmlContent"); if tag == None: return None; content = ""; for c in tag.contents: if Utils.isSoupStr(c): part = c; part = part.strip(); if "全本小说" in part: Log.I("[W] ignore line " + str(part)); continue; if len(part) > 0: content += part + "\n"; return content;
def test(self): self.createTable() bookInfoModel = BookInfoModel() bookInfoModel.bookId = "123456" bookInfoModel.title = "斗破苍穹" bookInfoModel.author = "天蚕土豆" bookInfoModel.category = "玄幻" bookInfoModel.des = "斗气大法" bookInfoModel.setUniqueKey() Log.D(" --- toDict = " + str(bookInfoModel.toDict(True))) # self.bookInfoDelete(bookInfoModel); self.insert(bookInfoModel) bookInfoModel.title = "武动乾坤" bookInfoModel.setUniqueKey() self.update(bookInfoModel)
def pushContent(self, content, fileuniquekey, toDir, complete): outDir = self.filepath if toDir != None: outDir = outDir + os.sep + toDir file = outDir + os.sep + fileuniquekey ret = None tmpFile = None try: tmpFile = Options.shared.tmpPath + os.sep + ( toDir != None and str(toDir) + "-" or "") + fileuniquekey tmpFileFd = open(tmpFile, "w") tmpFileFd.write(content) tmpFileFd.flush() tmpFileFd.close() token = self.qn.upload_token(self.bk, file, 3600) ret, info = put_file(token, file, tmpFile) except Exception, e: Log.Exc(e)
def bookInfo(self, bookPageSoup, bookMuluSoup): Log.I("[I] on get bookInfo "); model = BookInfoModel(); #检查书页中的tlj标签 tLJTags = Utils.findAllClassTag(bookPageSoup, "div", "tLJ"); for tag in tLJTags: self._checkTLJTag(tag, model); #字数 find = Utils.findAll(r"已写了(\d+)字", str(bookPageSoup)); if find and len(find) > 0: model.wordsCount = find[0]; #检查目录页中的数据 metaTags = bookMuluSoup.find_all(lambda t: t.name == "meta" and t.has_attr("property") and Utils.isMatch("og:.+?\"", t["property"]) != None); for tag in metaTags: self._checkMetaTag(tag, model); return model;