def incrementLogsDate(): try: startDate = config.OVERRUSTLE_LOGS_START config.OVERRUSTLE_LOGS_START = config.OVERRUSTLE_LOGS_START + datetime.timedelta(days=1) logging.info("[DGG-OR-LOGS] Incremented OverRustle Logs date, old value was " + str(startDate) + ", new value is " + str(config.OVERRUSTLE_LOGS_START)) except Exception as err: processor.printError(err, "overrustleLogsParser.incrementLogsDate()")
def parseLogs(bulk): try: array = bulk.strip().splitlines() totalRecords = len(array) for i,line in enumerate(array): start = time.time() line = unicode(line, "utf8") dirtyDatetime = line.split("UTC") restOfRecord = dirtyDatetime[1] date = (dirtyDatetime[0])[1::] colon = restOfRecord.index(':') endBracket = restOfRecord.index(']') username = restOfRecord[endBracket + 2:colon] message = restOfRecord[colon + 1::] date = date.strip() username = username.strip() message = message.strip() logging.info("[DGG-OR-LOGS] (" + str(i + 1) + "/" + str(totalRecords) + ") Sent: " + str(date) + " " + str(username) + ": " + str(message)) if not database.checkLogsExist(date, username, message): database.insertLogs(date, username, message) stop = time.time() delta = stop - start logging.debug("[TIME] Logs parsing time = " + str(delta)) except Exception as err: processor.printError(err, "overrustleLogsParser.parseLogs()")
def processLogs(): try: #config.OVERRUSTLE_LOGS_START = getLogsStartDate() config.OVERRUSTLE_LOGS_START = datetime.datetime.strptime(config.OVERRUSTLE_LOGS_START, "%Y-%m-%d %H:%M:%S") logsStartDate = config.OVERRUSTLE_LOGS_START logsEndDate = datetime.datetime.now() while logsStartDate <= logsEndDate: logsurl = createLogsURL() res = openLogsURL(logsurl) parseLogs(res) incrementLogsDate() except Exception as err: processor.printError(err, "overrustleLogsParser.processLogs()")
def createLogsURL(): try: baseURL = "http://dgg.overrustlelogs.net/Destinygg%20chatlog/" logsDate = config.OVERRUSTLE_LOGS_START year = logsDate.year month = '{:02d}'.format(logsDate.month) day = '{:02d}'.format(logsDate.day) monthName = monthDictionary[str(month)] finURL = baseURL + str(monthName) + "%20" + str(year) + "/" + str(year) + "-" + str(month) + "-" + str(day) + ".txt" logging.info("[DGG-OR-LOGS] dgg.overrustlelogs.com URL being used: " + str(finURL)) return finURL except Exception as err: processor.printError(err, "overrustleLogsParser.createLogsURL()")
def startDggBot(): try: initializePlatform() except Exception as err: processor.printError(err, "initializePlatform()") logging.basicConfig(level=logging.INFO, filename=config.LOG_LOCATION, filemode="a+", format="%(asctime)-15s %(levelname)-8s %(message)s") try: logging.info("[INIT] Using " + str(config.DATABASE_LOCATION) + " as database location.") logging.info("[INIT] Using " + str(config.LOG_LOCATION) + " as log location.") logging.info("[INIT] Using " + str(config.TOKEN_LOCATION) + " as token location.") processor.getDggEmotes() except Exception as err: processor.printError(err, "processor.getDggEmotes()") try: database.initializeDB() except Exception as err: processor.printError(err, "database.initializeDB()") #try: # if not config.SKIP_OVERRUSTLE: # logThread = Thread(target = overrustleLogsParser.processLogs(), args=()) # logThread.start() #except Exception as err: # processor.printError(err, "overrustleLogsParser.processLogs()") try: websock.initializeConnection(config.HOST, config.TOKEN) except Exception as err: processor.printError(err, "websock.initializeConnection()")
def sendMsg(msg, user, ws, msgType): try: if msg == config.LASTMSG: logging.info("[SEND] Duplicate text caught, padding message to avoid filtering.") msg = msg + " " + random.choice(config.EMOTES) if msgType.lower() == "public": formattedMsg = 'MSG {"data":"' + msg + '"}' ws.send(formattedMsg) if msgType.lower() == "private": formattedMsg = 'PRIVMSG {"nick":"' + user + '","data":"' + msg + '"}' try: ws.send(formattedMsg) except Exception as err: print("sendMsg error: " + str(err)) config.LASTMSG = msg logging.debug("[SEND] Message sent! Message is '" + str(formattedMsg) + "'") except Exception as err: processor.printError(err, "websock.sendMsg()")
def on_message(ws, msg): try: msg = unicode(msg, 'utf8') if "ERR" in msg: if "banned" in msg: logging.critical("[CHAT] The account you are attempting to connect to Destiny.gg chat with is banned.") return else: logging.error("[CHAT] Error received from Destiny.gg chat: " + str(msg)) return parsedMsg = processor.parse_chat(msg) cmd = parsedMsg["command"] if (cmd == "JOIN" or cmd == "PING" or cmd == "QUIT") or "command" not in parsedMsg: return if "connectioncount" in parsedMsg: connectionsNumber = str(parsedMsg["connectioncount"]) logging.info("[RCVD] Number of chatters currently online is " + connectionsNumber) elif cmd == "MSG": userNick = processor.sanitizeString("".join(parsedMsg["nick"])) userMessage = processor.sanitizeString("".join(parsedMsg["data"])) userFlair = parsedMsg["features"] realTimestamp = processor.convertUnixTimeToDateTime(parsedMsg["timestamp"]) logging.info("[RCVD] " + userNick + ": " + userMessage) formattedFlairs = processor.formatFlair(userFlair) emoteCheck = processor.checkEmote(userNick, userMessage) msgChars = len(userMessage) database.sendDB(userNick, formattedFlairs, emoteCheck, msgChars, realTimestamp, userMessage) command.checkForCommand(userNick, userMessage, ws, "public") elif cmd == "PRIVMSG": userNick = str(parsedMsg["nick"]) userMessage = str(parsedMsg["data"]) logging.info("[PRIVMSG] " + userNick + ": " + userMessage) command.checkForCommand(userNick, userMessage, ws, "private") except Exception as err: processor.printError(err, "websock.on_message()")
logging.info("[DGG-OR-LOGS] dgg.overrustlelogs.com URL being used: " + str(finURL)) return finURL except Exception as err: processor.printError(err, "overrustleLogsParser.createLogsURL()") def openLogsURL(url): try: url = str(url) results = "" while not results: results = urllib2.urlopen(url, timeout=5).read() return results except urllib2.HTTPError, e: logging.error("[DGG-OR-LOGS] Error fetching OverRustle logs via HTTP: " + str(e.fp.read())) except Exception as err: processor.printError(err, "overrustleLogsParser.openLogsURL()") def parseLogs(bulk): try: array = bulk.strip().splitlines() totalRecords = len(array) for i,line in enumerate(array): start = time.time() line = unicode(line, "utf8") dirtyDatetime = line.split("UTC") restOfRecord = dirtyDatetime[1] date = (dirtyDatetime[0])[1::] colon = restOfRecord.index(':') endBracket = restOfRecord.index(']') username = restOfRecord[endBracket + 2:colon] message = restOfRecord[colon + 1::]