def run(bot, settings): # Start logging tl.log = logger.init_logger( logger_name="redball.bots." + threading.current_thread().name, log_to_console=str( settings.get("Logging", {}).get("LOG_TO_CONSOLE", True)).lower() == "true", log_to_file=str(settings.get("Logging", {}).get("LOG_TO_FILE", True)).lower() == "true", log_path=redball.LOG_PATH, log_file="{}.log".format(threading.current_thread().name), file_log_level=settings.get("Logging", {}).get("FILE_LOG_LEVEL"), log_retention=settings.get("Logging", {}).get("LOG_RETENTION", 7), console_log_level=settings.get("Logging", {}).get("CONSOLE_LOG_LEVEL"), clear_first=True, propagate=False, ) tl.log.debug("Bot received settings: {}".format(settings)) # Initialize vars and do one-time setup steps i = 0 # This is used inside the sample loop to log 'still alive' entries every 10 cycles while True: # This loop keeps the bot running if (redball.SIGNAL is None and not bot.STOP ): # Make sure the main thread hasn't sent a stop command # THIS IS WHERE YOU DO YOUR THING: i = i + 1 if i == 10: # Log that you're still running every 10 minutes tl.log.debug("Still alive...") i = 0 time.sleep(1) else: # If main thread has said to stop, we stop! tl.log.info("Bot {} (id={}) exiting...".format(bot.name, bot.id)) break # Exit the infinite loop to stop the bot
def run(bot=None, settings=None): global log log = logger.init_logger( logger_name="redball.bots." + threading.current_thread().name, log_to_console=settings.get("Logging", {}).get("LOG_TO_CONSOLE", True), log_to_file=settings.get("Logging", {}).get("LOG_TO_FILE", True), log_path=redball.LOG_PATH, log_file="{}.log".format(threading.current_thread().name), file_log_level=settings.get("Logging", {}).get("FILE_LOG_LEVEL", "DEBUG"), console_log_level=settings.get("Logging", {}).get("CONSOLE_LOG_LEVEL", "INFO"), clear_first=True, propagate=settings.get("Logging", {}).get("PROPAGATE", False), ) mlbbot = StatBot(settings) mlbbot.run(bot)
def run(bot, settings): global log # Make logger available to the whole module # Start logging log = logger.init_logger(logger_name=threading.current_thread().name, log_to_console=str(settings.get('Logging',{}).get('LOG_TO_CONSOLE',True)).lower()=='true', log_to_file=str(settings.get('Logging',{}).get('LOG_TO_FILE',True)).lower()=='true', log_path=redball.LOG_PATH, log_file='{}.log'.format(threading.current_thread().name), file_log_level=settings.get('Logging',{}).get('FILE_LOG_LEVEL'), console_log_level=settings.get('Logging',{}).get('CONSOLE_LOG_LEVEL'), clear_first=True, cploggers=False, propagate=False) log.debug('Bot received settings: {}'.format(settings)) # Initialize vars and do one-time setup steps i = 0 # This is used inside the sample loop to log 'still alive' entries every 10 cycles while True: # This loop keeps the bot running if redball.SIGNAL is None and not bot.STOP: # Make sure the main thread hasn't sent a stop command # THIS IS WHERE YOU DO YOUR THING: i = i+1 if i == 10: # Log that you're still running every 10 minutes log.debug('Still alive...') i = 0 time.sleep(1) else: # If main thread has said to stop, we stop! log.info('Bot {} (id={}) exiting...'.format(bot.name, bot.id)) break # Exit the infinite loop to stop the bot
def run(bot, settings): # Start logging tl.log = logger.init_logger( logger_name=f"redball.bots.{threading.current_thread().name}", log_to_console=settings.get("Logging", {}).get("LOG_TO_CONSOLE", True), log_to_file=settings.get("Logging", {}).get("LOG_TO_FILE", True), log_path=redball.LOG_PATH, log_file=f"{threading.current_thread().name}.log", file_log_level=settings.get("Logging", {}).get("FILE_LOG_LEVEL"), log_retention=settings.get("Logging", {}).get("LOG_RETENTION", 7), console_log_level=settings.get("Logging", {}).get("CONSOLE_LOG_LEVEL"), clear_first=True, propagate=False, ) tl.log.debug(f"Bot received settings: {settings}") # Initialize vars and do one-time setup steps dbTable = settings.get("Database").get("dbTablePrefix", "") + "posts" sub = settings.get("Bot", {}).get("SUBREDDIT") audit = settings.get("Bot", {}).get("REPORT_ONLY", False) useContentHash = settings.get("Bot", {}).get("USE_CONTENT_HASH", False) historicalPosts = int(settings.get("Bot", {}).get("HISTORICAL_POSTS", 100)) pauseAfter = int(settings.get("Bot", {}).get("PAUSE_AFTER", 5)) ignoreDomains = [ x.strip() for x in settings.get("Bot", {}) .get("IGNORE_DOMAINS", "i.redd.it,v.redd.it") .split(",") ] removalCommentText = settings.get("Bot", {}).get( "REMOVAL_COMMENT_TEXT", "This post has been automatically removed as a duplicate of [[(title)]((link))]. If you believe this is a mistake, please send a message to the subreddit moderators.", ) reportText = settings.get("Bot", {}).get("REPORT_TEXT", "Duplicate: (link)") postCache = {} ignoredPostIdCache = [] tl.log.info("Initializing Reddit API...") with redball.REDDIT_AUTH_LOCKS[str(bot.redditAuth)]: try: r = praw.Reddit( client_id=settings["Reddit Auth"]["reddit_clientId"], client_secret=settings["Reddit Auth"]["reddit_clientSecret"], token_manager=bot.reddit_auth_token_manager, user_agent=f"redball Duplicate Link Removal Bot - https://github.com/toddrob99/redball/ v{__version__}", ) except Exception as e: tl.log.error( f"Error authenticating with Reddit. Ensure the bot has a valid Reddit Auth selected (with Refresh Token) and try again. Error message: {e}" ) raise try: if "identity" in r.auth.scopes(): tl.log.info(f"Authorized Reddit user: {r.user.me()}") except Exception as e: tl.log.error( f"Reddit authentication failure. Ensure the bot has a valid Reddit Auth selected (with Refresh Token and relevant scopes selected) and try again. Error message: {e}" ) raise try: db = sqlite3.connect( os.path.join( settings["Database"]["dbPath"], settings["Database"]["dbFile"] ), timeout=30, ) """Local sqlite database to store info about processed posts""" db.execute("PRAGMA journal_mode = off;") except sqlite3.Error as e: tl.log.error(f"Error connecting to database: {e}") raise dbc = db.cursor() dbc.execute( f"""CREATE TABLE IF NOT EXISTS {dbTable} ( submissionId text PRIMARY KEY, url text, canonical text, contentHash text, urls text, dateCreated text, dateRemoved text );""" ) dbc.execute( f"SELECT submissionId, url, canonical, contentHash, urls, dateCreated, dateRemoved FROM {dbTable} ORDER BY dateCreated ASC LIMIT {historicalPosts};" ) pids = dbc.fetchall() for pid in pids: post = { "submissionId": pid[0], "url": pid[1], "canonical": pid[2], "contentHash": pid[3], "urls": deserialize(pid[4]), "dateCreated": pid[5], "dateRemoved": pid[6], } postCache.update({post["submissionId"]: post}) tl.log.debug(f"Loaded {len(postCache)} post(s) from db.") tl.log.info(f"Monitoring posts in the following subreddit(s): {sub}...") while True: # This loop keeps the bot running if ( redball.SIGNAL is None and not bot.STOP ): # Make sure the main thread hasn't sent a stop command try: tl.log.debug("Checking for new posts...") for newPost in r.subreddit(sub).stream.submissions( pause_after=pauseAfter ): if newPost is None: break if ( newPost.id in postCache.keys() or newPost.id in ignoredPostIdCache ): continue if not newPost.is_self and (newPost.url.startswith("/")): checkUrl = f"https://reddit.com{newPost.url}" tl.log.debug( f"Detected relative URL, checking URL for ignored domains with https://reddit.com prepended: [{checkUrl}]" ) else: checkUrl = None if newPost.is_self: tl.log.debug(f"Post [{newPost.id}] is a self post--skipping.") ignoredPostIdCache.append(newPost.id) continue elif newPost.is_reddit_media_domain: tl.log.debug( f"Post [{newPost.id}] is a reddit media domain--skipping." ) ignoredPostIdCache.append(newPost.id) continue elif next( ( True for y in ignoreDomains if y in (checkUrl if checkUrl else newPost.url) ), False, ): tl.log.debug( f"Post [{newPost.id}] has an ignored domain [{newPost.url}]--skipping." ) ignoredPostIdCache.append(newPost.id) q = None else: try: if newPost.is_gallery: tl.log.debug( f"Post [{newPost.id}] is a gallery--skipping." ) ignoredPostIdCache.append(newPost.id) continue except AttributeError: pass tl.log.debug( f"Post [{newPost.id}] has a non-ignored domain link [{newPost.url}]..." ) this = getUrls(newPost) foundActiveDupeSubmissionId = None foundRemovedDupeSubmissionId = None matchingSubmissionGenerator = ( k for k, v in postCache.items() if k != newPost.id and ( next( (True for i in this["urls"] if i in v["urls"]), False, ) or ( useContentHash and ( this["contentHash"] == v["contentHash"] and v["contentHash"] is not None # url GET likely failed if None ) ) ) ) while matchingSubmissionId := next( matchingSubmissionGenerator, None ): matchingSubmission = r.submission(matchingSubmissionId) if not matchingSubmission.author: # Matching submission is deleted! tl.log.info( f"Matching prior submission [{matchingSubmission.id}] is deleted, ignoring." ) continue if not matchingSubmission.is_robot_indexable: # Matching submission is removed (not deleted since the last condition was not met) tl.log.info( f"Matching prior submission [{matchingSubmission.id}] is removed, ignoring unless no other dupes are found." ) foundRemovedDupeSubmissionId = matchingSubmissionId continue if newPost.id != matchingSubmissionId: # Matching submission found, not deleted or removed tl.log.info( f"New submission [{newPost.id}] identified as duplicate of submission [{matchingSubmission.id}]" ) foundActiveDupeSubmissionId = matchingSubmissionId break if foundActiveDupeSubmissionId or foundRemovedDupeSubmissionId: # Duplicate identified dupePost = r.submission( foundActiveDupeSubmissionId if foundActiveDupeSubmissionId else foundRemovedDupeSubmissionId ) if audit or not foundActiveDupeSubmissionId: # Report the post if audit and not foundActiveDupeSubmissionId: tl.log.info( f"Reporting new submission [{newPost.id}] due to audit mode (and prior post [{dupePost.id}] is removed)..." ) elif audit: tl.log.info( f"Reporting new submission [{newPost.id}] due to audit mode..." ) elif foundRemovedDupeSubmissionId: tl.log.info( f"Reporting new submission [{newPost.id}] since prior post [{dupePost.id}] is removed..." ) try: parsedReportText = reportText.replace( "(title)", dupePost.title ).replace("(link)", dupePost.shortlink) if not foundActiveDupeSubmissionId: parsedReportText += " (prior post is removed)" newPost.report(parsedReportText) except Exception as e: tl.log.error( f"Error reporting submission [{newPost.id}]: {e}" ) else: # Remove the post tl.log.info( f"Removing new post [{newPost.id}] as a duplicate of [{dupePost.id}]..." ) try: newPost.mod.remove( mod_note=f"Removed as duplicate of [{dupePost.id}]" ) except Exception as e: tl.log.error( f"Error removing submission [{newPost.id}]: {e}" ) # Reply to the post try: parsedReplyText = removalCommentText.replace( "(title)", dupePost.title ).replace("(link)", dupePost.shortlink) removalReply = newPost.reply(parsedReplyText) removalReply.mod.distinguish(sticky=True) except Exception as e: tl.log.error( f"Error submitting reply to submission [{newPost.id}]: {e}" ) q = f"INSERT OR IGNORE INTO {dbTable} (submissionId, url, canonical, contentHash, urls, dateCreated, dateRemoved) VALUES (?, ?, ?, ?, ?, ?, ?);" qa = ( newPost.id, newPost.url, this["canonical"], this["contentHash"], serialize(this["urls"]), newPost.created_utc, time.time(), ) else: # No duplicate found tl.log.debug( f"No duplicates found for submission [{newPost.id}]" ) q = f"INSERT OR IGNORE INTO {dbTable} (submissionId, url, canonical, contentHash, urls, dateCreated) VALUES (?, ?, ?, ?, ?, ?);" qa = ( newPost.id, newPost.url, this["canonical"], this["contentHash"], serialize(this["urls"]), newPost.created_utc, ) if q: try: dbc.execute(q, qa) db.commit() tl.log.debug("Inserted record to db.") except Exception as e: tl.log.error(f"Error inserting into database: {e}") postCache.update({newPost.id: this}) except Exception as e: tl.log.error( f"Sleeping for 10 seconds and then continuing after exception: {e}" ) time.sleep(10) else: # If main thread has said to stop, we stop! tl.log.info(f"Bot {bot.name} (id={bot.id}) exiting...") break # Exit the infinite loop to stop the bot
def run(bot, settings): # Start logging tl.log = logger.init_logger( logger_name="redball.bots." + threading.current_thread().name, log_to_console=settings.get("Logging", {}).get("LOG_TO_CONSOLE", True), log_to_file=settings.get("Logging", {}).get("LOG_TO_FILE", True), log_path=redball.LOG_PATH, log_file="{}.log".format(threading.current_thread().name), file_log_level=settings.get("Logging", {}).get("FILE_LOG_LEVEL"), log_retention=settings.get("Logging", {}).get("LOG_RETENTION", 7), console_log_level=settings.get("Logging", {}).get("CONSOLE_LOG_LEVEL"), clear_first=True, propagate=settings.get("Logging", {}).get("PROPAGATE", False), ) tl.log.debug("Bot received settings: {}".format(settings)) # Initialize vars and do one-time setup steps dbTable = settings.get("Database").get("dbTablePrefix", "") + "comments" keyResp = settings.get("Keyword Response", {}) reqName = str(settings.get("Bot", {}).get("REQ_NAME", "True")).lower() == "true" sub = settings.get("Bot", {}).get("SUBREDDIT") delThreshold = int(settings.get("Bot", {}).get("DEL_THRESHOLD", -2)) historicalDays = int(settings.get("Bot", {}).get("HISTORICAL_DAYS", 1)) pauseAfter = int(settings.get("Bot", {}).get("PAUSE_AFTER", 5)) replyFooter = settings.get("Bot", {}).get( "REPLY_FOOTER", "^^I ^^am ^^a ^^bot. ^^Downvoted ^^replies ^^will ^^be ^^deleted. ^^[[source/doc](https://github.com/toddrob99/redball/wiki/Comment-Response-Bot)] ^^[[feedback](https://reddit.com/r/redball/)]", ) comments = {} tl.log.info("Initializing Reddit API...") with redball.REDDIT_AUTH_LOCKS[str(bot.redditAuth)]: try: r = praw.Reddit( client_id=settings["Reddit Auth"]["reddit_clientId"], client_secret=settings["Reddit Auth"]["reddit_clientSecret"], token_manager=bot.reddit_auth_token_manager, user_agent= "redball Comment Response Bot - https://github.com/toddrob99/redball/ v{}" .format(__version__), ) except Exception as e: tl.log.error( "Error authenticating with Reddit. Ensure the bot has a valid Reddit Auth selected (with Refresh Token) and try again. Error message: {}" .format(e)) raise try: if "identity" in r.auth.scopes(): tl.log.info("Authorized Reddit user: {}".format(r.user.me())) elif str(reqName).lower() == "true": tl.log.error( "Reddit auth does not have `identity` scope authorized; cannot identify bot name in comments as required." ) raise Exception( "Reddit auth does not have `identity` scope authorized; cannot identify bot name in comments as required." ) except Exception as e: tl.log.error( "Reddit authentication failure. Ensure the bot has a valid Reddit Auth selected (with Refresh Token and relevant scopes selected) and try again. Error message: {}" .format(e)) raise try: db = sqlite3.connect( os.path.join(settings["Database"]["dbPath"], settings["Database"]["dbFile"]), timeout=30, ) """Local sqlite database to store info about processed comments""" db.execute("PRAGMA journal_mode = off;") # db.set_trace_callback(print) except sqlite3.Error as e: tl.log.error("Error connecting to database: {}".format(e)) raise dbc = db.cursor() dbc.execute("""CREATE TABLE IF NOT EXISTS {} ( comment_id text PRIMARY KEY, sub text NOT NULL, author text NOT NULL, post text NOT NULL, date text NOT NULL, kw text, errors text, reply text, removed integer );""".format(dbTable)) dbc.execute( "SELECT comment_id, date, reply, errors, removed FROM {} ORDER BY date DESC LIMIT 500;" .format(dbTable)) cids = dbc.fetchall() for cid in cids: comments.update({ cid[0]: { "date": cid[1], "reply": r.comment(cid[2]) if cid[2] else None, "errors": cid[3], "removed": cid[4], "historical": True, } }) tl.log.debug("Loaded {} comments from db.".format(len(comments))) tl.log.info( "Monitoring comments in the following subreddit(s): {}...".format(sub)) while True: # This loop keeps the bot running if (redball.SIGNAL is None and not bot.STOP ): # Make sure the main thread hasn't sent a stop command for comment in r.subreddit(sub).stream.comments( pause_after=pauseAfter): if bot and bot.STOP: tl.log.info( "Received stop signal! Closing DB connection...") # Close DB connection before exiting try: db.close() except sqlite3.Error as e: tl.log.error( "Error closing database connection: {}".format(e)) return if not comment: break # take a break to delete downvoted replies replyText = "" if (any(k for k in keyResp if k.lower() in comment.body.lower()) and ((reqName and str(r.user.me()).lower() in comment.body.lower()) or not reqName) and comment.author != r.user.me()): if comment.id in comments.keys(): tl.log.debug("Already processed comment {}".format( comment.id)) continue elif ( comment.created_utc <= time.time() - 60 * 60 * 24 * historicalDays - 3600 ): # Add 1 hour buffer to ensure recent comments are processed tl.log.debug( "Stream returned comment {} which is older than the HISTORICAL_DAYS setting ({}), ignoring..." .format(comment.id, historicalDays)) comments.update({ comment.id: { "sub": comment.subreddit, "author": comment.author, "post": comment.submission, "date": time.time(), "kw": [], "errors": [], } }) continue comments.update({ comment.id: { "sub": comment.subreddit, "author": comment.author, "post": comment.submission, "date": time.time(), "kw": [], "errors": [], } }) dbc.execute( "insert or ignore into {} (comment_id, sub, author, post, date) values (?, ?, ?, ?, ?);" .format(dbTable), ( str(comment.id), str(comment.subreddit), str(comment.author), str(comment.submission), str(comment.created_utc), ), ) tl.log.debug("({}) {} - {}: {}".format( comment.subreddit, comment.id, comment.author, comment.body)) for k in (k for k in keyResp.keys() if k.lower() in comment.body.lower()): comments[comment.id]["kw"].append(k) if replyText != "": replyText += "\n\n" replyText += keyResp[k] if replyText != "": try: latest_reply = comment.reply(replyText + "\n\n" + replyFooter) comments[comment.id].update( {"reply": latest_reply}) latest_reply.disable_inbox_replies() tl.log.info( "Replied with comment id {} and disabled inbox replies." .format(latest_reply)) dbc.execute( "update {} set kw=?,reply=? where comment_id=?;" .format(dbTable), ( str(comments[comment.id]["kw"]), str(latest_reply), str(comment.id), ), ) except Exception as e: tl.log.error( "Error replying to comment or disabling inbox replies: {}" .format(e)) comments[comment.id]["errors"].append( "Error submitting comment or disabling inbox replies: {}" .format(e)) if len(comments[comment.id].get("errors")): dbc.execute( "update {} set errors=? where comment_id=?;". format(dbTable), (str(comments[comment.id].get("errors")), str(comment.id)), ) db.commit() tl.log.debug("Checking for downvotes on {} replies...".format( sum(1 for x in comments if comments[x].get("reply") and not comments[x].get("removed") and float(comments[x].get("date")) >= time.time() - 60 * 60 * 24 * historicalDays - 3600))) for x in (x for x in comments if comments[x].get("reply") and not comments[x].get("removed") and float(comments[x].get("date")) >= time.time() - 60 * 60 * 24 * historicalDays - 3600): # print('Submission: {}, reply: {}'.format(r.comment(x).submission, comments[x]['reply'])) try: comments[x]["reply"].refresh() except praw.exceptions.ClientException as e: print( "Error refreshing attributes for comment reply {}: {}". format(comments[x]["reply"], e)) if "comment does not appear to be in the comment tree" in str( e): comments[x].update({"removed": time.time()}) dbc.execute( "update {} set removed=? where comment_id=?;". format(dbTable), (str(comments[x].get("removed")), str(x)), ) if not comments[x].get("removed"): if comments[x]["reply"].score <= delThreshold: tl.log.info( "Deleting comment {} with score ({}) at or below threshold ({})..." .format( comments[x]["reply"], comments[x]["reply"].score, delThreshold, )) try: comments[x]["reply"].delete() comments[x].update({"removed": time.time()}) dbc.execute( "update {} set removed=? where comment_id=?;". format(dbTable), (str(comments[x].get("removed")), str(x)), ) db.commit() except Exception as e: tl.log.error( "Error deleting downvoted comment: {}".format( e)) comments[x]["errors"].append( "Error deleting downvoted comment: {}".format( e)) limits = r.auth.limits if limits.get("remaining") < 60: tl.log.warning( "Approaching Reddit API rate limit, sleeping for a minute... {}" .format(limits)) time.sleep(60) else: tl.log.debug("Reddit API limits: {}".format(limits)) else: # If main thread has said to stop, we stop! tl.log.info("Bot {} (id={}) exiting...".format(bot.name, bot.id)) break # Exit the infinite loop to stop the bot # Close DB connection before exiting try: tl.log.info("Closing DB connection.") db.close() except sqlite3.Error as e: tl.log.error("Error closing database connection: {}".format(e))
def run(self): self.log = logger.init_logger( logger_name="redball.bots." + threading.current_thread().name, log_to_console=self.settings.get("Logging", {}).get("LOG_TO_CONSOLE", True), log_to_file=self.settings.get("Logging", {}).get("LOG_TO_FILE", True), log_path=redball.LOG_PATH, log_file="{}.log".format(threading.current_thread().name), file_log_level=self.settings.get("Logging", {}).get("FILE_LOG_LEVEL"), log_retention=self.settings.get("Logging", {}).get("LOG_RETENTION", 7), console_log_level=self.settings.get("Logging", {}).get("CONSOLE_LOG_LEVEL"), clear_first=True, propagate=False, ) self.log.debug( f"Sidebar Updater Bot v{__version__} received settings: {self.settings}. Template path: {self.BOT_TEMPLATE_PATH}" ) # Initialize Reddit API connection self.init_reddit() # Initialize scheduler if "SCHEDULER" in vars(self.bot): # Scheduler already exists, maybe bot restarted sch_jobs = self.bot.SCHEDULER.get_jobs() self.log.warning( f"Scheduler already exists on bot startup with the following job(s): {sch_jobs}" ) # Remove all jobs and shut down so we can start fresh for x in sch_jobs: x.remove() try: self.bot.SCHEDULER.shutdown() except SchedulerNotRunningError as e: self.log.debug(f"Could not shut down scheduler because: {e}") self.bot.SCHEDULER = BackgroundScheduler( timezone=tzlocal.get_localzone() if str(tzlocal.get_localzone()) != "local" else "America/New_York") self.bot.SCHEDULER.start() self.bot.detailedState = { "summary": { "text": "Starting up, please wait 1 minute...", "html": "Starting up, please wait 1 minute...", "markdown": "Starting up, please wait 1 minute...", } } # Initialize detailed state to a wait message # Start a scheduled task to update self.bot.detailedState every minute self.bot.SCHEDULER.add_job( self.bot_state, "interval", name=f"bot-{self.bot.id}-statusUpdateTask", id=f"bot-{self.bot.id}-statusUpdateTask", minutes=1, replace_existing=True, ) if sport := self.settings.get("Bot", {}).get("SPORT"): self.log.debug(f"Bot set to sport: {sport}") self.sport = sport