def get_picture(url, fname, timeout_sec=20, max_size=1024 * 1024 * 20): if "imgur.com" in url: url = proc_imgur_url(url) req = requests.get(url, stream=True) req.raise_for_status() start = utcnow() content = bytes() size = 0 # Get a chunk for chunk in req.iter_content(1024): # If download time exceeds the given timeout, exit if utcnow() - start > timeout_sec: print("Image %s took too long to download" % url) return None content += chunk size += len(chunk) # If the size eceeds the given maximum size, exit if size > max_size: print("Image %s is too large" % url) return None # Save the file save_data = open(fname, "wb") save_data.write(content) return fname
def new_report(item, author, body): """ Check if an item can be added in the report feeder """ def add_item(): if item.id not in report_cmds: report_cmds[item.id] = {} if "/created_utc" not in report_cmds[item.id]: report_cmds[item.id]["/created_utc"] = item.created_utc if author not in report_cmds[item.id]: report_cmds[item.id][author] = [body] else: report_cmds[item.id][author].append(body) report_cmds.sync() if not author: return # If the item is older than a week, ignore it if utcnow() - item.created_utc > timedata.SEC_IN_WEEK: return # Clean up items older than a week for key, val in list(report_cmds.items()): if utcnow() - val["/created_utc"] > timedata.SEC_IN_WEEK: del report_cmds[key] report_cmds.sync() new_item = False # If the item wasn't there before, add it if item.id not in report_cmds: new_item = True add_item() # If the reporter was not there before, add it if not new_item and author not in report_cmds[item.id]: new_item = True add_item() # If the report reason was not there before, add it if not new_item and body not in report_cmds[item.id][author]: new_item = True add_item() if new_item: report_feeder(report(item, get_user(author), body))
def __init__(self, reason, thing, author=None): self.reason = reason self.report_author = author self.author = thing.author self.created_utc = utils.utcnow() self.id = thing.id self.permalink = thing.permalink self.subreddit = thing.subreddit
def __init__(self, wiki): self.author = wiki.author self.content = wiki.content self.revision_date = wiki.revision_date self.recent_edit = False # Check if it's a recent edit if utils.utcnow() - self.revision_date < RECENT_EDIT_LIMIT: self.recent_edit = True
def tick(period, trigger): while True: # Sleep for the given period time.sleep(period) tnow = utcnow() try: trigger(tnow) except: import traceback traceback.print_exc()
def new_modlog_item(item): if item.id not in modlog_hist: modlog_hist[item.id] = item.created_utc modlog_feeder(modlog(item)) to_delete = [] for key, val in modlog_hist.items(): if utcnow() - val > timedata.SEC_IN_DAY * 7: to_delete.append(key) for key in to_delete: del modlog_hist[key]
def update_content(self): """ Update page content and return True if changed, False otherwise """ changed = False self.content = self.sub.wiki(self.wiki_page).get_content() if self.content != self.old_content: changed = True self.old_content = self.content self.last_update = utils.utcnow() return changed
def __init__(self, subreddit_name, author_name, title, body=None, url=None): self.id = base36.dumps(FakeSubmission.crt_id) self.shortlink = "https://redd.it/" + self.id self.permalink = self.shortlink self.created_utc = utils.utcnow() self.body = body self.url = url self.deleted = False self.selftext = "" if body: self.selftext = body self.domain = "self" if self.url: self.domain = self.url.split("//")[1].split("/")[0].replace( "www.", "") self.author = get_user(author_name) self.title = title self.is_crosspostable = True self.link_flair_text = None # Add submission to subreddit self.subreddit = get_subreddit(subreddit_name) self.subreddit.add_submission(self) # Add to global cache and increment submission id cache_submissions[self.id] = self FakeSubmission.crt_id += 1 cache_info["t3_%s" % self.id] = self self.reports = [] self.comments = [] self.flairs = None # Create a flair instance for each submission if self.subreddit.sub_flairs: self.flairs = self.FakeFlair(self.subreddit.sub_flairs, self) self.mod = FakeSubmission.mod(self) # Announce the bot that there is a new submission new_all_sub(self)
def __init__(self, author_name, target_author, action, details, description, subreddit_instance): self.mod = get_user(author_name) self.target_author = target_author self.action = action self.description = description self.details = details self.subreddit = subreddit_instance.display_name self.target_permalink = None # Set id self.id = FakeModLog.id FakeModLog.id += 1 # Set created self.created_utc = utils.utcnow()
def __init__(self, reddit_wiki): try: self.content = reddit_wiki.content_md except: self.content = "" # Set permissions to mod only try: reddit_wiki.mod.update(listed=False, permlevel=2) except: pass self.subreddit_name = reddit_wiki.subreddit.display_name self.name = reddit_wiki.name try: self.author = user(reddit_wiki.revision_by) except: self.author = "" try: self.revision_date = int(reddit_wiki.revision_date) except: self.revision_date = 0 # Check if the subreddit has a wiki storage if self.subreddit_name not in wiki_storages: logger.debug("Adding %s/%s to wiki storage" % (self.subreddit_name, self.name)) wiki_storages[self.subreddit_name] = dsdict( self.subreddit_name, "wiki_cache") storage = wiki_storages[self.subreddit_name] storage[self.name] = { "store_date": utcnow(), "subreddit": self.subreddit_name, "name": self.name, "content": self.content, "author": str(self.author), "revision_date": self.revision_date } storage.sync()
def add_hook(self, func, last_exec=utils.utcnow()): # Create an unique copy for this container func = copy.copy(func) if func.ctype == callback_type.SUB: self.callbacks_subs.append(func) elif func.ctype == callback_type.COM: self.callbacks_coms.append(func) elif func.ctype == callback_type.MLG: self.callbacks_mlog.append(func) elif func.ctype == callback_type.PER: self.callbacks_peri.append(func) if func.last_exec == 0: func.set_last_exec(last_exec) elif func.ctype == callback_type.ONL: # If callback is of type on load, call it now self.to_call(func, False, {**self.extra_args}) elif func.ctype == callback_type.ONS: self.callbacks_onstart.append(func) else: logger.error("Unhandled function type: " + str(func.ctype))
def wiki(self, name, force_live=False): if force_live: logger.debug("Getting a fresh copy of %s/%s" % (self.display_name, name)) self.wikis[name] = wiki(backend.get_wiki(self._raw, name)) logger.debug("[%s] Access wiki: %s" % (self, name)) if str(self) not in wiki_storages: wiki_storages[str(self)] = dsdict(str(self), "wiki_cache") # Check if there is a copy of the wiki stored if name in wiki_storages[str(self)] and \ utcnow() - wiki_storages[str(self)][name]["store_date"] < COLD_WIKI_LIMIT: logger.debug("Getting stored copy of %s/%s" % (self.display_name, name)) self.wikis[name] = wiki_stored(wiki_storages[str(self)][name]) else: logger.debug("Getting a fresh copy of %s/%s" % (self.display_name, name)) self.wikis[name] = wiki(backend.get_wiki(self._raw, name)) return self.wikis[name]
def __init__(self, bot_inst, master_subreddit, path_list=None, bot_config={}, db_params={}): """ Class that manages plugins from a given list of paths. :param bot_inst: bot instance :param master_subreddit: subreddit where core bot configuration is stored :param path_list: list of folders relative to the location from where to load plugins :param bot_config: bot config data :param db_params: how to log in to the psql server """ self.modules = [] self.plugin_threads = [] self.per_last_exec = {} self.bot = bot_inst self.config = bot_config self.plugin_args = {} self.inbox_cmds = {} self.master_sub = get_subreddit(master_subreddit) # Save the given watched subreddits self.given_watched_subs = {} for sub in get_moderated_subs(): self.given_watched_subs[sub] = True self.watched_subs = dict(self.given_watched_subs) self.db = None if db_params: print("Connecting to DB") # Create DB connection self.db = db_data( "postgresql+psycopg2://{user}:{password}@{host}/{database}".format(**db_params)) # Fill the standard parameter list self.plugin_args["plugin_manager"] = self self.plugin_args["reddit"] = self self.plugin_args["config"] = self.config self.plugin_args["db"] = self.db self.plugin_args["bot_owner"] = get_user( self.config.get("owner", "owner")) # Set start time self.start_time = utils.utcnow() print("[%d] Getting dispatchers" % utils.utcnow()) self.dispatchers = {} self.dispatchers[DISPATCH_ANY] = DispatchAll(self.plugin_args) # Get notifications from the hook module callbacks.append(self.add_plugin_function) print("[%d] Loading plugins" % utils.utcnow()) # Load plugins for path in path_list: self.load_plugins(path) print("[%d] Running on start plugins" % utils.utcnow()) self.dispatchers[DISPATCH_ANY].run_on_start(False) print("[%d] Creating periodic thread" % utils.utcnow()) # Create the periodic thread to trigger periodic events start_tick(1.0, self.periodic_func) print("[%d] Startup done!" % utils.utcnow()) # Start watching subreddits watch_all( self.feed_sub, self.feed_comms, self.feed_inbox, self.feed_reports, self.feed_modlog)
def do_tick(): time_trigger(utils.utcnow())
def expired(self): if utcnow() - self.last_check > self.interval: return True return False
def mark_updated(self): self.last_check = utcnow()
def set_content(self, content, author): self.content = content self.revision_by = get_user(author) self.revision_date = int(utils.utcnow())
def new_post(submission, reddit, subreddit): # Skip self posts if submission.is_self: return # Get wiki configuration if subreddit.display_name not in wiki_config: logger.debug("[%s] Not in wiki config %s" % (submission.shortlink, subreddit.display_name)) return config = wiki_config[subreddit.display_name] logger.debug("[%s] New post submitted with title: %s" % (submission.shortlink, submission.title)) # Get current time tnow = utcnow() # Don't take action on old posts if tnow - submission.created_utc > MAX_ACTION_TIME: logger.debug("[%s] Skipped because it's too old" % (submission.shortlink)) return # Check if the domain is in the configured list domain_valid = False for domain in config.domains: if not submission.url: break if domain in submission.url: domain_valid = True break if not domain_valid: return # Check if the user should be ignored if submission.author.name.lower() in config.ignore_users: return # Get the title and clean it article_title = clean_title(get_title(submission.url)) if len(article_title) == 0: return # Clean the submission title too submission_title = clean_title(submission.title) if len(submission_title) == 0: return logger.debug("[%s] Checking for editorialization\n\t%s\n\t%s" % (submission.shortlink, article_title, submission_title)) overlap_factor = calc_overlap_avg(article_title, submission_title) logger.debug("[%s] Calculated editorialized factor %f" % (submission.shortlink, overlap_factor)) # If there is too little overlap, report it if overlap_factor < config.minimum_overlap_percent: submission.report("Possible editorialization with a factor of %.2f%%" % (overlap_factor))
def __init__(self, subreddit, plugin): self.sub = subreddit self.subreddit_name = subreddit.display_name self.wiki_page = plugin.wiki_page self.content = "" try: self.content = self.sub.wiki(self.wiki_page).get_content() except prawcore.exceptions.NotFound: logger.debug( "Subreddit %s does not contain wiki %s. Creating it." % (subreddit.display_name, self.wiki_page)) self.sub.wiki[self.wiki_page].edit(EMPTY_WIKI) self.content = EMPTY_WIKI self.old_content = self.content self.last_update = utils.utcnow() self.refresh_interval = plugin.refresh_interval # not used self.notifier = plugin.wiki_change_notifier self.description = plugin.description self.raw_documentation = plugin.documentation self.documentation = None # Provide extra args for upper classes self.args = {} # Create location for small storage self.storage = dsdict(self.subreddit_name, self.wiki_page) self.args["storage"] = self.storage # Initialize wiki r/w self.mode = "" if "r" in plugin.mode: self.mode += "r" if "w" in plugin.mode: self.mode += "w" # Pre generate documentation string if self.raw_documentation: self.documentation = WatchedWiki.DOC_BEGIN split_doc = self.raw_documentation.strip().split("\n") self.documentation += "\n".join( (" ## " + line).rstrip() for line in split_doc) self.documentation += WatchedWiki.DOC_END # Set documentation if not already present self.content = self.content.replace("\r", "") if self.documentation and self.documentation.strip( ) not in self.content: begin = self.content.replace("\r", "").find(WatchedWiki.DOC_BEGIN) end = self.content.replace("\r", "").find(WatchedWiki.DOC_END) if begin != -1 and end != -1: self.content = self.content[0:begin] + self.documentation + \ self.content[end + len(WatchedWiki.DOC_END):] else: self.content = self.documentation + self.content self.set_content(self.content, with_documentation=False)
def new_post(submission, storage, reddit, subreddit): if "subs" not in storage: storage["subs"] = {} # Get wiki configuration if subreddit.display_name not in wiki_config: logger.debug("[%s] Not in wiki config %s" % (submission.shortlink, subreddit.display_name)) return config = wiki_config[subreddit.display_name] # Check if the user should be ignored if submission.author.name.lower() in config.ignore_users: return logger.debug("[%s] New post submitted with title: %s" % (submission.shortlink, submission.title)) # Get current time tnow = utcnow() # Don't take action on old posts if tnow - submission.created_utc > MAX_ACTION_TIME: logger.debug("[%s] Skipped because it's too old" % (submission.shortlink)) return if submission.shortlink in storage["subs"]: logger.debug("[%s] Submission already added" % (submission.shortlink)) return # Clean the title cleaned_title = clean_title(submission.title, config.minimum_word_length) if len(cleaned_title) < config.minimum_nb_words: logger.debug("[%s] Title is too short" % (submission.shortlink)) return # Check against old titles for post in storage["subs"].values(): # Calculate two-way overlap factor logger.debug("[%s] Checking\n\t%s\n\t%s" % (submission.shortlink, cleaned_title, post["filtered"])) overlap_factor = calc_overlap_avg(cleaned_title, post["filtered"]) logger.debug("[%s] Calculated repost factor %f" % (submission.shortlink, overlap_factor)) if overlap_factor > config.min_overlap_percent: post_sub = reddit.get_submission(url=post["shortlink"]) # Did the author remove it? if post_sub.author == None: continue # Was it removed? elif post_sub.is_crosspostable == False: continue logger.debug( "[%s] Reporting as dupe for %s / factor %f" % (submission.shortlink, post["shortlink"], overlap_factor)) submission.report( "Possible repost of %s, with a factor of %.2f%%" % (post["shortlink"], overlap_factor)) return # Add new element new = {} new["original"] = submission.title new["filtered"] = cleaned_title new["shortlink"] = submission.shortlink new["created_utc"] = submission.created_utc # Eliminate old submissions for post in list(storage["subs"].values()): if tnow - post["created_utc"] > MAX_AGE: del storage["subs"][post["shortlink"]] storage["subs"][submission.shortlink] = new storage.sync()