def get_picture(url, fname, timeout_sec=20, max_size=1024 * 1024 * 20):
if "imgur.com" in url:
url = proc_imgur_url(url)
req = requests.get(url, stream=True)
req.raise_for_status()
start = utcnow()
content = bytes()
size = 0
# Get a chunk
for chunk in req.iter_content(1024):
# If download time exceeds the given timeout, exit
if utcnow() - start > timeout_sec:
print("Image %s took too long to download" % url)
return None
content += chunk
size += len(chunk)
# If the size eceeds the given maximum size, exit
if size > max_size:
print("Image %s is too large" % url)
return None
# Save the file
save_data = open(fname, "wb")
save_data.write(content)
return fname
def new_report(item, author, body):
"""
Check if an item can be added in the report feeder
"""
def add_item():
if item.id not in report_cmds:
report_cmds[item.id] = {}
if "/created_utc" not in report_cmds[item.id]:
report_cmds[item.id]["/created_utc"] = item.created_utc
if author not in report_cmds[item.id]:
report_cmds[item.id][author] = [body]
else:
report_cmds[item.id][author].append(body)
report_cmds.sync()
if not author:
return
# If the item is older than a week, ignore it
if utcnow() - item.created_utc > timedata.SEC_IN_WEEK:
return
# Clean up items older than a week
for key, val in list(report_cmds.items()):
if utcnow() - val["/created_utc"] > timedata.SEC_IN_WEEK:
del report_cmds[key]
report_cmds.sync()
new_item = False
# If the item wasn't there before, add it
if item.id not in report_cmds:
new_item = True
add_item()
# If the reporter was not there before, add it
if not new_item and author not in report_cmds[item.id]:
new_item = True
add_item()
# If the report reason was not there before, add it
if not new_item and body not in report_cmds[item.id][author]:
new_item = True
add_item()
if new_item:
report_feeder(report(item, get_user(author), body))
def __init__(self, reason, thing, author=None):
self.reason = reason
self.report_author = author
self.author = thing.author
self.created_utc = utils.utcnow()
self.id = thing.id
self.permalink = thing.permalink
self.subreddit = thing.subreddit
def __init__(self, wiki):
self.author = wiki.author
self.content = wiki.content
self.revision_date = wiki.revision_date
self.recent_edit = False
# Check if it's a recent edit
if utils.utcnow() - self.revision_date < RECENT_EDIT_LIMIT:
self.recent_edit = True
def tick(period, trigger):
while True:
# Sleep for the given period
time.sleep(period)
tnow = utcnow()
try:
trigger(tnow)
except:
import traceback
traceback.print_exc()
def new_modlog_item(item):
if item.id not in modlog_hist:
modlog_hist[item.id] = item.created_utc
modlog_feeder(modlog(item))
to_delete = []
for key, val in modlog_hist.items():
if utcnow() - val > timedata.SEC_IN_DAY * 7:
to_delete.append(key)
for key in to_delete:
del modlog_hist[key]
def update_content(self):
"""
Update page content and return True if changed, False otherwise
"""
changed = False
self.content = self.sub.wiki(self.wiki_page).get_content()
if self.content != self.old_content:
changed = True
self.old_content = self.content
self.last_update = utils.utcnow()
return changed
def __init__(self,
subreddit_name,
author_name,
title,
body=None,
url=None):
self.id = base36.dumps(FakeSubmission.crt_id)
self.shortlink = "https://redd.it/" + self.id
self.permalink = self.shortlink
self.created_utc = utils.utcnow()
self.body = body
self.url = url
self.deleted = False
self.selftext = ""
if body:
self.selftext = body
self.domain = "self"
if self.url:
self.domain = self.url.split("//")[1].split("/")[0].replace(
"www.", "")
self.author = get_user(author_name)
self.title = title
self.is_crosspostable = True
self.link_flair_text = None
# Add submission to subreddit
self.subreddit = get_subreddit(subreddit_name)
self.subreddit.add_submission(self)
# Add to global cache and increment submission id
cache_submissions[self.id] = self
FakeSubmission.crt_id += 1
cache_info["t3_%s" % self.id] = self
self.reports = []
self.comments = []
self.flairs = None
# Create a flair instance for each submission
if self.subreddit.sub_flairs:
self.flairs = self.FakeFlair(self.subreddit.sub_flairs, self)
self.mod = FakeSubmission.mod(self)
# Announce the bot that there is a new submission
new_all_sub(self)
def __init__(self, author_name, target_author, action, details,
description, subreddit_instance):
self.mod = get_user(author_name)
self.target_author = target_author
self.action = action
self.description = description
self.details = details
self.subreddit = subreddit_instance.display_name
self.target_permalink = None
# Set id
self.id = FakeModLog.id
FakeModLog.id += 1
# Set created
self.created_utc = utils.utcnow()
def __init__(self, reddit_wiki):
try:
self.content = reddit_wiki.content_md
except:
self.content = ""
# Set permissions to mod only
try:
reddit_wiki.mod.update(listed=False, permlevel=2)
except:
pass
self.subreddit_name = reddit_wiki.subreddit.display_name
self.name = reddit_wiki.name
try:
self.author = user(reddit_wiki.revision_by)
except:
self.author = ""
try:
self.revision_date = int(reddit_wiki.revision_date)
except:
self.revision_date = 0
# Check if the subreddit has a wiki storage
if self.subreddit_name not in wiki_storages:
logger.debug("Adding %s/%s to wiki storage" %
(self.subreddit_name, self.name))
wiki_storages[self.subreddit_name] = dsdict(
self.subreddit_name, "wiki_cache")
storage = wiki_storages[self.subreddit_name]
storage[self.name] = {
"store_date": utcnow(),
"subreddit": self.subreddit_name,
"name": self.name,
"content": self.content,
"author": str(self.author),
"revision_date": self.revision_date
}
storage.sync()
def add_hook(self, func, last_exec=utils.utcnow()):
# Create an unique copy for this container
func = copy.copy(func)
if func.ctype == callback_type.SUB:
self.callbacks_subs.append(func)
elif func.ctype == callback_type.COM:
self.callbacks_coms.append(func)
elif func.ctype == callback_type.MLG:
self.callbacks_mlog.append(func)
elif func.ctype == callback_type.PER:
self.callbacks_peri.append(func)
if func.last_exec == 0:
func.set_last_exec(last_exec)
elif func.ctype == callback_type.ONL:
# If callback is of type on load, call it now
self.to_call(func, False, {**self.extra_args})
elif func.ctype == callback_type.ONS:
self.callbacks_onstart.append(func)
else:
logger.error("Unhandled function type: " + str(func.ctype))
def wiki(self, name, force_live=False):
if force_live:
logger.debug("Getting a fresh copy of %s/%s" %
(self.display_name, name))
self.wikis[name] = wiki(backend.get_wiki(self._raw, name))
logger.debug("[%s] Access wiki: %s" % (self, name))
if str(self) not in wiki_storages:
wiki_storages[str(self)] = dsdict(str(self), "wiki_cache")
# Check if there is a copy of the wiki stored
if name in wiki_storages[str(self)] and \
utcnow() - wiki_storages[str(self)][name]["store_date"] < COLD_WIKI_LIMIT:
logger.debug("Getting stored copy of %s/%s" %
(self.display_name, name))
self.wikis[name] = wiki_stored(wiki_storages[str(self)][name])
else:
logger.debug("Getting a fresh copy of %s/%s" %
(self.display_name, name))
self.wikis[name] = wiki(backend.get_wiki(self._raw, name))
return self.wikis[name]
def __init__(self,
bot_inst,
master_subreddit,
path_list=None,
bot_config={},
db_params={}):
"""
Class that manages plugins from a given list of paths.
:param bot_inst: bot instance
:param master_subreddit: subreddit where core bot configuration is stored
:param path_list: list of folders relative to the location from where to load plugins
:param bot_config: bot config data
:param db_params: how to log in to the psql server
"""
self.modules = []
self.plugin_threads = []
self.per_last_exec = {}
self.bot = bot_inst
self.config = bot_config
self.plugin_args = {}
self.inbox_cmds = {}
self.master_sub = get_subreddit(master_subreddit)
# Save the given watched subreddits
self.given_watched_subs = {}
for sub in get_moderated_subs():
self.given_watched_subs[sub] = True
self.watched_subs = dict(self.given_watched_subs)
self.db = None
if db_params:
print("Connecting to DB")
# Create DB connection
self.db = db_data(
"postgresql+psycopg2://{user}:{password}@{host}/{database}".format(**db_params))
# Fill the standard parameter list
self.plugin_args["plugin_manager"] = self
self.plugin_args["reddit"] = self
self.plugin_args["config"] = self.config
self.plugin_args["db"] = self.db
self.plugin_args["bot_owner"] = get_user(
self.config.get("owner", "owner"))
# Set start time
self.start_time = utils.utcnow()
print("[%d] Getting dispatchers" % utils.utcnow())
self.dispatchers = {}
self.dispatchers[DISPATCH_ANY] = DispatchAll(self.plugin_args)
# Get notifications from the hook module
callbacks.append(self.add_plugin_function)
print("[%d] Loading plugins" % utils.utcnow())
# Load plugins
for path in path_list:
self.load_plugins(path)
print("[%d] Running on start plugins" % utils.utcnow())
self.dispatchers[DISPATCH_ANY].run_on_start(False)
print("[%d] Creating periodic thread" % utils.utcnow())
# Create the periodic thread to trigger periodic events
start_tick(1.0, self.periodic_func)
print("[%d] Startup done!" % utils.utcnow())
# Start watching subreddits
watch_all(
self.feed_sub,
self.feed_comms,
self.feed_inbox,
self.feed_reports,
self.feed_modlog)
def do_tick():
time_trigger(utils.utcnow())
def expired(self):
if utcnow() - self.last_check > self.interval:
return True
return False
def mark_updated(self):
self.last_check = utcnow()
def set_content(self, content, author):
self.content = content
self.revision_by = get_user(author)
self.revision_date = int(utils.utcnow())
def new_post(submission, reddit, subreddit):
# Skip self posts
if submission.is_self:
return
# Get wiki configuration
if subreddit.display_name not in wiki_config:
logger.debug("[%s] Not in wiki config %s" %
(submission.shortlink, subreddit.display_name))
return
config = wiki_config[subreddit.display_name]
logger.debug("[%s] New post submitted with title: %s" %
(submission.shortlink, submission.title))
# Get current time
tnow = utcnow()
# Don't take action on old posts
if tnow - submission.created_utc > MAX_ACTION_TIME:
logger.debug("[%s] Skipped because it's too old" %
(submission.shortlink))
return
# Check if the domain is in the configured list
domain_valid = False
for domain in config.domains:
if not submission.url:
break
if domain in submission.url:
domain_valid = True
break
if not domain_valid:
return
# Check if the user should be ignored
if submission.author.name.lower() in config.ignore_users:
return
# Get the title and clean it
article_title = clean_title(get_title(submission.url))
if len(article_title) == 0:
return
# Clean the submission title too
submission_title = clean_title(submission.title)
if len(submission_title) == 0:
return
logger.debug("[%s] Checking for editorialization\n\t%s\n\t%s" %
(submission.shortlink, article_title, submission_title))
overlap_factor = calc_overlap_avg(article_title, submission_title)
logger.debug("[%s] Calculated editorialized factor %f" %
(submission.shortlink, overlap_factor))
# If there is too little overlap, report it
if overlap_factor < config.minimum_overlap_percent:
submission.report("Possible editorialization with a factor of %.2f%%" %
(overlap_factor))
def __init__(self, subreddit, plugin):
self.sub = subreddit
self.subreddit_name = subreddit.display_name
self.wiki_page = plugin.wiki_page
self.content = ""
try:
self.content = self.sub.wiki(self.wiki_page).get_content()
except prawcore.exceptions.NotFound:
logger.debug(
"Subreddit %s does not contain wiki %s. Creating it." %
(subreddit.display_name, self.wiki_page))
self.sub.wiki[self.wiki_page].edit(EMPTY_WIKI)
self.content = EMPTY_WIKI
self.old_content = self.content
self.last_update = utils.utcnow()
self.refresh_interval = plugin.refresh_interval # not used
self.notifier = plugin.wiki_change_notifier
self.description = plugin.description
self.raw_documentation = plugin.documentation
self.documentation = None
# Provide extra args for upper classes
self.args = {}
# Create location for small storage
self.storage = dsdict(self.subreddit_name, self.wiki_page)
self.args["storage"] = self.storage
# Initialize wiki r/w
self.mode = ""
if "r" in plugin.mode:
self.mode += "r"
if "w" in plugin.mode:
self.mode += "w"
# Pre generate documentation string
if self.raw_documentation:
self.documentation = WatchedWiki.DOC_BEGIN
split_doc = self.raw_documentation.strip().split("\n")
self.documentation += "\n".join(
(" ## " + line).rstrip() for line in split_doc)
self.documentation += WatchedWiki.DOC_END
# Set documentation if not already present
self.content = self.content.replace("\r", "")
if self.documentation and self.documentation.strip(
) not in self.content:
begin = self.content.replace("\r", "").find(WatchedWiki.DOC_BEGIN)
end = self.content.replace("\r", "").find(WatchedWiki.DOC_END)
if begin != -1 and end != -1:
self.content = self.content[0:begin] + self.documentation + \
self.content[end + len(WatchedWiki.DOC_END):]
else:
self.content = self.documentation + self.content
self.set_content(self.content, with_documentation=False)
def new_post(submission, storage, reddit, subreddit):
if "subs" not in storage:
storage["subs"] = {}
# Get wiki configuration
if subreddit.display_name not in wiki_config:
logger.debug("[%s] Not in wiki config %s" %
(submission.shortlink, subreddit.display_name))
return
config = wiki_config[subreddit.display_name]
# Check if the user should be ignored
if submission.author.name.lower() in config.ignore_users:
return
logger.debug("[%s] New post submitted with title: %s" %
(submission.shortlink, submission.title))
# Get current time
tnow = utcnow()
# Don't take action on old posts
if tnow - submission.created_utc > MAX_ACTION_TIME:
logger.debug("[%s] Skipped because it's too old" %
(submission.shortlink))
return
if submission.shortlink in storage["subs"]:
logger.debug("[%s] Submission already added" % (submission.shortlink))
return
# Clean the title
cleaned_title = clean_title(submission.title, config.minimum_word_length)
if len(cleaned_title) < config.minimum_nb_words:
logger.debug("[%s] Title is too short" % (submission.shortlink))
return
# Check against old titles
for post in storage["subs"].values():
# Calculate two-way overlap factor
logger.debug("[%s] Checking\n\t%s\n\t%s" %
(submission.shortlink, cleaned_title, post["filtered"]))
overlap_factor = calc_overlap_avg(cleaned_title, post["filtered"])
logger.debug("[%s] Calculated repost factor %f" %
(submission.shortlink, overlap_factor))
if overlap_factor > config.min_overlap_percent:
post_sub = reddit.get_submission(url=post["shortlink"])
# Did the author remove it?
if post_sub.author == None:
continue
# Was it removed?
elif post_sub.is_crosspostable == False:
continue
logger.debug(
"[%s] Reporting as dupe for %s / factor %f" %
(submission.shortlink, post["shortlink"], overlap_factor))
submission.report(
"Possible repost of %s, with a factor of %.2f%%" %
(post["shortlink"], overlap_factor))
return
# Add new element
new = {}
new["original"] = submission.title
new["filtered"] = cleaned_title
new["shortlink"] = submission.shortlink
new["created_utc"] = submission.created_utc
# Eliminate old submissions
for post in list(storage["subs"].values()):
if tnow - post["created_utc"] > MAX_AGE:
del storage["subs"][post["shortlink"]]
storage["subs"][submission.shortlink] = new
storage.sync()