def process_comment(reddit_comment: Comment):
"""
Process a single PRAW Comment. Adds it to the database if it didn't previously exist as well as parent comments
and the thread it belongs to.
"""
comment = comment_service.get_comment_by_id(reddit_comment.id)
if comment:
# Update our record of the comment if necessary.
comment_service.update_comment(comment, reddit_comment)
return
author_name = reddit_comment.author.name if reddit_comment.author is not None else "[deleted]"
logger.info(
f"Processing comment {reddit_comment.id} - /u/{author_name} (post {reddit_comment.submission.id})"
)
# Post needs to exist before we can add a comment for it, start with that.
post = post_service.get_post_by_id(reddit_comment.submission.id)
if not post:
post_service.add_post(reddit_comment.submission)
# Since all comments will reference a parent if it exists, add all parent comments first.
logger.debug(f"Saving parent comments of {reddit_comment.id}")
comment_service.add_comment_parent_tree(reddit, reddit_comment)
logger.debug(f"Saving comment {reddit_comment.id}")
comment = comment_service.add_comment(reddit_comment)
logger.debug(f"Finished processing {comment.id}")
def insert():
title = request.form['title']
body = request.form['body']
img = request.files.get('img')
publish_date = request.form['publish_date']
author_id = request.form['author_id']
requires_login = request.form['requires_login']
post_service.add_post(title, body, publish_date, img, requires_login,
author_id)
return Response(status=201)
def process_post(submission: Submission):
"""
Process a single PRAW Submission. Adds it to the database if it didn't previously exist, updates post if necessary.
"""
# If we've already saved the post and sent it to Discord, no need to do anything (likely upon restart).
post = post_service.get_post_by_id(submission.id)
if post and post.sent_to_feed:
logger.debug(f"Already processed, skipping post {submission.id}")
return
author_name = submission.author.name if submission.author is not None else "[deleted]"
logger.info(
f"Processing post {submission.id} - /u/{author_name} - {submission.link_flair_text}"
)
if post:
post = post_service.update_post(post, submission)
else:
post = post_service.add_post(submission)
send_new_submission_message(submission)
post.sent_to_feed = True
base_data_service.update(post)
logger.debug(f"Finished processing {submission.id}")
def add_comment_parent_tree(reddit: Reddit, reddit_comment: Comment):
"""
Starting with the comment that's the *parent* of the specified comment (non-inclusive),
recursively crawl up the tree and add all of them to the database.
Stops when it reaches a comment that already exists in the database or upon reaching the root.
Needs improvements for efficiency.
"""
# Could do this with recursive calls to this function, but
# I don't know how deep reddit comment chains are allowed to get.
# So instead we need to keep a stack of comments so we can insert
# them in the correct order, root first then down the chain.
# This is necessary because the parent_id of each needs to already exist.
comment_stack = []
# At the start of each loop, if we're at the top comment of the tree there will be no parents to add.
# parent_id will return a submission for top level comments, so check is_root instead.
while not reddit_comment.is_root:
parent_id = reddit_comment.parent_id.split("t1_")[1]
parent_exists = get_comment_by_id(parent_id)
# Once we reach a child where the parent already exists, we can stop adding new comments up the chain.
if parent_exists:
break
# Parent now becomes the base comment, then create a model for it (but don't insert yet).
reddit_comment = reddit.comment(id=parent_id)
comment = _create_comment_model(reddit_comment)
comment_stack.append(comment)
# Insert the author into the database if they don't exist yet.
if reddit_comment.author is not None and not user_service.get_user(
reddit_comment.author.name):
user_service.add_user(reddit_comment.author)
# Insert post into the database if it doesn't exist yet.
if not post_service.get_post_by_id(reddit_comment.submission.id):
post_service.add_post(reddit_comment.submission)
# Reverse the order that we're iterating through the stack for inserting, last->first.
for comment in comment_stack[::-1]:
_comment_data.insert(comment, error_on_conflict=False)
def add_comment(reddit_comment: Comment) -> CommentModel:
"""
Parses some basic information for a comment and adds it to the database.
Creates author and post if necessary.
This also assumes its parent comment is already created, call
add_comment_parent_tree first if necessary.
"""
comment = _create_comment_model(reddit_comment)
# Insert the author into the database if they don't exist yet.
if reddit_comment.author is not None and not user_service.get_user(
reddit_comment.author.name):
user_service.add_user(reddit_comment.author)
# Insert post into the database if it doesn't exist yet.
if not post_service.get_post_by_id(reddit_comment.submission.id):
post_service.add_post(reddit_comment.submission)
new_comment = _comment_data.insert(comment, error_on_conflict=False)
return new_comment
def save_post_and_comments(reddit_submission: Submission):
"""
Saves a single reddit post and its comments to the database.
"""
post_name = reddit_submission.permalink
# Ensure post is in the database first.
post_service.add_post(reddit_submission)
logger.info(
f"Loading {reddit_submission.num_comments} comments on {post_name}")
# Load all comments
retry_count = 0
while True:
try:
retry_count += 1
if retry_count > 3:
logger.info(f"Unable to load all comments for {post_name}")
return
reddit_submission.comments.replace_more(limit=None)
break
except Exception:
logger.exception("Handling replace_more exception")
time.sleep(5)
logger.info(f"Processing comments on {post_name}")
index = -1
for index, reddit_comment in enumerate(reddit_submission.comments.list()):
# Since all comments will reference a parent if it exists, add all parent comments first.
logger.debug(f"Saving parent comments of {reddit_comment.id}")
comment_service.add_comment_parent_tree(reddit, reddit_comment)
logger.debug(f"Saving comment {reddit_comment.id}")
comment_service.add_comment(reddit_comment)
if (index + 1) % 500 == 0:
logger.info(f"Completed {index + 1} comments on {post_name}")
logger.info(f"Finished processing {post_name}, total {index + 1} comments")
def add_frontpage_post(reddit_post: Submission, snapshot: SnapshotModel,
rank: int) -> SnapshotFrontpageModel:
"""Adds the specified post ranking for the snapshot. Also inserts the post itself if necessary."""
post_id = reddit_post.id
post = post_service.get_post_by_id(post_id)
# Add or update post as necessary.
if not post:
logger.debug(f"Saving post {post_id}")
post = post_service.add_post(reddit_post)
else:
post = post_service.update_post(post, reddit_post)
frontpage_model = SnapshotFrontpageModel()
frontpage_model.post_id = post.id
frontpage_model.snapshot_id = snapshot.id
frontpage_model.rank = rank
frontpage_model.score = reddit_post.score
saved_frontpage_model = _snapshot_data.insert(frontpage_model)
return saved_frontpage_model
def parse_mod_action(mod_action: ModAction):
"""
Process a single PRAW ModAction. Assumes that reddit and subreddit are already instantiated by
one of the two entry points (monitor_stream or load_archive).
"""
# Check if we've already processed this mod action, do nothing if so.
mod_action_id = mod_action.id.replace("ModAction_", "")
if mod_action_service.get_mod_action_by_id(mod_action_id):
logger.debug(f"Already processed, skipping mod action {mod_action_id}")
return
logger.info(
f"Processing mod action {mod_action_id}: {mod_action.mod.name} - "
f"{mod_action.action} - {mod_action.target_fullname}"
)
# If there's an action by an unknown moderator or admin, make a note of it and check to see
# if they should be added to the mod list.
send_notification = False
if mod_action.action in mod_constants.MOD_ACTIONS_ALWAYS_NOTIFY:
send_notification = True
if mod_action.mod.name not in active_mods:
# Add them to the database if necessary.
mod_user = user_service.get_user(mod_action.mod.name)
if not mod_user:
mod_user = user_service.add_user(mod_action.mod)
# We'd normally send a notification for all actions from non-mods, but temporary mutes expiring
# always come from reddit and we don't really care about those.
# Similarly, crowd control removals as those are filtered to the mod queue.
if not (
(mod_action.mod.name == "reddit" and mod_action.action == "unmuteuser")
or (
mod_action.mod.name == "reddit"
and mod_action.action in ("removecomment", "removelink")
and mod_action.details == "Crowd Control"
)
):
send_notification = True
# For non-admin cases, check to see if they're a [new] mod of the subreddit and refresh the list if so.
if mod_action.mod not in ("Anti-Evil Operations", "reddit"):
logger.info(f"Unknown mod found: {mod_action.mod.name}")
if mod_user.username in subreddit.moderator():
logger.debug(f"Updating mod status for {mod_user}")
mod_user.moderator = True
base_data_service.update(mod_user)
_get_moderators()
# See if the user targeted by this action exists in the system, add them if not.
# Bans and similar user-focused actions independent of posts/comments will also have
# a target_fullname value (t2_...) but won't be necessary to check after this.
if mod_action.target_author:
user = user_service.get_user(mod_action.target_author)
if not user:
logger.debug(f"Saving user {mod_action.target_author}")
user = user_service.add_user(reddit.redditor(name=mod_action.target_author))
# For bans and unbans, update the user in the database.
if mod_action.action == "banuser":
# Weirdly this returns a ListingGenerator so we have to iterate over it; there should only be one though.
# If the user isn't banned, the loop won't execute.
for ban_user in subreddit.banned(redditor=user.username):
# Permanent if days_left is None
if ban_user.days_left is None:
user.banned_until = "infinity"
# days_left will show 0 if they were banned for 1 day a few seconds ago; it seems like it rounds down
# based on the time of the ban occurring, so we can safely assume that even if the ban happened
# a few seconds before getting to this point, we should add an extra day onto the reported number.
else:
ban_start = datetime.fromtimestamp(mod_action.created_utc, tz=timezone.utc)
user.banned_until = ban_start + timedelta(days=ban_user.days_left + 1)
break
base_data_service.update(user)
elif mod_action.action == "unbanuser":
user.banned_until = None
base_data_service.update(user)
elif mod_action.action == "removemoderator":
logger.debug(f"Updating mod status for {user}")
user.moderator = False
base_data_service.update(user)
_get_moderators()
# See if the post targeted by this action exists in the system, add it if not.
if mod_action.target_fullname and mod_action.target_fullname.startswith("t3_"):
post_id = mod_action.target_fullname.split("_")[1]
post = post_service.get_post_by_id(post_id)
reddit_post = reddit.submission(id=post_id)
# Add or update post as necessary.
if not post:
logger.debug(f"Saving post {post_id}")
post = post_service.add_post(reddit_post)
else:
post = post_service.update_post(post, reddit_post)
# If the user deleted their text post, the mod action still has the post body that we can save in place.
if post.deleted and post.body == "[deleted]" and post.body != mod_action.target_body:
post.body = mod_action.target_body
base_data_service.update(post)
# See if the comment targeted by this action *and its post* exist in the system, add either if not.
if mod_action.target_fullname and mod_action.target_fullname.startswith("t1_"):
comment_id = mod_action.target_fullname.split("_")[1]
comment = comment_service.get_comment_by_id(comment_id)
reddit_comment = reddit.comment(id=comment_id)
if not comment:
# Post needs to exist before we can add a comment for it, start with that.
post = post_service.get_post_by_id(reddit_comment.submission.id)
if not post:
post_service.add_post(reddit_comment.submission)
# Since all comments will reference a parent if it exists, add all parent comments first.
logger.debug(f"Saving parent comments of {comment_id}")
comment_service.add_comment_parent_tree(reddit, reddit_comment)
logger.debug(f"Saving comment {comment_id}")
comment = comment_service.add_comment(reddit_comment)
else:
# Update our record of the comment if necessary.
comment = comment_service.update_comment(comment, reddit_comment)
# If the user deleted their comment, the mod action still has the body that we can save in place.
if comment.deleted and comment.body != mod_action.target_body:
comment.body = mod_action.target_body
base_data_service.update(comment)
logger.debug(f"Saving mod action {mod_action_id}")
mod_action = mod_action_service.add_mod_action(mod_action)
if send_notification:
send_discord_message(mod_action)