def update_comments(self, submission, submission_dbid, subreddit_dbid):
# Submission sometimes doesn't have comment attr?
# Maybe if we've already seen this submission and gone through comments?
try:
comments = layer_comments(submission.comments)
except AttributeError: # No comments
return
for layer, comment_list in comments.iteritems():
for comment in comment_list:
try:
self.add_comment(comment, layer, submission_dbid, subreddit_dbid)
except mongo.errors.DuplicateKeyError:
continue
def add_submission(self, submission, subreddit_name, follow_link=False):
subreddit = self.subreddit_exists(subreddit_name)
if subreddit:
subreddit_id = subreddit.get("_id")
print "found subreddit:", subreddit_id
# If the comment has an author (could have been deleted),
# take care of updating or adding user information
if submission.author:
print submission.author
# If the submission's author already exists, update
# the author's information.
auth = self.user_exists(submission.author.id)
if auth:
auth_id = auth.get("_id")
print self.update_user(auth_id, submission.subreddit, "submissions")
# Otherwise create a new user object
else:
self.add_user(submission.author)
author = submission.author.fullname
else:
print "No author"
author = None
# If the submission already exists, update comments
sub = self.submission_exists(submission)
if sub:
print "Submission exists-updating"
submission_id = sub.get("_id")
self.update_comments(submission, submission_id, subreddit_id)
# Otherwise create submission object and add all the current comments
else:
print "Adding new submission"
document = {"subreddit_id": subreddit_id,
"submission_title": submission.title,
"submission_text": submission.selftext,
"karma": submission.ups,
"downvotes": submission.downs,
"num_comments": submission.num_comments,
"flair": submission.link_flair_text,
"url": submission.url,
"praw_id": submission.id,
"praw_fullid": submission.fullname,
"created": submission.created,
"author": author
}
# First add the submission to the db to make sure we have it
submission_id = self.submission_collection.insert(document)
self.subreddit_collection.update({"_id": subreddit_id},
{"$set": {"last_update": int(datetime.today().strftime("%s"))}})
# Next add the comments, if they're available
try:
comments = layer_comments(submission.comments)
for layer, comment_list in comments.iteritems():
for comment in comment_list:
self.add_comment(comment, layer, submission_id, subreddit_id)
# Sometimes submissions don't have comment attribute
except AttributeError:
pass
# Now follow link if this was specified and if
# submission contains a link to follow
if follow_link and submission.url:
content = links.scrape_link(submission.url, subreddit_name)
self.add_link_content(submission_id, str(content))
return submission_id
else:
raise errors.MissingError("Subreddit %s does not exist in the database." % subreddit_name)