def _make_comment_item(self, item):
"""
Make comment item.
Calls previously defined private method:
self._make_zip_dict()
Parameters
----------
item: PRAW object
PRAW comment item
Returns
-------
redditor_item: dict
Dictionary for a Redditor's comment
"""
edit_date = item.edited \
if str(item.edited).isalpha() \
else convert_time(item.edited)
items = [
"comment",
convert_time(item.created_utc), item.score, item.body,
item.parent_id, item.link_id, edit_date, item.stickied,
item.submission.selftext, item.submission.subreddit.display_name
]
return self._make_zip_dict(items, self._comment_titles)
def make_submission(self, include_all, submission):
"""
Make a submission object.
Calls a previously defined public method:
self.make_subreddit()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
include_all: boolean
Boolean to determine whether the "type" field should be included
submission: PRAW submission object
Returns
-------
submission_object: dict
Dictionary containing submission metadata
"""
submission_object = {
"author": "u/" + submission.author.name \
if hasattr(submission.author, "name") \
else "[deleted]",
"created_utc": convert_time(submission.created_utc),
"distinguished": submission.distinguished,
"edited": submission.edited \
if submission.edited == False \
else convert_time(submission.edited),
"id": submission.id,
"is_original_content": submission.is_original_content,
"is_self": submission.is_self,
"link_flair_text": submission.link_flair_text,
"locked": submission.locked,
"name": submission.name,
"nsfw": submission.over_18,
"num_comments": submission.num_comments,
"permalink": submission.permalink,
"score": submission.score,
"selftext": submission.selftext,
"spoiler": submission.spoiler,
"stickied": submission.stickied,
"title": submission.title,
"upvote_ratio": submission.upvote_ratio,
"url": submission.url
}
if include_all:
submission_object["subreddit"] = self.make_subreddit(submission.subreddit)
submission_object["type"] = "submission"
submission_object = dict(sorted(submission_object.items()))
return submission_object
def make_comment(self, comment, include_all):
"""
Make a comment item.
Calls previously defined public method:
self.make_submission()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
comment: PRAW comment object
include_all: boolean
Boolean to determine whether the "type" field should be included
Returns
-------
redditor_item: dict
Dictionary containing comment metadata
"""
comment_object = {
"author": "u/" + comment.author.name \
if hasattr(comment.author, "name") \
else "[deleted]",
"body": comment.body,
"body_html": comment.body_html,
"created_utc": convert_time(comment.created_utc),
"distinguished": comment.distinguished,
"edited": comment.edited \
if comment.edited == False \
else convert_time(comment.edited),
"id": comment.id,
"is_submitter": comment.is_submitter,
"link_id": comment.link_id,
"parent_id": comment.parent_id,
"score": comment.score,
"stickied": comment.stickied,
}
if include_all:
comment_object["submission"] = self.make_submission(include_all, comment.submission)
comment_object["subreddit_id"] = comment.subreddit_id
comment_object["type"] = "comment"
comment_object = dict(sorted(comment_object.items()))
return comment_object
def _fix_attributes(self, comment):
"""
If applicable, handle deleted Redditors or edited time.
Calls a method from an external module:
Global.convert_time()
Parameters
----------
comment: PRAW object
Returns
-------
author_name: str
edit_date: str
"""
try:
author_name = comment.author.name
except AttributeError:
author_name = "[deleted]"
edit_date = comment.edited \
if str(comment.edited).isalpha() \
else convert_time(comment.edited)
return author_name, edit_date
def add_comment(self, comment):
"""
Add list of dictionary of comments attributes to use when sorting.
Calls methods from an external module:
Global.make_none_dict()
Global.convert_time()
Parameters
----------
comment: PRAW object
Returns
-------
comment_object: dict
Dictionary for comment attribute
"""
comment_object = make_none_dict(self._titles)
author_name, edit_date = self._fix_attributes(comment)
comment_attributes = [
comment.parent_id, comment.id, author_name,
convert_time(comment.created_utc), comment.score, comment.body,
edit_date, comment.is_submitter, comment.stickied
]
for title, attribute in zip(self._titles, comment_attributes):
comment_object[title] = attribute
return comment_object
def get_rules(subreddit):
"""
Return post requirements and Subreddit rules.
Parameters
----------
subreddit: PRAW Subreddit object
Returns
-------
post_requirements: dict
Dictionary containing Subreddit's post requirements
rules: list[str]
List containing Subreddit rules
"""
rules = [{
"created_utc": convert_time(rule.created_utc),
"description": rule.description,
"kind": rule.kind,
"priority": rule.priority,
"short_name": rule.short_name,
"violation_reason": rule.violation_reason,
} for rule in subreddit.rules]
return subreddit.post_requirements(), rules
def _get_user_info(self, skeleton, user):
"""
Get Redditor account information.
Calls a previously defined private method:
self._info_titles()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
skeleton: dict
Dictionary containing all Redditor data
user: PRAW Redditor object
Returns
-------
None
"""
user_info_titles = self._info_titles
user_info = [
user.name, user.fullname, user.id,
convert_time(user.created_utc), user.comment_karma,
user.link_karma, user.is_employee, user.is_friend, user.is_mod,
user.is_gold
]
for info_title, user_item in zip(user_info_titles, user_info):
skeleton["data"]["information"][info_title] = user_item
def _make_submission_item(self, item):
"""
Make submission item.
Calls previously defined private method:
self._make_zip_dict()
Parameters
----------
item: PRAW object
PRAW submission item
Returns
-------
redditor_item: dict
Dictionary for a Redditor's submission
"""
items = [
"submission", item.title,
convert_time(item.created), item.score, item.upvote_ratio, item.id,
item.over_18, item.subreddit.display_name, item.selftext
]
return self._make_zip_dict(items, self._submission_titles)
def make_subreddit(self, subreddit):
"""
Make a Subreddit object.
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
subreddit: PRAW Subreddit object
Returns
-------
subreddit_object: dict
Dictionary containing Subreddit metadata
"""
return {
"can_assign_link_flair": subreddit.can_assign_link_flair,
"can_assign_user_flair": subreddit.can_assign_user_flair,
"created_utc": convert_time(subreddit.created_utc),
"description": subreddit.description,
"description_html": subreddit.description_html,
"display_name": subreddit.display_name,
"id": subreddit.id,
"name": subreddit.name,
"nsfw": subreddit.over18,
"public_description": subreddit.public_description,
"spoilers_enabled": subreddit.spoilers_enabled,
"subscribers": subreddit.subscribers,
"user_is_banned": subreddit.user_is_banned,
"user_is_moderator": subreddit.user_is_moderator,
"user_is_subscriber": subreddit.user_is_subscriber
}
def wrapper(reddit):
user_limits = function(reddit)
logging.info("RATE LIMIT DISPLAYED.")
logging.info("Remaining requests: %s" %
int(user_limits["remaining"]))
logging.info("Used requests: %s" % user_limits["used"])
logging.info("")
if int(user_limits["remaining"]) == 0:
Errors.l_title(convert_time(user_limits["reset_timestamp"]))
logging.critical(
"RATE LIMIT REACHED. RATE LIMIT WILL RESET AT %s." %
convert_time(user_limits["reset_timestamp"]))
logging.critical("ABORTING URS.\n")
quit()
return user_limits
def _make_submission(submission):
"""
Format submission metadata.
Parameters
----------
submission: PRAW submission object
Returns
-------
submission_dict: dict
Dictionary containing submission metadata
"""
return {
"author": "u/" + submission.author.name \
if hasattr(submission.author, "name") \
else "[deleted]",
"created_utc": convert_time(submission.created_utc),
"distinguished": submission.distinguished,
"edited": submission.edited \
if submission.edited == False \
else convert_time(submission.edited),
"id": submission.id,
"is_original_content": submission.is_original_content,
"is_self": submission.is_self,
"link_flair_text": submission.link_flair_text,
"locked": submission.locked,
"name": submission.name,
"num_comments": submission.num_comments,
"nsfw": submission.over_18,
"permalink": submission.permalink,
"score": submission.score,
"selftext": submission.selftext,
"spoiler": submission.spoiler,
"stickied": submission.stickied,
"title": submission.title,
"upvote_ratio": submission.upvote_ratio,
"url": submission.url
}
def make_comment(self, comment):
"""
Make a comment item.
Calls previously defined public method:
self.make_submission()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
comment: PRAW comment object
Returns
-------
redditor_item: dict
Dictionary containing comment metadata
"""
return {
"type": "comment",
"body": comment.body,
"body_html": comment.body_html,
"created_utc": convert_time(comment.created_utc),
"distinguished": comment.distinguished,
"edited": comment.edited \
if comment.edited == False \
else convert_time(comment.edited),
"id": comment.id,
"is_submitter": comment.is_submitter,
"link_id": comment.link_id,
"parent_id": comment.parent_id,
"score": comment.score,
"stickied": comment.stickied,
"submission": self.make_submission(comment.submission),
"subreddit_id": comment.subreddit_id
}
def create(comment):
"""
Create the comment object.
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
comment: PRAW comment object
Returns
-------
comment_object: dict
Dictionary containing comment metadata
"""
return {
"author": "u/" + comment.author.name \
if hasattr(comment.author, "name") \
else "[deleted]",
"body": comment.body,
"body_html": comment.body_html,
"created_utc": convert_time(comment.created_utc),
"distinguished": comment.distinguished,
"edited": comment.edited \
if comment.edited == False \
else convert_time(comment.edited),
"id": comment.id,
"is_submitter": comment.is_submitter,
"link_id": comment.link_id,
"parent_id": comment.parent_id,
"score": comment.score,
"stickied": comment.stickied
}
def _get_user_info(redditor, skeleton):
"""
Get Redditor account information.
Calls a previously defined public method:
GetInteractions._get_trophies()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
redditor: PRAW Redditor object
skeleton: dict
Dictionary containing all Redditor data
Returns
-------
None
"""
try:
skeleton["data"]["information"] = {
"is_suspended": redditor.is_suspended,
"name": redditor.name
}
except AttributeError:
skeleton["data"]["information"] = {
"comment_karma": redditor.comment_karma,
"created_utc": convert_time(redditor.created_utc),
"fullname": redditor.fullname,
"has_verified_email": redditor.has_verified_email,
"icon_img": redditor.icon_img,
"id": redditor.id,
"is_employee": redditor.is_employee,
"is_friend": redditor.is_friend,
"is_mod": redditor.is_mod,
"is_gold": redditor.is_gold,
"link_karma": redditor.link_karma,
"name": redditor.name,
"subreddit": redditor.subreddit,
"trophies": GetInteractions._get_trophies(redditor)
}
def make_multireddit(self, multireddit):
"""
Make a multireddit item.
Calls a previously defined public method:
self.make_subreddit()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
multireddit: PRAW multireddit object
Returns
-------
multireddit_object: dict
Dictionary containing multireddit metadata
"""
multireddit_object = {
"can_edit": multireddit.can_edit,
"copied_from": multireddit.copied_from,
"created_utc": convert_time(multireddit.created_utc),
"description_html": multireddit.description_html,
"description_md": multireddit.description_md,
"display_name": multireddit.display_name,
"name": multireddit.name,
"nsfw": multireddit.over_18,
"subreddits": [],
"visibility": multireddit.visibility
}
if multireddit.subreddits:
for subreddit in multireddit.subreddits:
subreddit = self.make_subreddit(subreddit)
multireddit_object["subreddits"].append(subreddit)
return multireddit_object
def _get_data(self, post):
"""
Get post data.
Calls previously defined private method:
self._fix_edit_date()
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
post: PRAW submission object
Returns
-------
post_data: list
List containing post metadata
"""
edited = self._fix_edit_date(post)
post_data = [
post.title,
post.link_flair_text,
convert_time(post.created),
post.score,
post.upvote_ratio,
post.id,
edited,
post.locked,
post.over_18,
post.spoiler,
post.stickied,
post.url,
post.num_comments,
post.selftext
]
return post_data
def get_rules(subreddit):
"""
Return post requirements and Subreddit rules.
Parameters
----------
subreddit: PRAW Subreddit object
Returns
-------
post_requirements: dict
Dictionary containing Subreddit's post requirements
rules: list[str]
List containing Subreddit rules
"""
rules = [rule_list for rule, rule_list in subreddit.rules().items() if rule == "rules"]
for rule in rules[0]:
rule["created_utc"] = convert_time(rule["created_utc"])
return subreddit.post_requirements(), rules[0]
def _fix_edit_date(self, post):
"""
Fix "Edited?" date.
Calls a public method from an external module:
Global.convert_time()
Parameters
----------
post: PRAW submission object
Returns
-------
fixed_date: boolean or str
Boolean or string:
Boolean indicating the post was not edited
String denoting an edited date
"""
return post.edited \
if str(post.edited).isalpha() \
else str(convert_time(post.edited))
def _make_json_skeleton(args, limit, submission, url):
"""
Create a skeleton for JSON export. Include scrape details at the top.
Parameters
----------
args: Namespace
Namespace object containing all arguments that were defined in the CLI
limit: str
Integer of string type denoting n_results or RAW format
submission: PRAW submission object
url: str
String denoting submission URL
Returns
-------
skeleton: dict
Dictionary containing scrape settings and all scrape data
"""
metadata_status = Status(
"Extracted submission metadata.",
"Extracting submission metadata.",
"white"
)
metadata_status.start()
skeleton = {
"scrape_settings": {
"n_results": int(limit) \
if int(limit) > 0 \
else "all",
"style": "structured" \
if not args.raw \
else "raw",
"url": url
},
"data": {
"submission_metadata": {
"author": "u/" + submission.author.name \
if hasattr(submission.author, "name") \
else "[deleted]",
"created_utc": convert_time(submission.created_utc),
"distinguished": submission.distinguished,
"edited": submission.edited \
if submission.edited == False \
else convert_time(submission.edited),
"is_original_content": submission.is_original_content,
"is_self": submission.is_self,
"link_flair_text": submission.link_flair_text,
"locked": submission.locked,
"num_comments": submission.num_comments,
"nsfw": submission.over_18,
"permalink": submission.permalink,
"score": submission.score,
"selftext": submission.selftext,
"spoiler": submission.spoiler,
"stickied": submission.stickied,
"subreddit": submission.subreddit.display_name,
"title": submission.title,
"upvote_ratio": submission.upvote_ratio
},
"comments": None
}
}
metadata_status.succeed()
return skeleton
