def _make_comment_item(self, item): """ Make comment item. Calls previously defined private method: self._make_zip_dict() Parameters ---------- item: PRAW object PRAW comment item Returns ------- redditor_item: dict Dictionary for a Redditor's comment """ edit_date = item.edited \ if str(item.edited).isalpha() \ else convert_time(item.edited) items = [ "comment", convert_time(item.created_utc), item.score, item.body, item.parent_id, item.link_id, edit_date, item.stickied, item.submission.selftext, item.submission.subreddit.display_name ] return self._make_zip_dict(items, self._comment_titles)
def make_submission(self, include_all, submission): """ Make a submission object. Calls a previously defined public method: self.make_subreddit() Calls a public method from an external module: Global.convert_time() Parameters ---------- include_all: boolean Boolean to determine whether the "type" field should be included submission: PRAW submission object Returns ------- submission_object: dict Dictionary containing submission metadata """ submission_object = { "author": "u/" + submission.author.name \ if hasattr(submission.author, "name") \ else "[deleted]", "created_utc": convert_time(submission.created_utc), "distinguished": submission.distinguished, "edited": submission.edited \ if submission.edited == False \ else convert_time(submission.edited), "id": submission.id, "is_original_content": submission.is_original_content, "is_self": submission.is_self, "link_flair_text": submission.link_flair_text, "locked": submission.locked, "name": submission.name, "nsfw": submission.over_18, "num_comments": submission.num_comments, "permalink": submission.permalink, "score": submission.score, "selftext": submission.selftext, "spoiler": submission.spoiler, "stickied": submission.stickied, "title": submission.title, "upvote_ratio": submission.upvote_ratio, "url": submission.url } if include_all: submission_object["subreddit"] = self.make_subreddit(submission.subreddit) submission_object["type"] = "submission" submission_object = dict(sorted(submission_object.items())) return submission_object
def make_comment(self, comment, include_all): """ Make a comment item. Calls previously defined public method: self.make_submission() Calls a public method from an external module: Global.convert_time() Parameters ---------- comment: PRAW comment object include_all: boolean Boolean to determine whether the "type" field should be included Returns ------- redditor_item: dict Dictionary containing comment metadata """ comment_object = { "author": "u/" + comment.author.name \ if hasattr(comment.author, "name") \ else "[deleted]", "body": comment.body, "body_html": comment.body_html, "created_utc": convert_time(comment.created_utc), "distinguished": comment.distinguished, "edited": comment.edited \ if comment.edited == False \ else convert_time(comment.edited), "id": comment.id, "is_submitter": comment.is_submitter, "link_id": comment.link_id, "parent_id": comment.parent_id, "score": comment.score, "stickied": comment.stickied, } if include_all: comment_object["submission"] = self.make_submission(include_all, comment.submission) comment_object["subreddit_id"] = comment.subreddit_id comment_object["type"] = "comment" comment_object = dict(sorted(comment_object.items())) return comment_object
def _fix_attributes(self, comment): """ If applicable, handle deleted Redditors or edited time. Calls a method from an external module: Global.convert_time() Parameters ---------- comment: PRAW object Returns ------- author_name: str edit_date: str """ try: author_name = comment.author.name except AttributeError: author_name = "[deleted]" edit_date = comment.edited \ if str(comment.edited).isalpha() \ else convert_time(comment.edited) return author_name, edit_date
def add_comment(self, comment): """ Add list of dictionary of comments attributes to use when sorting. Calls methods from an external module: Global.make_none_dict() Global.convert_time() Parameters ---------- comment: PRAW object Returns ------- comment_object: dict Dictionary for comment attribute """ comment_object = make_none_dict(self._titles) author_name, edit_date = self._fix_attributes(comment) comment_attributes = [ comment.parent_id, comment.id, author_name, convert_time(comment.created_utc), comment.score, comment.body, edit_date, comment.is_submitter, comment.stickied ] for title, attribute in zip(self._titles, comment_attributes): comment_object[title] = attribute return comment_object
def get_rules(subreddit): """ Return post requirements and Subreddit rules. Parameters ---------- subreddit: PRAW Subreddit object Returns ------- post_requirements: dict Dictionary containing Subreddit's post requirements rules: list[str] List containing Subreddit rules """ rules = [{ "created_utc": convert_time(rule.created_utc), "description": rule.description, "kind": rule.kind, "priority": rule.priority, "short_name": rule.short_name, "violation_reason": rule.violation_reason, } for rule in subreddit.rules] return subreddit.post_requirements(), rules
def _get_user_info(self, skeleton, user): """ Get Redditor account information. Calls a previously defined private method: self._info_titles() Calls a public method from an external module: Global.convert_time() Parameters ---------- skeleton: dict Dictionary containing all Redditor data user: PRAW Redditor object Returns ------- None """ user_info_titles = self._info_titles user_info = [ user.name, user.fullname, user.id, convert_time(user.created_utc), user.comment_karma, user.link_karma, user.is_employee, user.is_friend, user.is_mod, user.is_gold ] for info_title, user_item in zip(user_info_titles, user_info): skeleton["data"]["information"][info_title] = user_item
def _make_submission_item(self, item): """ Make submission item. Calls previously defined private method: self._make_zip_dict() Parameters ---------- item: PRAW object PRAW submission item Returns ------- redditor_item: dict Dictionary for a Redditor's submission """ items = [ "submission", item.title, convert_time(item.created), item.score, item.upvote_ratio, item.id, item.over_18, item.subreddit.display_name, item.selftext ] return self._make_zip_dict(items, self._submission_titles)
def make_subreddit(self, subreddit): """ Make a Subreddit object. Calls a public method from an external module: Global.convert_time() Parameters ---------- subreddit: PRAW Subreddit object Returns ------- subreddit_object: dict Dictionary containing Subreddit metadata """ return { "can_assign_link_flair": subreddit.can_assign_link_flair, "can_assign_user_flair": subreddit.can_assign_user_flair, "created_utc": convert_time(subreddit.created_utc), "description": subreddit.description, "description_html": subreddit.description_html, "display_name": subreddit.display_name, "id": subreddit.id, "name": subreddit.name, "nsfw": subreddit.over18, "public_description": subreddit.public_description, "spoilers_enabled": subreddit.spoilers_enabled, "subscribers": subreddit.subscribers, "user_is_banned": subreddit.user_is_banned, "user_is_moderator": subreddit.user_is_moderator, "user_is_subscriber": subreddit.user_is_subscriber }
def wrapper(reddit): user_limits = function(reddit) logging.info("RATE LIMIT DISPLAYED.") logging.info("Remaining requests: %s" % int(user_limits["remaining"])) logging.info("Used requests: %s" % user_limits["used"]) logging.info("") if int(user_limits["remaining"]) == 0: Errors.l_title(convert_time(user_limits["reset_timestamp"])) logging.critical( "RATE LIMIT REACHED. RATE LIMIT WILL RESET AT %s." % convert_time(user_limits["reset_timestamp"])) logging.critical("ABORTING URS.\n") quit() return user_limits
def _make_submission(submission): """ Format submission metadata. Parameters ---------- submission: PRAW submission object Returns ------- submission_dict: dict Dictionary containing submission metadata """ return { "author": "u/" + submission.author.name \ if hasattr(submission.author, "name") \ else "[deleted]", "created_utc": convert_time(submission.created_utc), "distinguished": submission.distinguished, "edited": submission.edited \ if submission.edited == False \ else convert_time(submission.edited), "id": submission.id, "is_original_content": submission.is_original_content, "is_self": submission.is_self, "link_flair_text": submission.link_flair_text, "locked": submission.locked, "name": submission.name, "num_comments": submission.num_comments, "nsfw": submission.over_18, "permalink": submission.permalink, "score": submission.score, "selftext": submission.selftext, "spoiler": submission.spoiler, "stickied": submission.stickied, "title": submission.title, "upvote_ratio": submission.upvote_ratio, "url": submission.url }
def make_comment(self, comment): """ Make a comment item. Calls previously defined public method: self.make_submission() Calls a public method from an external module: Global.convert_time() Parameters ---------- comment: PRAW comment object Returns ------- redditor_item: dict Dictionary containing comment metadata """ return { "type": "comment", "body": comment.body, "body_html": comment.body_html, "created_utc": convert_time(comment.created_utc), "distinguished": comment.distinguished, "edited": comment.edited \ if comment.edited == False \ else convert_time(comment.edited), "id": comment.id, "is_submitter": comment.is_submitter, "link_id": comment.link_id, "parent_id": comment.parent_id, "score": comment.score, "stickied": comment.stickied, "submission": self.make_submission(comment.submission), "subreddit_id": comment.subreddit_id }
def create(comment): """ Create the comment object. Calls a public method from an external module: Global.convert_time() Parameters ---------- comment: PRAW comment object Returns ------- comment_object: dict Dictionary containing comment metadata """ return { "author": "u/" + comment.author.name \ if hasattr(comment.author, "name") \ else "[deleted]", "body": comment.body, "body_html": comment.body_html, "created_utc": convert_time(comment.created_utc), "distinguished": comment.distinguished, "edited": comment.edited \ if comment.edited == False \ else convert_time(comment.edited), "id": comment.id, "is_submitter": comment.is_submitter, "link_id": comment.link_id, "parent_id": comment.parent_id, "score": comment.score, "stickied": comment.stickied }
def _get_user_info(redditor, skeleton): """ Get Redditor account information. Calls a previously defined public method: GetInteractions._get_trophies() Calls a public method from an external module: Global.convert_time() Parameters ---------- redditor: PRAW Redditor object skeleton: dict Dictionary containing all Redditor data Returns ------- None """ try: skeleton["data"]["information"] = { "is_suspended": redditor.is_suspended, "name": redditor.name } except AttributeError: skeleton["data"]["information"] = { "comment_karma": redditor.comment_karma, "created_utc": convert_time(redditor.created_utc), "fullname": redditor.fullname, "has_verified_email": redditor.has_verified_email, "icon_img": redditor.icon_img, "id": redditor.id, "is_employee": redditor.is_employee, "is_friend": redditor.is_friend, "is_mod": redditor.is_mod, "is_gold": redditor.is_gold, "link_karma": redditor.link_karma, "name": redditor.name, "subreddit": redditor.subreddit, "trophies": GetInteractions._get_trophies(redditor) }
def make_multireddit(self, multireddit): """ Make a multireddit item. Calls a previously defined public method: self.make_subreddit() Calls a public method from an external module: Global.convert_time() Parameters ---------- multireddit: PRAW multireddit object Returns ------- multireddit_object: dict Dictionary containing multireddit metadata """ multireddit_object = { "can_edit": multireddit.can_edit, "copied_from": multireddit.copied_from, "created_utc": convert_time(multireddit.created_utc), "description_html": multireddit.description_html, "description_md": multireddit.description_md, "display_name": multireddit.display_name, "name": multireddit.name, "nsfw": multireddit.over_18, "subreddits": [], "visibility": multireddit.visibility } if multireddit.subreddits: for subreddit in multireddit.subreddits: subreddit = self.make_subreddit(subreddit) multireddit_object["subreddits"].append(subreddit) return multireddit_object
def _get_data(self, post): """ Get post data. Calls previously defined private method: self._fix_edit_date() Calls a public method from an external module: Global.convert_time() Parameters ---------- post: PRAW submission object Returns ------- post_data: list List containing post metadata """ edited = self._fix_edit_date(post) post_data = [ post.title, post.link_flair_text, convert_time(post.created), post.score, post.upvote_ratio, post.id, edited, post.locked, post.over_18, post.spoiler, post.stickied, post.url, post.num_comments, post.selftext ] return post_data
def get_rules(subreddit): """ Return post requirements and Subreddit rules. Parameters ---------- subreddit: PRAW Subreddit object Returns ------- post_requirements: dict Dictionary containing Subreddit's post requirements rules: list[str] List containing Subreddit rules """ rules = [rule_list for rule, rule_list in subreddit.rules().items() if rule == "rules"] for rule in rules[0]: rule["created_utc"] = convert_time(rule["created_utc"]) return subreddit.post_requirements(), rules[0]
def _fix_edit_date(self, post): """ Fix "Edited?" date. Calls a public method from an external module: Global.convert_time() Parameters ---------- post: PRAW submission object Returns ------- fixed_date: boolean or str Boolean or string: Boolean indicating the post was not edited String denoting an edited date """ return post.edited \ if str(post.edited).isalpha() \ else str(convert_time(post.edited))
def _make_json_skeleton(args, limit, submission, url): """ Create a skeleton for JSON export. Include scrape details at the top. Parameters ---------- args: Namespace Namespace object containing all arguments that were defined in the CLI limit: str Integer of string type denoting n_results or RAW format submission: PRAW submission object url: str String denoting submission URL Returns ------- skeleton: dict Dictionary containing scrape settings and all scrape data """ metadata_status = Status( "Extracted submission metadata.", "Extracting submission metadata.", "white" ) metadata_status.start() skeleton = { "scrape_settings": { "n_results": int(limit) \ if int(limit) > 0 \ else "all", "style": "structured" \ if not args.raw \ else "raw", "url": url }, "data": { "submission_metadata": { "author": "u/" + submission.author.name \ if hasattr(submission.author, "name") \ else "[deleted]", "created_utc": convert_time(submission.created_utc), "distinguished": submission.distinguished, "edited": submission.edited \ if submission.edited == False \ else convert_time(submission.edited), "is_original_content": submission.is_original_content, "is_self": submission.is_self, "link_flair_text": submission.link_flair_text, "locked": submission.locked, "num_comments": submission.num_comments, "nsfw": submission.over_18, "permalink": submission.permalink, "score": submission.score, "selftext": submission.selftext, "spoiler": submission.spoiler, "stickied": submission.stickied, "subreddit": submission.subreddit.display_name, "title": submission.title, "upvote_ratio": submission.upvote_ratio }, "comments": None } } metadata_status.succeed() return skeleton