def is_bot_banned(sub_name: Text, reddit: Reddit) -> Optional[bool]: """ Check if bot is banned on a given sub :rtype: bool :param subreddit: Sub to check :return: bool """ subreddit = reddit.subreddit(sub_name) if not subreddit: log.error('Failed to locate subreddit %s', sub_name) return None banned = False try: sub = subreddit.submit('ban test', selftext='ban test') sub.delete() except Forbidden: banned = True except APIException as e: if e.error_type == 'SUBREDDIT_NOTALLOWED': banned = True if banned: log.info('Bot is banned from %s', subreddit.display_name) else: log.info('Bot is allowed on %s', subreddit.display_name) return banned
def bot_has_permission(sub_name: Text, permission_name: Text, reddit: Reddit) -> Optional[bool]: log.debug('Checking if bot has %s permission in %s', permission_name, sub_name) subreddit = reddit.subreddit(sub_name) if not subreddit: log.error('Failed to locate subreddit %s', sub_name) return None try: for mod in subreddit.moderator(): if mod.name == 'RepostSleuthBot': if 'all' in mod.mod_permissions: log.debug('Bot has All permissions in %s', subreddit.display_name) return True elif permission_name.lower() in mod.mod_permissions: log.debug('Bot has %s permission in %s', permission_name, subreddit.display_name) return True else: log.debug('Bot does not have %s permission in %s', permission_name, subreddit.display_name) return False log.error('Bot is not mod on %s', subreddit.display_name) return None except (Forbidden, NotFound): return None
def download_file(url: str, output_dir: str) -> Tuple[str, str]: """ Take a URL to a video, download the file and return the path to the audio :param url: URL of video """ ops = { 'postprocessors': [{ 'key': 'FFmpegExtractAudio' }], 'outtmpl': os.path.join(output_dir, '%(id)s.%(ext)s'), 'keepvideo': True } ydl = YoutubeDL(ops) try: ydl.download([url]) except Exception as e: log.error('Failed to download %s', url) shutil.rmtree(output_dir) raise audio_file = None video_file = None video_exts = ['.mp4'] audio_exts = ['.m4a', '.mp3'] for f in os.listdir(output_dir): if os.path.splitext(f)[1] in video_exts: video_file = os.path.join(output_dir, f) elif os.path.splitext(f)[1] in audio_exts: audio_file = os.path.join(output_dir, f) return audio_file, video_file
def _send_private_message(self, user: Redditor, message_body, subject: Text = 'Repost Check', source: Text = None, post_id: Text = None, comment_id: Text = None) -> NoReturn: if not user: log.error('No user provided to send private message') return try: start_time = perf_counter() user.message(subject, message_body) self._record_api_event( float(round(perf_counter() - start_time, 2)), 'private_message', self.reddit.reddit.auth.limits['remaining']) log.info('Sent PM to %s. ', user.name) except Exception as e: log.exception('Failed to send PM to %s', user.name, exc_info=True) raise self._save_private_message( BotPrivateMessage(subject=subject, body=message_body, in_response_to_post=post_id, in_response_to_comment=comment_id, triggered_from=source, recipient=user.name))
def monitor_for_summons(self, subreddits: str = 'all'): """ Monitors the subreddits set in the config for comments containing the summoning string """ log.info('Starting praw summons monitor for subs %s', subreddits) while True: try: for comment in self.reddit.subreddit( subreddits).stream.comments(): if comment is None: continue if self.check_for_summons(comment.body, '\?repost'): if comment.author.name.lower() in [ 'sneakpeekbot', 'automoderator' ]: continue self._save_summons(comment) except ResponseException as e: if e.response.status_code == 429: log.error('IP Rate limit hit. Waiting') time.sleep(60) continue except Exception as e: if 'code: 429' in str(e): log.error('Too many requests from IP. Waiting') time.sleep(60) continue log.exception('Praw summons thread died', exc_info=True)
def on_get_search_by_url(self, req: Request, resp: Response): image_match_percent = req.get_param_as_int('image_match_percent', required=False, default=None) target_meme_match_percent = req.get_param_as_int('target_meme_match_percent', required=False, default=None) same_sub = req.get_param_as_bool('same_sub', required=False, default=False) only_older = req.get_param_as_bool('only_older', required=False, default=False) meme_filter = req.get_param_as_bool('meme_filter', required=False, default=False) filter_crossposts = req.get_param_as_bool('filter_crossposts', required=False, default=True) filter_author = req.get_param_as_bool('filter_author', required=False, default=True) url = req.get_param('url', required=True) filter_dead_matches = req.get_param_as_bool('filter_dead_matches', required=False, default=False) target_days_old = req.get_param_as_int('target_days_old', required=False, default=0) try: search_results = self.image_svc.check_image( url, target_match_percent=image_match_percent, target_meme_match_percent=target_meme_match_percent, meme_filter=meme_filter, same_sub=same_sub, date_cutoff=target_days_old, only_older_matches=only_older, filter_crossposts=filter_crossposts, filter_dead_matches=filter_dead_matches, filter_author=filter_author, max_matches=500, max_depth=-1, source='api' ) except NoIndexException: log.error('No available index for image repost check. Trying again later') raise HTTPServiceUnavailable('Search API is not available.', 'The search API is not currently available') print(search_results.search_times.to_dict()) resp.body = json.dumps(search_results, cls=ImageRepostWrapperEncoder)
def on_post(self, req: Request, resp: Response, subreddit: Text): log.info('Attempting to create monitored sub %s', subreddit) try: self.reddit.subreddit(subreddit).mod.accept_invite() except APIException as e: if e.error_type == 'NO_INVITE_FOUND': log.error('No open invite to %s', subreddit) raise HTTPInternalServerError( f'No available invite for {subreddit}', f'We were unable to find a ' f'pending mod invote for r/{subreddit}') else: log.exception('Problem accepting invite', exc_info=True) raise HTTPInternalServerError( f'Unknown error accepting mod invite for r/{subreddit}', f'Unknown error accepting mod invite for r/{subreddit}. Please contact us' ) except Exception as e: log.exception('Failed to accept invite', exc_info=True) raise HTTPInternalServerError( f'Unknown error accepting mod invite for r/{subreddit}', f'Unknown error accepting mod invite for r/{subreddit}. Please contact us' ) with self.uowm.start() as uow: existing = uow.monitored_sub.get_by_sub(subreddit) if existing: resp.body = json.dumps(existing.to_dict()) return monitored_sub = create_monitored_sub_in_db(subreddit, uow) resp.body = json.dumps(monitored_sub.to_dict())
def process_image_post( post: Post, hash_api) -> Tuple[Post, RedditImagePost, RedditImagePostCurrent]: if 'imgur' not in post.url: try: # Make sure URL is still valid r = requests.head(post.url) except ConnectionError as e: log.error('Post %s: Failed to verify image URL at %s', post.post_id, post.url) raise if r.status_code != 200: if r.status_code == 404: log.error('Post %s: Image no longer exists %s: %s', post.post_id, r.status_code, post.url) raise ImageRemovedException( f'Post {post.post_id} has been deleted') else: log.debug('Bad status code from image URL %s', r.status_code) raise InvalidImageUrlException( f'Issue getting image url: {post.url} - Status Code {r.status_code}' ) log.info('%s - Post %s: Hashing with URL: %s', os.getpid(), post.post_id, post.url) if hash_api: log.debug('Post %s: Using hash API: %s', post.post_id, hash_api) set_image_hashes_api(post, hash_api) else: log.debug('Post %s: Using local hashing', post.post_id) set_image_hashes(post) return create_image_posts(post)
def _process_watch_request(self, msg: Message) -> NoReturn: """ Process someone that wants to active a watch from top posts :param msg: message """ if not msg.replies: return if 'yes' in msg.replies[0].body.lower(): post_id_search = re.search(r'(?:https://redd.it/)([A-Za-z0-9]{6})', msg.body) if not post_id_search: log.error('Failed to get post ID from watch offer message') return post_id = post_id_search.group(1) with self.uowm.start() as uow: existing_watch = uow.repostwatch.find_existing_watch( msg.dest.name, post_id) if existing_watch: log.info('Existing watch found for post %s by user %s', post_id, msg.dest.name) return uow.repostwatch.add( RepostWatch(post_id=post_id, user=msg.dest.name, source='Top Post')) uow.commit() log.info('Created post watch on %s for %s. Source: Top Post', post_id, msg.author.name) self.response_handler.reply_to_private_message( msg, WATCH_ENABLED)
def _set_match_post(self, match: ImageSearchMatch, historical: bool = True) -> Optional[ImageSearchMatch]: """ Take a search match, lookup the Post object and attach to match :param match: Match object :param historical: Is the match from the historical index """ with self.uowm.start() as uow: # Hit the correct table if historical or current if historical: original_image_post = uow.image_post.get_by_id( match.index_match_id) else: original_image_post = uow.image_post_current.get_by_id( match.index_match_id) if not original_image_post: log.error( 'Failed to lookup original match post. ID %s - Historical: %s', match.index_match_id, historical) return match_post = uow.posts.get_by_post_id(original_image_post.post_id) if not match_post: log.error('Failed to find original reddit_post for match') return match.post = match_post return match
def check_meme_template_potential_votes(uowm: UnitOfWorkManager) -> NoReturn: with uowm.start() as uow: potential_templates = uow.meme_template_potential.get_all() for potential_template in potential_templates: if potential_template.vote_total >= 10: existing_template = uow.meme_template.get_by_post_id(potential_template.post_id) if existing_template: log.info('Meme template already exists for %s. Removing', potential_template.post_id) uow.meme_template_potential.remove(potential_template) uow.commit() return log.info('Post %s received %s votes. Creating meme template', potential_template.post_id, potential_template.vote_total) post = uow.posts.get_by_post_id(potential_template.post_id) try: meme_hashes = get_image_hashes(post.searched_url, hash_size=32) except Exception as e: log.error('Failed to get meme hash for %s', post.post_id) return meme_template = MemeTemplate( dhash_h=post.dhash_h, dhash_256=meme_hashes['dhash_h'], post_id=post.post_id ) uow.meme_template.add(meme_template) uow.meme_template_potential.remove(potential_template) elif potential_template.vote_total <= -10: log.info('Removing potential template with at least 10 negative votes') uow.meme_template_potential.remove(potential_template) else: continue uow.commit()
def save_unknown_post(self, post_id: Text) -> Optional[Post]: """ If we received a request on a post we haven't ingest save it :rtype: Optional[Post] :param post_id: Submission ID :return: Post object """ submission = self.reddit.submission(post_id) try: post = pre_process_post(submission_to_post(submission), self.uowm, None) except InvalidImageUrlException: return except Forbidden: log.error('Failed to download post %s, appears we are banned', post_id) return if not post or post.post_type != 'image': log.error( 'Problem ingesting post. Either failed to save or it is not an image' ) return return post
def _process_comment(self, bot_comment: BotComment): reddit_comment = self._get_comment_data(bot_comment.perma_link) if not reddit_comment: log.error('Failed to locate comment %s', bot_comment.comment_id) return bot_comment.karma = reddit_comment['ups'] if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold: log.info('Comment %s has karma of %s. Removing', bot_comment.comment_id, bot_comment.karma) if self.notification_svc: self.notification_svc.send_notification( f'Removing comment with {bot_comment.karma} karma. https://reddit.com{bot_comment.perma_link}', subject='Removing Downvoted Comment') comment = self.reddit.comment(bot_comment.comment_id) try: comment.delete() except Exception as e: log.exception('Failed to delete comment %s', bot_comment.comment_id, exc_info=True) bot_comment.needs_review = True bot_comment.active = False elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold: log.info('Comment %s has karma of %s. Flagging for review', bot_comment.comment_id, bot_comment.karma) bot_comment.needs_review = True
def build_sub_comment(self, monitored_sub: MonitoredSub, search_results: SearchResults, **kwargs) -> Text: """ Take a given MonitoredSub and attempt to build their customer message templates using the search results. If the final formatting of the template fails, it will revert to the default response template :rtype: Text :param monitored_sub: MonitoredSub to get template from :param search_results: Set of search results :param kwargs: Args to pass along to default comment builder :return: """ if len(search_results.matches) > 0: message = monitored_sub.repost_response_template else: message = monitored_sub.oc_response_template if not message: return self.build_default_comment(search_results, **kwargs) try: return self.build_default_comment(search_results, message, **kwargs) except KeyError: log.error('Custom repost template for %s has a bad slug: %s', monitored_sub.name, monitored_sub.repost_response_template) return self.build_default_comment(search_results, **kwargs)
def update_wiki_config_from_database(self, monitored_sub: MonitoredSub, wiki_page: WikiPage = None, notify: bool = False) -> bool: """ Sync the database settings to a given monitor sub's wiki page :param notify: Send notification when config is loaded :param wiki_page: Wiki Page object :param monitored_sub: Monitored Sub to update :rtype: bool :return: bool if config was successfully loaded """ subreddit = self.reddit.subreddit(monitored_sub.name) if not wiki_page: wiki_page = subreddit.wiki[self.config.wiki_config_name] new_config = self._create_wiki_config_from_database(monitored_sub) if not new_config: log.error('Failed to generate new config for %s', monitored_sub.name) return False self._update_wiki_page(wiki_page, new_config) wiki_page = subreddit.wiki[ 'repost_sleuth_config'] # Force refresh so we can get latest revision ID self._create_revision(wiki_page) self._set_config_validity(wiki_page.revision_id, True) if notify: self._notify_successful_load(subreddit) return True
def generate_thumbnails_from_file(video_file: str, total_thumbs: int = 20) -> int: result = {'duration': None, 'thumbs': []} try: probe = ffmpeg.probe(video_file) except Exception as e: log.error('Failed to probe video: %s', video_file) raise duration = float(probe['format']['duration']) interval = float(probe['format']['duration']) / total_thumbs count = 1 seek = interval log.info('Video length is %s seconds. Grabbing thumb every %s seconds', duration, interval) output_dir = os.path.split(video_file)[0] while count <= total_thumbs: print(str(count)) try: (ffmpeg.input(video_file, ss=seek).filter('scale', 720, -1).output( os.path.join(output_dir, '{}.png'.format(str(count))), vframes=1).overwrite_output().run(capture_stdout=True, capture_stderr=True)) except ffmpeg.Error as e: print(e.stderr.decode(), file=sys.stderr) seek += interval count += 1 return duration
def _final_meme_filter(self, searched_hash: Text, matches: List[ImageSearchMatch], target_hamming) -> List[ImageSearchMatch]: results = [] log.debug('MEME FILTER - Filtering %s matches', len(matches)) if len(matches) == 0: return matches for match in matches: try: match_hash = self._get_meme_hash(match.post.url) except Exception as e: log.error('Failed to get meme hash for %s', match.post.id) continue h_distance = hamming(searched_hash, match_hash) if h_distance > target_hamming: log.info( 'Meme Hamming Filter Reject - Target: %s Actual: %s - %s', target_hamming, h_distance, f'https://redd.it/{match.post.post_id}') continue log.debug('Match found: %s - H:%s', f'https://redd.it/{match.post.post_id}', h_distance) match.hamming_distance = h_distance match.hash_size = len(searched_hash) results.append(match) return results
def _offer_watch(self, submission: Submission) -> NoReturn: """ Offer to add watch to OC post :param search: """ if not self.config.top_post_offer_watch: log.debug('Top Post Offer Watch Disabled') return log.info('Offer watch to %s on post %s', submission.author.name, submission.id) with self.uowm.start() as uow: existing_response = uow.bot_private_message.get_by_user_source_and_post( submission.author.name, 'toppost', submission.id) if existing_response: log.info('Already sent a message to %s', submission.author.name) return try: self.response_handler.send_private_message( submission.author, TOP_POST_WATCH_BODY.format( shortlink=f'https://redd.it/{submission.id}'), subject=TOP_POST_WATCH_SUBJECT, source='toppost', post_id=submission.id) except APIException as e: if e.error_type == 'NOT_WHITELISTED_BY_USER_MESSAGE': log.error('Not whitelisted API error') else: log.exception('Unknown error sending PM to %s', submission.author.name, exc_info=True)
def check_for_repost(self, post: Post) -> Optional[SearchResults]: """ Take a given post and check if it's a repost :rtype: SearchResults :param post: Post obj :return: Search results """ if post.post_type == 'image': try: return self.image_service.check_image( post.url, post=post, ) except NoIndexException: log.error( 'No available index for image repost check. Trying again later' ) return elif post.post_type == 'link': search_results = get_link_reposts(post.url, self.uowm, get_default_link_search_settings( self.config), post=post, get_total=True) return filter_search_results( search_results, reddit=self.reddit, uitl_api=f'{self.config.util_api}/maintenance/removed') else: log.info( f'Post {post.post_id} is a {post.post_type} post. Skipping') return
def _process_comment(self, bot_comment: BotComment): reddit_comment = self._get_comment_data(bot_comment.perma_link) if not reddit_comment: log.error('Failed to locate comment %s', bot_comment.comment_id) return bot_comment.karma = reddit_comment['ups'] if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold: log.info('Comment %s has karma of %s. Removing', bot_comment.comment_id, bot_comment.karma) comment = self.reddit.comment(bot_comment.comment_id) try: comment.delete() except Exception as e: log.exception('Failed to delete comment %s', bot_comment.comment_id, exc_info=True) bot_comment.needs_review = True bot_comment.active = False elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold: log.info('Comment %s has karma of %s. Flagging for review', bot_comment.comment_id, bot_comment.karma) bot_comment.needs_review = True
def _reply_to_comment(self, comment_id: Text, comment_body: Text, subreddit: Text = None) -> Optional[Comment]: """ Post a given reply to a given comment ID :rtype: Optional[Comment] :param comment_id: ID of comment to reply to :param comment_body: Body of the comment to leave in reply :return: """ comment = self.reddit.comment(comment_id) if not comment: log.error('Failed to find comment %s', comment_id) return try: start_time = perf_counter() reply_comment = comment.reply(comment_body) self._record_api_event( float(round(perf_counter() - start_time, 2)), 'reply_to_comment', self.reddit.reddit.auth.limits['remaining'] ) self._log_response(reply_comment) log.info('Left comment at: https://reddit.com%s', reply_comment.permalink) return reply_comment except Forbidden: log.exception('Forbidden to respond to comment %s', comment_id, exc_info=False) # If we get Forbidden there's a chance we don't have hte comment data to get subreddit if subreddit: self._save_banned_sub(subreddit) raise except AssertionError: log.exception('Problem leaving comment', exc_info=True) raise
def _reply_to_submission(self, submission_id: str, comment_body) -> Optional[Comment]: submission = self.reddit.submission(submission_id) if not submission: log.error('Failed to get submission %s', submission_id) return try: start_time = perf_counter() comment = submission.reply(comment_body) self._record_api_event( float(round(perf_counter() - start_time, 2)), 'reply_to_submission', self.reddit.reddit.auth.limits['remaining'] ) log.info('Left comment at: https://reddit.com%s', comment.permalink) log.debug(comment_body) self._log_response(comment) return comment except APIException as e: if e.error_type == 'RATELIMIT': log.exception('Reddit rate limit') raise RateLimitException('Hit rate limit') else: log.exception('Unknown error type of APIException', exc_info=True) raise except Forbidden: self._save_banned_sub(submission.subreddit.display_name) except Exception: log.exception('Unknown exception leaving comment on post https://redd.it/%s', submission_id, exc_info=True) raise
def send_mod_mail(self, subreddit_name: Text, subject: Text, body: Text, triggered_from: Text = None) -> NoReturn: """ Send a modmail message :rtype: NoReturn :param subreddit_name: name of subreddit :param subject: Message Subject :param body: Message Body """ subreddit = self.reddit.subreddit(subreddit_name) if not subreddit: log.error('Failed to get Subreddit %s when attempting to send modmail') return try: subreddit.message(subject, body) self._save_private_message( BotPrivateMessage( subject=subject, body=body, triggered_from=triggered_from, recipient=subreddit_name ) ) except RedditAPIException: log.exception('Problem sending modmail message', exc_info=True)
def parse_root_command(self, command: str): if not command: log.error('Got empty command. Returning repost') return 'repost' parser = ArgumentParserThrow() parser.add_argument('command', default=None, choices=['repost', 'watch', 'unwatch']) options, args = parser.parse_known_args(command.split(' ')) return options.command
def get_subscribers(sub_name: Text, reddit: Reddit) -> Optional[int]: subreddit = reddit.subreddit(sub_name) try: return subreddit.subscribers except Forbidden: log.error('Failed to get subscribers, Forbidden %s', sub_name) return except NotFound: log.error('Failed to get subscribers, not found %s', sub_name) return
def run_update(self): print('[Scheduled Job] Stats Update Starting') self.get_all_stats() output = self.build_template() wiki = self.reddit.subreddit('RepostSleuthBot').wiki['stats'] try: wiki.edit(output) except BadRequest: log.error('Failed to update wiki page') print('[Scheduled Job] Stats Update Ending')
def _sticky_reply(self, monitored_sub: MonitoredSub, comment: Comment) -> NoReturn: if monitored_sub.sticky_comment: try: comment.mod.distinguish(sticky=True) log.info('Made comment %s sticky', comment.id) except Forbidden: log.error('Failed to sticky comment, no permissions') except Exception as e: log.exception('Failed to sticky comment', exc_info=True)
def target_hash(self) -> Text: """ Returns the hash to be searched. This allows us to work with just a URL or a full post :return: hash """ if self._target_hash: return self._target_hash log.error('No target hash set, attempting to get') hashes = get_image_hashes(self.checked_url, hash_size=16) self._target_hash = hashes['dhash_h'] return self._target_hash
def _send_to_hook(self, payload: Dict): try: r = requests.post(self.hook, headers={'Content-Type': 'application/json'}, json=payload) except (ConnectionError, Timeout): log.error('Failed to send discord notification') return if r.status_code != 204: log.error('Unexpected status code %s from Discord webhook: %s', r.status_code, r.text)
def _lock_comment(self, monitored_sub: MonitoredSub, comment: Comment) -> NoReturn: if monitored_sub.lock_response_comment: log.info('Attempting to lock comment %s on subreddit %s', comment.id, monitored_sub.name) try: comment.mod.lock() log.info('Locked comment') except Forbidden: log.error('Failed to lock comment, no permission') except Exception as e: log.exception('Failed to lock comment', exc_info=True)