def is_bot_banned(sub_name: Text, reddit: Reddit) -> Optional[bool]:
"""
Check if bot is banned on a given sub
:rtype: bool
:param subreddit: Sub to check
:return: bool
"""
subreddit = reddit.subreddit(sub_name)
if not subreddit:
log.error('Failed to locate subreddit %s', sub_name)
return None
banned = False
try:
sub = subreddit.submit('ban test', selftext='ban test')
sub.delete()
except Forbidden:
banned = True
except APIException as e:
if e.error_type == 'SUBREDDIT_NOTALLOWED':
banned = True
if banned:
log.info('Bot is banned from %s', subreddit.display_name)
else:
log.info('Bot is allowed on %s', subreddit.display_name)
return banned
def bot_has_permission(sub_name: Text, permission_name: Text,
reddit: Reddit) -> Optional[bool]:
log.debug('Checking if bot has %s permission in %s', permission_name,
sub_name)
subreddit = reddit.subreddit(sub_name)
if not subreddit:
log.error('Failed to locate subreddit %s', sub_name)
return None
try:
for mod in subreddit.moderator():
if mod.name == 'RepostSleuthBot':
if 'all' in mod.mod_permissions:
log.debug('Bot has All permissions in %s',
subreddit.display_name)
return True
elif permission_name.lower() in mod.mod_permissions:
log.debug('Bot has %s permission in %s', permission_name,
subreddit.display_name)
return True
else:
log.debug('Bot does not have %s permission in %s',
permission_name, subreddit.display_name)
return False
log.error('Bot is not mod on %s', subreddit.display_name)
return None
except (Forbidden, NotFound):
return None
def download_file(url: str, output_dir: str) -> Tuple[str, str]:
"""
Take a URL to a video, download the file and return the path to the audio
:param url: URL of video
"""
ops = {
'postprocessors': [{
'key': 'FFmpegExtractAudio'
}],
'outtmpl': os.path.join(output_dir, '%(id)s.%(ext)s'),
'keepvideo': True
}
ydl = YoutubeDL(ops)
try:
ydl.download([url])
except Exception as e:
log.error('Failed to download %s', url)
shutil.rmtree(output_dir)
raise
audio_file = None
video_file = None
video_exts = ['.mp4']
audio_exts = ['.m4a', '.mp3']
for f in os.listdir(output_dir):
if os.path.splitext(f)[1] in video_exts:
video_file = os.path.join(output_dir, f)
elif os.path.splitext(f)[1] in audio_exts:
audio_file = os.path.join(output_dir, f)
return audio_file, video_file
def _send_private_message(self,
user: Redditor,
message_body,
subject: Text = 'Repost Check',
source: Text = None,
post_id: Text = None,
comment_id: Text = None) -> NoReturn:
if not user:
log.error('No user provided to send private message')
return
try:
start_time = perf_counter()
user.message(subject, message_body)
self._record_api_event(
float(round(perf_counter() - start_time, 2)),
'private_message', self.reddit.reddit.auth.limits['remaining'])
log.info('Sent PM to %s. ', user.name)
except Exception as e:
log.exception('Failed to send PM to %s', user.name, exc_info=True)
raise
self._save_private_message(
BotPrivateMessage(subject=subject,
body=message_body,
in_response_to_post=post_id,
in_response_to_comment=comment_id,
triggered_from=source,
recipient=user.name))
def monitor_for_summons(self, subreddits: str = 'all'):
"""
Monitors the subreddits set in the config for comments containing the summoning string
"""
log.info('Starting praw summons monitor for subs %s', subreddits)
while True:
try:
for comment in self.reddit.subreddit(
subreddits).stream.comments():
if comment is None:
continue
if self.check_for_summons(comment.body, '\?repost'):
if comment.author.name.lower() in [
'sneakpeekbot', 'automoderator'
]:
continue
self._save_summons(comment)
except ResponseException as e:
if e.response.status_code == 429:
log.error('IP Rate limit hit. Waiting')
time.sleep(60)
continue
except Exception as e:
if 'code: 429' in str(e):
log.error('Too many requests from IP. Waiting')
time.sleep(60)
continue
log.exception('Praw summons thread died', exc_info=True)
def on_get_search_by_url(self, req: Request, resp: Response):
image_match_percent = req.get_param_as_int('image_match_percent', required=False, default=None)
target_meme_match_percent = req.get_param_as_int('target_meme_match_percent', required=False, default=None)
same_sub = req.get_param_as_bool('same_sub', required=False, default=False)
only_older = req.get_param_as_bool('only_older', required=False, default=False)
meme_filter = req.get_param_as_bool('meme_filter', required=False, default=False)
filter_crossposts = req.get_param_as_bool('filter_crossposts', required=False, default=True)
filter_author = req.get_param_as_bool('filter_author', required=False, default=True)
url = req.get_param('url', required=True)
filter_dead_matches = req.get_param_as_bool('filter_dead_matches', required=False, default=False)
target_days_old = req.get_param_as_int('target_days_old', required=False, default=0)
try:
search_results = self.image_svc.check_image(
url,
target_match_percent=image_match_percent,
target_meme_match_percent=target_meme_match_percent,
meme_filter=meme_filter,
same_sub=same_sub,
date_cutoff=target_days_old,
only_older_matches=only_older,
filter_crossposts=filter_crossposts,
filter_dead_matches=filter_dead_matches,
filter_author=filter_author,
max_matches=500,
max_depth=-1,
source='api'
)
except NoIndexException:
log.error('No available index for image repost check. Trying again later')
raise HTTPServiceUnavailable('Search API is not available.', 'The search API is not currently available')
print(search_results.search_times.to_dict())
resp.body = json.dumps(search_results, cls=ImageRepostWrapperEncoder)
def on_post(self, req: Request, resp: Response, subreddit: Text):
log.info('Attempting to create monitored sub %s', subreddit)
try:
self.reddit.subreddit(subreddit).mod.accept_invite()
except APIException as e:
if e.error_type == 'NO_INVITE_FOUND':
log.error('No open invite to %s', subreddit)
raise HTTPInternalServerError(
f'No available invite for {subreddit}',
f'We were unable to find a '
f'pending mod invote for r/{subreddit}')
else:
log.exception('Problem accepting invite', exc_info=True)
raise HTTPInternalServerError(
f'Unknown error accepting mod invite for r/{subreddit}',
f'Unknown error accepting mod invite for r/{subreddit}. Please contact us'
)
except Exception as e:
log.exception('Failed to accept invite', exc_info=True)
raise HTTPInternalServerError(
f'Unknown error accepting mod invite for r/{subreddit}',
f'Unknown error accepting mod invite for r/{subreddit}. Please contact us'
)
with self.uowm.start() as uow:
existing = uow.monitored_sub.get_by_sub(subreddit)
if existing:
resp.body = json.dumps(existing.to_dict())
return
monitored_sub = create_monitored_sub_in_db(subreddit, uow)
resp.body = json.dumps(monitored_sub.to_dict())
def process_image_post(
post: Post,
hash_api) -> Tuple[Post, RedditImagePost, RedditImagePostCurrent]:
if 'imgur' not in post.url:
try: # Make sure URL is still valid
r = requests.head(post.url)
except ConnectionError as e:
log.error('Post %s: Failed to verify image URL at %s',
post.post_id, post.url)
raise
if r.status_code != 200:
if r.status_code == 404:
log.error('Post %s: Image no longer exists %s: %s',
post.post_id, r.status_code, post.url)
raise ImageRemovedException(
f'Post {post.post_id} has been deleted')
else:
log.debug('Bad status code from image URL %s', r.status_code)
raise InvalidImageUrlException(
f'Issue getting image url: {post.url} - Status Code {r.status_code}'
)
log.info('%s - Post %s: Hashing with URL: %s', os.getpid(), post.post_id,
post.url)
if hash_api:
log.debug('Post %s: Using hash API: %s', post.post_id, hash_api)
set_image_hashes_api(post, hash_api)
else:
log.debug('Post %s: Using local hashing', post.post_id)
set_image_hashes(post)
return create_image_posts(post)
def _process_watch_request(self, msg: Message) -> NoReturn:
"""
Process someone that wants to active a watch from top posts
:param msg: message
"""
if not msg.replies:
return
if 'yes' in msg.replies[0].body.lower():
post_id_search = re.search(r'(?:https://redd.it/)([A-Za-z0-9]{6})',
msg.body)
if not post_id_search:
log.error('Failed to get post ID from watch offer message')
return
post_id = post_id_search.group(1)
with self.uowm.start() as uow:
existing_watch = uow.repostwatch.find_existing_watch(
msg.dest.name, post_id)
if existing_watch:
log.info('Existing watch found for post %s by user %s',
post_id, msg.dest.name)
return
uow.repostwatch.add(
RepostWatch(post_id=post_id,
user=msg.dest.name,
source='Top Post'))
uow.commit()
log.info('Created post watch on %s for %s. Source: Top Post',
post_id, msg.author.name)
self.response_handler.reply_to_private_message(
msg, WATCH_ENABLED)
def _set_match_post(self,
match: ImageSearchMatch,
historical: bool = True) -> Optional[ImageSearchMatch]:
"""
Take a search match, lookup the Post object and attach to match
:param match: Match object
:param historical: Is the match from the historical index
"""
with self.uowm.start() as uow:
# Hit the correct table if historical or current
if historical:
original_image_post = uow.image_post.get_by_id(
match.index_match_id)
else:
original_image_post = uow.image_post_current.get_by_id(
match.index_match_id)
if not original_image_post:
log.error(
'Failed to lookup original match post. ID %s - Historical: %s',
match.index_match_id, historical)
return
match_post = uow.posts.get_by_post_id(original_image_post.post_id)
if not match_post:
log.error('Failed to find original reddit_post for match')
return
match.post = match_post
return match
def check_meme_template_potential_votes(uowm: UnitOfWorkManager) -> NoReturn:
with uowm.start() as uow:
potential_templates = uow.meme_template_potential.get_all()
for potential_template in potential_templates:
if potential_template.vote_total >= 10:
existing_template = uow.meme_template.get_by_post_id(potential_template.post_id)
if existing_template:
log.info('Meme template already exists for %s. Removing', potential_template.post_id)
uow.meme_template_potential.remove(potential_template)
uow.commit()
return
log.info('Post %s received %s votes. Creating meme template', potential_template.post_id, potential_template.vote_total)
post = uow.posts.get_by_post_id(potential_template.post_id)
try:
meme_hashes = get_image_hashes(post.searched_url, hash_size=32)
except Exception as e:
log.error('Failed to get meme hash for %s', post.post_id)
return
meme_template = MemeTemplate(
dhash_h=post.dhash_h,
dhash_256=meme_hashes['dhash_h'],
post_id=post.post_id
)
uow.meme_template.add(meme_template)
uow.meme_template_potential.remove(potential_template)
elif potential_template.vote_total <= -10:
log.info('Removing potential template with at least 10 negative votes')
uow.meme_template_potential.remove(potential_template)
else:
continue
uow.commit()
def save_unknown_post(self, post_id: Text) -> Optional[Post]:
"""
If we received a request on a post we haven't ingest save it
:rtype: Optional[Post]
:param post_id: Submission ID
:return: Post object
"""
submission = self.reddit.submission(post_id)
try:
post = pre_process_post(submission_to_post(submission), self.uowm,
None)
except InvalidImageUrlException:
return
except Forbidden:
log.error('Failed to download post %s, appears we are banned',
post_id)
return
if not post or post.post_type != 'image':
log.error(
'Problem ingesting post. Either failed to save or it is not an image'
)
return
return post
def _process_comment(self, bot_comment: BotComment):
reddit_comment = self._get_comment_data(bot_comment.perma_link)
if not reddit_comment:
log.error('Failed to locate comment %s', bot_comment.comment_id)
return
bot_comment.karma = reddit_comment['ups']
if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold:
log.info('Comment %s has karma of %s. Removing',
bot_comment.comment_id, bot_comment.karma)
if self.notification_svc:
self.notification_svc.send_notification(
f'Removing comment with {bot_comment.karma} karma. https://reddit.com{bot_comment.perma_link}',
subject='Removing Downvoted Comment')
comment = self.reddit.comment(bot_comment.comment_id)
try:
comment.delete()
except Exception as e:
log.exception('Failed to delete comment %s',
bot_comment.comment_id,
exc_info=True)
bot_comment.needs_review = True
bot_comment.active = False
elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold:
log.info('Comment %s has karma of %s. Flagging for review',
bot_comment.comment_id, bot_comment.karma)
bot_comment.needs_review = True
def build_sub_comment(self, monitored_sub: MonitoredSub,
search_results: SearchResults, **kwargs) -> Text:
"""
Take a given MonitoredSub and attempt to build their customer message templates using the search results.
If the final formatting of the template fails, it will revert to the default response template
:rtype: Text
:param monitored_sub: MonitoredSub to get template from
:param search_results: Set of search results
:param kwargs: Args to pass along to default comment builder
:return:
"""
if len(search_results.matches) > 0:
message = monitored_sub.repost_response_template
else:
message = monitored_sub.oc_response_template
if not message:
return self.build_default_comment(search_results, **kwargs)
try:
return self.build_default_comment(search_results, message,
**kwargs)
except KeyError:
log.error('Custom repost template for %s has a bad slug: %s',
monitored_sub.name,
monitored_sub.repost_response_template)
return self.build_default_comment(search_results, **kwargs)
def update_wiki_config_from_database(self,
monitored_sub: MonitoredSub,
wiki_page: WikiPage = None,
notify: bool = False) -> bool:
"""
Sync the database settings to a given monitor sub's wiki page
:param notify: Send notification when config is loaded
:param wiki_page: Wiki Page object
:param monitored_sub: Monitored Sub to update
:rtype: bool
:return: bool if config was successfully loaded
"""
subreddit = self.reddit.subreddit(monitored_sub.name)
if not wiki_page:
wiki_page = subreddit.wiki[self.config.wiki_config_name]
new_config = self._create_wiki_config_from_database(monitored_sub)
if not new_config:
log.error('Failed to generate new config for %s',
monitored_sub.name)
return False
self._update_wiki_page(wiki_page, new_config)
wiki_page = subreddit.wiki[
'repost_sleuth_config'] # Force refresh so we can get latest revision ID
self._create_revision(wiki_page)
self._set_config_validity(wiki_page.revision_id, True)
if notify:
self._notify_successful_load(subreddit)
return True
def generate_thumbnails_from_file(video_file: str,
total_thumbs: int = 20) -> int:
result = {'duration': None, 'thumbs': []}
try:
probe = ffmpeg.probe(video_file)
except Exception as e:
log.error('Failed to probe video: %s', video_file)
raise
duration = float(probe['format']['duration'])
interval = float(probe['format']['duration']) / total_thumbs
count = 1
seek = interval
log.info('Video length is %s seconds. Grabbing thumb every %s seconds',
duration, interval)
output_dir = os.path.split(video_file)[0]
while count <= total_thumbs:
print(str(count))
try:
(ffmpeg.input(video_file, ss=seek).filter('scale', 720, -1).output(
os.path.join(output_dir, '{}.png'.format(str(count))),
vframes=1).overwrite_output().run(capture_stdout=True,
capture_stderr=True))
except ffmpeg.Error as e:
print(e.stderr.decode(), file=sys.stderr)
seek += interval
count += 1
return duration
def _final_meme_filter(self, searched_hash: Text,
matches: List[ImageSearchMatch],
target_hamming) -> List[ImageSearchMatch]:
results = []
log.debug('MEME FILTER - Filtering %s matches', len(matches))
if len(matches) == 0:
return matches
for match in matches:
try:
match_hash = self._get_meme_hash(match.post.url)
except Exception as e:
log.error('Failed to get meme hash for %s', match.post.id)
continue
h_distance = hamming(searched_hash, match_hash)
if h_distance > target_hamming:
log.info(
'Meme Hamming Filter Reject - Target: %s Actual: %s - %s',
target_hamming, h_distance,
f'https://redd.it/{match.post.post_id}')
continue
log.debug('Match found: %s - H:%s',
f'https://redd.it/{match.post.post_id}', h_distance)
match.hamming_distance = h_distance
match.hash_size = len(searched_hash)
results.append(match)
return results
def _offer_watch(self, submission: Submission) -> NoReturn:
"""
Offer to add watch to OC post
:param search:
"""
if not self.config.top_post_offer_watch:
log.debug('Top Post Offer Watch Disabled')
return
log.info('Offer watch to %s on post %s', submission.author.name,
submission.id)
with self.uowm.start() as uow:
existing_response = uow.bot_private_message.get_by_user_source_and_post(
submission.author.name, 'toppost', submission.id)
if existing_response:
log.info('Already sent a message to %s', submission.author.name)
return
try:
self.response_handler.send_private_message(
submission.author,
TOP_POST_WATCH_BODY.format(
shortlink=f'https://redd.it/{submission.id}'),
subject=TOP_POST_WATCH_SUBJECT,
source='toppost',
post_id=submission.id)
except APIException as e:
if e.error_type == 'NOT_WHITELISTED_BY_USER_MESSAGE':
log.error('Not whitelisted API error')
else:
log.exception('Unknown error sending PM to %s',
submission.author.name,
exc_info=True)
def check_for_repost(self, post: Post) -> Optional[SearchResults]:
"""
Take a given post and check if it's a repost
:rtype: SearchResults
:param post: Post obj
:return: Search results
"""
if post.post_type == 'image':
try:
return self.image_service.check_image(
post.url,
post=post,
)
except NoIndexException:
log.error(
'No available index for image repost check. Trying again later'
)
return
elif post.post_type == 'link':
search_results = get_link_reposts(post.url,
self.uowm,
get_default_link_search_settings(
self.config),
post=post,
get_total=True)
return filter_search_results(
search_results,
reddit=self.reddit,
uitl_api=f'{self.config.util_api}/maintenance/removed')
else:
log.info(
f'Post {post.post_id} is a {post.post_type} post. Skipping')
return
def _process_comment(self, bot_comment: BotComment):
reddit_comment = self._get_comment_data(bot_comment.perma_link)
if not reddit_comment:
log.error('Failed to locate comment %s', bot_comment.comment_id)
return
bot_comment.karma = reddit_comment['ups']
if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold:
log.info('Comment %s has karma of %s. Removing',
bot_comment.comment_id, bot_comment.karma)
comment = self.reddit.comment(bot_comment.comment_id)
try:
comment.delete()
except Exception as e:
log.exception('Failed to delete comment %s',
bot_comment.comment_id,
exc_info=True)
bot_comment.needs_review = True
bot_comment.active = False
elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold:
log.info('Comment %s has karma of %s. Flagging for review',
bot_comment.comment_id, bot_comment.karma)
bot_comment.needs_review = True
def _reply_to_comment(self, comment_id: Text, comment_body: Text, subreddit: Text = None) -> Optional[Comment]:
"""
Post a given reply to a given comment ID
:rtype: Optional[Comment]
:param comment_id: ID of comment to reply to
:param comment_body: Body of the comment to leave in reply
:return:
"""
comment = self.reddit.comment(comment_id)
if not comment:
log.error('Failed to find comment %s', comment_id)
return
try:
start_time = perf_counter()
reply_comment = comment.reply(comment_body)
self._record_api_event(
float(round(perf_counter() - start_time, 2)),
'reply_to_comment',
self.reddit.reddit.auth.limits['remaining']
)
self._log_response(reply_comment)
log.info('Left comment at: https://reddit.com%s', reply_comment.permalink)
return reply_comment
except Forbidden:
log.exception('Forbidden to respond to comment %s', comment_id, exc_info=False)
# If we get Forbidden there's a chance we don't have hte comment data to get subreddit
if subreddit:
self._save_banned_sub(subreddit)
raise
except AssertionError:
log.exception('Problem leaving comment', exc_info=True)
raise
def _reply_to_submission(self, submission_id: str, comment_body) -> Optional[Comment]:
submission = self.reddit.submission(submission_id)
if not submission:
log.error('Failed to get submission %s', submission_id)
return
try:
start_time = perf_counter()
comment = submission.reply(comment_body)
self._record_api_event(
float(round(perf_counter() - start_time, 2)),
'reply_to_submission',
self.reddit.reddit.auth.limits['remaining']
)
log.info('Left comment at: https://reddit.com%s', comment.permalink)
log.debug(comment_body)
self._log_response(comment)
return comment
except APIException as e:
if e.error_type == 'RATELIMIT':
log.exception('Reddit rate limit')
raise RateLimitException('Hit rate limit')
else:
log.exception('Unknown error type of APIException', exc_info=True)
raise
except Forbidden:
self._save_banned_sub(submission.subreddit.display_name)
except Exception:
log.exception('Unknown exception leaving comment on post https://redd.it/%s', submission_id, exc_info=True)
raise
def send_mod_mail(self, subreddit_name: Text, subject: Text, body: Text, triggered_from: Text = None) -> NoReturn:
"""
Send a modmail message
:rtype: NoReturn
:param subreddit_name: name of subreddit
:param subject: Message Subject
:param body: Message Body
"""
subreddit = self.reddit.subreddit(subreddit_name)
if not subreddit:
log.error('Failed to get Subreddit %s when attempting to send modmail')
return
try:
subreddit.message(subject, body)
self._save_private_message(
BotPrivateMessage(
subject=subject,
body=body,
triggered_from=triggered_from,
recipient=subreddit_name
)
)
except RedditAPIException:
log.exception('Problem sending modmail message', exc_info=True)
def parse_root_command(self, command: str):
if not command:
log.error('Got empty command. Returning repost')
return 'repost'
parser = ArgumentParserThrow()
parser.add_argument('command', default=None, choices=['repost', 'watch', 'unwatch'])
options, args = parser.parse_known_args(command.split(' '))
return options.command
def get_subscribers(sub_name: Text, reddit: Reddit) -> Optional[int]:
subreddit = reddit.subreddit(sub_name)
try:
return subreddit.subscribers
except Forbidden:
log.error('Failed to get subscribers, Forbidden %s', sub_name)
return
except NotFound:
log.error('Failed to get subscribers, not found %s', sub_name)
return
def run_update(self):
print('[Scheduled Job] Stats Update Starting')
self.get_all_stats()
output = self.build_template()
wiki = self.reddit.subreddit('RepostSleuthBot').wiki['stats']
try:
wiki.edit(output)
except BadRequest:
log.error('Failed to update wiki page')
print('[Scheduled Job] Stats Update Ending')
def _sticky_reply(self, monitored_sub: MonitoredSub,
comment: Comment) -> NoReturn:
if monitored_sub.sticky_comment:
try:
comment.mod.distinguish(sticky=True)
log.info('Made comment %s sticky', comment.id)
except Forbidden:
log.error('Failed to sticky comment, no permissions')
except Exception as e:
log.exception('Failed to sticky comment', exc_info=True)
def target_hash(self) -> Text:
"""
Returns the hash to be searched. This allows us to work with just a URL or a full post
:return: hash
"""
if self._target_hash:
return self._target_hash
log.error('No target hash set, attempting to get')
hashes = get_image_hashes(self.checked_url, hash_size=16)
self._target_hash = hashes['dhash_h']
return self._target_hash
def _send_to_hook(self, payload: Dict):
try:
r = requests.post(self.hook,
headers={'Content-Type': 'application/json'},
json=payload)
except (ConnectionError, Timeout):
log.error('Failed to send discord notification')
return
if r.status_code != 204:
log.error('Unexpected status code %s from Discord webhook: %s',
r.status_code, r.text)
def _lock_comment(self, monitored_sub: MonitoredSub,
comment: Comment) -> NoReturn:
if monitored_sub.lock_response_comment:
log.info('Attempting to lock comment %s on subreddit %s',
comment.id, monitored_sub.name)
try:
comment.mod.lock()
log.info('Locked comment')
except Forbidden:
log.error('Failed to lock comment, no permission')
except Exception as e:
log.exception('Failed to lock comment', exc_info=True)
