def _enable_watch(self, summons: Summons) -> NoReturn:
response = SummonsResponse(summons=summons)
with self.uowm.start() as uow:
existing_watch = uow.repostwatch.find_existing_watch(
summons.requestor, summons.post_id)
if existing_watch:
if not existing_watch.enabled:
log.info(
'Found existing watch that is disabled. Enabling watch %s',
existing_watch.id)
existing_watch.enabled = True
response.message = WATCH_ENABLED
uow.commit()
self._send_response(response)
return
else:
response.message = WATCH_ALREADY_ENABLED
self._send_response(response)
return
repost_watch = RepostWatch(post_id=summons.post_id,
user=summons.requestor,
enabled=True)
with self.uowm.start() as uow:
uow.repostwatch.add(repost_watch)
try:
uow.commit()
response.message = WATCH_ENABLED
except Exception as e:
log.exception('Failed save repost watch', exc_info=True)
response.message = 'An error prevented me from creating a watch on this post. Please try again'
self._send_response(response)
def _send_private_message(self,
user: Redditor,
message_body,
subject: Text = 'Repost Check',
source: Text = None,
post_id: Text = None,
comment_id: Text = None) -> NoReturn:
if not user:
log.error('No user provided to send private message')
return
try:
start_time = perf_counter()
user.message(subject, message_body)
self._record_api_event(
float(round(perf_counter() - start_time, 2)),
'private_message', self.reddit.reddit.auth.limits['remaining'])
log.info('Sent PM to %s. ', user.name)
except Exception as e:
log.exception('Failed to send PM to %s', user.name, exc_info=True)
raise
self._save_private_message(
BotPrivateMessage(subject=subject,
body=message_body,
in_response_to_post=post_id,
in_response_to_comment=comment_id,
triggered_from=source,
recipient=user.name))
def _process_comment(self, bot_comment: BotComment):
reddit_comment = self._get_comment_data(bot_comment.perma_link)
if not reddit_comment:
log.error('Failed to locate comment %s', bot_comment.comment_id)
return
bot_comment.karma = reddit_comment['ups']
if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold:
log.info('Comment %s has karma of %s. Removing',
bot_comment.comment_id, bot_comment.karma)
if self.notification_svc:
self.notification_svc.send_notification(
f'Removing comment with {bot_comment.karma} karma. https://reddit.com{bot_comment.perma_link}',
subject='Removing Downvoted Comment')
comment = self.reddit.comment(bot_comment.comment_id)
try:
comment.delete()
except Exception as e:
log.exception('Failed to delete comment %s',
bot_comment.comment_id,
exc_info=True)
bot_comment.needs_review = True
bot_comment.active = False
elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold:
log.info('Comment %s has karma of %s. Flagging for review',
bot_comment.comment_id, bot_comment.karma)
bot_comment.needs_review = True
def monitor_for_summons(self, subreddits: str = 'all'):
"""
Monitors the subreddits set in the config for comments containing the summoning string
"""
log.info('Starting praw summons monitor for subs %s', subreddits)
while True:
try:
for comment in self.reddit.subreddit(
subreddits).stream.comments():
if comment is None:
continue
if self.check_for_summons(comment.body, '\?repost'):
if comment.author.name.lower() in [
'sneakpeekbot', 'automoderator'
]:
continue
self._save_summons(comment)
except ResponseException as e:
if e.response.status_code == 429:
log.error('IP Rate limit hit. Waiting')
time.sleep(60)
continue
except Exception as e:
if 'code: 429' in str(e):
log.error('Too many requests from IP. Waiting')
time.sleep(60)
continue
log.exception('Praw summons thread died', exc_info=True)
def send_mod_mail(self, subreddit_name: Text, subject: Text, body: Text, triggered_from: Text = None) -> NoReturn:
"""
Send a modmail message
:rtype: NoReturn
:param subreddit_name: name of subreddit
:param subject: Message Subject
:param body: Message Body
"""
subreddit = self.reddit.subreddit(subreddit_name)
if not subreddit:
log.error('Failed to get Subreddit %s when attempting to send modmail')
return
try:
subreddit.message(subject, body)
self._save_private_message(
BotPrivateMessage(
subject=subject,
body=body,
triggered_from=triggered_from,
recipient=subreddit_name
)
)
except RedditAPIException:
log.exception('Problem sending modmail message', exc_info=True)
def _process_comment(self, bot_comment: BotComment):
reddit_comment = self._get_comment_data(bot_comment.perma_link)
if not reddit_comment:
log.error('Failed to locate comment %s', bot_comment.comment_id)
return
bot_comment.karma = reddit_comment['ups']
if bot_comment.karma <= self.config.bot_comment_karma_remove_threshold:
log.info('Comment %s has karma of %s. Removing',
bot_comment.comment_id, bot_comment.karma)
comment = self.reddit.comment(bot_comment.comment_id)
try:
comment.delete()
except Exception as e:
log.exception('Failed to delete comment %s',
bot_comment.comment_id,
exc_info=True)
bot_comment.needs_review = True
bot_comment.active = False
elif bot_comment.karma <= self.config.bot_comment_karma_flag_threshold:
log.info('Comment %s has karma of %s. Flagging for review',
bot_comment.comment_id, bot_comment.karma)
bot_comment.needs_review = True
def _reply_to_comment(self, comment_id: Text, comment_body: Text, subreddit: Text = None) -> Optional[Comment]:
"""
Post a given reply to a given comment ID
:rtype: Optional[Comment]
:param comment_id: ID of comment to reply to
:param comment_body: Body of the comment to leave in reply
:return:
"""
comment = self.reddit.comment(comment_id)
if not comment:
log.error('Failed to find comment %s', comment_id)
return
try:
start_time = perf_counter()
reply_comment = comment.reply(comment_body)
self._record_api_event(
float(round(perf_counter() - start_time, 2)),
'reply_to_comment',
self.reddit.reddit.auth.limits['remaining']
)
self._log_response(reply_comment)
log.info('Left comment at: https://reddit.com%s', reply_comment.permalink)
return reply_comment
except Forbidden:
log.exception('Forbidden to respond to comment %s', comment_id, exc_info=False)
# If we get Forbidden there's a chance we don't have hte comment data to get subreddit
if subreddit:
self._save_banned_sub(subreddit)
raise
except AssertionError:
log.exception('Problem leaving comment', exc_info=True)
raise
def _remove_post(self, monitored_sub: MonitoredSub,
submission: Submission) -> NoReturn:
"""
Check if given sub wants posts removed. Remove is enabled
@param monitored_sub: Monitored sub
@param submission: Submission to remove
"""
if monitored_sub.remove_repost:
if not monitored_sub.removal_reason:
log.error(
'Sub %s does not have a removal reason set. Cannot remove',
monitored_sub.name)
return
try:
removal_reason_id = self._get_removal_reason_id(
monitored_sub.removal_reason, submission.subreddit)
if not removal_reason_id:
log.error('Failed to get Removal Reason ID from reason %s',
monitored_sub.removal_reason)
return
submission.mod.remove(reason_id=removal_reason_id)
log.error(
'[%s][%s] - Failed to remove post using reason ID %s. Likely a bad reasons ID',
monitored_sub.name, submission.id,
monitored_sub.removal_reason_id)
submission.mod.remove()
except Forbidden:
log.error(
'Failed to remove post https://redd.it/%s, no permission',
submission.id)
except Exception as e:
log.exception('Failed to remove submission https://redd.it/%s',
submission.id,
exc_info=True)
def _offer_watch(self, submission: Submission) -> NoReturn:
"""
Offer to add watch to OC post
:param search:
"""
if not self.config.top_post_offer_watch:
log.debug('Top Post Offer Watch Disabled')
return
log.info('Offer watch to %s on post %s', submission.author.name,
submission.id)
with self.uowm.start() as uow:
existing_response = uow.bot_private_message.get_by_user_source_and_post(
submission.author.name, 'toppost', submission.id)
if existing_response:
log.info('Already sent a message to %s', submission.author.name)
return
try:
self.response_handler.send_private_message(
submission.author,
TOP_POST_WATCH_BODY.format(
shortlink=f'https://redd.it/{submission.id}'),
subject=TOP_POST_WATCH_SUBJECT,
source='toppost',
post_id=submission.id)
except APIException as e:
if e.error_type == 'NOT_WHITELISTED_BY_USER_MESSAGE':
log.error('Not whitelisted API error')
else:
log.exception('Unknown error sending PM to %s',
submission.author.name,
exc_info=True)
def on_post(self, req: Request, resp: Response, subreddit: Text):
log.info('Attempting to create monitored sub %s', subreddit)
try:
self.reddit.subreddit(subreddit).mod.accept_invite()
except APIException as e:
if e.error_type == 'NO_INVITE_FOUND':
log.error('No open invite to %s', subreddit)
raise HTTPInternalServerError(
f'No available invite for {subreddit}',
f'We were unable to find a '
f'pending mod invote for r/{subreddit}')
else:
log.exception('Problem accepting invite', exc_info=True)
raise HTTPInternalServerError(
f'Unknown error accepting mod invite for r/{subreddit}',
f'Unknown error accepting mod invite for r/{subreddit}. Please contact us'
)
except Exception as e:
log.exception('Failed to accept invite', exc_info=True)
raise HTTPInternalServerError(
f'Unknown error accepting mod invite for r/{subreddit}',
f'Unknown error accepting mod invite for r/{subreddit}. Please contact us'
)
with self.uowm.start() as uow:
existing = uow.monitored_sub.get_by_sub(subreddit)
if existing:
resp.body = json.dumps(existing.to_dict())
return
monitored_sub = create_monitored_sub_in_db(subreddit, uow)
resp.body = json.dumps(monitored_sub.to_dict())
def send_notification(self, msg: Text, **kwargs) -> NoReturn:
for agent in self.notification_agents:
log.info('Sending notification to %s', agent.name)
log.debug(msg)
try:
agent.send(msg, **kwargs)
except Exception as e:
log.exception('Failed to send notification', exc_info=True)
def _sticky_reply(self, monitored_sub: MonitoredSub,
comment: Comment) -> NoReturn:
if monitored_sub.sticky_comment:
try:
comment.mod.distinguish(sticky=True)
log.info('Made comment %s sticky', comment.id)
except Forbidden:
log.error('Failed to sticky comment, no permissions')
except Exception as e:
log.exception('Failed to sticky comment', exc_info=True)
def notify_watch(self, watches: List[Dict[SearchMatch, RepostWatch]], repost: Post):
repost_watch_notify(watches, self.reddit, self.response_handler, repost)
with self.uowm.start() as uow:
for w in watches:
w['watch'].last_detection = func.utc_timestamp()
uow.repostwatch.update(w['watch'])
try:
uow.commit()
except Exception as e:
log.exception('Failed to save repost watch %s', w['watch'].id, exc_info=True)
def _mark_post_as_comment_left(self, post: Post):
try:
with self.uowm.start() as uow:
post.left_comment = True
uow.posts.update(post)
uow.commit()
except Exception as e:
log.exception('Failed to mark post %s as checked',
post.id,
exc_info=True)
def generate_img_by_file(path: str) -> Image:
try:
img = Image.open(path)
except (HTTPError, ConnectionError, OSError, DecompressionBombError,
UnicodeEncodeError) as e:
log.exception('Failed to convert image %s. Error: %s ', path, str(e))
raise ImageConversioinException(str(e))
return img if img else None
def monitor_for_mentions(self):
bad_mentions = []
while True:
try:
for comment in self.reddit.inbox.mentions():
if comment.created_utc < datetime.utcnow().timestamp(
) - 86400:
log.debug('Skipping old mention. Created at %s',
datetime.fromtimestamp(comment.created_utc))
continue
if comment.author.name.lower() in [
'sneakpeekbot', 'automoderator'
]:
continue
if comment.id in bad_mentions:
continue
with self.uowm.start() as uow:
existing_summons = uow.summons.get_by_comment_id(
comment.id)
if existing_summons:
log.debug('Skipping existing mention %s',
comment.id)
continue
summons = Summons(
post_id=comment.submission.id,
comment_id=comment.id,
comment_body=comment.body.replace('\\', ''),
summons_received_at=datetime.fromtimestamp(
comment.created_utc),
requestor=comment.author.name,
subreddit=comment.subreddit.display_name)
uow.summons.add(summons)
try:
uow.commit()
except DataError as e:
log.error('SQLAlchemy Data error saving comment')
bad_mentions.append(comment.id)
continue
except ResponseException as e:
if e.response.status_code == 429:
log.error('IP Rate limit hit. Waiting')
time.sleep(60)
continue
except AssertionError as e:
if 'code: 429' in str(e):
log.error('Too many requests from IP. Waiting')
time.sleep(60)
return
except Exception as e:
log.exception('Mention monitor failed', exc_info=True)
time.sleep(20)
def link_repost_check(self, posts, ):
with self.uowm.start() as uow:
for post in posts:
"""
if post.url_hash == '540f1167d27dcca2ea2772443beb5c79':
continue
"""
if post.url_hash in self.link_blacklist:
log.info('Skipping blacklisted URL hash %s', post.url_hash)
continue
log.debug('Checking URL for repost: %s', post.url_hash)
search_results = get_link_reposts(post.url, self.uowm, get_default_link_search_settings(self.config),
post=post)
if len(search_results.matches) > 10000:
log.info('Link hash %s shared %s times. Adding to blacklist', post.url_hash, len(search_results.matches))
self.link_blacklist.append(post.url_hash)
self.notification_svc.send_notification(f'URL has been shared {len(search_results.matches)} times. Adding to blacklist. \n\n {post.url}')
search_results = filter_search_results(
search_results,
uitl_api=f'{self.config.util_api}/maintenance/removed'
)
search_results.search_times.stop_timer('total_search_time')
log.info('Link Query Time: %s', search_results.search_times.query_time)
if not search_results.matches:
log.debug('Not matching linkes for post %s', post.post_id)
post.checked_repost = True
uow.posts.update(post)
uow.commit()
continue
log.info('Found %s matching links', len(search_results.matches))
log.info('Creating Link Repost. Post %s is a repost of %s', post.post_id, search_results.matches[0].post.post_id)
repost_of = search_results.matches[0].post
new_repost = LinkRepost(post_id=post.post_id, repost_of=repost_of.post_id, author=post.author, source='ingest', subreddit=post.subreddit)
repost_of.repost_count += 1
post.checked_repost = True
uow.posts.update(post)
uow.link_repost.add(new_repost)
try:
uow.commit()
self.event_logger.save_event(RepostEvent(event_type='repost_found', status='success',
repost_of=search_results.matches[0].post.post_id,
post_type=post.post_type))
except IntegrityError as e:
uow.rollback()
log.exception('Error saving link repost', exc_info=True)
self.event_logger.save_event(RepostEvent(event_type='repost_found', status='error',
repost_of=search_results.matches[0].post.post_id,
post_type=post.post_type))
self.event_logger.save_event(
BatchedEvent(event_type='repost_check', status='success', count=len(posts), post_type='link'))
def _report_submission(self, monitored_sub: MonitoredSub,
submission: Submission,
report_msg: Text) -> NoReturn:
if not monitored_sub.report_reposts:
return
log.info('Reporting post %s on %s', f'https://redd.it/{submission.id}',
monitored_sub.name)
try:
submission.report(report_msg)
except Exception as e:
log.exception('Failed to report submission', exc_info=True)
def _create_checked_post(self, post: Post):
try:
with self.uowm.start() as uow:
uow.monitored_sub_checked.add(
MonitoredSubChecks(post_id=post.post_id,
subreddit=post.subreddit))
uow.commit()
except Exception as e:
log.exception('Failed to create checked post for submission %s',
post.post_id,
exc_info=True)
def save_image_repost_result(search_results: ImageSearchResults,
uowm: UnitOfWorkManager,
high_match_check: bool = False,
source: Text = 'unknown') -> NoReturn:
"""
Take a found repost and save to the database
:param source: What triggered this search
:rtype: NoReturn
:param high_match_check: Perform a high match meme check.
:param search_results: Set of search results
:param uowm: Unit of Work Manager
:return:None
"""
with uowm.start() as uow:
search_results.checked_post.checked_repost = True
if not search_results.matches:
log.debug('Post %s has no matches',
search_results.checked_post.post_id)
uow.posts.update(search_results.checked_post)
uow.commit()
return
# This is used for ingest repost checking. If a meme template gets created, it intentionally throws a
# IngestHighMatchMeme exception. This will cause celery to retry the task so the newly created meme template
# gets used
if high_match_check:
check_for_high_match_meme(
search_results,
uowm) # This intentionally throws if we create a meme template
log.info('Creating repost. Post %s is a repost of %s',
search_results.checked_post.url,
search_results.matches[0].post.url)
new_repost = ImageRepost(
post_id=search_results.checked_post.post_id,
repost_of=search_results.matches[0].post.post_id,
hamming_distance=search_results.matches[0].hamming_distance,
annoy_distance=search_results.matches[0].annoy_distance,
author=search_results.checked_post.author,
search_id=search_results.logged_search.id
if search_results.logged_search else None,
subreddit=search_results.checked_post.subreddit,
source=source)
uow.image_repost.add(new_repost)
uow.posts.update(search_results.checked_post)
try:
uow.commit()
except Exception as e:
log.exception('Failed to save image repost', exc_info=True)
def _lock_comment(self, monitored_sub: MonitoredSub,
comment: Comment) -> NoReturn:
if monitored_sub.lock_response_comment:
log.info('Attempting to lock comment %s on subreddit %s',
comment.id, monitored_sub.name)
try:
comment.mod.lock()
log.info('Locked comment')
except Forbidden:
log.error('Failed to lock comment, no permission')
except Exception as e:
log.exception('Failed to lock comment', exc_info=True)
def _save_private_message(self, bot_message: BotPrivateMessage) -> NoReturn:
"""
Save a private message to the database
:param bot_message: BotMessage obj
"""
try:
with self.uowm.start() as uow:
uow.bot_private_message.add(bot_message)
uow.commit()
except Exception as e:
# TODO - Get specific exc
log.exception('Failed to save private message to DB', exc_info=True)
def _create_wiki_page(self, subreddit: Subreddit):
log.info('Creating config wiki page for %s', subreddit.display_name)
try:
subreddit.wiki.create(self.config.wiki_config_name,
json.dumps(DEFAULT_CONFIG_VALUES))
except NotFound:
log.exception('Failed to create wiki page', exc_info=False)
raise
self.notification_svc.send_notification(
f'Created new config for {subreddit.display_name}',
subject='Created new config from template')
def _lock_post(self, monitored_sub: MonitoredSub,
submission: Submission) -> NoReturn:
if monitored_sub.lock_post:
try:
submission.mod.lock()
except Forbidden:
log.error(
'Failed to lock post https://redd.it/%s, no permission',
submission.id)
except Exception as e:
log.exception('Failed to lock submission https://redd.it/%s',
submission.id,
exc_info=True)
def _mark_post_as_oc(self, monitored_sub: MonitoredSub,
submission: Submission):
if monitored_sub.mark_as_oc:
try:
submission.mod.set_original_content()
except Forbidden:
log.error(
'Failed to set post OC https://redd.it/%s, no permission',
submission.id)
except Exception as e:
log.exception('Failed to set post OC https://redd.it/%s',
submission.id,
exc_info=True)
def reply_to_private_message(self, message: Message,
body: Text) -> NoReturn:
log.debug('Replying to private message from %s with subject %s',
message.dest.name, message.subject)
try:
message.reply(body)
self._save_private_message(
BotPrivateMessage(subject=message.subject,
body=body,
triggered_from='inbox_reply',
recipient=message.dest.name))
except RedditAPIException:
log.exception('Problem replying to private message', exc_info=True)
def _save_response(self, response: SummonsResponse):
with self.uowm.start() as uow:
summons = uow.summons.get_by_id(response.summons.id)
if summons:
summons.comment_reply = response.message
summons.summons_replied_at = datetime.utcnow()
summons.comment_reply_id = response.comment_reply_id
try:
uow.commit()
log.debug('Committed summons response to database')
except InternalError:
log.exception('Failed to save response to summons',
exc_info=True)
def run(self):
while True:
try:
with self.uowm.start() as uow:
monitored_subs = uow.monitored_sub.get_all()
for sub in monitored_subs:
if not sub.active:
log.debug('Sub %s is disabled', sub.name)
continue
self._check_sub(sub)
log.info('Sleeping until next run')
time.sleep(60)
except Exception as e:
log.exception('Sub monitor service crashed', exc_info=True)
def _delete_mention(self, comment_id: Text) -> NoReturn:
log.info('Attempting to delete mention %s', comment_id)
comment = self.reddit.comment(comment_id)
if not comment:
log.error('Failed to load comment %s', comment_id)
return
try:
comment.mod.remove()
log.info('Removed mention %s', comment_id)
except Exception as e:
log.exception('Failed to delete comment %s',
comment_id,
exc_info=True)
return
def _log_search(
self,
search_results: ImageSearchResults,
source: str,
used_current_index: bool,
used_historical_index: bool,
) -> ImageSearchResults:
image_search = ImageSearch(
post_id=search_results.checked_post.post_id
if search_results.checked_post else 'url',
used_historical_index=used_historical_index,
used_current_index=used_current_index,
target_hamming_distance=search_results.target_hamming_distance,
target_annoy_distance=search_results.search_settings.
target_annoy_distance,
same_sub=search_results.search_settings.same_sub,
max_days_old=search_results.search_settings.max_days_old,
filter_dead_matches=search_results.search_settings.
filter_dead_matches,
only_older_matches=search_results.search_settings.
only_older_matches,
meme_filter=search_results.search_settings.meme_filter,
meme_template_used=search_results.meme_template.id
if search_results.meme_template else None,
search_time=search_results.search_times.total_search_time,
index_search_time=search_results.search_times.index_search_time,
total_filter_time=search_results.search_times.total_filter_time,
target_title_match=search_results.search_settings.
target_title_match,
matches_found=len(search_results.matches),
source=source,
subreddit=search_results.checked_post.subreddit
if search_results.checked_post else 'url',
search_results=create_search_result_json(search_results),
target_image_meme_match=search_results.search_settings.
target_meme_match_percent,
target_image_match=search_results.search_settings.
target_match_percent)
with self.uowm.start() as uow:
uow.image_search.add(image_search)
try:
uow.commit()
search_results.logged_search = image_search
except Exception as e:
log.exception('Failed to save image search', exc_info=False)
return search_results
