def check_for_repost(self, post: Post) -> Optional[SearchResults]:
"""
Take a given post and check if it's a repost
:rtype: SearchResults
:param post: Post obj
:return: Search results
"""
if post.post_type == 'image':
try:
return self.image_service.check_image(
post.url,
post=post,
)
except NoIndexException:
log.error(
'No available index for image repost check. Trying again later'
)
return
elif post.post_type == 'link':
search_results = get_link_reposts(post.url,
self.uowm,
get_default_link_search_settings(
self.config),
post=post,
get_total=True)
return filter_search_results(
search_results,
reddit=self.reddit,
uitl_api=f'{self.config.util_api}/maintenance/removed')
else:
log.info(
f'Post {post.post_id} is a {post.post_type} post. Skipping')
return
def process_link_repost_request(self,
summons: Summons,
post: Post,
monitored_sub: MonitoredSub = None):
response = SummonsResponse(summons=summons)
search_results = get_link_reposts(post.url,
self.uowm,
get_default_link_search_settings(
self.config),
post=post,
get_total=True)
search_results = filter_search_results(
search_results,
reddit=self.reddit.reddit,
uitl_api=f'{self.config.util_api}/maintenance/removed')
if not monitored_sub:
response.message = self.response_builder.build_default_comment(
search_results, signature=False)
else:
response.message = self.response_builder.build_sub_comment(
monitored_sub, search_results, signature=False)
if search_results.matches:
save_link_repost(post, search_results.matches[0].post, self.uowm,
'summons')
self._send_response(response)
def link_repost_check(self, posts, ):
with self.uowm.start() as uow:
for post in posts:
"""
if post.url_hash == '540f1167d27dcca2ea2772443beb5c79':
continue
"""
if post.url_hash in self.link_blacklist:
log.info('Skipping blacklisted URL hash %s', post.url_hash)
continue
log.debug('Checking URL for repost: %s', post.url_hash)
search_results = get_link_reposts(post.url, self.uowm, get_default_link_search_settings(self.config),
post=post)
if len(search_results.matches) > 10000:
log.info('Link hash %s shared %s times. Adding to blacklist', post.url_hash, len(search_results.matches))
self.link_blacklist.append(post.url_hash)
self.notification_svc.send_notification(f'URL has been shared {len(search_results.matches)} times. Adding to blacklist. \n\n {post.url}')
search_results = filter_search_results(
search_results,
uitl_api=f'{self.config.util_api}/maintenance/removed'
)
search_results.search_times.stop_timer('total_search_time')
log.info('Link Query Time: %s', search_results.search_times.query_time)
if not search_results.matches:
log.debug('Not matching linkes for post %s', post.post_id)
post.checked_repost = True
uow.posts.update(post)
uow.commit()
continue
log.info('Found %s matching links', len(search_results.matches))
log.info('Creating Link Repost. Post %s is a repost of %s', post.post_id, search_results.matches[0].post.post_id)
repost_of = search_results.matches[0].post
new_repost = LinkRepost(post_id=post.post_id, repost_of=repost_of.post_id, author=post.author, source='ingest', subreddit=post.subreddit)
repost_of.repost_count += 1
post.checked_repost = True
uow.posts.update(post)
uow.link_repost.add(new_repost)
try:
uow.commit()
self.event_logger.save_event(RepostEvent(event_type='repost_found', status='success',
repost_of=search_results.matches[0].post.post_id,
post_type=post.post_type))
except IntegrityError as e:
uow.rollback()
log.exception('Error saving link repost', exc_info=True)
self.event_logger.save_event(RepostEvent(event_type='repost_found', status='error',
repost_of=search_results.matches[0].post.post_id,
post_type=post.post_type))
self.event_logger.save_event(
BatchedEvent(event_type='repost_check', status='success', count=len(posts), post_type='link'))
def _filter_results_for_reposts(self,
search_results: ImageSearchResults,
sort_by='created') -> ImageSearchResults:
"""
Take a list of matches and filter out posts that are not reposts.
This is done via distance checking, creation date, crosspost
:param checked_post: The post we're finding matches for
:param search_results: A cleaned list of matches
:param target_hamming_distance: Hamming cutoff for matches
:param target_annoy_distance: Annoy cutoff for matches
:rtype: List[ImageSearchMatch]
"""
log.debug('Starting result filters with %s matches',
len(search_results.matches))
search_results.matches = list(
filter(filter_no_dhash, search_results.matches))
search_results = filter_search_results(
search_results,
reddit=self.reddit,
uitl_api=f'{self.config.util_api}/maintenance/removed')
closest_match = get_closest_image_match(search_results.matches,
check_url=True)
if closest_match and closest_match.hamming_match_percent > 40: # TODO - Move to config
search_results.closest_match = closest_match
if search_results.closest_match and search_results.meme_template:
match_hash = self._get_meme_hash(
search_results.closest_match.post.url)
search_results.closest_match.hamming_distance = hamming(
search_results.meme_hash, match_hash)
search_results.closest_match.hash_size = len(match_hash)
# Has to be after closest match so we don't drop closest
search_results.matches = list(
filter(
annoy_distance_filter(
search_results.search_settings.target_annoy_distance),
search_results.matches))
search_results.matches = list(
filter(
hamming_distance_filter(
search_results.target_hamming_distance),
search_results.matches))
if search_results.meme_template:
search_results.search_times.start_timer('meme_filter_time')
search_results.matches = self._final_meme_filter(
search_results.meme_hash, search_results.matches,
search_results.target_meme_hamming_distance)
search_results.search_times.stop_timer('meme_filter_time')
search_results.matches = sort_reposts(search_results.matches,
sort_by=sort_by)
for match in search_results.matches:
log.debug('Match found: %s - A:%s H:%s P:%s',
f'https://redd.it/{match.post.post_id}',
round(match.annoy_distance, 5), match.hamming_distance,
f'{match.hamming_match_percent}%')
return search_results
def test_filter_search_results_hit_all_filters(self):
search_results = get_image_search_results_multi_match()
search_results.search_settings.filter_same_author = True
search_results.search_settings.filter_crossposts = True
search_results.search_settings.only_older_matches = True
search_results.search_settings.same_sub = True
search_results.search_settings.target_title_match = None
search_results.search_settings.max_days_old = 4
search_results.checked_post.author = 'barry'
search_results.checked_post.subreddit = 'sub1'
search_results.checked_post.post_id = '1111'
search_results.checked_post.created_at = datetime.utcfromtimestamp(
1573995250)
matches = []
# Dropped by same author
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=1,
author='barry',
post_id='abc123',
created_at=datetime.strptime('2019-01-28 05:20:03',
'%Y-%m-%d %H:%M:%S')), 10,
10, 32))
# Dropped by crosspost
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=2,
author='steve',
post_id='123abc',
crosspost_parent='abc',
created_at=datetime.strptime('2019-06-28 05:20:03',
'%Y-%m-%d %H:%M:%S')), 10,
10, 32))
# Dropped by only older
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=3,
author='steve',
post_id='3333',
title='some normal title',
created_at=datetime.utcfromtimestamp(1574081650)), 10,
0.250, 32))
# Dropped by same sub
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=4,
author='steve',
post_id='4444',
title='some normal title',
subreddit='sub2',
created_at=datetime.utcfromtimestamp(1573908850)), 10,
0.250, 32))
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=5,
author='steve',
post_id='5555',
title='some normal title',
subreddit='sub1',
created_at=datetime.utcfromtimestamp(1573988200)), 10,
0.250, 32))
# Dropped by same post
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=6,
post_id='1111',
title='some normal title',
subreddit='sub1',
created_at=datetime.utcfromtimestamp(1573908850)), 10,
0.250, 32))
matches.append(
ImageSearchMatch(
'test.com', 1,
Post(id=7,
post_id='6666',
title='some normal title',
subreddit='sub1',
created_at=datetime.utcfromtimestamp(1573908850)), 10,
0.250, 32))
search_results.matches = matches
with patch('redditrepostsleuth.core.util.repost_filters.datetime'
) as mock_date:
mock_date.utcnow.return_value = datetime.utcfromtimestamp(
1574360460)
r = filter_search_results(search_results)
self.assertEqual(1, len(search_results.matches))
self.assertEqual('5555', r.matches[0].post.post_id)
print('')
