def filter_dead_urls_remote(util_api: Text,
matches: List[SearchMatch]) -> List[SearchMatch]:
"""
Batch checking a list of matches to see if the associated links have been removed.
This function is using our utility API that runs on a Pool of VMs so we can check matches at high volume
We are piggy backing the util API I use to clean deleted posts from the database.
API response is:
[
{'id': '12345', 'action': 'remove'
]
Several actions can be returned. However, we're only interested in the remove since that results from the post's
URL returning a 404
:param util_api: API URL to call to get results
:param matches: List of matches
:return: List of filtered matches
"""
results = []
match_urls = [{
'id': match.post.post_id,
'url': match.post.url
} for match in matches]
alive_urls = batch_check_urls(match_urls, util_api)
for url in alive_urls:
match = next((x for x in matches if x.post.post_id == url['id']), None)
if match:
results.append(match)
return results
def test_batch_check_urls_valid_call(self):
urls = [
{'url': 'example.com', 'id': '1'},
{'url': 'example.com', 'id': '2'},
{'url': 'example.com', 'id': '3'},
{'url': 'example.com', 'id': '4'},
{'url': 'example.com', 'id': '5'},
]
api_return = [
{'url': 'example.com', 'id': '1', 'action': 'remove'},
{'url': 'example.com', 'id': '2', 'action': 'update'},
{'url': 'example.com', 'id': '3', 'action': 'update'},
{'url': 'example.com', 'id': '4', 'action': 'remove'},
{'url': 'example.com', 'id': '5', 'action': 'update'},
]
with mock.patch('redditrepostsleuth.core.util.helpers.requests.post') as mock_post:
mock_post.return_value = Mock(status_code=200, text=json.dumps(api_return))
res = batch_check_urls(urls, 'test.com')
self.assertEqual(3, len(res))
self.assertEqual('2', res[0]['id'])
self.assertEqual('3', res[1]['id'])
self.assertEqual('5', res[2]['id'])
def test_batch_check_urls_valid_bad_connection_error(self):
urls = [
{'url': 'example.com', 'id': '1'},
{'url': 'example.com', 'id': '2'},
{'url': 'example.com', 'id': '3'},
{'url': 'example.com', 'id': '4'},
{'url': 'example.com', 'id': '5'},
]
with mock.patch('redditrepostsleuth.core.util.helpers.requests.post') as mock_post:
mock_post.side_effect = ConnectionError()
res = batch_check_urls(urls, 'test.com')
self.assertEqual(5, len(res))
def test_batch_check_urls_valid_bad_status(self):
urls = [
{'url': 'example.com', 'id': '1'},
{'url': 'example.com', 'id': '2'},
{'url': 'example.com', 'id': '3'},
{'url': 'example.com', 'id': '4'},
{'url': 'example.com', 'id': '5'},
]
with mock.patch('redditrepostsleuth.core.util.helpers.requests.post') as mock_post:
mock_post.return_value = Mock(status_code=500, text=json.dumps(urls))
res = batch_check_urls(urls, 'test.com')
self.assertEqual(5, len(res))
def check_if_watched_post_is_active(self, watches: List[RepostWatch]):
urls_to_check = []
with self.uowm.start() as uow:
for watch in watches:
post = uow.posts.get_by_post_id(watch.post_id)
if not post:
continue
urls_to_check.append({'url': post.url, 'id': str(watch.id)})
active_urls = batch_check_urls(
urls_to_check, f'{self.config.util_api}/maintenance/removed')
for watch in watches:
if not next(
(x for x in active_urls if x['id'] == str(watch.id)), None):
log.info('Removing watch %s', watch.id)
uow.repostwatch.remove(watch)
uow.commit()