def test_multiple_imgur_rate_limit_exceeded_with_timeout_dict(
self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(
status_code=429, error_message='error')
img_mock.credits = {'UserRemaining': None}
increment = Const.TIMEOUT_INCREMENT
ie = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
ie.extract_content()
self.assertEqual(ExtractorUtils.time_limit_dict[type(ie).__name__],
increment)
increment += Const.TIMEOUT_INCREMENT
ie = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
ie.extract_content()
self.assertEqual(ExtractorUtils.time_limit_dict[type(ie).__name__],
increment)
increment += Const.TIMEOUT_INCREMENT
ie = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
ie.extract_content()
self.assertEqual(ExtractorUtils.time_limit_dict[type(ie).__name__],
increment)
def test_extract_content_assignment_direct(self, es_mock):
post = MockObjects.get_mock_post_vidble()
post.url = self.extracted_single_url_explore
ve = VidbleExtractor(post, MockObjects.get_blank_user())
ve.extract_content()
es_mock.assert_called()
def test_imgur_does_not_exist_error(self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(status_code=404, error_message='error')
ie = ImgurExtractor(self.get_single_post(), MockObjects.get_blank_user())
ie.extract_content()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Content does not exist' in failed_post.status)
self.assertFalse(len(ie.failed_extracts_to_save) > 0)
def test_extract_content_assignment_single_explore(self, es_mock):
post = MockObjects.get_mock_post_vidble()
post.url = 'https://vidble.com/explore/XOwqxH6Xz9'
ve = VidbleExtractor(post, MockObjects.get_blank_user())
ve.extract_content()
es_mock.assert_called()
def test_extract_content_assignment_direct(self, es_mock):
post = MockObjects.get_mock_post_gfycat()
post.url += '.webm'
ge = GfycatExtractor(post, MockObjects.get_blank_user())
ge.extract_content()
es_mock.assert_called()
def test_extract_content_assignment_direct(self, ex_mock):
post = MockObjects.get_generic_mock_post()
post.url = 'https://imgur.com/fb2yRj0.jpg'
ie = ImgurExtractor(post, MockObjects.get_blank_user())
ie.extract_content()
ex_mock.assert_called()
def test_failed_connection(self, es_mock):
es_mock.side_effect = ConnectionError()
ge = GfycatExtractor(MockObjects.get_mock_post_gfycat(),
MockObjects.get_blank_user())
ge.extract_content()
failed_post = ge.failed_extract_posts[0]
self.assertTrue('Failed to locate content' in failed_post.status)
def test_extract_video_with_audio(self):
post = MockObjects.get_mock_post_reddit_video()
post.is_video = True
fallback_url = post.url + '/DASH_2_4_M?source=fallback'
post.media = {'reddit_video': {'fallback_url': fallback_url}}
re = RedditVideoExtractor(post, MockObjects.get_blank_user())
re.extract_content()
self.assertEqual(2, len(re.extracted_content))
vid_content = re.extracted_content[0]
self.assertEqual(fallback_url, vid_content.url)
self.assertEqual('PublicFreakout', vid_content.subreddit)
self.assertEqual(
'C:/Users/Gorgoth/Downloads/JohnEveryman/abcde(video).mp4',
vid_content.filename)
audio_content = re.extracted_content[1]
self.assertEqual(post.url + '/audio', audio_content.url)
self.assertEqual('PublicFreakout', audio_content.subreddit)
self.assertEqual(
'C:/Users/Gorgoth/Downloads/JohnEveryman/abcde(audio).mp3',
audio_content.filename)
self.assertEqual(0, len(re.failed_extract_posts))
self.assertEqual(1, len(VideoMerger.videos_to_merge))
def test_imgur_over_capacity_error(self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(status_code=500, error_message='Over capacity')
ie = ImgurExtractor(self.get_single_post(), MockObjects.get_blank_user())
ie.extract_content()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('over capacity' in failed_post.status)
self.assertTrue(len(ie.failed_extracts_to_save) > 0)
def test_nsfw_filter_include(self):
post_filter = PostFilter()
user = MockObjects.get_blank_user()
post = MockPrawPost(over_18=True)
self.assertTrue(post_filter.nsfw_filter(post, user))
post = MockPrawPost(over_18=False)
self.assertTrue(post_filter.nsfw_filter(post, user))
def test_extract_direct_unknown_ext(self, ):
post = MockObjects.get_generic_mock_post()
post.url = 'https://i.imgur.com/fb2yRj0.foo'
ie = ImgurExtractor(post, MockObjects.get_blank_user())
ie.extract_direct_link()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Unrecognized extension' in failed_post.status)
def test_extract_single(self, j_mock):
dir_url = 'https://giant.gfycat.com/KindlyElderlyCony.webm'
j_mock.return_value = {'gfyItem': {'webmUrl': dir_url}}
ge = GfycatExtractor(MockObjects.get_mock_post_gfycat(), MockObjects.get_blank_user())
ge.extract_single()
j_mock.assert_called_with('https://gfycat.com/cajax/get/KindlyElderlyCony')
self.check_output(ge, dir_url)
def test_extract_content_assignment_single_explore(self, es_mock):
post = MockObjects.get_mock_post_vidble()
post.url = post.url.replace('show', 'explore')
ve = VidbleExtractor(post, MockObjects.get_blank_user())
ve.extract_content()
es_mock.assert_called()
def test_filter_content(self):
user = MockObjects.get_blank_user()
ex = Extractor(user)
img_content = MockObjects.create_content(user, None, None)
vid_content = MockObjects.create_content(user, None, None)
vid_content.file_ext = '.mp4'
self.assertTrue(ex.filter_content(img_content))
self.assertTrue(ex.filter_content(vid_content))
user.download_images = False
self.assertFalse(ex.filter_content(img_content))
self.assertTrue(ex.filter_content(vid_content))
user.download_images = True
self.assertTrue(ex.filter_content(img_content))
self.assertTrue(ex.filter_content(vid_content))
user.download_videos = False
self.assertTrue(ex.filter_content(img_content))
self.assertFalse(ex.filter_content(vid_content))
user.download_videos = True
user.previous_downloads.append(img_content.url)
self.assertFalse(ex.filter_content(img_content))
user.avoid_duplicates = False
self.assertTrue(ex.filter_content(img_content))
def test_imgur_client_unknown_error_on_setup(self, img_mock, util_mock):
img_mock.side_effect = RuntimeError()
ie = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Failed to connect to imgur.com' in failed_post.status)
self.assertTrue(len(ie.failed_extracts_to_save) > 0)
def test_extract_content_assignment_direct_mislinked(
self, img_mock, ex_mock):
ie = ImgurExtractor(self.get_direct_post_mislinked(),
MockObjects.get_blank_user())
ie.extract_content()
ex_mock.assert_called()
def test_extract_album(self, img_mock):
img_mock.get_album_images.return_value = [MockClientResponse(link=url) for url in self.url_extract_dict['ALBUM']]
ie = ImgurExtractor(self.get_album_post(), MockObjects.get_blank_user())
ie.extract_album()
self.assertTrue(self.check_img_album_output(ie.extracted_content))
self.assertTrue(len(ie.failed_extract_posts) == 0)
def test_imgur_rate_limit_exceeded_credit_dict_is_null(self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(status_code=429, error_message='error')
img_mock.credits = {'UserRemaining': None}
ie = ImgurExtractor(self.get_single_post(), MockObjects.get_blank_user())
ie.extract_content()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('rate limit exceeded' in failed_post.status)
self.assertTrue(len(ie.failed_extracts_to_save) > 0)
def test_extract_content_explore_direct(self, es_mock):
post = MockObjects.get_mock_post_vidble()
post.url = 'https://vidble.com/explore/XOwqxH6Xz9.jpg'
ve = VidbleExtractor(post, MockObjects.get_blank_user())
ve.extract_content()
self.assertEqual('https://vidble.com/XOwqxH6Xz9.jpg', ve.url)
es_mock.assert_called()
def test_imgur_client_connection_error_unknown(self, img_mock, util_mock):
img_mock.side_effect = ImgurClientError(status_code=900,
error_message='Unknown')
ie = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Unknown imgur connection error' in failed_post.status)
self.assertTrue(len(ie.failed_extracts_to_save) > 0)
def test_imgur_failed_to_locate_general(self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(status_code=403, error_message='error')
img_mock.credits = {'ClientRemaining': 900}
ie = ImgurExtractor(self.get_single_post(), MockObjects.get_blank_user())
ie.extract_content()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Failed to locate content' in failed_post.status)
self.assertFalse(len(ie.failed_extracts_to_save) > 0)
def test_imgur_no_credit_error(self, img_mock):
img_mock.get_image.side_effect = ImgurClientError(status_code=403, error_message='error')
img_mock.credits = {'ClientRemaining': 0}
ie = ImgurExtractor(self.get_single_post(), MockObjects.get_blank_user())
ie.extract_content()
failed_post = ie.failed_extract_posts[0]
self.assertTrue('Not enough imgur credits to extract post' in failed_post.status)
self.assertTrue(len(ie.failed_extracts_to_save) > 0)
def test_date_limit_last_download_time(self):
post_filter = PostFilter()
user = MockObjects.get_blank_user()
user.date_limit = MOCK_DATE_LIMIT
post = MockPrawPost(created=MOCK_DATE_LIMIT + 1000)
self.assertTrue(post_filter.date_filter(post, user))
post = MockPrawPost(created=MOCK_DATE_LIMIT - 1000)
self.assertFalse(post_filter.date_filter(post, user))
def test_extract_question(self):
post = MockObjects.get_generic_mock_post()
post.url = 'https://imgur.com/fb2yRj0.jpg?1'
ie = ImgurExtractor(post, MockObjects.get_blank_user())
ie.extract_direct_link()
content = ie.extracted_content[0]
self.assertEqual('https://imgur.com/fb2yRj0.jpg', content.url)
def test_handle_content(self, ex_mock):
mock_user = MockObjects.get_blank_user()
content_list = ['Failed extract one']
content_list.extend(
MockObjects.get_downloadable_content_list(mock_user))
ex_mock.extracted_content = content_list
ex = Extractor(mock_user)
ex.handle_content(ex_mock)
self.assertTrue('Failed extract one' in mock_user.failed_extracts)
self.assertTrue(len(mock_user.content) == 1)
mock_user = MockObjects.get_blank_user()
mock_user.avoid_duplicates = False
ex = Extractor(mock_user)
ex.handle_content(ex_mock)
self.assertTrue(len(mock_user.content) == 5)
def test_imgur_does_not_exist_error(self, img_mock):
img_mock.side_effect = ImgurError(status_code=404)
extractor = ImgurExtractor(self.get_single_post(),
MockObjects.get_blank_user())
extractor.extract_content()
failed_post = extractor.failed_extract_posts[0]
self.assertEqual("Content does not exist.", failed_post.status)
self.assertFalse(len(extractor.failed_extracts_to_save) > 0)
def test_extract_single_show(self, s_mock):
s_mock.return_value = self.get_single_soup_show()
ve = VidbleExtractor(MockObjects.get_mock_post_vidble(),
MockObjects.get_blank_user())
ve.extract_single()
content = ve.extracted_content[0]
self.assertEqual(self.extracted_single_url_show, content.url)
self.check_output(content)
self.assertTrue(len(ve.failed_extract_posts) == 0)
def test_extract_album(self, img_mock):
print(img_mock)
img_mock.return_value = self.url_extract_dict['ALBUM']
ie = ImgurExtractor(self.get_album_post(),
MockObjects.get_blank_user())
ie.extract_album()
img_mock.assert_called_with('Bi63r')
self.assertTrue(self.check_img_album_output(ie.extracted_content))
self.assertTrue(len(ie.failed_extract_posts) == 0)
def test_extract_video_failed_to_find_url(self, rv_mock):
rv_mock.return_value = None
post = MockObjects.get_mock_post_reddit_video()
post.is_video = True
re = RedditVideoExtractor(post, MockObjects.get_blank_user())
re.extract_content()
self.assertEqual(1, len(re.failed_extract_posts))
self.assertEqual(0, len(re.extracted_content))
def test_unsupported_domain(self):
user = MockObjects.get_blank_user()
ex = Extractor(user)
post = MockObjects.get_generic_mock_post()
post.url = 'https://invalid-url.com/a/34ndkoij'
ex.extract(post)
failed_post = user.failed_extracts[0]
self.assertTrue(
failed_post.status.startswith(
'Failed to extract post: Url domain not supported'))
