def test_filter_download_submitter_download_author_only(self):
user = get_user(extract_comments=CommentDownload.DOWNLOAD,
download_comments=CommentDownload.DOWNLOAD_ONLY_AUTHOR,
download_comment_content=CommentDownload.DOWNLOAD)
comment = MagicMock()
comment.is_submitter = True
self.assertTrue(self.filter.filter_download(comment, user))
def test_nsfw_filter_exclude(self):
user = get_user(download_nsfw=NsfwFilter.EXCLUDE)
submission = MockPrawSubmission(over_18=False)
self.assertTrue(self.sub_filter.nsfw_filter(submission, user))
submission.over_18 = True
self.assertFalse(self.sub_filter.nsfw_filter(submission, user))
def test_filter_extraction_submitter_do_not_extract_do_download(self):
user = get_user(extract_comments=CommentDownload.DO_NOT_DOWNLOAD,
download_comments=CommentDownload.DOWNLOAD,
download_comment_content=CommentDownload.DOWNLOAD)
comment = MagicMock()
comment.is_submitter = True
self.assertTrue(self.filter.filter_extraction(comment, user))
def test_nsfw_filter_only(self):
user = get_user(download_nsfw=NsfwFilter.ONLY)
submission = MockPrawSubmission(over_18=True)
self.assertTrue(self.sub_filter.nsfw_filter(submission, user))
submission.over_18 = False
self.assertFalse(self.sub_filter.nsfw_filter(submission, user))
def test_filter_content_download_non_submitter_download_all(self):
user = get_user(extract_comments=CommentDownload.DOWNLOAD,
download_comments=CommentDownload.DOWNLOAD,
download_comment_content=CommentDownload.DOWNLOAD)
comment = MagicMock()
comment.is_submitter = False
self.assertTrue(self.filter.filter_content_download(comment, user))
def test_get_submissions_with_new_pinned_posts_restricted_download(self, get_raw_submissions, reddit_utils):
allowed_subreddit = get_subreddit(name='allowed')
setattr(allowed_subreddit, 'display_name', 'allowed')
forbidden_subreddit = get_subreddit(name='forbidden')
setattr(forbidden_subreddit, 'display_name', 'forbidden')
user = get_user(absolute_date_limit=self.now - timedelta(days=10))
mock_submissions = []
for x in range(2):
mock_submissions.append(MockPrawSubmission(created=self.now - timedelta(days=x), pinned=True,
subreddit=allowed_subreddit))
for x in range(2, 20):
mock_submissions.append(MockPrawSubmission(created=self.now - timedelta(days=x),
subreddit=allowed_subreddit))
for x in range(2, 20):
mock_submissions.append(MockPrawSubmission(created=self.now - timedelta(days=x),
subreddit=forbidden_subreddit))
get_raw_submissions.return_value = self.submission_generator(mock_submissions)
self.assertEqual(38, len(mock_submissions))
download_runner = DownloadRunner()
download_runner.filter_subreddits = True
download_runner.validated_subreddits.append(allowed_subreddit.display_name)
submissions = download_runner.get_submissions(None, user)
self.assertEqual(10, len(submissions))
pinned = 0
for sub in submissions:
self.assertGreater(sub.created, user.absolute_date_limit.timestamp())
self.assertNotEqual(sub.subreddit, forbidden_subreddit)
if sub.pinned:
pinned += 1
self.assertEqual(2, pinned)
def test_filter_extraction_non_submitter_download_all_author_only(self):
user = get_user(
extract_comments=CommentDownload.DOWNLOAD_ONLY_AUTHOR,
download_comments=CommentDownload.DOWNLOAD_ONLY_AUTHOR,
download_comment_content=CommentDownload.DOWNLOAD_ONLY_AUTHOR)
comment = MagicMock()
comment.is_submitter = False
self.assertFalse(self.filter.filter_extraction(comment, user))
def test_filter_file_type_video_cross_contamination(self):
user = get_user(download_videos=False)
post = get_post(author=user, significant=user)
ext = 'mp4'
f = self.content_filter.filter_file_type(post, ext)
self.assertFalse(f)
self.assertTrue(self.content_filter.filter_file_type(post, 'jpg'))
self.assertTrue(self.content_filter.filter_file_type(post, 'webm'))
def test_date_filter_absolute_limit(self):
limit = datetime.now() - timedelta(days=10)
date = datetime.now() - timedelta(days=5)
user = get_user(absolute_date_limit=limit, date_limit=None)
submission = MockPrawSubmission(created=date)
self.assertTrue(self.sub_filter.date_filter(submission, user))
submission.created = (date - timedelta(days=20)).timestamp()
self.assertFalse(self.sub_filter.date_filter(submission, user))
def test_score_filter_no_limit_restriction(self):
user = get_user(post_score_limit_operator=LimitOperator.NO_LIMIT,
post_score_limit=1000)
submission = MockPrawSubmission(score=2000)
f = self.sub_filter.score_filter(submission, user)
self.assertTrue(f)
submission.score = 500
self.assertTrue(self.sub_filter.score_filter(submission, user))
def test_filter_extraction_submitter_download_content_submitter_only_no_others(
self):
user = get_user(
extract_comments=CommentDownload.DO_NOT_DOWNLOAD,
download_comments=CommentDownload.DO_NOT_DOWNLOAD,
download_comment_content=CommentDownload.DOWNLOAD_ONLY_AUTHOR)
comment = MagicMock()
comment.is_submitter = True
self.assertTrue(self.filter.filter_extraction(comment, user))
def test_score_filter_less_than(self):
user = get_user(post_score_limit_operator=LimitOperator.LESS_THAN,
post_score_limit=1000)
submission = MockPrawSubmission(score=500)
self.assertTrue(self.sub_filter.score_filter(submission, user))
submission.score = 1000
self.assertTrue(self.sub_filter.score_filter(submission, user))
submission.score = 2000
self.assertFalse(self.sub_filter.score_filter(submission, user))
def test_filter_duplicate_url_already_in_db_do_not_avoid_duplicates(self):
db = DatabaseHandler(in_memory=True)
with db.get_scoped_session() as session:
user = get_user(avoid_duplicates=False)
content = get_content(user=user)
post = content.post
session.add(content)
session.commit()
self.assertEqual(post.significant_reddit_object, user)
f = self.content_filter.filter_duplicate(post, content.url)
self.assertTrue(f)
def test_filter_by_score_no_limit_restriction(self):
user = get_user(comment_score_limit_operator=LimitOperator.NO_LIMIT,
comment_score_limit=1000)
comment = MagicMock()
comment.score = 2000
self.assertTrue(self.filter.filter_score_limit(comment, user))
comment.score = 1000
self.assertTrue(self.filter.filter_score_limit(comment, user))
comment.score = 500
self.assertTrue(self.filter.filter_score_limit(comment, user))
def test_filter_by_score_less_than(self):
user = get_user(comment_score_limit_operator=LimitOperator.LESS_THAN,
comment_score_limit=1000)
comment = MagicMock()
comment.score = 2000
self.assertFalse(self.filter.filter_score_limit(comment, user))
comment.score = 1000
self.assertTrue(self.filter.filter_score_limit(comment, user))
comment.score = 500
self.assertTrue(self.filter.filter_score_limit(comment, user))
def test_get_submissions_with_no_stickied_posts(self, get_raw_submissions, reddit_utils):
user = get_user(absolute_date_limit=self.now - timedelta(days=10))
mock_submissions = []
for x in range(20):
mock_submissions.append(MockPrawSubmission(created=self.now - timedelta(days=x)))
get_raw_submissions.return_value = self.submission_generator(mock_submissions)
self.assertEqual(20, len(mock_submissions))
download_runner = DownloadRunner()
submissions = download_runner.get_submissions(None, user)
self.assertEqual(10, len(submissions))
for sub in submissions:
self.assertGreater(sub.created, user.absolute_date_limit.timestamp())
def test_filter_file_type_gifs_allowed(self):
user = get_user(download_gifs=True)
post = get_post(author=user, significant=user)
ext = 'webm'
f = self.content_filter.filter_file_type(post, ext)
self.assertTrue(f)
def test_filter_file_type_image_not_allowed(self):
user = get_user(download_images=False)
post = get_post(author=user, significant=user)
ext = 'jpg'
f = self.content_filter.filter_file_type(post, ext)
self.assertFalse(f)