def test_regex_list_filter_function_with_regex(self):
fil = IgnoreFilter(self.ignore_list_contents + [self.png_regex],
is_regex=True)
test_list = ['123456.png', '123456.jpg',
'6789.png'] # only 1 jpg; 2 pngs
filtered = tuple(fil.filter(test_list))
self.assertEqual(len(filtered), 1)
def test_normal_list_filter_function(self):
"""Test that the filter method is the normal one"""
fil = IgnoreFilter(self.ignore_list_contents, is_regex=False)
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = tuple(fil.filter(not_downloaded))
self.assertNotIn(self.ignore_list_contents, filtered)
def test_regex_list_filter_function_no_regex(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=True)
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = tuple(fil.filter(not_downloaded))
# self.assertNotIn(self.ignore_list_contents, filtered)
for ignore in self.ignore_list_contents:
self.assertNotIn(ignore, filtered)
def test_regex_list_filter_function_with_no_regex(self):
# fil = IgnoreFilter(self.ignore_list_contents + [self.png_regex], is_regex=True)
ignore = ['123456.png', '123456.jpg']
fil = IgnoreFilter(ignore, is_regex=True)
test_list = [
'123456.png', '123456.jpg', '6789.png', 'abceasdf.png', 'fff.webm'
]
filtered = tuple(fil.filter(test_list))
# self.assertEqual(len(filtered), 1)
for file in ignore:
with self.subTest(f=file):
self.assertNotIn(file, filtered)
self.assertEqual(len(filtered), len(test_list) - len(ignore))
def test_save_filter_list_no_source_path_error_raised_and_not_saved(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
fil.source_path = None
os.remove(self.ignore_list_path)
with self.assertRaises(ValueError):
fil.save()
self.assertFalse(os.path.exists(self.ignore_list_path))
def test_filter_with_ignore_list(self):
# ignore_list = ['http://i.4cdn.org/wg/1507921740712.jpg', 'https://i.4cdn.org/wg/1506628360792.png']
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = list(
IgnoreFilter.filter_with_ignore_list(not_downloaded,
self.ignore_list_contents))
self.assertNotIn(self.ignore_list_contents, filtered)
self.assertEqual(len(filtered),
len(not_downloaded) - len(self.ignore_list_contents))
def test_instantiate_with_regex_list(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=True)
self.assertNotEqual(fil.filter_list, self.ignore_list_contents)
def test_instantiate_with_normal_list(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=False)
self.assertEqual(fil.filter_list, self.ignore_list_contents)
def test_source_path(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
self.assertEqual(fil.source_path, self.ignore_list_path)
self.assertTrue(os.path.exists(fil.source_path))
def setUp(self):
self.ignore_list_contents = SOME_THREAD_FILE_URLS
self.fil = IgnoreFilter(self.ignore_list_contents)
self.png_regex = r'\w+\.png'
class IgnoreFilterFilteringListsTestCase(unittest.TestCase):
def setUp(self):
self.ignore_list_contents = SOME_THREAD_FILE_URLS
self.fil = IgnoreFilter(self.ignore_list_contents)
self.png_regex = r'\w+\.png'
def test_filter_with_ignore_list_containing_regexp_length(self):
ignore_list = self.ignore_list_contents
regexp_ignore_list = self.fil.convert_filter_list_to_regex_list(
ignore_list)
self.assertEqual(len(regexp_ignore_list),
len(self.ignore_list_contents
)) # check length same as list + png regex
def test_filter_with_ignore_list_containing_regexp(self):
ignore_list = self.ignore_list_contents + [self.png_regex]
# ignore_list = self.fil.load_filter(self.ignore_list_path)
regexp_ignore_list = self.fil.convert_filter_list_to_regex_list(
ignore_list)
for i in range(len(regexp_ignore_list)):
with self.subTest(i=i):
self.assertTrue(not isinstance(regexp_ignore_list[i], str))
def test_regexp_filter_list(self):
ignore_list = self.ignore_list_contents + [self.png_regex]
regexp_ignore_list = self.fil.convert_filter_list_to_regex_list(
ignore_list)
test_list = ['123456.png', '123456.jpg', '6789.png']
result = tuple(
self.fil.filter_with_regexp_list(test_list, regexp_ignore_list))
self.assertEqual(len(result), 1)
def test_filter_with_ignore_list(self):
# ignore_list = ['http://i.4cdn.org/wg/1507921740712.jpg', 'https://i.4cdn.org/wg/1506628360792.png']
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = list(
IgnoreFilter.filter_with_ignore_list(not_downloaded,
self.ignore_list_contents))
self.assertNotIn(self.ignore_list_contents, filtered)
self.assertEqual(len(filtered),
len(not_downloaded) - len(self.ignore_list_contents))
def test_instantiate_with_normal_list(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=False)
self.assertEqual(fil.filter_list, self.ignore_list_contents)
def test_normal_list_filter_function(self):
"""Test that the filter method is the normal one"""
fil = IgnoreFilter(self.ignore_list_contents, is_regex=False)
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = tuple(fil.filter(not_downloaded))
self.assertNotIn(self.ignore_list_contents, filtered)
def test_instantiate_with_regex_list(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=True)
self.assertNotEqual(fil.filter_list, self.ignore_list_contents)
def test_regex_list_filter_function_no_regex(self):
fil = IgnoreFilter(self.ignore_list_contents, is_regex=True)
downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME))
not_downloaded = list(downloader.get_links_not_downloaded())
filtered = tuple(fil.filter(not_downloaded))
# self.assertNotIn(self.ignore_list_contents, filtered)
for ignore in self.ignore_list_contents:
self.assertNotIn(ignore, filtered)
def test_regex_list_filter_function_with_regex(self):
fil = IgnoreFilter(self.ignore_list_contents + [self.png_regex],
is_regex=True)
test_list = ['123456.png', '123456.jpg',
'6789.png'] # only 1 jpg; 2 pngs
filtered = tuple(fil.filter(test_list))
self.assertEqual(len(filtered), 1)
def test_regex_list_filter_function_with_no_regex(self):
# fil = IgnoreFilter(self.ignore_list_contents + [self.png_regex], is_regex=True)
ignore = ['123456.png', '123456.jpg']
fil = IgnoreFilter(ignore, is_regex=True)
test_list = [
'123456.png', '123456.jpg', '6789.png', 'abceasdf.png', 'fff.webm'
]
filtered = tuple(fil.filter(test_list))
# self.assertEqual(len(filtered), 1)
for file in ignore:
with self.subTest(f=file):
self.assertNotIn(file, filtered)
self.assertEqual(len(filtered), len(test_list) - len(ignore))
def test_save_filter_list_no_source_path_error_raised(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
fil.source_path = None
with self.assertRaises(ValueError):
fil.save()
def test_save_filter_list_default_path(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
os.remove(self.ignore_list_path)
fil.save() # No path
self.assertTrue(os.path.exists(self.ignore_list_path))
def test_save_filter_list_custom_path(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
fil.save(self.fake_path) # Custom path
self.assertTrue(os.path.exists(self.fake_path))
def setUp(self):
self.downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME), TMP_DIRECTORY)
self.downloader.ifilter = IgnoreFilter(SOME_THREAD_FILE_URLS) # Just a normal one, without regular expressions
def setUp(self):
self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.thread_dir.name)
self.ignore = [os.path.basename(url) for url in ('http://i.4cdn.org/wg/1489266876258.png', 'http://i.4cdn.org/wg/1489266748255.jpg')]
self.ifilter = IgnoreFilter(self.ignore, is_regex=True)
self.downloader.ifilter = self.ifilter
def test_load_filter_list_regex(self):
fil = IgnoreFilter.load_filter(self.ignore_list_path)
# self.assertEqual(fil.filter_list, self.ignore_list_contents)
self.assertIsNotNone(fil)
self.assertIsNotNone(fil.filter_list)
self.assertTrue(not isinstance(fil.filter_list[0], str))