def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
#self.destination_directory = os.path.expanduser('~/Downloads/TestDownloadThread/')
self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.destination_directory = self.thread_dir.name
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
self.download_files()
class BatchDownloaderDownloadingTestCase(unittest.TestCase):
GET_NUM_FILES = 5
def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
#self.destination_directory = os.path.expanduser('~/Downloads/TestDownloadThread/')
self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.destination_directory = self.thread_dir.name
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
self.download_files()
def download_files(self):
for url in self.linkser.get_all_file_links()[:self.GET_NUM_FILES]:
download_path = utilities.download_file(url, self.downloader.destination_folder)
assert os.path.exists(download_path)
def test_files_downloaded(self):
downloaded = self.downloader.get_files_downloaded()
self.assertEqual(len(downloaded), self.GET_NUM_FILES)
def test_files_not_downloaded_gen(self):
not_downloaded = self.downloader.links_not_downloaded()
self.assertIsNotNone(not_downloaded)
not_downloaded_tuple = tuple(not_downloaded)
self.assertEqual(len(not_downloaded_tuple), len(self.downloader.files_to_download) - len(self.downloader.get_files_downloaded()))
def test_compare_downloaded(self):
not_downloaded = tuple(self.downloader.get_links_not_downloaded())
self.assertEqual(len(not_downloaded), len(self.linkser.get_all_file_links()) - self.GET_NUM_FILES)
def setUp(self):
BatchDownloader.DBG_DOWNLOAD = True
self.expired_thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.alive_thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
# create_test_environment(self.expired_thread_dir.name, 0, EXPIRED_THREAD_URL)
# LinksRetriever now raises error when it reaches a 404'd thread
# self.dead_downloader = BatchDownloader(LinksRetriever(EXPIRED_THREAD_URL), self.expired_thread_dir.name)
self.alive_downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.alive_thread_dir.name)
def test_add_recent_exceed(self):
"""The idea is that when the number of recent threads exceed the config's maximum,
the list pop the first element off, before adding the new one in"""
self.assertEqual(len(self.climan.recent_threads),
self.climan.DEFAULT_MAX_RECENT_THREADS)
downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL),
self.alive_thread_dir.name)
self.climan.recent_threads_add(downloader)
# Still the same length
self.assertEqual(len(self.climan.recent_threads),
self.climan.DEFAULT_MAX_RECENT_THREADS)
# The alive thread is the last element
self.assertEqual(self.climan.recent_threads[-1],
os.path.abspath(self.alive_thread_dir.name))
def create_test_environment(dirname,
num_files_to_download=0,
url=TEST_THREAD_FILENAME):
utilities.create_directory_tree(dirname)
# Create pickle
downloader = BatchDownloader(LinksRetriever(url), dirname)
downloader.pickle_details()
# Create filter
ifilter = utilities.IgnoreFilter(['\w+\.png'], is_regex=True)
ifilter.save(os.path.join(dirname, downloader.IGNORE_LIST_FILENAME))
# Download the first/top 3 images from the thread
for url in downloader.links()[:num_files_to_download]:
utilities.download_file(url, dirname)
def cli_download_to_directory(self, args):
"""
Start downloading thread using a namespace arg
:param args: Namespace
:return: string containing destination folder of downloaded thread
"""
links_retriever = LinksRetriever(args.url)
if args.dir is None:
thread_dir = os.path.join(self.DEFAULT_DOWNLOAD_DIR,
links_retriever.board_initials,
links_retriever.thread_id)
args.dir = thread_dir
downloader = BatchDownloader(links_retriever, args.dir)
self.downloader_start(downloader)
self.recent_threads_add(downloader)
self.save_config()
return downloader.destination_folder
def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
self.download_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.destination_directory = self.download_dir.name
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
def setUp(self):
self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.thread_dir.name)
self.ignore = [os.path.basename(url) for url in ('http://i.4cdn.org/wg/1489266876258.png', 'http://i.4cdn.org/wg/1489266748255.jpg')]
self.ifilter = IgnoreFilter(self.ignore, is_regex=True)
self.downloader.ifilter = self.ifilter
def test_instantiate_with_destination_folder(self):
getter = LinksRetriever(TEST_THREAD_FILENAME)
destination_dir = os.path.expanduser('~/Downloads/')
downloader = BatchDownloader(getter, destination_dir)
self.assertEqual(downloader.destination_folder, destination_dir)
def setUp(self):
self.downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME), TMP_DIRECTORY)
self.downloader.ifilter = IgnoreFilter(SOME_THREAD_FILE_URLS) # Just a normal one, without regular expressions
def test_instantiate_with_link_retriever(self):
getter = LinksRetriever(TEST_THREAD_FILENAME)
downloader = BatchDownloader(getter)
self.assertEqual(downloader.links_retriever, getter)
def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
self.destination_directory = os.path.expanduser(TMP_DIRECTORY)
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
self.downloader.pickle_details()