def setUp(self): self.linkser = LinksRetriever(THREAD_URL) #self.destination_directory = os.path.expanduser('~/Downloads/TestDownloadThread/') self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.destination_directory = self.thread_dir.name self.downloader = BatchDownloader(self.linkser, self.destination_directory) self.download_files()
class BatchDownloaderDownloadingTestCase(unittest.TestCase): GET_NUM_FILES = 5 def setUp(self): self.linkser = LinksRetriever(THREAD_URL) #self.destination_directory = os.path.expanduser('~/Downloads/TestDownloadThread/') self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.destination_directory = self.thread_dir.name self.downloader = BatchDownloader(self.linkser, self.destination_directory) self.download_files() def download_files(self): for url in self.linkser.get_all_file_links()[:self.GET_NUM_FILES]: download_path = utilities.download_file(url, self.downloader.destination_folder) assert os.path.exists(download_path) def test_files_downloaded(self): downloaded = self.downloader.get_files_downloaded() self.assertEqual(len(downloaded), self.GET_NUM_FILES) def test_files_not_downloaded_gen(self): not_downloaded = self.downloader.links_not_downloaded() self.assertIsNotNone(not_downloaded) not_downloaded_tuple = tuple(not_downloaded) self.assertEqual(len(not_downloaded_tuple), len(self.downloader.files_to_download) - len(self.downloader.get_files_downloaded())) def test_compare_downloaded(self): not_downloaded = tuple(self.downloader.get_links_not_downloaded()) self.assertEqual(len(not_downloaded), len(self.linkser.get_all_file_links()) - self.GET_NUM_FILES)
def setUp(self): BatchDownloader.DBG_DOWNLOAD = True self.expired_thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.alive_thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY) # create_test_environment(self.expired_thread_dir.name, 0, EXPIRED_THREAD_URL) # LinksRetriever now raises error when it reaches a 404'd thread # self.dead_downloader = BatchDownloader(LinksRetriever(EXPIRED_THREAD_URL), self.expired_thread_dir.name) self.alive_downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.alive_thread_dir.name)
def test_add_recent_exceed(self): """The idea is that when the number of recent threads exceed the config's maximum, the list pop the first element off, before adding the new one in""" self.assertEqual(len(self.climan.recent_threads), self.climan.DEFAULT_MAX_RECENT_THREADS) downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.alive_thread_dir.name) self.climan.recent_threads_add(downloader) # Still the same length self.assertEqual(len(self.climan.recent_threads), self.climan.DEFAULT_MAX_RECENT_THREADS) # The alive thread is the last element self.assertEqual(self.climan.recent_threads[-1], os.path.abspath(self.alive_thread_dir.name))
def create_test_environment(dirname, num_files_to_download=0, url=TEST_THREAD_FILENAME): utilities.create_directory_tree(dirname) # Create pickle downloader = BatchDownloader(LinksRetriever(url), dirname) downloader.pickle_details() # Create filter ifilter = utilities.IgnoreFilter(['\w+\.png'], is_regex=True) ifilter.save(os.path.join(dirname, downloader.IGNORE_LIST_FILENAME)) # Download the first/top 3 images from the thread for url in downloader.links()[:num_files_to_download]: utilities.download_file(url, dirname)
def cli_download_to_directory(self, args): """ Start downloading thread using a namespace arg :param args: Namespace :return: string containing destination folder of downloaded thread """ links_retriever = LinksRetriever(args.url) if args.dir is None: thread_dir = os.path.join(self.DEFAULT_DOWNLOAD_DIR, links_retriever.board_initials, links_retriever.thread_id) args.dir = thread_dir downloader = BatchDownloader(links_retriever, args.dir) self.downloader_start(downloader) self.recent_threads_add(downloader) self.save_config() return downloader.destination_folder
def setUp(self): self.linkser = LinksRetriever(THREAD_URL) self.download_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.destination_directory = self.download_dir.name self.downloader = BatchDownloader(self.linkser, self.destination_directory)
def setUp(self): self.thread_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.downloader = BatchDownloader(LinksRetriever(STICKY_THREAD_URL), self.thread_dir.name) self.ignore = [os.path.basename(url) for url in ('http://i.4cdn.org/wg/1489266876258.png', 'http://i.4cdn.org/wg/1489266748255.jpg')] self.ifilter = IgnoreFilter(self.ignore, is_regex=True) self.downloader.ifilter = self.ifilter
def test_instantiate_with_destination_folder(self): getter = LinksRetriever(TEST_THREAD_FILENAME) destination_dir = os.path.expanduser('~/Downloads/') downloader = BatchDownloader(getter, destination_dir) self.assertEqual(downloader.destination_folder, destination_dir)
def setUp(self): self.downloader = BatchDownloader(LinksRetriever(TEST_THREAD_FILENAME), TMP_DIRECTORY) self.downloader.ifilter = IgnoreFilter(SOME_THREAD_FILE_URLS) # Just a normal one, without regular expressions
def test_instantiate_with_link_retriever(self): getter = LinksRetriever(TEST_THREAD_FILENAME) downloader = BatchDownloader(getter) self.assertEqual(downloader.links_retriever, getter)
def setUp(self): self.linkser = LinksRetriever(THREAD_URL) self.destination_directory = os.path.expanduser(TMP_DIRECTORY) self.downloader = BatchDownloader(self.linkser, self.destination_directory) self.downloader.pickle_details()