class BatchDownloaderTestCase(unittest.TestCase): # pass def setUp(self): self.linkser = LinksRetriever(THREAD_URL) self.download_dir = TemporaryDirectory(dir=TMP_DIRECTORY) self.destination_directory = self.download_dir.name self.downloader = BatchDownloader(self.linkser, self.destination_directory) #self.thread_download_directory = os.path.join(self.destination_directory, BatchDownloader.THREAD_SAVE_NAME) # def tearDown(self): # # Delete all downloaded files # utilities.delete_directory_tree(self.destination_directory) def test_files_to_download(self): pass ''' In unit tests, downloaded shouldn't really download. Instead, it should just present a list of things to be downloaded, or the paths of the files that have been downloaded. ''' def test_download_html(self): self.downloader.save_html() # Downloads the HTML to the destination self.assertTrue(os.path.exists(os.path.join(self.destination_directory, BatchDownloader.THREAD_SAVE_NAME))) def test_pickle_details_save_exists(self): self.downloader.pickle_details() self.assertTrue(os.path.exists(os.path.join(self.destination_directory, BatchDownloader.THREAD_DETAILS_FILENAME)))
def test_load_details(self): down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name) details = down.construct_details_dict() down.pickle_details() loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path()) self.assertTrue(isinstance(loaded, dict)) self.assertEqual(loaded, details)
def test_compare_details(self): down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name) details = down.construct_details_dict() down.pickle_details() loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path()) self.assertEqual(loaded['last-modified'], details['last-modified']) self.assertEqual(loaded['url'], details['url']) self.assertEqual(loaded['thread_alive'], details['thread_alive'])
def test_pickle_details_custom_details(self): download_dir = TemporaryDirectory(dir=TMP_DIRECTORY) custom_details = {'last-modified':'123456', 'thread_alive': False, 'url':THREAD_URL} down = BatchDownloader(self.linkser, download_dir.name) #details = down.construct_details_dict() down.pickle_details(custom_details) loaded = BatchDownloader.load_details_into_dict(down.get_details_path()) self.assertEqual(loaded['last-modified'], custom_details['last-modified']) self.assertEqual(loaded['url'], custom_details['url']) self.assertEqual(loaded['thread_alive'], custom_details['thread_alive'])
def create_test_environment(dirname, num_files_to_download=0, url=TEST_THREAD_FILENAME): utilities.create_directory_tree(dirname) # Create pickle downloader = BatchDownloader(LinksRetriever(url), dirname) downloader.pickle_details() # Create filter ifilter = utilities.IgnoreFilter(['\w+\.png'], is_regex=True) ifilter.save(os.path.join(dirname, downloader.IGNORE_LIST_FILENAME)) # Download the first/top 3 images from the thread for url in downloader.links()[:num_files_to_download]: utilities.download_file(url, dirname)
class BatchDownloaderDetailsTestCase(unittest.TestCase): def setUp(self): self.linkser = LinksRetriever(THREAD_URL) self.destination_directory = os.path.expanduser(TMP_DIRECTORY) self.downloader = BatchDownloader(self.linkser, self.destination_directory) self.downloader.pickle_details() def test_construct_details_dict(self): details = self.downloader.construct_details_dict() keys = details.keys() self.assertIn('last-modified', keys) self.assertTrue(isinstance(details['last-modified'], str)) self.assertIn('url', keys) self.assertTrue(isinstance(details['url'], str)) self.assertIn('thread_alive', keys) self.assertTrue(isinstance(details['thread_alive'], bool)) def test_load_details(self): down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name) details = down.construct_details_dict() down.pickle_details() loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path()) self.assertTrue(isinstance(loaded, dict)) self.assertEqual(loaded, details) def test_compare_details(self): down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name) details = down.construct_details_dict() down.pickle_details() loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path()) self.assertEqual(loaded['last-modified'], details['last-modified']) self.assertEqual(loaded['url'], details['url']) self.assertEqual(loaded['thread_alive'], details['thread_alive']) def test_pickle_details_custom_details(self): download_dir = TemporaryDirectory(dir=TMP_DIRECTORY) custom_details = {'last-modified':'123456', 'thread_alive': False, 'url':THREAD_URL} down = BatchDownloader(self.linkser, download_dir.name) #details = down.construct_details_dict() down.pickle_details(custom_details) loaded = BatchDownloader.load_details_into_dict(down.get_details_path()) self.assertEqual(loaded['last-modified'], custom_details['last-modified']) self.assertEqual(loaded['url'], custom_details['url']) self.assertEqual(loaded['thread_alive'], custom_details['thread_alive'])