class BatchDownloaderTestCase(unittest.TestCase):
# pass
def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
self.download_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
self.destination_directory = self.download_dir.name
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
#self.thread_download_directory = os.path.join(self.destination_directory, BatchDownloader.THREAD_SAVE_NAME)
# def tearDown(self):
# # Delete all downloaded files
# utilities.delete_directory_tree(self.destination_directory)
def test_files_to_download(self):
pass
'''
In unit tests, downloaded shouldn't really download.
Instead, it should just present a list of things to be downloaded,
or the paths of the files that have been downloaded.
'''
def test_download_html(self):
self.downloader.save_html() # Downloads the HTML to the destination
self.assertTrue(os.path.exists(os.path.join(self.destination_directory, BatchDownloader.THREAD_SAVE_NAME)))
def test_pickle_details_save_exists(self):
self.downloader.pickle_details()
self.assertTrue(os.path.exists(os.path.join(self.destination_directory, BatchDownloader.THREAD_DETAILS_FILENAME)))
def test_load_details(self):
down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name)
details = down.construct_details_dict()
down.pickle_details()
loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path())
self.assertTrue(isinstance(loaded, dict))
self.assertEqual(loaded, details)
def test_compare_details(self):
down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name)
details = down.construct_details_dict()
down.pickle_details()
loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path())
self.assertEqual(loaded['last-modified'], details['last-modified'])
self.assertEqual(loaded['url'], details['url'])
self.assertEqual(loaded['thread_alive'], details['thread_alive'])
def test_pickle_details_custom_details(self):
download_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
custom_details = {'last-modified':'123456', 'thread_alive': False, 'url':THREAD_URL}
down = BatchDownloader(self.linkser, download_dir.name)
#details = down.construct_details_dict()
down.pickle_details(custom_details)
loaded = BatchDownloader.load_details_into_dict(down.get_details_path())
self.assertEqual(loaded['last-modified'], custom_details['last-modified'])
self.assertEqual(loaded['url'], custom_details['url'])
self.assertEqual(loaded['thread_alive'], custom_details['thread_alive'])
def create_test_environment(dirname,
num_files_to_download=0,
url=TEST_THREAD_FILENAME):
utilities.create_directory_tree(dirname)
# Create pickle
downloader = BatchDownloader(LinksRetriever(url), dirname)
downloader.pickle_details()
# Create filter
ifilter = utilities.IgnoreFilter(['\w+\.png'], is_regex=True)
ifilter.save(os.path.join(dirname, downloader.IGNORE_LIST_FILENAME))
# Download the first/top 3 images from the thread
for url in downloader.links()[:num_files_to_download]:
utilities.download_file(url, dirname)
class BatchDownloaderDetailsTestCase(unittest.TestCase):
def setUp(self):
self.linkser = LinksRetriever(THREAD_URL)
self.destination_directory = os.path.expanduser(TMP_DIRECTORY)
self.downloader = BatchDownloader(self.linkser, self.destination_directory)
self.downloader.pickle_details()
def test_construct_details_dict(self):
details = self.downloader.construct_details_dict()
keys = details.keys()
self.assertIn('last-modified', keys)
self.assertTrue(isinstance(details['last-modified'], str))
self.assertIn('url', keys)
self.assertTrue(isinstance(details['url'], str))
self.assertIn('thread_alive', keys)
self.assertTrue(isinstance(details['thread_alive'], bool))
def test_load_details(self):
down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name)
details = down.construct_details_dict()
down.pickle_details()
loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path())
self.assertTrue(isinstance(loaded, dict))
self.assertEqual(loaded, details)
def test_compare_details(self):
down = BatchDownloader(self.linkser, TemporaryDirectory(dir=TMP_DIRECTORY).name)
details = down.construct_details_dict()
down.pickle_details()
loaded = BatchDownloader.load_details_into_dict(self.downloader.get_details_path())
self.assertEqual(loaded['last-modified'], details['last-modified'])
self.assertEqual(loaded['url'], details['url'])
self.assertEqual(loaded['thread_alive'], details['thread_alive'])
def test_pickle_details_custom_details(self):
download_dir = TemporaryDirectory(dir=TMP_DIRECTORY)
custom_details = {'last-modified':'123456', 'thread_alive': False, 'url':THREAD_URL}
down = BatchDownloader(self.linkser, download_dir.name)
#details = down.construct_details_dict()
down.pickle_details(custom_details)
loaded = BatchDownloader.load_details_into_dict(down.get_details_path())
self.assertEqual(loaded['last-modified'], custom_details['last-modified'])
self.assertEqual(loaded['url'], custom_details['url'])
self.assertEqual(loaded['thread_alive'], custom_details['thread_alive'])
