def test_file_scheme(self):
"""
In this test, we're making sure that file:// URLs work and is reported as succeeded
when the path is valid.
"""
# Test
config = DownloaderConfig(max_concurrent=1)
downloader = HTTPCurlDownloader(config)
request_list = self._file_download_requests()[:1]
listener = AggregatingEventListener()
downloader.event_listener = listener
downloader.download(request_list)
# Verify
self.assertEqual(len(listener.succeeded_reports), 1)
self.assertEqual(len(listener.failed_reports), 0)
self.assertTrue(os.path.exists(request_list[0].destination))
# verify the downloaded file matches
path_in = urlparse.urlparse(request_list[0].url).path
fp = open(path_in)
original_content = fp.read()
fp.close()
fp = open(request_list[0].destination)
destination_content = fp.read()
fp.close()
self.assertEqual(original_content, destination_content)
def test_calls__process_completed_download_err_list(self, _process_completed_download):
"""
In this test, we assert that download() calls _process_completed_download() correctly when pycurl
reports a completed download through the error list.
"""
# If we set max_concurrent to 1, it's easy to deterministically find the mocked curl for assertions
# later
config = DownloaderConfig(max_concurrent=1)
curl_downloader = HTTPCurlDownloader(config)
request_list = self._download_requests()[:1]
curl_downloader.download(request_list)
mock_easy_handle = pycurl.Curl.mock_objs[0]
mock_multi_handle = pycurl.CurlMulti.mock_objs[0]
self.assertEqual(_process_completed_download.call_count, 1)
args = _process_completed_download.mock_calls[0][1]
# There should be four args, since there were errors
self.assertEqual(len(args), 4)
# Now let's assert that the arguments were correct
self.assertEqual(args[0], mock_easy_handle)
self.assertEqual(args[1], mock_multi_handle)
# There should be no free handles, since there was only one and it's being reported on
self.assertEqual(args[2], [])
# Assert that the error condition was passed
self.assertEqual(args[3], {'code': 999, 'message': 'ERROR!'})
def main():
test_names = _get_test_names()
for name in test_names:
url_list = TESTS[name]()
print '%s: download %d files from %s' % (name.upper(), len(url_list), url_list[0].rsplit('/', 1)[0])
config = DownloaderConfig()
download_dir = tempfile.mkdtemp(prefix=name+'-')
request_list = [DownloadRequest(url, os.path.join(download_dir, _filename_from_url(url))) for url in url_list]
downloader = HTTPCurlDownloader(config, TestDownloadEventListener())
downloader.download(request_list)
def test_is_canceled_false(self):
"""
In this test, we leave the is_canceled boolean unset on the downloader, and we verify that the main
loop executes once. Because our pycurl mocks "download" the entire file in one go, it will only
execute one time, which means we can simply count that the select() call was made exactly once.
"""
config = DownloaderConfig()
curl_downloader = HTTPCurlDownloader(config)
request_list = self._download_requests()[:1]
curl_downloader.download(request_list)
mock_multi_curl = pycurl.CurlMulti.mock_objs[0]
# The call_count on the select() should be 1 since our pycurl Mock "downloads" the file in one go
self.assertEqual(mock_multi_curl.select.call_count, 1)
def test_is_canceled_true(self):
"""
In this test, we set the is_canceled boolean on the downloader, and we verify that the main loop
does not execute.
"""
config = DownloaderConfig()
curl_downloader = HTTPCurlDownloader(config)
# Let's go ahead and set the cancellation flag, so the loop should not execute
curl_downloader.cancel()
request_list = self._download_requests()[:1]
curl_downloader.download(request_list)
mock_multi_curl = pycurl.CurlMulti.mock_objs[0]
# Because we cancelled the download, the call_count on the select() should be 0
self.assertEqual(mock_multi_curl.select.call_count, 0)
def main():
test_names = _get_test_names()
for name in test_names:
url_list = TESTS[name]()
print '%s: download %d files from %s' % (name.upper(), len(url_list),
url_list[0].rsplit('/', 1)[0])
config = DownloaderConfig()
download_dir = tempfile.mkdtemp(prefix=name + '-')
request_list = [
DownloadRequest(
url, os.path.join(download_dir, _filename_from_url(url)))
for url in url_list
]
downloader = HTTPCurlDownloader(config, TestDownloadEventListener())
downloader.download(request_list)
def test_file_scheme_with_invalid_path(self):
"""
In this test, we're making sure that file:// URLs work and is reported as failed
when the path is invalid.
"""
# Test
config = DownloaderConfig(max_concurrent=1)
downloader = HTTPCurlDownloader(config)
request_list = self._file_download_requests()[:1]
request_list[0].url += 'BADPATHBADPATHBADPATH' # booger up the path
listener = AggregatingEventListener()
downloader.event_listener = listener
downloader.download(request_list)
# Verify
self.assertEqual(len(listener.succeeded_reports), 0)
self.assertEqual(len(listener.failed_reports), 1)
report = listener.failed_reports[0]
self.assertEqual(report.bytes_downloaded, 0)
self.assertEqual(report.error_report['response_code'], 0)
def demo(demo_name):
downloader_config = DownloaderConfig(max_concurrent=None)
downloader = HTTPCurlDownloader(downloader_config, DemoEventListener())
storage_dir = mkdtemp(prefix=demo_name)
url_list = URLS_MAP[demo_name]()
request_list = requests_from_urls(storage_dir, url_list)
print demo_name.upper(), 'Demo'
print 'downloading %d files to %s' % (len(url_list), storage_dir)
print '=' * 80
start_time = datetime.now()
report_list = downloader.download(request_list)
run_time = datetime.now() - start_time
print '%s downloaded %d files: %s' % (demo_name, len(report_list), str(run_time))
