def demo(demo_name): downloader_config = DownloaderConfig(max_concurrent=None) downloader = HTTPCurlDownloader(downloader_config, DemoEventListener()) storage_dir = mkdtemp(prefix=demo_name) url_list = URLS_MAP[demo_name]() request_list = requests_from_urls(storage_dir, url_list) print demo_name.upper(), 'Demo' print 'downloading %d files to %s' % (len(url_list), storage_dir) print '=' * 80 start_time = datetime.now() report_list = downloader.download(request_list) run_time = datetime.now() - start_time print '%s downloaded %d files: %s' % (demo_name, len(report_list), str(run_time))
def main(): test_names = _get_test_names() for name in test_names: url_list = TESTS[name]() print '%s: download %d files from %s' % (name.upper(), len(url_list), url_list[0].rsplit('/', 1)[0]) config = DownloaderConfig() download_dir = tempfile.mkdtemp(prefix=name + '-') request_list = [ DownloadRequest( url, os.path.join(download_dir, _filename_from_url(url))) for url in url_list ] downloader = HTTPCurlDownloader(config, TestDownloadEventListener()) downloader.download(request_list)