def test_download_multi_file(self): config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) request_list = self._download_requests() downloader.download(request_list) # this is really just testing my mock curl objects, but it's nice to know for file_name, file_size in zip(self.file_list, self.file_sizes): input_file = os.path.join(self.data_dir, file_name) input_file_size = os.stat(input_file)[6] output_file = os.path.join(self.storage_dir, file_name) output_file_size = os.stat(output_file)[6] self.assertEqual(input_file_size, file_size) self.assertEqual(output_file_size, file_size) # does check that close() was called properly mock_curl_multi = pycurl.CurlMulti.mock_objs[0] self.assertEqual(mock_curl_multi.perform.call_count, 1) num_unused_curl_objs = max(0, curl_backend.DEFAULT_MAX_CONCURRENT - len(self.file_list)) unused_mock_curl_objs = pycurl.Curl.mock_objs[:num_unused_curl_objs] for mock_curl in unused_mock_curl_objs: self.assertEqual(mock_curl.perform.call_count, 0) used_mock_curl_objs = pycurl.Curl.mock_objs[num_unused_curl_objs:] for mock_curl in used_mock_curl_objs: self.assertEqual(mock_curl.perform.call_count, 1)
def __init__(self, sync_conduit, config): self.sync_conduit = sync_conduit self._remove_missing_units = config.get(constants.CONFIG_REMOVE_MISSING_UNITS, default=False) self._repo_url = encode_unicode(config.get(constants.CONFIG_FEED_URL)) self._validate_downloads = config.get(constants.CONFIG_VALIDATE_DOWNLOADS, default=True) # Cast our config parameters to the correct types and use them to build a Downloader max_speed = config.get(constants.CONFIG_MAX_SPEED) if max_speed is not None: max_speed = float(max_speed) num_threads = config.get(constants.CONFIG_NUM_THREADS) if num_threads is not None: num_threads = int(num_threads) else: num_threads = constants.DEFAULT_NUM_THREADS downloader_config = { 'max_speed': max_speed, 'num_threads': num_threads, 'ssl_client_cert': config.get(constants.CONFIG_SSL_CLIENT_CERT), 'ssl_client_key': config.get(constants.CONFIG_SSL_CLIENT_KEY), 'ssl_ca_cert': config.get(constants.CONFIG_SSL_CA_CERT), 'ssl_verify_host': 1, 'ssl_verify_peer': 1, 'proxy_url': config.get(constants.CONFIG_PROXY_URL), 'proxy_port': config.get(constants.CONFIG_PROXY_PORT), 'proxy_user': config.get(constants.CONFIG_PROXY_USER), 'proxy_password': config.get(constants.CONFIG_PROXY_PASSWORD)} downloader_config = DownloaderConfig(protocol='https', **downloader_config) # We will pass self as the event_listener, so that we can receive the callbacks in this class self.downloader = factory.get_downloader(downloader_config, self) self.progress_report = SyncProgressReport(sync_conduit)
def test_download_destination_file_like_object(self): """ We want to assert that we can download URLs to file-like objects, and not just to filesystem paths. """ config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) destination_file = StringIO() request_list = [ DownloadRequest(urljoin('http://localhost:8088/', self.data_dir + '/' + self.file_list[0]), destination_file)] downloader.download(request_list) with open(os.path.join(self.data_dir, self.file_list[0])) as expected_data_file: expected_data = expected_data_file.read() destination_file.seek(0) destination_file_data = destination_file.read() # The destination_file should be safe to close now (This actually does test that the # downloader hasn't already closed the file, because closing a file twice is an error.) destination_file.close() self.assertEqual(len(destination_file_data), len(expected_data)) self.assertEqual(destination_file_data, expected_data)
def test_download_multiple(self): config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) request_list = self._download_requests() try: downloader.download(request_list) except Exception, e: self.fail(str(e))
def test_https_factory(self): config = DownloaderConfig('https') downloader = download_factory.get_downloader(config) self.assertTrue(isinstance(downloader, curl_backend.HTTPSCurlDownloadBackend)) ssl_working_dir = downloader.ssl_working_dir self.assertTrue(os.path.exists(ssl_working_dir)) del downloader self.assertFalse(os.path.exists(ssl_working_dir))
def test_download_even_listener(self): config = DownloaderConfig('http') listener = MockEventListener() downloader = download_factory.get_downloader(config, listener) request_list = self._download_requests()[:1] downloader.download(request_list) self.assertEqual(listener.batch_started.call_count, 1) self.assertEqual(listener.batch_finished.call_count, 1) self.assertEqual(listener.download_started.call_count, 1) self.assertNotEqual(listener.download_progress.call_count, 0) # not sure how many times self.assertEqual(listener.download_succeeded.call_count, 1) self.assertEqual(listener.download_failed.call_count, 0)
def test_https_download(self): config = DownloaderConfig('https') downloader = download_factory.get_downloader(config) for attr in ('ssl_working_dir', 'ssl_ca_cert', 'ssl_client_cert', 'ssl_client_key'): self.assertTrue(hasattr(downloader, attr)) request_list = self._download_requests('https')[:1] downloader.download(request_list) mock_curl = pycurl.Curl.mock_objs[-1] # curl objects are used from the end self.assertEqual(mock_curl.setopt.call_count, 10) # dangerous as this could easily change
def test_download_single(self): config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) request_list = self._download_requests()[:1] downloader.download(request_list) input_file_name = self.file_list[0] input_file_size = self.file_sizes[0] output_file_path = os.path.join(self.storage_dir, input_file_name) output_file_size = os.stat(output_file_path)[6] self.assertEqual(input_file_size, output_file_size)
def test_download_event_listener(self): config = DownloaderConfig('http') listener = MockEventListener() downloader = download_factory.get_downloader(config, listener) request_list = self._download_requests()[:1] downloader.download(request_list) self.assertEqual(listener.batch_started.call_count, 1) self.assertEqual(listener.batch_finished.call_count, 1) self.assertEqual(listener.download_started.call_count, 1) self.assertEqual(listener.download_progress.call_count, 2) # this one only tests the mock curl self.assertEqual(listener.download_succeeded.call_count, 1) self.assertEqual(listener.download_failed.call_count, 0)
def main(): test_names = _get_test_names() for name in test_names: url_list = TESTS[name]() print '%s: download %d files from %s' % (name.upper(), len(url_list), url_list[0].rsplit('/', 1)[0]) config = DownloaderConfig('http') download_dir = tempfile.mkdtemp(prefix=name+'-') request_list = [DownloadRequest(url, os.path.join(download_dir, _filename_from_url(url))) for url in url_list] downloader = download_factory.get_downloader(config, TestDownloadEventListener()) downloader.download(request_list)
def test_round_trip(self): # Test manifest = Manifest() units = [] for i in range(0, self.NUM_UNITS): units.append({i:i+1}) manifest.write(self.tmp_dir, units) cfg = DownloaderConfig('http') downloader = factory.get_downloader(cfg) manifest = Manifest() path = os.path.join(self.tmp_dir, Manifest.FILE_NAME) url = 'file://%s' % path units_in = manifest.read(url, downloader) # Verify self.verify(units, units_in)
def demo(demo_name): downloader_config = DownloaderConfig(protocol='http', max_concurrent=None) downloader = download_factory.get_downloader(downloader_config, DemoEventListener()) storage_dir = mkdtemp(prefix=demo_name) url_list = URLS_MAP[demo_name]() request_list = requests_from_urls(storage_dir, url_list) print demo_name.upper(), 'Demo' print 'downloading %d files to %s' % (len(url_list), storage_dir) print '=' * 80 start_time = datetime.now() report_list = downloader.download(request_list) run_time = datetime.now() - start_time print '%s downloaded %d files: %s' % (demo_name, len(report_list), str(run_time))
def test_read(self): # Setup units = [] for i in range(0, self.NUM_UNITS): units.append({i:i+1}) s = json.dumps(units) path = os.path.join(self.tmp_dir, Manifest.FILE_NAME) fp = gzip.open(path, 'wb') fp.write(s) fp.close() # Test cfg = DownloaderConfig('http') downloader = factory.get_downloader(cfg) manifest = Manifest() path = os.path.join(self.tmp_dir, Manifest.FILE_NAME) url = 'file://%s' % path units_in = manifest.read(url, downloader) # Verify self.verify(units, units_in)
def _downloader(self, config): """ Get a configured downloader. The integration between the importer configuration and the download package happens here. The https downloader may be used for both http and https so always chosen for simplicity. :param config: The importer configuration. :param config: pulp.plugins.config.PluginCallConfiguration :return: A configured downloader :rtype: pulp.common.download.backends.base.DownloadBackend """ ssl = config.get('ssl', {}) conf = DownloaderConfig( 'https', ssl_ca_cert_path=self._safe_str(ssl.get('ca_cert')), ssl_client_cert_path=self._safe_str(ssl.get('client_cert')), ssl_verify_host=0, ssl_verify_peer=0) downloader = factory.get_downloader(conf) return downloader
def test_publisher(self): # setup units = [] for n in range(0, 3): fn = 'test_%d' % n path = os.path.join(self.unit_dir, fn) fp = open(path, 'w') fp.write(fn) fp.close() unit = {'type_id':'unit', 'unit_key':{'n':n}, 'storage_path':path} units.append(unit) # test # publish repo_id = 'test_repo' base_url = 'file://' publish_dir = os.path.join(self.tmpdir, 'nodes/repos') virtual_host = (publish_dir, publish_dir) p = HttpPublisher(base_url, virtual_host, repo_id) p.publish(units) # verify conf = DownloaderConfig('http') downloader = factory.get_downloader(conf) manifest_path = p.manifest_path() manifest = Manifest() url = 'file://'+manifest_path units = manifest.read(url, downloader) n = 0 for unit in units: file_content = 'test_%d' % n _download = unit['_download'] url = _download['url'] self.assertEqual(url.rsplit('/', 1)[0],'/'.join((base_url, publish_dir[1:], repo_id, 'content'))) path = url.split('//', 1)[1] self.assertTrue(os.path.islink(path)) f = open(path) s = f.read() f.close() self.assertEqual(s, file_content) self.assertEqual(unit['unit_key']['n'], n) n += 1
def test_publisher(self): # setup units = [] for n in range(0, 3): fn = "test_%d" % n path = os.path.join(self.unit_dir, fn) fp = open(path, "w") fp.write(fn) fp.close() unit = {"type_id": "unit", "unit_key": {"n": n}, "storage_path": path} units.append(unit) # test # publish repo_id = "test_repo" base_url = "file://" publish_dir = os.path.join(self.tmpdir, "citrus/repos") virtual_host = (publish_dir, publish_dir) p = HttpPublisher(base_url, virtual_host, repo_id) p.publish(units) # verify conf = DownloaderConfig("http") downloader = factory.get_downloader(conf) manifest_path = p.manifest_path() manifest = Manifest() url = "file://" + manifest_path units = manifest.read(url, downloader) n = 0 for unit in units: file_content = "test_%d" % n _download = unit["_download"] url = _download["url"] self.assertEqual(url.rsplit("/", 1)[0], "/".join((base_url, publish_dir[1:], repo_id, "content"))) path = url.split("//", 1)[1] self.assertTrue(os.path.islink(path)) f = open(path) s = f.read() f.close() self.assertEqual(s, file_content) self.assertEqual(unit["unit_key"]["n"], n) n += 1
def test_download_single_file(self): config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) request_list = self._download_requests()[:1] downloader.download(request_list) self.assertEqual(len(pycurl.CurlMulti.mock_objs), 1) self.assertEqual(len(pycurl.Curl.mock_objs), curl_backend.DEFAULT_MAX_CONCURRENT) mock_multi_curl = pycurl.CurlMulti.mock_objs[0] self.assertEqual(mock_multi_curl.setopt.call_count, 2) # dangerous as this could easily change self.assertEqual(mock_multi_curl.add_handle.call_count, 1) self.assertEqual(mock_multi_curl.select.call_count, 1) self.assertEqual(mock_multi_curl.perform.call_count, 1) self.assertEqual(mock_multi_curl.info_read.call_count, 1) self.assertEqual(mock_multi_curl.remove_handle.call_count, 1) mock_curl = pycurl.Curl.mock_objs[-1] # curl objects are used from back of the list self.assertEqual(mock_curl.setopt.call_count, 8) # also dangerous for the same reasons self.assertEqual(mock_curl.perform.call_count, 1)
def test_publish(self): # Setup self.populate() pulp_conf.set('server', 'storage_dir', self.parentfs) # Test dist = NodesHttpDistributor() repo = Repository(self.REPO_ID) conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) dist.publish_repo(repo, conduit, self.dist_conf()) # Verify conf = DownloaderConfig('http') downloader = factory.get_downloader(conf) manifest = Manifest() pub = dist.publisher(repo, self.dist_conf()) url = '/'.join((pub.base_url, pub.manifest_path())) units = list(manifest.read(url, downloader)) self.assertEqual(len(units), self.NUM_UNITS) for n in range(0, self.NUM_UNITS): unit = units[n] created = self.units[n] for p, v in unit['metadata'].items(): if p.startswith('_'): continue self.assertEqual(created[p], v)
def test_http_factory(self): config = DownloaderConfig('http') downloader = download_factory.get_downloader(config) self.assertTrue(isinstance(downloader, curl_backend.HTTPCurlDownloadBackend))