def test_download_multi_file(self):
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
request_list = self._download_requests()
downloader.download(request_list)
# this is really just testing my mock curl objects, but it's nice to know
for file_name, file_size in zip(self.file_list, self.file_sizes):
input_file = os.path.join(self.data_dir, file_name)
input_file_size = os.stat(input_file)[6]
output_file = os.path.join(self.storage_dir, file_name)
output_file_size = os.stat(output_file)[6]
self.assertEqual(input_file_size, file_size)
self.assertEqual(output_file_size, file_size) # does check that close() was called properly
mock_curl_multi = pycurl.CurlMulti.mock_objs[0]
self.assertEqual(mock_curl_multi.perform.call_count, 1)
num_unused_curl_objs = max(0, curl_backend.DEFAULT_MAX_CONCURRENT - len(self.file_list))
unused_mock_curl_objs = pycurl.Curl.mock_objs[:num_unused_curl_objs]
for mock_curl in unused_mock_curl_objs:
self.assertEqual(mock_curl.perform.call_count, 0)
used_mock_curl_objs = pycurl.Curl.mock_objs[num_unused_curl_objs:]
for mock_curl in used_mock_curl_objs:
self.assertEqual(mock_curl.perform.call_count, 1)
def __init__(self, sync_conduit, config):
self.sync_conduit = sync_conduit
self._remove_missing_units = config.get(constants.CONFIG_REMOVE_MISSING_UNITS, default=False)
self._repo_url = encode_unicode(config.get(constants.CONFIG_FEED_URL))
self._validate_downloads = config.get(constants.CONFIG_VALIDATE_DOWNLOADS, default=True)
# Cast our config parameters to the correct types and use them to build a Downloader
max_speed = config.get(constants.CONFIG_MAX_SPEED)
if max_speed is not None:
max_speed = float(max_speed)
num_threads = config.get(constants.CONFIG_NUM_THREADS)
if num_threads is not None:
num_threads = int(num_threads)
else:
num_threads = constants.DEFAULT_NUM_THREADS
downloader_config = {
'max_speed': max_speed, 'num_threads': num_threads,
'ssl_client_cert': config.get(constants.CONFIG_SSL_CLIENT_CERT),
'ssl_client_key': config.get(constants.CONFIG_SSL_CLIENT_KEY),
'ssl_ca_cert': config.get(constants.CONFIG_SSL_CA_CERT), 'ssl_verify_host': 1,
'ssl_verify_peer': 1, 'proxy_url': config.get(constants.CONFIG_PROXY_URL),
'proxy_port': config.get(constants.CONFIG_PROXY_PORT),
'proxy_user': config.get(constants.CONFIG_PROXY_USER),
'proxy_password': config.get(constants.CONFIG_PROXY_PASSWORD)}
downloader_config = DownloaderConfig(protocol='https', **downloader_config)
# We will pass self as the event_listener, so that we can receive the callbacks in this class
self.downloader = factory.get_downloader(downloader_config, self)
self.progress_report = SyncProgressReport(sync_conduit)
def test_download_destination_file_like_object(self):
"""
We want to assert that we can download URLs to file-like objects, and not just to
filesystem paths.
"""
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
destination_file = StringIO()
request_list = [
DownloadRequest(urljoin('http://localhost:8088/',
self.data_dir + '/' + self.file_list[0]),
destination_file)]
downloader.download(request_list)
with open(os.path.join(self.data_dir, self.file_list[0])) as expected_data_file:
expected_data = expected_data_file.read()
destination_file.seek(0)
destination_file_data = destination_file.read()
# The destination_file should be safe to close now (This actually does test that the
# downloader hasn't already closed the file, because closing a file twice is an error.)
destination_file.close()
self.assertEqual(len(destination_file_data), len(expected_data))
self.assertEqual(destination_file_data, expected_data)
def test_download_multiple(self):
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
request_list = self._download_requests()
try:
downloader.download(request_list)
except Exception, e:
self.fail(str(e))
def test_https_factory(self):
config = DownloaderConfig('https')
downloader = download_factory.get_downloader(config)
self.assertTrue(isinstance(downloader, curl_backend.HTTPSCurlDownloadBackend))
ssl_working_dir = downloader.ssl_working_dir
self.assertTrue(os.path.exists(ssl_working_dir))
del downloader
self.assertFalse(os.path.exists(ssl_working_dir))
def test_download_even_listener(self):
config = DownloaderConfig('http')
listener = MockEventListener()
downloader = download_factory.get_downloader(config, listener)
request_list = self._download_requests()[:1]
downloader.download(request_list)
self.assertEqual(listener.batch_started.call_count, 1)
self.assertEqual(listener.batch_finished.call_count, 1)
self.assertEqual(listener.download_started.call_count, 1)
self.assertNotEqual(listener.download_progress.call_count, 0) # not sure how many times
self.assertEqual(listener.download_succeeded.call_count, 1)
self.assertEqual(listener.download_failed.call_count, 0)
def test_https_download(self):
config = DownloaderConfig('https')
downloader = download_factory.get_downloader(config)
for attr in ('ssl_working_dir', 'ssl_ca_cert', 'ssl_client_cert', 'ssl_client_key'):
self.assertTrue(hasattr(downloader, attr))
request_list = self._download_requests('https')[:1]
downloader.download(request_list)
mock_curl = pycurl.Curl.mock_objs[-1] # curl objects are used from the end
self.assertEqual(mock_curl.setopt.call_count, 10) # dangerous as this could easily change
def test_download_single(self):
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
request_list = self._download_requests()[:1]
downloader.download(request_list)
input_file_name = self.file_list[0]
input_file_size = self.file_sizes[0]
output_file_path = os.path.join(self.storage_dir, input_file_name)
output_file_size = os.stat(output_file_path)[6]
self.assertEqual(input_file_size, output_file_size)
def test_download_event_listener(self):
config = DownloaderConfig('http')
listener = MockEventListener()
downloader = download_factory.get_downloader(config, listener)
request_list = self._download_requests()[:1]
downloader.download(request_list)
self.assertEqual(listener.batch_started.call_count, 1)
self.assertEqual(listener.batch_finished.call_count, 1)
self.assertEqual(listener.download_started.call_count, 1)
self.assertEqual(listener.download_progress.call_count, 2) # this one only tests the mock curl
self.assertEqual(listener.download_succeeded.call_count, 1)
self.assertEqual(listener.download_failed.call_count, 0)
def main():
test_names = _get_test_names()
for name in test_names:
url_list = TESTS[name]()
print '%s: download %d files from %s' % (name.upper(), len(url_list), url_list[0].rsplit('/', 1)[0])
config = DownloaderConfig('http')
download_dir = tempfile.mkdtemp(prefix=name+'-')
request_list = [DownloadRequest(url, os.path.join(download_dir, _filename_from_url(url))) for url in url_list]
downloader = download_factory.get_downloader(config, TestDownloadEventListener())
downloader.download(request_list)
def test_round_trip(self):
# Test
manifest = Manifest()
units = []
for i in range(0, self.NUM_UNITS):
units.append({i:i+1})
manifest.write(self.tmp_dir, units)
cfg = DownloaderConfig('http')
downloader = factory.get_downloader(cfg)
manifest = Manifest()
path = os.path.join(self.tmp_dir, Manifest.FILE_NAME)
url = 'file://%s' % path
units_in = manifest.read(url, downloader)
# Verify
self.verify(units, units_in)
def demo(demo_name):
downloader_config = DownloaderConfig(protocol='http', max_concurrent=None)
downloader = download_factory.get_downloader(downloader_config, DemoEventListener())
storage_dir = mkdtemp(prefix=demo_name)
url_list = URLS_MAP[demo_name]()
request_list = requests_from_urls(storage_dir, url_list)
print demo_name.upper(), 'Demo'
print 'downloading %d files to %s' % (len(url_list), storage_dir)
print '=' * 80
start_time = datetime.now()
report_list = downloader.download(request_list)
run_time = datetime.now() - start_time
print '%s downloaded %d files: %s' % (demo_name, len(report_list), str(run_time))
def test_read(self):
# Setup
units = []
for i in range(0, self.NUM_UNITS):
units.append({i:i+1})
s = json.dumps(units)
path = os.path.join(self.tmp_dir, Manifest.FILE_NAME)
fp = gzip.open(path, 'wb')
fp.write(s)
fp.close()
# Test
cfg = DownloaderConfig('http')
downloader = factory.get_downloader(cfg)
manifest = Manifest()
path = os.path.join(self.tmp_dir, Manifest.FILE_NAME)
url = 'file://%s' % path
units_in = manifest.read(url, downloader)
# Verify
self.verify(units, units_in)
def _downloader(self, config):
"""
Get a configured downloader.
The integration between the importer configuration and the
download package happens here. The https downloader may be
used for both http and https so always chosen for simplicity.
:param config: The importer configuration.
:param config: pulp.plugins.config.PluginCallConfiguration
:return: A configured downloader
:rtype: pulp.common.download.backends.base.DownloadBackend
"""
ssl = config.get('ssl', {})
conf = DownloaderConfig(
'https',
ssl_ca_cert_path=self._safe_str(ssl.get('ca_cert')),
ssl_client_cert_path=self._safe_str(ssl.get('client_cert')),
ssl_verify_host=0,
ssl_verify_peer=0)
downloader = factory.get_downloader(conf)
return downloader
def test_publisher(self):
# setup
units = []
for n in range(0, 3):
fn = 'test_%d' % n
path = os.path.join(self.unit_dir, fn)
fp = open(path, 'w')
fp.write(fn)
fp.close()
unit = {'type_id':'unit', 'unit_key':{'n':n}, 'storage_path':path}
units.append(unit)
# test
# publish
repo_id = 'test_repo'
base_url = 'file://'
publish_dir = os.path.join(self.tmpdir, 'nodes/repos')
virtual_host = (publish_dir, publish_dir)
p = HttpPublisher(base_url, virtual_host, repo_id)
p.publish(units)
# verify
conf = DownloaderConfig('http')
downloader = factory.get_downloader(conf)
manifest_path = p.manifest_path()
manifest = Manifest()
url = 'file://'+manifest_path
units = manifest.read(url, downloader)
n = 0
for unit in units:
file_content = 'test_%d' % n
_download = unit['_download']
url = _download['url']
self.assertEqual(url.rsplit('/', 1)[0],'/'.join((base_url, publish_dir[1:], repo_id, 'content')))
path = url.split('//', 1)[1]
self.assertTrue(os.path.islink(path))
f = open(path)
s = f.read()
f.close()
self.assertEqual(s, file_content)
self.assertEqual(unit['unit_key']['n'], n)
n += 1
def test_publisher(self):
# setup
units = []
for n in range(0, 3):
fn = "test_%d" % n
path = os.path.join(self.unit_dir, fn)
fp = open(path, "w")
fp.write(fn)
fp.close()
unit = {"type_id": "unit", "unit_key": {"n": n}, "storage_path": path}
units.append(unit)
# test
# publish
repo_id = "test_repo"
base_url = "file://"
publish_dir = os.path.join(self.tmpdir, "citrus/repos")
virtual_host = (publish_dir, publish_dir)
p = HttpPublisher(base_url, virtual_host, repo_id)
p.publish(units)
# verify
conf = DownloaderConfig("http")
downloader = factory.get_downloader(conf)
manifest_path = p.manifest_path()
manifest = Manifest()
url = "file://" + manifest_path
units = manifest.read(url, downloader)
n = 0
for unit in units:
file_content = "test_%d" % n
_download = unit["_download"]
url = _download["url"]
self.assertEqual(url.rsplit("/", 1)[0], "/".join((base_url, publish_dir[1:], repo_id, "content")))
path = url.split("//", 1)[1]
self.assertTrue(os.path.islink(path))
f = open(path)
s = f.read()
f.close()
self.assertEqual(s, file_content)
self.assertEqual(unit["unit_key"]["n"], n)
n += 1
def test_download_single_file(self):
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
request_list = self._download_requests()[:1]
downloader.download(request_list)
self.assertEqual(len(pycurl.CurlMulti.mock_objs), 1)
self.assertEqual(len(pycurl.Curl.mock_objs), curl_backend.DEFAULT_MAX_CONCURRENT)
mock_multi_curl = pycurl.CurlMulti.mock_objs[0]
self.assertEqual(mock_multi_curl.setopt.call_count, 2) # dangerous as this could easily change
self.assertEqual(mock_multi_curl.add_handle.call_count, 1)
self.assertEqual(mock_multi_curl.select.call_count, 1)
self.assertEqual(mock_multi_curl.perform.call_count, 1)
self.assertEqual(mock_multi_curl.info_read.call_count, 1)
self.assertEqual(mock_multi_curl.remove_handle.call_count, 1)
mock_curl = pycurl.Curl.mock_objs[-1] # curl objects are used from back of the list
self.assertEqual(mock_curl.setopt.call_count, 8) # also dangerous for the same reasons
self.assertEqual(mock_curl.perform.call_count, 1)
def test_publish(self):
# Setup
self.populate()
pulp_conf.set('server', 'storage_dir', self.parentfs)
# Test
dist = NodesHttpDistributor()
repo = Repository(self.REPO_ID)
conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
dist.publish_repo(repo, conduit, self.dist_conf())
# Verify
conf = DownloaderConfig('http')
downloader = factory.get_downloader(conf)
manifest = Manifest()
pub = dist.publisher(repo, self.dist_conf())
url = '/'.join((pub.base_url, pub.manifest_path()))
units = list(manifest.read(url, downloader))
self.assertEqual(len(units), self.NUM_UNITS)
for n in range(0, self.NUM_UNITS):
unit = units[n]
created = self.units[n]
for p, v in unit['metadata'].items():
if p.startswith('_'):
continue
self.assertEqual(created[p], v)
def test_http_factory(self):
config = DownloaderConfig('http')
downloader = download_factory.get_downloader(config)
self.assertTrue(isinstance(downloader, curl_backend.HTTPCurlDownloadBackend))
