def _test_one_semaphore_for_each_host(self, mirror1, mirror2, mirror3,
probe_function):
"""Check that we create one semaphore per host when probing the given
mirrors using the given probe_function.
mirror1.base_url and mirror2.base_url must be on the same host while
mirror3.base_url must be on a different one.
The given probe_function must be either probe_cdimage_mirror or
probe_archive_mirror.
"""
request_manager = RequestManager()
mirror1_host = URI(mirror1.base_url).host
mirror2_host = URI(mirror2.base_url).host
mirror3_host = URI(mirror3.base_url).host
probe_function(mirror1, StringIO(), [], logging)
# Since we have a single mirror to probe we need to have a single
# DeferredSemaphore with a limit of PER_HOST_REQUESTS, to ensure we
# don't issue too many simultaneous connections on that host.
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror1_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# Note that our multi_lock contains another semaphore to control the
# overall number of requests.
self.assertEquals(multi_lock.overall_lock.limit, OVERALL_REQUESTS)
probe_function(mirror2, StringIO(), [], logging)
# Now we have two mirrors to probe, but they have the same hostname,
# so we'll still have a single semaphore in host_semaphores.
self.assertEquals(mirror2_host, mirror1_host)
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror2_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
probe_function(mirror3, StringIO(), [], logging)
# This third mirror is on a separate host, so we'll have a second
# semaphore added to host_semaphores.
self.failUnless(mirror3_host != mirror1_host)
self.assertEquals(len(request_manager.host_locks), 2)
multi_lock = request_manager.host_locks[mirror3_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# When using an http_proxy, even though we'll actually connect to the
# proxy, we'll use the mirror's host as the key to find the semaphore
# that should be used
orig_proxy = os.getenv('http_proxy')
os.environ['http_proxy'] = 'http://squid.internal:3128/'
probe_function(mirror3, StringIO(), [], logging)
self.assertEquals(len(request_manager.host_locks), 2)
restore_http_proxy(orig_proxy)
def _test_one_semaphore_for_each_host(
self, mirror1, mirror2, mirror3, probe_function):
"""Check that we create one semaphore per host when probing the given
mirrors using the given probe_function.
mirror1.base_url and mirror2.base_url must be on the same host while
mirror3.base_url must be on a different one.
The given probe_function must be either probe_cdimage_mirror or
probe_archive_mirror.
"""
request_manager = RequestManager()
mirror1_host = URI(mirror1.base_url).host
mirror2_host = URI(mirror2.base_url).host
mirror3_host = URI(mirror3.base_url).host
probe_function(mirror1, StringIO(), [], logging)
# Since we have a single mirror to probe we need to have a single
# DeferredSemaphore with a limit of PER_HOST_REQUESTS, to ensure we
# don't issue too many simultaneous connections on that host.
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror1_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# Note that our multi_lock contains another semaphore to control the
# overall number of requests.
self.assertEquals(multi_lock.overall_lock.limit, OVERALL_REQUESTS)
probe_function(mirror2, StringIO(), [], logging)
# Now we have two mirrors to probe, but they have the same hostname,
# so we'll still have a single semaphore in host_semaphores.
self.assertEquals(mirror2_host, mirror1_host)
self.assertEquals(len(request_manager.host_locks), 1)
multi_lock = request_manager.host_locks[mirror2_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
probe_function(mirror3, StringIO(), [], logging)
# This third mirror is on a separate host, so we'll have a second
# semaphore added to host_semaphores.
self.failUnless(mirror3_host != mirror1_host)
self.assertEquals(len(request_manager.host_locks), 2)
multi_lock = request_manager.host_locks[mirror3_host]
self.assertEquals(multi_lock.host_lock.limit, PER_HOST_REQUESTS)
# When using an http_proxy, even though we'll actually connect to the
# proxy, we'll use the mirror's host as the key to find the semaphore
# that should be used
orig_proxy = os.getenv('http_proxy')
os.environ['http_proxy'] = 'http://squid.internal:3128/'
probe_function(mirror3, StringIO(), [], logging)
self.assertEquals(len(request_manager.host_locks), 2)
restore_http_proxy(orig_proxy)
def tearDown(self):
restore_http_proxy(self.orig_proxy)
return self.listening_port.stopListening()
def tearDown(self):
restore_http_proxy(self.orig_proxy)
return self.listening_port.stopListening()