def test_integration(self):
monitor = DomainMonitor("google.com")
self.assertEqual(monitor.domain, "google.com")
self.assertEqual(
monitor.stats(), {
'acquisitions_processed': 0,
'available': 0,
'avg_resp_time': float('inf'),
'indices': {}
})
proxy_a = ProxyProps('http', 'localhost', 8888, 0.1, 'us',
'transparent')
proxy_b = ProxyProps('http', 'localhost', 8889, 0.2)
monitor.register(proxy_a)
monitor.register(proxy_b)
acquired = monitor.acquire('us')
self.assertEqual(acquired, str(proxy_a))
self.assertIsNone(monitor.acquire('us'))
self.assertEqual(monitor.acquire(), str(proxy_b))
monitor.release(acquired, 0.01)
acquired = monitor.acquire('us')
self.assertEqual(acquired, str(proxy_a))
monitor.release(acquired, 0.01)
monitor.delist(str(proxy_a))
def test_average_response_time(self):
monitor = DomainMonitor("google.com")
monitor.register(
ProxyProps('http', 'localhost', 8888, 0.1, 'us', 'transparent'))
monitor.register(
ProxyProps('http', 'localhost', 8889, 0.3, 'us', 'transparent'))
self.assertEqual(monitor.average_response_time(), 0.2)
def setUp(self):
self.domain_monitor = DomainMonitor('google.com')
proxy_a = ProxyProps('http', 'proxy-a', 8888, 0.1)
proxy_b = ProxyProps('http', 'proxy-b', 8888, 0.1)
self.domain_monitor.register(proxy_a)
self.domain_monitor.register(proxy_b)
self.proxy_strs = {str(proxy_a), str(proxy_b)}
def test_zeros_bug(self):
monitor = DomainMonitor("google.com")
a = ProxyProps('http', 'localhost', 8888, 0.0)
b = ProxyProps('http', 'localhost', 9000, 0.0)
monitor.register(a)
monitor.register(b)
for _ in range(10):
monitor.release(monitor.acquire(), 0.0)
self.assertTrue(True) # I.E. Finishes
def test_proxy_props(self):
proto, host, port = 'http', 'localhost', 8888
a = ProxyProps(proto, host, port, 1.0, 'us', 'high')
self.assertEqual(str(a), "http://localhost:8888".upper())
self.assertEqual(str(a), repr(a))
for args in product([proto, 'https'], [host, 'me'], [port, 8888]):
if args == (proto, host, port): continue
b = ProxyProps(*args, 1.0, 'us', 'high')
self.assertNotEqual(a, b)
c = ProxyProps(proto, host, port, 2.0, 'ca', 'low')
self.assertEqual(a, c)
async def test_acquire_waiting(self):
broker = Broker(self.domain_monitor)
proxy_dup = ProxyProps('http', 'proxy-a', 8888, 0.1)
# Consume both proxies.
proxy_a = await broker.acquire()
proxy_b = await broker.acquire()
self.assertEqual(broker.stats()['available'], 0)
# Create a task that will wait for availability.
acquired = []
async def acquire_post_clock():
acquired.append(await broker.acquire())
post_clock = self.loop.create_task(acquire_post_clock())
# Fast forward thirty seconds, it should still be waiting.
await self.advance(THIRTY_SECONDS)
self.assertFalse(post_clock.done())
self.assertEqual(broker.stats()['available'], 0)
# Now, forward past the max wait time.
# Two proxies were returned. One was acquired.
await self.advance(THIRTY_SECONDS + 1)
self.assertTrue(post_clock.done())
self.assertEqual(broker.stats()['available'], 1)
self.assertIn(acquired[0], self.proxy_strs)
def test_stochastic_sampling(self):
monitor = DomainMonitor("google.com")
a = ProxyProps('http', 'localhost', 8888, 0.1)
b = ProxyProps('http', 'localhost', 9000, 0.2)
counts = {str(x): 0 for x in [a, b]}
inc = {str(x): x.resp_time for x in [a, b]}
monitor.register(a)
monitor.register(b)
for _ in range(100):
x = monitor.acquire()
counts[str(x)] += 1
monitor.release(x, inc[str(x)])
self.assertGreater(counts[str(a)], counts[str(b)])
async def test_register_deregister(self):
broker = Broker(self.domain_monitor)
proxy = ProxyProps('http', 'proxy-c', 8888, 0.1)
broker.register(proxy)
self.assertEqual(broker.stats()['available'], 3)
broker.delist(str(proxy))
self.assertEqual(broker.stats()['available'], 2)
def get_app(self, loop):
proxies = ProxyCollection()
a = ProxyProps('http', 'proxy-a', 8888, 0.1)
b = ProxyProps('http', 'proxy-b', 8888, 0.1)
proxies.register_proxy(a.to_dict())
proxies.register_proxy(b.to_dict())
brokerage = Brokerage(proxies, broker_opts={'loop': loop})
app = RESTProxyBroker(proxy_collection=proxies,
brokerage=brokerage,
loop=loop)._app
return app
def register_proxy(self, proxy):
proxy = ProxyProps(**proxy)
self._proxies[str(proxy)] = proxy
for monitor in self._monitors.values():
monitor.register(proxy)
LOGGER.info("ProxyCollection registering %s", str(proxy))
async def test_wont_reregister(self):
broker = Broker(self.domain_monitor)
proxy_dup = ProxyProps('http', 'proxy-a', 8888, 0.1)
broker.register(proxy_dup)
self.assertEqual(broker.stats()['available'], 2)