def _fp_coroutine(self, timeout=1, wait_time=120):
"""A coroutine to continuosly yield free proxies
It takes back the proxies that stopped working and marks it as dirty.
"""
freeproxy = FreeProxy(rand=False, timeout=timeout)
if not hasattr(self, '_dirty_freeproxies'):
self._dirty_freeproxies = set()
all_proxies = freeproxy.get_proxy_list()
all_proxies.reverse() # Try the older proxies first
t1 = time.time()
while (time.time() - t1 < wait_time):
proxy = all_proxies.pop()
if not all_proxies:
all_proxies = freeproxy.get_proxy_list()
if proxy in self._dirty_freeproxies:
continue
proxies = {'http': proxy, 'https': proxy}
proxy_works = self._check_proxy(proxies)
if proxy_works:
dirty_proxy = (yield proxy)
t1 = time.time()
else:
dirty_proxy = proxy
self._dirty_freeproxies.add(dirty_proxy)
def test_empty_proxy_list(self):
test = FreeProxy()
test.get_proxy_list = MagicMock(return_value=[])
self.assertEqual("There are no working proxies at this time.", test.get())
def test_invalid_proxy(self):
test = FreeProxy()
test.get_proxy_list = MagicMock(return_value=['111.111.11:2222'])
self.assertEqual("There are no working proxies at this time.", test.get())
def test_empty_proxy_list(self):
test = FreeProxy()
test.get_proxy_list = MagicMock(return_value=[])
with self.assertRaises(RuntimeError):
test.get()
def test_anonym_filter(self):
test1 = FreeProxy()
cnt1 = len(test1.get_proxy_list())
test2 = FreeProxy(anonym=True)
cnt2 = len(test2.get_proxy_list())
self.assertTrue(cnt2 < cnt1)
def test_invalid_proxy(self):
test = FreeProxy()
test.get_proxy_list = MagicMock(return_value=['111.111.11:2222'])
with self.assertRaises(RuntimeError):
test.get()
