def _fp_coroutine(self, timeout=1, wait_time=120): """A coroutine to continuosly yield free proxies It takes back the proxies that stopped working and marks it as dirty. """ freeproxy = FreeProxy(rand=False, timeout=timeout) if not hasattr(self, '_dirty_freeproxies'): self._dirty_freeproxies = set() all_proxies = freeproxy.get_proxy_list() all_proxies.reverse() # Try the older proxies first t1 = time.time() while (time.time() - t1 < wait_time): proxy = all_proxies.pop() if not all_proxies: all_proxies = freeproxy.get_proxy_list() if proxy in self._dirty_freeproxies: continue proxies = {'http': proxy, 'https': proxy} proxy_works = self._check_proxy(proxies) if proxy_works: dirty_proxy = (yield proxy) t1 = time.time() else: dirty_proxy = proxy self._dirty_freeproxies.add(dirty_proxy)
def test_empty_proxy_list(self): test = FreeProxy() test.get_proxy_list = MagicMock(return_value=[]) self.assertEqual("There are no working proxies at this time.", test.get())
def test_invalid_proxy(self): test = FreeProxy() test.get_proxy_list = MagicMock(return_value=['111.111.11:2222']) self.assertEqual("There are no working proxies at this time.", test.get())
def test_empty_proxy_list(self): test = FreeProxy() test.get_proxy_list = MagicMock(return_value=[]) with self.assertRaises(RuntimeError): test.get()
def test_anonym_filter(self): test1 = FreeProxy() cnt1 = len(test1.get_proxy_list()) test2 = FreeProxy(anonym=True) cnt2 = len(test2.get_proxy_list()) self.assertTrue(cnt2 < cnt1)
def test_invalid_proxy(self): test = FreeProxy() test.get_proxy_list = MagicMock(return_value=['111.111.11:2222']) with self.assertRaises(RuntimeError): test.get()