def test_RebroWeeblyParser(self): with HTTMock(rebro_weebly_mock): proxy_provider = RebroWeeblyParser('ReBro', 'http://rebro.weebly.com') proxy_list = proxy_provider.parse_proxyList() proxy_list_addr = [] for proxy in proxy_list: proxy_list_addr.append(proxy.get_address()) self.assertEqual(proxy_list_addr, rebro_weebly_expected)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5): self.userAgent = UserAgentManager() self.logger = logging.getLogger() self.logger.addHandler(handler) self.logger.setLevel(0) ##### # Each of the classes below implements a specific URL Parser ##### parsers = list([]) parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout)) parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout)) parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)) parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout)) self.logger.debug("=== Initialized Proxy Parsers ===") for i in range(len(parsers)): self.logger.debug("\t {0}".format(parsers[i].__str__())) self.logger.debug("=================================") self.sustain = sustain self.parsers = parsers self.proxy_list = web_proxy_list for i in range(len(parsers)): try: self.proxy_list += parsers[i].parse_proxyList() except ReadTimeout: self.logger.warning("Proxy Parser: '{}' TimedOut!".format(parsers[i].url)) self.current_proxy = self.randomize_proxy()
def __init__(self, timeout=1.0, bandwidth=10.0): # Each of the entries implements a specific URL Parser self.parsers = dict() self.parsers['rebro'] = RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout) self.parsers['prem'] = PremProxyParser('Prem', 'https://premproxy.com', timeout=timeout) self.parsers['freeproxy'] = FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout) self.parsers['proxyforeu'] = ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', bandwidth=bandwidth, timeout=timeout)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5): self.userAgent = UserAgentManager() self.logger = logging.getLogger() self.logger.addHandler(handler) self.logger.setLevel(0) ##### # Each of the classes below implements a specific URL Parser ##### parsers = list([]) parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout)) parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout)) parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)) parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout)) self.logger.debug("=== Initialized Proxy Parsers ===")
def test_RebroWeeblyParser(self): with HTTMock(rebro_weebly_mock): proxy_provider = RebroWeeblyParser('http://rebro.weebly.com') proxy_list = proxy_provider.parse_proxyList() self.assertEqual(proxy_list, rebro_weebly_expected)