def test_RebroWeeblyParser(self):
with HTTMock(rebro_weebly_mock):
proxy_provider = RebroWeeblyParser('ReBro', 'http://rebro.weebly.com')
proxy_list = proxy_provider.parse_proxyList()
proxy_list_addr = []
for proxy in proxy_list:
proxy_list_addr.append(proxy.get_address())
self.assertEqual(proxy_list_addr, rebro_weebly_expected)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
for i in range(len(parsers)):
self.logger.debug("\t {0}".format(parsers[i].__str__()))
self.logger.debug("=================================")
self.sustain = sustain
self.parsers = parsers
self.proxy_list = web_proxy_list
for i in range(len(parsers)):
try:
self.proxy_list += parsers[i].parse_proxyList()
except ReadTimeout:
self.logger.warning("Proxy Parser: '{}' TimedOut!".format(parsers[i].url))
self.current_proxy = self.randomize_proxy()
def __init__(self, timeout=1.0, bandwidth=10.0):
# Each of the entries implements a specific URL Parser
self.parsers = dict()
self.parsers['rebro'] = RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)
self.parsers['prem'] = PremProxyParser('Prem', 'https://premproxy.com', timeout=timeout)
self.parsers['freeproxy'] = FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout)
self.parsers['proxyforeu'] = ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php',
bandwidth=bandwidth, timeout=timeout)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
def test_RebroWeeblyParser(self):
with HTTMock(rebro_weebly_mock):
proxy_provider = RebroWeeblyParser('http://rebro.weebly.com')
proxy_list = proxy_provider.parse_proxyList()
self.assertEqual(proxy_list, rebro_weebly_expected)
