def test_FreeProxyParser(self):
with HTTMock(free_proxy_mock):
proxy_provider = FreeProxyParser('FreeProxy', 'http://free-proxy-list.net')
proxy_list = proxy_provider.parse_proxyList()
proxy_list_addr = []
for proxy in proxy_list:
proxy_list_addr.append(proxy.get_address())
self.assertEqual(proxy_list_addr, free_proxy_expected)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
for i in range(len(parsers)):
self.logger.debug("\t {0}".format(parsers[i].__str__()))
self.logger.debug("=================================")
self.sustain = sustain
self.parsers = parsers
self.proxy_list = web_proxy_list
for i in range(len(parsers)):
try:
self.proxy_list += parsers[i].parse_proxyList()
except ReadTimeout:
self.logger.warning("Proxy Parser: '{}' TimedOut!".format(parsers[i].url))
self.current_proxy = self.randomize_proxy()
def __init__(self, web_proxy_list=[], sustain=False, timeout=5, protocol=Protocol.HTTP):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
#parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout)) <--doesn't work anymore
#parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)) <--doesn't work anymore
parsers.append(PremProxyParser('PremProxy', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
for i in range(len(parsers)):
self.logger.debug("\t {0}".format(parsers[i].__str__()))
self.logger.debug("=================================")
self.sustain = sustain
self.parsers = parsers
self.proxy_list = web_proxy_list
for parser in parsers:
try:
size = len(self.proxy_list)
self.proxy_list += parser.parse_proxyList()
self.logger.debug('Added {} proxies from {}'.format(len(self.proxy_list)-size, parser.id))
except ReadTimeout:
self.logger.warning("Proxy Parser: '{}' TimedOut!".format(parser.url))
self.logger.debug('Total proxies = '+str(len(self.proxy_list)))
# filtering the list of available proxies according to user preferences
self.proxy_list = [p for p in self.proxy_list if protocol in p.protocols]
self.logger.debug('Filtered proxies = '+str(len(self.proxy_list)))
self.current_proxy = self.randomize_proxy()
def __init__(self, timeout=1.0, bandwidth=10.0):
# Each of the entries implements a specific URL Parser
self.parsers = dict()
self.parsers['rebro'] = RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout)
self.parsers['prem'] = PremProxyParser('Prem', 'https://premproxy.com', timeout=timeout)
self.parsers['freeproxy'] = FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout)
self.parsers['proxyforeu'] = ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php',
bandwidth=bandwidth, timeout=timeout)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
def test_FreeProxyParser(self):
with HTTMock(free_proxy_mock):
proxy_provider = FreeProxyParser('http://free-proxy-list.net')
proxy_list = proxy_provider.parse_proxyList()
self.assertEqual(proxy_list, free_proxy_expected)