def test_SemairProxyParser(self):
with HTTMock(samair_mock):
proxy_provider = SamairProxyParser('Samair',
'https://www.premproxy.com')
proxy_list = proxy_provider.parse_proxyList()
proxy_list_addr = []
for proxy in proxy_list:
proxy_list_addr.append(proxy.get_address())
for item in samair_expected:
self.assertTrue(item in proxy_list_addr)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
for i in range(len(parsers)):
self.logger.debug("\t {0}".format(parsers[i].__str__()))
self.logger.debug("=================================")
self.sustain = sustain
self.parsers = parsers
self.proxy_list = web_proxy_list
for i in range(len(parsers)):
try:
self.proxy_list += parsers[i].parse_proxyList()
except ReadTimeout:
self.logger.warning("Proxy Parser: '{}' TimedOut!".format(parsers[i].url))
self.current_proxy = self.randomize_proxy()
def __init__(self, timeout=1.0, bandwidth=10.0):
# Each of the entries implements a specific URL Parser
self.parsers = dict()
self.parsers['rebro'] = RebroWeeblyParser('ReBro',
'http://rebro.weebly.com',
timeout=timeout)
self.parsers['samair'] = SamairProxyParser('Samair',
'https://premproxy.com',
timeout=timeout)
self.parsers['freeproxy'] = FreeProxyParser(
'FreeProxy', 'http://free-proxy-list.net', timeout=timeout)
self.parsers['proxyforeu'] = ProxyForEuParser(
'ProxyForEU',
'http://proxyfor.eu/geo.php',
bandwidth=bandwidth,
timeout=timeout)
def __init__(self, web_proxy_list=[], sustain=False, timeout=5):
self.userAgent = UserAgentManager()
self.logger = logging.getLogger()
self.logger.addHandler(handler)
self.logger.setLevel(0)
#####
# Each of the classes below implements a specific URL Parser
#####
parsers = list([])
parsers.append(FreeProxyParser('FreeProxy', 'http://free-proxy-list.net', timeout=timeout))
parsers.append(ProxyForEuParser('ProxyForEU', 'http://proxyfor.eu/geo.php', 1.0, timeout=timeout))
parsers.append(RebroWeeblyParser('ReBro', 'http://rebro.weebly.com', timeout=timeout))
parsers.append(SamairProxyParser('Samair', 'https://premproxy.com', timeout=timeout))
self.logger.debug("=== Initialized Proxy Parsers ===")
def test_SemairProxyParser(self):
with HTTMock(samair_mock):
proxy_provider = SamairProxyParser(
'http://www.samair.ru/proxy/time-01.htm')
proxy_list = proxy_provider.parse_proxyList()
self.assertEqual(proxy_list, samair_expected)
def test_SemairProxyParser(self):
with HTTMock(samair_mock):
proxy_provider = SamairProxyParser('https://www.premproxy.com')
proxy_list = proxy_provider.parse_proxyList()
for item in samair_expected:
self.assertTrue(item in proxy_list)