def initialize(self):
"""
Start threads to get proxies
:return: Array list proxies
"""
self.log('Looking at aliveproxy.com...', flush=True)
results = []
threads = [
ThreadManager(target=self.get,
args=('high-anonymity-proxy-list/', )),
ThreadManager(target=self.get, args=('anonymous-proxy-list/', )),
ThreadManager(target=self.get, args=('transparent-proxy-list/', ))
]
for thread in threads:
thread.start()
try:
for thread in threads:
results.append(thread.join())
except KeyboardInterrupt:
self.log('Ctrl-C caught, exiting', 'WARNING', True)
sys.exit(1)
result = [x for i in results for x in i]
self.log('Total at aliveproxy.com: \033[93m{}'.format(len(result)))
return [x for i in results for x in i]
def initialize(self):
"""
Start threads to get proxies
:return: Array list proxies
"""
self.log('Looking at cool-proxy.net...', flush=True)
results = []
threads = []
pages = self.pages()
pages_per_threads = (pages[i:i + 3] for i in xrange(0, len(pages), 3))
for pgs in pages_per_threads:
threads.append(ThreadManager(target=self.get, args=(pgs, )))
for thread in threads:
thread.start()
try:
for thread in threads:
results.append(thread.join())
except KeyboardInterrupt:
self.log('Ctrl-C caught, exiting', 'WARNING', True)
sys.exit(1)
result = [x for i in results for x in i]
self.log('Total at cool-proxy.net: \033[93m{}'.format(len(result)))
return result
def initialize(self):
"""
Start threads to get proxies
:return: Array list proxies
"""
self.log('Looking at gatherproxy.com...', flush=True)
results = []
threads = []
proxy_types = ['elite', 'anonymous', 'transparent']
for proxy_type in proxy_types:
pages = self.pages(proxy_type)
pages_per_threads = (pages[i:i + 20]
for i in xrange(0, len(pages), 20))
for pgs in pages_per_threads:
threads.append(
ThreadManager(target=self.get, args=(
proxy_type,
pgs,
)))
for thread in threads:
thread.start()
try:
for thread in threads:
results.append(thread.join())
except KeyboardInterrupt:
self.log('Ctrl-C caught, exiting', 'WARNING', True)
sys.exit(1)
result = [x for i in results for x in i]
self.log('Total at gatherproxy.com: \033[93m{}'.format(len(result)))
return result
def initialize(self):
"""
Start threads to get proxies
:return: Array list proxies
"""
self.log('Looking at freeproxylists.com...', flush=True)
results = []
pages = [
'elite.html', 'anonymous.html', 'non-anonymous.html', 'https.html',
'standard.html', 'us.html', 'uk.html', 'ca.html', 'fr.html'
]
for page in pages:
threads = []
links = self.pages(page)
links_per_threads = (links[i:i + 5]
for i in xrange(0, len(links), 5))
for lks in links_per_threads:
threads.append(ThreadManager(target=self.get, args=(lks, )))
for thread in threads:
thread.start()
try:
for thread in threads:
results.append(thread.join())
except KeyboardInterrupt:
self.log('Ctrl-C caught, exiting', 'WARNING', True)
sys.exit(1)
result = [x for i in results for x in i]
self.log('Total at freeproxylists.com: \033[93m{}'.format(len(result)))
return result