def __init__(self, userGiven=False, flag=None, log=False, proxyType=None):
self._FLAG = flag
self._UG = userGiven
if log:
self._FILEDESC = open(log, 'w')
if not self._UG:
self.scraper = ProxyScraper()
self._proxies = self.scraper.getProxies()
for key in self._proxies.keys():
self.handleIPs(key)
else:
if self._UG.find(':') != -1:
# Single proxy mode
self._proxy = tuple(self._UG.split(':'))
self.checkSingle(proxyType)
else:
# Proxy list mode
self.lhandler = ListHandler(self._UG, proxyType)
self._proxies = self.lhandler.getProxies()
self.handleIPs(proxyType)
if log:
self._FILEDESC.close()
class ProxyTester(object):
# tests whether the give proxies are working
_proxies = { 'socks5' : [], 'socks4' : [], 'mixedSocks' : [], 'http' : [],
'unknown': [] }
# In case of user-given proxies 'mixedSocks' will handle all requests for either
# HTTP proxies, or unknown proxy types
_queues = {'socks5' : Queue.Queue(), 'socks4' : Queue.Queue(), 'mixedSocks' : Queue.Queue(),
'http' : Queue.Queue(), 'unknown' : Queue.Queue()}
_THREADS_PER_TYPE = 10
_FILEDESC = None
def __init__(self, userGiven = False, flag = None, log = False, proxyType = None):
self._FLAG = flag
self._UG = userGiven
if log:
self._FILEDESC = open(log, 'w')
if not self._UG:
self.scraper = ProxyScraper()
self._proxies = self.scraper.getProxies()
for key in self._proxies.keys():
self.handleIPs(key)
else:
if self._UG.find(':') != -1:
# Single proxy mode
self._proxy = tuple(self._UG.split(':'))
self.checkSingle(proxyType)
else:
# Proxy list mode
self.lhandler = ListHandler(self._UG, proxyType)
self._proxies = self.lhandler.getProxies()
self.handleIPs(proxyType)
if log:
self._FILEDESC.close()
def checkSingle(self, proxyType):
self._queues[proxyType].put(self._proxy)
worker = ProxyTestWorker(self._queues[proxyType], proxyType, self._FLAG, self._FILEDESC, True)
worker.daemon = True
worker.start()
self._queues[proxyType].join()
def handleIPs(self, proxyType):
isUserGiven = True if self._UG else False
for lst in self._proxies[proxyType]:
if isUserGiven:
self._queues[proxyType].put((lst[0], lst[1]))
else:
for ip, port in lst:
self._queues[proxyType].put((ip, port))
for i in range(self._THREADS_PER_TYPE):
worker = ProxyTestWorker(self._queues[proxyType], proxyType, self._FLAG, self._FILEDESC, isUserGiven)
worker.daemon = True
worker.start()
self._queues[proxyType].join()
def __init__(self, userGiven = False, flag = None, log = False, proxyType = None):
self._FLAG = flag
self._UG = userGiven
if log:
self._FILEDESC = open(log, 'w')
if not self._UG:
self.scraper = ProxyScraper()
self._proxies = self.scraper.getProxies()
for key in self._proxies.keys():
self.handleIPs(key)
else:
if self._UG.find(':') != -1:
# Single proxy mode
self._proxy = tuple(self._UG.split(':'))
self.checkSingle(proxyType)
else:
# Proxy list mode
self.lhandler = ListHandler(self._UG, proxyType)
self._proxies = self.lhandler.getProxies()
self.handleIPs(proxyType)
if log:
self._FILEDESC.close()
class ProxyTester(object):
# tests whether the give proxies are working
_proxies = {
'socks5': [],
'socks4': [],
'mixedSocks': [],
'http': [],
'unknown': []
}
# In case of user-given proxies 'mixedSocks' will handle all requests for either
# HTTP proxies, or unknown proxy types
_queues = {
'socks5': Queue.Queue(),
'socks4': Queue.Queue(),
'mixedSocks': Queue.Queue(),
'http': Queue.Queue(),
'unknown': Queue.Queue()
}
_THREADS_PER_TYPE = 10
_FILEDESC = None
def __init__(self, userGiven=False, flag=None, log=False, proxyType=None):
self._FLAG = flag
self._UG = userGiven
if log:
self._FILEDESC = open(log, 'w')
if not self._UG:
self.scraper = ProxyScraper()
self._proxies = self.scraper.getProxies()
for key in self._proxies.keys():
self.handleIPs(key)
else:
if self._UG.find(':') != -1:
# Single proxy mode
self._proxy = tuple(self._UG.split(':'))
self.checkSingle(proxyType)
else:
# Proxy list mode
self.lhandler = ListHandler(self._UG, proxyType)
self._proxies = self.lhandler.getProxies()
self.handleIPs(proxyType)
if log:
self._FILEDESC.close()
def checkSingle(self, proxyType):
self._queues[proxyType].put(self._proxy)
worker = ProxyTestWorker(self._queues[proxyType], proxyType,
self._FLAG, self._FILEDESC, True)
worker.daemon = True
worker.start()
self._queues[proxyType].join()
def handleIPs(self, proxyType):
isUserGiven = True if self._UG else False
for lst in self._proxies[proxyType]:
if isUserGiven:
self._queues[proxyType].put((lst[0], lst[1]))
else:
for ip, port in lst:
self._queues[proxyType].put((ip, port))
for i in range(self._THREADS_PER_TYPE):
worker = ProxyTestWorker(self._queues[proxyType], proxyType,
self._FLAG, self._FILEDESC, isUserGiven)
worker.daemon = True
worker.start()
self._queues[proxyType].join()
