class _ThreadChecker(Thread):
""" 多线程检测 """
def __init__(self, work_type, target_queue, thread_name):
Thread.__init__(self, name=thread_name)
self.work_type = work_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.target_queue = target_queue
self.conf = ConfigHandler()
def run(self):
self.log.info("{}ProxyCheck - {}: start".format(
self.work_type.title(), self.name))
while True:
try:
proxy = self.target_queue.get(block=False)
except Empty:
self.log.info("{}ProxyCheck - {}: complete".format(
self.work_type.title(), self.name))
break
proxy = DoValidator.validator(proxy, self.work_type)
if self.work_type == "raw":
self.__ifRaw(proxy)
else:
self.__ifUse(proxy)
self.target_queue.task_done()
def __ifRaw(self, proxy):
if proxy.last_status:
if self.proxy_handler.exists(proxy):
self.log.info('RawProxyCheck - {}: {} exist'.format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info('RawProxyCheck - {}: {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info('RawProxyCheck - {}: {} fail'.format(
self.name, proxy.proxy.ljust(23)))
def __ifUse(self, proxy):
if proxy.last_status:
self.log.info('UseProxyCheck - {}: {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
if proxy.fail_count > self.conf.maxFailCount:
self.log.info(
'UseProxyCheck - {}: {} fail, count {} delete'.format(
self.name, proxy.proxy.ljust(23), proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
'UseProxyCheck - {}: {} fail, count {} keep'.format(
self.name, proxy.proxy.ljust(23), proxy.fail_count))
self.proxy_handler.put(proxy)
class Checker(Thread):
"""
多线程检测代理是否可用
"""
def __init__(self, check_type, queue, thread_name):
Thread.__init__(self, name=thread_name)
self.type = check_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.queue = queue
self.conf = ConfigHandler()
def run(self):
self.log.info("ProxyCheck - {} : start".format(self.name))
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("ProxyCheck - {} : complete".format(self.name))
break
proxy = Proxy.createFromJson(proxy_json)
proxy = proxyCheck(proxy)
if self.type == "raw":
if proxy.last_status:
if self.proxy_handler.exists(proxy):
self.log.info('ProxyCheck - {} : {} exists'.format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info('ProxyCheck - {} : {} success'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info('ProxyCheck - {} : {} fail'.format(
self.name, proxy.proxy.ljust(23)))
else:
if proxy.last_status:
self.log.info('ProxyCheck - {} : {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.update(proxy)
else:
if proxy.fail_count > self.conf.maxFailCount:
self.log.info(
'ProxyCheck - {} : {} fail, count {} delete'.
format(self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
'ProxyCheck - {} : {} fail, count {} keep'.format(
self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.update(proxy)
self.queue.task_done()