def run(self):
self.log.info("UsefulProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_str = self.queue.get(block=False)
except Empty:
self.log.info("UsefulProxyCheck - {} : exit".format(
self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_str)
proxy_obj, status = checkProxyUseful(proxy_obj)
if status or proxy_obj.fail_count < FAIL_COUNT:
if self.db.exists(proxy_obj.proxy):
self.log.info(
'UsefulProxyCheck - {} : {} validation exists'.format(
self.name, proxy_obj.proxy.ljust(20)))
self.db.put(proxy_obj)
self.log.info(
'UsefulProxyCheck - {} : {} validation pass'.format(
self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info(
'UsefulProxyCheck - {} : {} validation fail'.format(
self.name, proxy_obj.proxy.ljust(20)))
self.db.delete(proxy_obj.proxy)
self.queue.task_done()
def run(self):
self.log.info("RawProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("RawProxyCheck - {} : exit".format(self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_json)
proxy_obj, status = checkProxyUseful(proxy_obj, self.origin_ips)
if status:
if self.db.exists(proxy_obj.proxy):
self.log.info(
'RawProxyCheck - {} : {} validation exists'.format(
self.name, proxy_obj.proxy.ljust(20)))
else:
self.db.put(proxy_obj)
self.log.info(
'RawProxyCheck - {} : {} validation pass'.format(
self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info(
'RawProxyCheck - {} : {} validation fail'.format(
self.name, proxy_obj.proxy.ljust(20)))
self.queue.task_done()