def __init__(self, work_type, target_queue, thread_name):
Thread.__init__(self, name=thread_name)
self.work_type = work_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.target_queue = target_queue
self.conf = ConfigHandler()
def __init__(self, check_type, queue, thread_name):
Thread.__init__(self, name=thread_name)
self.type = check_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.queue = queue
self.conf = ConfigHandler()
def runProxyCheck():
proxy_queue = Queue()
if not ProxyHandler().getAll():
runProxyFetch()
for proxy in ProxyHandler().getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def _runProxyCheck():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
_runProxyFetch()
else:
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def __runProxyCheck():
proxy_handler = ProxyHandler()
proxy_queue = Queue()
if proxy_handler.db.getCount().get("total",
0) < proxy_handler.conf.poolSizeMin:
__runProxyFetch()
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy)
Checker("use", proxy_queue)
class _ThreadChecker(Thread):
""" 多线程检测 """
def __init__(self, work_type, target_queue, thread_name):
Thread.__init__(self, name=thread_name)
self.work_type = work_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.target_queue = target_queue
self.conf = ConfigHandler()
def run(self):
self.log.info("{}ProxyCheck - {}: start".format(
self.work_type.title(), self.name))
while True:
try:
proxy = self.target_queue.get(block=False)
except Empty:
self.log.info("{}ProxyCheck - {}: complete".format(
self.work_type.title(), self.name))
break
proxy = DoValidator.validator(proxy, self.work_type)
if self.work_type == "raw":
self.__ifRaw(proxy)
else:
self.__ifUse(proxy)
self.target_queue.task_done()
def __ifRaw(self, proxy):
if proxy.last_status:
if self.proxy_handler.exists(proxy):
self.log.info('RawProxyCheck - {}: {} exist'.format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info('RawProxyCheck - {}: {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info('RawProxyCheck - {}: {} fail'.format(
self.name, proxy.proxy.ljust(23)))
def __ifUse(self, proxy):
if proxy.last_status:
self.log.info('UseProxyCheck - {}: {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
if proxy.fail_count > self.conf.maxFailCount:
self.log.info(
'UseProxyCheck - {}: {} fail, count {} delete'.format(
self.name, proxy.proxy.ljust(23), proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
'UseProxyCheck - {}: {} fail, count {} keep'.format(
self.name, proxy.proxy.ljust(23), proxy.fail_count))
self.proxy_handler.put(proxy)
class Checker(Thread):
"""
多线程检测代理是否可用
"""
def __init__(self, check_type, queue, thread_name):
Thread.__init__(self, name=thread_name)
self.type = check_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.queue = queue
self.conf = ConfigHandler()
def run(self):
self.log.info("ProxyCheck - {} : start".format(self.name))
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("ProxyCheck - {} : complete".format(self.name))
break
proxy = Proxy.createFromJson(proxy_json)
proxy = proxyCheck(proxy)
if self.type == "raw":
if proxy.last_status:
if self.proxy_handler.exists(proxy):
self.log.info('ProxyCheck - {} : {} exists'.format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info('ProxyCheck - {} : {} success'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info('ProxyCheck - {} : {} fail'.format(
self.name, proxy.proxy.ljust(23)))
else:
if proxy.last_status:
self.log.info('ProxyCheck - {} : {} pass'.format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.update(proxy)
else:
if proxy.fail_count > self.conf.maxFailCount:
self.log.info(
'ProxyCheck - {} : {} fail, count {} delete'.
format(self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
'ProxyCheck - {} : {} fail, count {} keep'.format(
self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.update(proxy)
self.queue.task_done()
def __init__(self, fetch_source, proxy_dict):
Thread.__init__(self)
self.fetch_source = fetch_source
self.proxy_dict = proxy_dict
self.fetcher = getattr(ProxyFetcher, fetch_source, None)
self.log = LogHandler("fetcher")
self.conf = ConfigHandler()
self.proxy_handler = ProxyHandler()
class Checker(Thread):
def __init__(self, check_type, queue, thread_name):
Thread.__init__(self, name=thread_name)
self.type = check_type
self.log = LogHelper("proxy_checker")
self.proxy_handler = ProxyHandler()
self.queue = queue
def run(self) -> None:
self.log.info("ProxyCheck - {} : start".format(self.name))
while True:
try:
proxy_json = self.queue.get(block=False)
except Exception:
self.log.info("ProxyCheck - {} : complete".format(self.name))
break
proxy = Proxy.createFromJson(proxy_json)
proxy = check_proxy(proxy)
if self.type == "raw":
if proxy.last_status:
if self.proxy_handler.exits(proxy):
self.log.info("ProxyCheck - {} : {} exits".format(
self.name, proxy.proxy.ljust(23)))
else:
self.log.info("ProxyCheck - {} : {} success".format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
self.log.info("ProxyCheck - {} : {} fail".format(
self.name, proxy.proxy.ljust(23)))
else:
if proxy.last_status:
self.log.info("ProxyCheck - {} : {} pass".format(
self.name, proxy.proxy.ljust(23)))
self.proxy_handler.put(proxy)
else:
if proxy.fail_count > config.MAX_FAIL_COUNT:
self.log.info(
"ProxyCheck - {} : {} fail, count {} delete".
format(self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.delete(proxy)
else:
self.log.info(
"ProxyCheck - {} : {} fail, count {} keep".format(
self.name, proxy.proxy.ljust(23),
proxy.fail_count))
self.proxy_handler.put(proxy)
self.queue.task_done()
def _runProxyFetch():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
# 当代理池中数量小于poolSizeMin时，再抓取
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)
-------------------------------------------------
"""
__author__ = 'JHao'
import platform
from werkzeug.wrappers import Response
from flask import Flask, jsonify, request
from util.six import iteritems
from helper.proxy import Proxy
from handler.proxyHandler import ProxyHandler
from handler.configHandler import ConfigHandler
app = Flask(__name__)
conf = ConfigHandler()
proxy_handler = ProxyHandler()
class JsonResponse(Response):
@classmethod
def force_type(cls, response, environ=None):
if isinstance(response, (dict, list)):
response = jsonify(response)
return super(JsonResponse, cls).force_type(response, environ)
app.response_class = JsonResponse
api_list = [{
"url": "/get",
def _runProxyFetch():
# proxy_queue = Queue()
proxy_handler = ProxyHandler()
for proxy in runFetcher():
proxy_handler.db.put(Proxy(proxy))
def __init__(self):
self.log = LogHandler(self.name)
self.conf = ConfigHandler()
self.proxy_handler = ProxyHandler()
def __init__(self):
self.name = 'fetcherHandler'
self.log = LogHelper(self.name)
self.proxy_handler = ProxyHandler()
def run_proxy_check():
proxy_queue = Queue()
for proxy in ProxyHandler().get_all():
proxy_queue.put(proxy.to_json)
run_checker('use', proxy_queue)
