def runProxyCheck():
proxy_queue = Queue()
if not ProxyHandler().getAll():
runProxyFetch()
for proxy in ProxyHandler().getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def __init__(self, check_type, queue, thread_name):
Thread.__init__(self, name=thread_name)
self.type = check_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.queue = queue
self.conf = ConfigHandler()
def __init__(self, work_type, target_queue, thread_name):
Thread.__init__(self, name=thread_name)
self.work_type = work_type
self.log = LogHandler("checker")
self.proxy_handler = ProxyHandler()
self.target_queue = target_queue
self.conf = ConfigHandler()
def __init__(self, fetch_source, proxy_dict):
Thread.__init__(self)
self.fetch_source = fetch_source
self.proxy_dict = proxy_dict
self.fetcher = getattr(ProxyFetcher, fetch_source, None)
self.log = LogHandler("fetcher")
self.conf = ConfigHandler()
self.proxy_handler = ProxyHandler()
def _runProxyCheck():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
_runProxyFetch()
else:
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def __runProxyCheck():
proxy_handler = ProxyHandler()
proxy_queue = Queue()
if proxy_handler.db.getCount().get("total",
0) < proxy_handler.conf.poolSizeMin:
__runProxyFetch()
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy)
Checker("use", proxy_queue)
def _runProxyFetch():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
# 当代理池中数量小于poolSizeMin时，再抓取
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)
-------------------------------------------------
"""
__author__ = 'JHao'
import platform
from werkzeug.wrappers import Response
from flask import Flask, jsonify, request
from util.six import iteritems
from helper.proxy import Proxy
from handler.proxyHandler import ProxyHandler
from handler.configHandler import ConfigHandler
app = Flask(__name__)
conf = ConfigHandler()
proxy_handler = ProxyHandler()
class JsonResponse(Response):
@classmethod
def force_type(cls, response, environ=None):
if isinstance(response, (dict, list)):
response = jsonify(response)
return super(JsonResponse, cls).force_type(response, environ)
app.response_class = JsonResponse
api_list = [{
"url": "/get",
def _runProxyFetch():
# proxy_queue = Queue()
proxy_handler = ProxyHandler()
for proxy in runFetcher():
proxy_handler.db.put(Proxy(proxy))
def __init__(self):
self.log = LogHandler(self.name)
self.conf = ConfigHandler()
self.proxy_handler = ProxyHandler()
def __init__(self):
self.name = 'fetcherHandler'
self.log = LogHelper(self.name)
self.proxy_handler = ProxyHandler()
def run_proxy_check():
proxy_queue = Queue()
for proxy in ProxyHandler().get_all():
proxy_queue.put(proxy.to_json)
run_checker('use', proxy_queue)