def runProxyCheck():
proxy_queue = Queue()
for proxy in ProxyHandler().getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def _runProxyFetch():
proxy_queue = Queue()
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)
def runProxyCheck():
proxy_queue = Queue()
dbclient = SqlClient()
for proxy in dbclient.getAll():
proxy_queue.put(proxy)
runChecker("use", proxy_queue)
def runProxyFetch():
proxy_queue = Queue()
for proxy in runFetcher():
proxy_queue.put(proxy)
runChecker("raw", proxy_queue)
def __runProxyFetch():
proxy_queue = Queue()
proxy_fetcher = Fetcher()
for proxy in proxy_fetcher.run():
proxy_queue.put(proxy)
Checker("raw", proxy_queue)
def _runProxyCheck():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
_runProxyFetch()
else:
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy.to_json)
runChecker("use", proxy_queue)
def __runProxyCheck():
proxy_handler = ProxyHandler()
proxy_queue = Queue()
if proxy_handler.db.getCount().get("total",
0) < proxy_handler.conf.poolSizeMin:
__runProxyFetch()
for proxy in proxy_handler.getAll():
proxy_queue.put(proxy)
Checker("use", proxy_queue)
def _runProxyFetch():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
# 当代理池中数量小于poolSizeMin时，再抓取
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)