def _runProxyFetch():
proxy_queue = Queue()
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)
def runProxyFetch():
proxy_queue = Queue()
for proxy in runFetcher():
proxy_queue.put(proxy)
runChecker("raw", proxy_queue)
def testMain():
proxy_queue = Queue()
for proxy in runFetcher():
proxy_queue.put(proxy)
# runChecker("raw", proxy_queue)
checker = Checker('raw', proxy_queue, f"thread_01")
checker.start()
checker.join()
def _runProxyFetch():
proxy_queue = Queue()
proxy_handler = ProxyHandler()
# 当代理池中数量小于poolSizeMin时，再抓取
if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
for proxy in runFetcher():
proxy_queue.put(Proxy(proxy).to_json)
runChecker("raw", proxy_queue)
def _runProxyFetch():
# proxy_queue = Queue()
proxy_handler = ProxyHandler()
for proxy in runFetcher():
proxy_handler.db.put(Proxy(proxy))
