Exemple #1
0
def runProxyCheck():
    proxy_queue = Queue()

    for proxy in ProxyHandler().getAll():
        proxy_queue.put(proxy.to_json)

    runChecker("use", proxy_queue)
def _runProxyFetch():
    proxy_queue = Queue()

    for proxy in runFetcher():
        proxy_queue.put(Proxy(proxy).to_json)

    runChecker("raw", proxy_queue)
Exemple #3
0
def runProxyCheck():
    proxy_queue = Queue()
    dbclient = SqlClient()
    for proxy in dbclient.getAll():
        proxy_queue.put(proxy)

    runChecker("use", proxy_queue)
Exemple #4
0
def runProxyFetch():
    proxy_queue = Queue()

    for proxy in runFetcher():
        proxy_queue.put(proxy)

    runChecker("raw", proxy_queue)
Exemple #5
0
def __runProxyFetch():
    proxy_queue = Queue()
    proxy_fetcher = Fetcher()

    for proxy in proxy_fetcher.run():
        proxy_queue.put(proxy)

    Checker("raw", proxy_queue)
def _runProxyCheck():
    proxy_queue = Queue()
    proxy_handler = ProxyHandler()
    if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
        _runProxyFetch()
    else:
        for proxy in proxy_handler.getAll():
            proxy_queue.put(proxy.to_json)
        runChecker("use", proxy_queue)
Exemple #7
0
def __runProxyCheck():
    proxy_handler = ProxyHandler()
    proxy_queue = Queue()
    if proxy_handler.db.getCount().get("total",
                                       0) < proxy_handler.conf.poolSizeMin:
        __runProxyFetch()
    for proxy in proxy_handler.getAll():
        proxy_queue.put(proxy)
    Checker("use", proxy_queue)
Exemple #8
0
def _runProxyFetch():
    proxy_queue = Queue()
    proxy_handler = ProxyHandler()
    # 当代理池中数量小于poolSizeMin时,再抓取
    if proxy_handler.db.getCount() < proxy_handler.conf.poolSizeMin:
        for proxy in runFetcher():
            proxy_queue.put(Proxy(proxy).to_json)

        runChecker("raw", proxy_queue)