def run(self): try: proxy_manager = ProxyManager() logger.info("Start proxy verify") while True: proxies = proxy_manager.proxy_verified_before(minutes=30, limit=1000) if not len(proxies): logger.info( 'Not proxy need to be verified! Sleep [ 5 ] minutes.') proxy_manager.close() break verifier = ProxyGeventVerifier() start = time.time() tasks = verifier.generate_tasks(proxies) logger.info('Created %d verify tasks' % len(proxies)) gevent.joinall(tasks) logger.info('Proxy Verify Using %d sec.' % (time.time() - start)) passed, failed = 0, 0 for proxy in proxies: if proxy.usable: passed += 1 proxy_manager.verify_passed(proxy) else: failed += 1 proxy_manager.verify_failed(proxy) proxy_manager.commit() logger.info('Valid Complete! %d / %d' % (passed, failed)) proxy_manager.remove_bad_proxy() except Exception as e: logger.exception(e)
def run(self): try: pm = ProxyManager() proxies = set() tasks = [] self.__gen_fetch_tasks(tasks, proxies) self.__wait_fetch(tasks) logger.info('Fetched %d proxies' % len(proxies)) proxies = self.__remove_exist_proxies(pm, proxies) if proxies: verify_tasks = self.__gen_gevent_tasks(proxies) self.__wait_for_gevent_tasks(verify_tasks) self.__write_verify_result(pm, proxies) pm.close() gc.collect() logger.info('ProxyModel Fetch Finished, wait for 10 min') except Exception as e: logger.exception(e)