def run(self):
try:
proxy_manager = ProxyManager()
logger.info("Start proxy verify")
while True:
proxies = proxy_manager.proxy_verified_before(minutes=30,
limit=1000)
if not len(proxies):
logger.info(
'Not proxy need to be verified! Sleep [ 5 ] minutes.')
proxy_manager.close()
break
verifier = ProxyGeventVerifier()
start = time.time()
tasks = verifier.generate_tasks(proxies)
logger.info('Created %d verify tasks' % len(proxies))
gevent.joinall(tasks)
logger.info('Proxy Verify Using %d sec.' %
(time.time() - start))
passed, failed = 0, 0
for proxy in proxies:
if proxy.usable:
passed += 1
proxy_manager.verify_passed(proxy)
else:
failed += 1
proxy_manager.verify_failed(proxy)
proxy_manager.commit()
logger.info('Valid Complete! %d / %d' % (passed, failed))
proxy_manager.remove_bad_proxy()
except Exception as e:
logger.exception(e)
def run(self):
try:
pm = ProxyManager()
proxies = set()
tasks = []
self.__gen_fetch_tasks(tasks, proxies)
self.__wait_fetch(tasks)
logger.info('Fetched %d proxies' % len(proxies))
proxies = self.__remove_exist_proxies(pm, proxies)
if proxies:
verify_tasks = self.__gen_gevent_tasks(proxies)
self.__wait_for_gevent_tasks(verify_tasks)
self.__write_verify_result(pm, proxies)
pm.close()
gc.collect()
logger.info('ProxyModel Fetch Finished, wait for 10 min')
except Exception as e:
logger.exception(e)