class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
判断是否达到了代理池限制
:return:
"""
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
print('获取器开始执行')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
print(proxy, type(proxy))
self.redis.add(proxy)
class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
判断数据库内容数量超过阈值
:return: （bool）
"""
if self.redis.count() > 10000:
return True
else:
return False
def run(self):
"""
运行爬虫程序
"""
print('开始获取代理')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
for proxy in proxies:
self.redis.add(proxy)
def addproxy():
print('开始添加代理')
lists = []
Crawler().getAllproxy(lists)
for list in lists:
mysql.add(list[0], list[1])
print('代理添加完成')
class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
return False
def run(self):
print('获取器开始执行')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__crawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
for proxy in proxies:
self.redis.add(proxy)
class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
Compare to the capacity of proxy pool.
"""
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
logger.debug('Getter is running.')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
# 获取代理
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
self.redis.add(proxy)
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
# -*- coding: utf-8 -*-
from ProxyPool.crawler import Crawler
c = Crawler()
proxies = c.crawl_89ip()
for proxy in proxies:
print(proxy)