def do_response():
redis = RedisClient(host=HOST, port=PORT, key=KEY)
RANDOM = str(redis.random())
RANDOM_MAX = str(redis.random_max())
ALL = redis.all()
all = ''
for ip in ALL:
all += str(ip) + "<br />"
ALL = all
ALL_MAX = redis.all_max()
all = ''
for ip in ALL_MAX:
all += str(ip) + "<br />"
ALL_MAX = all
COUNT = str(redis.count())
WRONG = 'Sorry,wrong name or wrong password,try again...'
if request.method == "GET":
name = request.args.get("name", "")
password = request.args.get("password", "")
method = request.args.get("method", "")
if not name:
return app.send_static_file("index.html")
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
if request.method == "POST":
name = request.form['name']
password = request.form['password']
method = request.form['method']
if name == "yooongchun" and password == "121561":
if method == "random":
return RANDOM
elif method == "random_max":
return RANDOM_MAX
elif method == "all":
return ALL
elif method == "count":
return COUNT
elif method == "all_max":
return ALL_MAX
else:
return WRONG
else:
return WRONG
class Tester(object):
def __init__(self):
self.redis = RedisClient()
async def test_single_proxy(self, proxy):
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('测试代理: {}'.format(real_proxy))
async with session.get(TEST_URL,
proxy=real_proxy,
allow_redirects=False,
timeout=10) as resp:
if resp.status in VALID_STATUS_CODE:
self.redis.max(proxy)
print('代理{} 可用'.format(real_proxy))
else:
self.redis.decrase(proxy)
print('代理{}的返回状态错误'.format(real_proxy))
except (ClientError,
aiohttp.client_exceptions.ClientConnectorError,
asyncio.TimeoutError, AttributeError):
self.redis.decrase(proxy)
print("代理{}请求异常".format(real_proxy))
def run(self):
'''
测试主函数
'''
print('开始测试')
try:
count = self.redis.count()
print('当前剩余{}个代理'.format(count))
for i in range(0, count, BATCH_TEST_SIZE):
start = i
stop = min(i + BATCH_TEST_SIZE, count)
print('测试测范围{}-{}'.format(start, stop))
test_proxies = self.redis.batch(start, stop)
loop = asyncio.get_event_loop()
tasks = [
self.test_single_proxy(proxy) for proxy in test_proxies
]
loop.run_until_complete(asyncio.wait(tasks))
sys.stdout.flush()
time.sleep(2)
except Exception as e:
print('测试异常{}'.format(e.args))
class API(object):
def __init__(self,
host="localhost",
port='6379',
key='Proxy',
moni_port=9999):
self.__redis = RedisClient(host=host,
port=port,
password=None,
key=key)
self.__port = moni_port
self.__host = host
self.__db_port = port
self.__key = key
def __message(self, method):
if method == "random":
return self.__redis.random()
elif method == "random_max":
return self.__redis.random_max()
elif method == "all":
return self.__redis.all()
elif method == "all_max":
return self.__redis.all_max()
elif method == "count":
return self.__redis.count()
else:
return "Sorry,wrong name or wrong password,try again..."
def run(self):
self.__set_api()
app.run(host="0.0.0.0", port=self.__port)
def __set_api(self):
global HOST
global PORT
global KEY
global MONI_PORT
HOST = self.__host
PORT = self.__db_port
KEY = self.__key
MONI_PORT = self.__port
class Getter(object):
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
'''
判断是否超出了抓取的限制
'''
if self.redis.count() >= settings.POOL_UPPER_THRESHOLD:
return True
return False
def run(self):
print('开始抓取')
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
sys.stdout.flush()
for proxy in proxies:
self.redis.add(proxy)
class Getter():
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def is_over_threshold(self):
"""
判断是否达到代理池限制
:return:
"""
if self.redis.count() >= POOL_UPPER_THRESHOLD:
return True
else:
return False
def run(self):
print("获取器开始执行")
if not self.is_over_threshold():
for callback_label in range(self.crawler.__CrawlFuncCount__):
callback = self.crawler.__CrawlFunc__[callback_label]
proxies = self.crawler.get_proxies(callback)
for proxy in proxies:
self.redis.add(proxy)
