def do_response(): redis = RedisClient(host=HOST, port=PORT, key=KEY) RANDOM = str(redis.random()) RANDOM_MAX = str(redis.random_max()) ALL = redis.all() all = '' for ip in ALL: all += str(ip) + "<br />" ALL = all ALL_MAX = redis.all_max() all = '' for ip in ALL_MAX: all += str(ip) + "<br />" ALL_MAX = all COUNT = str(redis.count()) WRONG = 'Sorry,wrong name or wrong password,try again...' if request.method == "GET": name = request.args.get("name", "") password = request.args.get("password", "") method = request.args.get("method", "") if not name: return app.send_static_file("index.html") if name == "yooongchun" and password == "121561": if method == "random": return RANDOM elif method == "random_max": return RANDOM_MAX elif method == "all": return ALL elif method == "count": return COUNT elif method == "all_max": return ALL_MAX else: return WRONG else: return WRONG if request.method == "POST": name = request.form['name'] password = request.form['password'] method = request.form['method'] if name == "yooongchun" and password == "121561": if method == "random": return RANDOM elif method == "random_max": return RANDOM_MAX elif method == "all": return ALL elif method == "count": return COUNT elif method == "all_max": return ALL_MAX else: return WRONG else: return WRONG
class Tester(object): def __init__(self): self.redis = RedisClient() async def test_single_proxy(self, proxy): conn = aiohttp.TCPConnector(verify_ssl=False) async with aiohttp.ClientSession(connector=conn) as session: try: if isinstance(proxy, bytes): proxy = proxy.decode('utf-8') real_proxy = 'http://' + proxy print('测试代理: {}'.format(real_proxy)) async with session.get(TEST_URL, proxy=real_proxy, allow_redirects=False, timeout=10) as resp: if resp.status in VALID_STATUS_CODE: self.redis.max(proxy) print('代理{} 可用'.format(real_proxy)) else: self.redis.decrase(proxy) print('代理{}的返回状态错误'.format(real_proxy)) except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError): self.redis.decrase(proxy) print("代理{}请求异常".format(real_proxy)) def run(self): ''' 测试主函数 ''' print('开始测试') try: count = self.redis.count() print('当前剩余{}个代理'.format(count)) for i in range(0, count, BATCH_TEST_SIZE): start = i stop = min(i + BATCH_TEST_SIZE, count) print('测试测范围{}-{}'.format(start, stop)) test_proxies = self.redis.batch(start, stop) loop = asyncio.get_event_loop() tasks = [ self.test_single_proxy(proxy) for proxy in test_proxies ] loop.run_until_complete(asyncio.wait(tasks)) sys.stdout.flush() time.sleep(2) except Exception as e: print('测试异常{}'.format(e.args))
class API(object): def __init__(self, host="localhost", port='6379', key='Proxy', moni_port=9999): self.__redis = RedisClient(host=host, port=port, password=None, key=key) self.__port = moni_port self.__host = host self.__db_port = port self.__key = key def __message(self, method): if method == "random": return self.__redis.random() elif method == "random_max": return self.__redis.random_max() elif method == "all": return self.__redis.all() elif method == "all_max": return self.__redis.all_max() elif method == "count": return self.__redis.count() else: return "Sorry,wrong name or wrong password,try again..." def run(self): self.__set_api() app.run(host="0.0.0.0", port=self.__port) def __set_api(self): global HOST global PORT global KEY global MONI_PORT HOST = self.__host PORT = self.__db_port KEY = self.__key MONI_PORT = self.__port
class Getter(object): def __init__(self): self.redis = RedisClient() self.crawler = Crawler() def is_over_threshold(self): ''' 判断是否超出了抓取的限制 ''' if self.redis.count() >= settings.POOL_UPPER_THRESHOLD: return True return False def run(self): print('开始抓取') if not self.is_over_threshold(): for callback_label in range(self.crawler.__CrawlFuncCount__): callback = self.crawler.__CrawlFunc__[callback_label] proxies = self.crawler.get_proxies(callback) sys.stdout.flush() for proxy in proxies: self.redis.add(proxy)
class Getter(): def __init__(self): self.redis = RedisClient() self.crawler = Crawler() def is_over_threshold(self): """ 判断是否达到代理池限制 :return: """ if self.redis.count() >= POOL_UPPER_THRESHOLD: return True else: return False def run(self): print("获取器开始执行") if not self.is_over_threshold(): for callback_label in range(self.crawler.__CrawlFuncCount__): callback = self.crawler.__CrawlFunc__[callback_label] proxies = self.crawler.get_proxies(callback) for proxy in proxies: self.redis.add(proxy)