def set_max(self, proxy):
"""
set proxy to max score
:param proxy:
:return:
"""
app_logger.info(f'{proxy} is valid, set to {PROXY_SCORE_MAX}')
maping = {proxy: PROXY_SCORE_MAX}
return self.db.zadd(REDIS_KEY, maping)
def run_getter(self, cycle=CYCLE_GETTER):
"""
定时开启爬虫补充代理
:param cycle:
:return:
"""
if not ENABLE_GETTER:
app_logger.info('getter not enabled, exit')
return
getter = Getproxies()
sch.add_job(getter.run, 'interval', seconds=cycle, id='run_getter')
sch.start()
def run_tester(self, cycle=CYCLE_TESTER):
"""
定时启动检测器
:param cycle:
:return:
"""
if not ENABLE_TESTER:
app_logger.info('tester not enabled, exit')
return
tester = Tester()
sch.add_job(tester.run, 'interval', seconds=cycle, id='run_tester')
sch.start()
def crawl(self):
"""
crawl main method
"""
for url in self.urls:
app_logger.info(f'fetching {url}')
# html = self.LOOP.run_until_complete(asyncio.gather(self._get_page(url)))
html = self.LOOP.run_until_complete(self._get_page(url))
# print('html', html)
for proxy in self.parse(html):
# app_logger.info(f'fetched proxy {proxy.string()} from {url}')
yield proxy
def decrease(self, proxy) -> int:
"""
decrease score of proxy, if small than PROXY_SCORE_MIN, delete it
:param proxy:
:return: new score
"""
score = self.db.zscore(REDIS_KEY, proxy)
if score and score > PROXY_SCORE_MIN:
app_logger.error(f'{proxy} current score {score}, decrease 1')
return self.db.zincrby(REDIS_KEY, -1, proxy)
else:
app_logger.info(f'{proxy} current score {score}, remove it')
self.db.zrem(REDIS_KEY, proxy)
def run(self):
"""
test main method
:return:
"""
# event loop of aiohttp
app_logger.info('stating tester...')
count = self.redis.count()
app_logger.debug(f'{count} proxies to test')
for i in range(0, count, TEST_BATCH):
# start end end offset
start, end = i, min(i + TEST_BATCH, count)
app_logger.debug(f'testing proxies from {start} to {end} indices')
proxies = self.redis.batch(start, end)
tasks = [self.test(proxy) for proxy in proxies]
# run tasks using event loop
self.loop.run_until_complete(asyncio.wait(tasks))
def run(self):
"""调度器启动"""
global tester_process, getter_process, server_process
try:
app_logger.info('starting proxypool...')
if ENABLE_TESTER:
tester_process = multiprocessing.Process(
target=self.run_tester)
app_logger.info(
f'starting tester, pid {tester_process.pid}...')
tester_process.start()
if ENABLE_GETTER:
getter_process = multiprocessing.Process(
target=self.run_getter)
# print(dir(getter_process),getter_process)
app_logger.info(f'starting getter, pid{getter_process.pid}...')
getter_process.start()
if ENABLE_SERVER:
server_process = multiprocessing.Process(
target=self.run_server)
app_logger.info(f'starting server, pid{server_process.pid}...')
server_process.start()
tester_process.join()
getter_process.join()
server_process.join()
except KeyboardInterrupt:
app_logger.info('received keyboard interrupt signal')
tester_process.terminate()
getter_process.terminate()
server_process.terminate()
finally:
# must call join method before calling is_alive
tester_process.join()
getter_process.join()
server_process.join()
app_logger.info(
f'tester is {"alive" if tester_process.is_alive() else "dead"}'
)
app_logger.info(
f'getter is {"alive" if getter_process.is_alive() else "dead"}'
)
app_logger.info(
f'server is {"alive" if server_process.is_alive() else "dead"}'
)
app_logger.info('proxy terminated')
def run_server(self):
"""web服务端开启"""
if not ENABLE_SERVER:
app_logger.info('server not enabled, exit')
return
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)