async def save_ip(self, ip: IPData):
await self.update_score(ip)
with await Redis.share() as redis:
if ip.http is True:
await redis.sadd(Config.REDIS_KEY_ABLE_HTTP, ip.to_str())
else:
await redis.srem(Config.REDIS_KEY_ABLE_HTTP, ip.to_str())
if ip.https is True:
await redis.sadd(Config.REDIS_KEY_ABLE_HTTPS, ip.to_str())
else:
await redis.srem(Config.REDIS_KEY_ABLE_HTTPS, ip.to_str())
# Rules check
for key, res in ip.rules.items():
if res is True:
await redis.sadd(Config.REDIS_KEY_ABLE_RULES % key, ip.to_str())
else:
await redis.srem(Config.REDIS_KEY_ABLE_RULES % key, ip.to_str())
# Delay pool
if ip.available():
delay_key = self.get_delay_key(ip.delay)
if delay_key:
await redis.sadd(delay_key, ip.to_str())
if ip.available():
await self.available_call(ip)
else:
await self.fail_call(ip)
async def https_check(self, ip: IPData, session) -> IPData:
"""
HTTPS 可用性检测
:return:
"""
try:
async with session.get(self.NORMAL_CHECK_URL.replace(
'http', 'https', 1),
proxy=ip.to_http()) as resp:
result = await resp.json()
if not result.get('origin'):
raise ValidationFailException()
ip.https = True
except Exception:
ip.https = False
return ip
async def http_check(self, ip: IPData, session) -> IPData:
"""
可用与匿名检测 毫秒
:param ip:
:return:
"""
time_spend = datetime.datetime.now()
try:
async with session.get(self.NORMAL_CHECK_URL,
proxy=ip.to_http()) as resp:
result = await resp.json()
if not result.get('origin'):
raise ValidationFailException()
time_spend = datetime.datetime.now() - time_spend
ip.delay = time_spend.total_seconds()
ip.http = True
except Exception:
ip.http = False
return ip
async def rules_check(self, ip: IPData, session) -> IPData:
"""
通过规则进行检测
:return:
"""
rules = {}
for rule in Config.RULES:
assert isinstance(rule, RuleData), 'Error rule format'
if not rule.enable:
continue
try:
async with session.get(rule.url, proxy=ip.to_http()) as resp:
result = await resp.text()
assert isinstance(result, str)
if rule.contains and result.find(rule.contains) < 0:
raise ValidationFailException()
rules[rule.key] = True
except Exception:
rules[rule.key] = False
ip.rules = rules
return ip
async def start_check(self):
with await Redis.share() as redis:
ip_str = await redis.blpop(Config.REDIS_KEY_CHECK_POOL)
ip_str = ip_str[1].decode()
Logger.info('[check] got ip %s' % ip_str)
Prometheus.IP_CHECK_TOTAL.inc(1)
ip = IPData.with_str(ip_str)
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(
Config.DEFAULT_REQUEST_CHECK_TIME_OUT)) as session:
ip = await self.http_check(ip, session)
ip = await self.https_check(ip, session)
ip = await self.rules_check(ip, session)
Logger.info(
'[check] Check result %s http %s https %s %s', ip.to_str(),
ip.http, ip.https,
" ".join(["%s %s" % (k, r) for k, r in ip.rules.items()]))
await IPSaver().save_ip(ip)
async def get_ips(cls,
http: bool = True,
https: bool = False,
delay: int = None,
rule: str = None):
keys = []
if http:
keys.append(Config.REDIS_KEY_ABLE_HTTP)
if https:
keys.append(Config.REDIS_KEY_ABLE_HTTPS)
if delay:
keys.append(Config.REDIS_KEY_NET_DELAY % delay)
if rule:
keys.append(Config.REDIS_KEY_ABLE_RULES % rule)
with await Redis.share() as redis:
ips = await redis.sinter(*keys)
ips = [IPData.with_str(ip.decode()) for ip in ips]
return ips
async def update_score(self, ip: IPData):
with await Redis.share() as redis:
score = await redis.zscore(Config.REDIS_KEY_IP_POOL, ip.to_str())
ip.score = score if score is not None else 0
async def available_call(self, ip: IPData):
if ip.score >= Config.DEFAULT_MAX_SCORE:
return
with await Redis.share() as redis:
await redis.zincrby(Config.REDIS_KEY_IP_POOL, Config.DEFAULT_INC_SCORE, ip.to_str())
async def fail_call(self, ip: IPData):
if ip.score <= Config.DEFAULT_MINI_SCORE:
return
with await Redis.share() as redis:
await redis.zincrby(Config.REDIS_KEY_IP_POOL, -Config.DEFAULT_DEC_SCORE, ip.to_str())