def __validProxy(self):
"""
验证代理
:return:
"""
while True:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
if validUsefulProxy(each_proxy):
# 成功计数器加1
self.db.inckey(each_proxy, 1)
self.log.debug(
'validProxy_b: {} validation pass'.format(each_proxy))
else:
# 失败计数器减一
self.db.inckey(each_proxy, -1)
# self.db.delete(each_proxy)
self.log.info(
'validProxy_b: {} validation fail'.format(each_proxy))
value = self.db.getvalue(each_proxy)
if value and int(value) < -5:
# 计数器小于-5删除该代理
self.db.delete(each_proxy)
self.log.info('validProxy_a running normal')
def _proxyFilter(self):
for item in self.all_proxies:
if validUsefulProxy(item):
self.vaild_proxies.append(item)
self.roper.addworkin(item)
else:
self.invalid_proxies.append(item)
def _handler(self):
try:
if (self.roper.getCacheNum() > 0):
proxy = self.roper.getRandomUntrust()
print(proxy)
self.roper.remCache(proxy)
if validUsefulProxy(proxy, mode='out'):
self.roper.addWorkin(proxy)
else:
time.sleep(1)
return
except Exception as ex:
time.sleep(2)
def _handler(self):
try:
if (self.rdb.scard(self.cachename) > 0):
proxy = self.rdb.srandmember(self.cachename, 1)[0].decode('utf-8')
self.rdb.srem(self.cachename, proxy)
num_retries = 3
for i in range(0, num_retries):
if validUsefulProxy(proxy):
self.rdb.sadd(self.workinname, proxy)
return
else:
time.sleep(1)
return
except Exception as ex:
time.sleep(2)
def _handler(self):
try:
num = self.rdb.scard(self.workinname)
if (num > 0):
proxy = self.rdb.srandmember(self.workinname,
1)[0].decode('utf-8')
num_retries = 3
for i in range(0, num_retries):
if validUsefulProxy(proxy):
if (num <= 3):
time.sleep(3)
return
self.rdb.srem(self.workinname, proxy)
else:
time.sleep(1)
except Exception as ex:
time.sleep(1)
def _handler(self):
try:
num = self.roper.getWorkinNum()
if (num > 0):
proxy = self.roper.getRandomUsable()
print(proxy)
num_retries = 3
for i in range(0, num_retries):
if validUsefulProxy(proxy, mode='out'):
if (num <= 3):
time.sleep(3)
return
self.roper.remWorkin(proxy)
else:
time.sleep(1)
except Exception as ex:
time.sleep(1)
def validProxy(self):
"""
验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s start validProxy_a' % time.ctime())
exist_proxy = self.db.getAll()
while raw_proxy:
if validUsefulProxy(raw_proxy) and (raw_proxy not in exist_proxy):
self.db.changeTable(self.useful_proxy_queue)
self.db.put(raw_proxy)
self.log.info('validProxy_a: %s validation pass' % raw_proxy)
else:
self.log.debug('validProxy_a: %s validation fail' % raw_proxy)
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s validProxy_a complete' % time.ctime())
from util.utilFunction import validUsefulProxy
print(validUsefulProxy('141.196.72.166:8080'))