class ProxyRefreshSchedule(ProxyManager):
"""
代理定时刷新
"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('refresh_schedule')
def validProxy(self):
"""
验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s start validProxy_a' % time.ctime())
# exist_proxy = self.db.getAll()
while raw_proxy:
self.db.changeTable(self.useful_proxy_queue)
exist_proxy = self.db.getAll()
if validUsefulProxy(raw_proxy) and (raw_proxy not in exist_proxy):
# self.db.changeTable(self.useful_proxy_queue)
self.db.put(raw_proxy)
self.log.info('validProxy_a: %s validation pass' % raw_proxy)
else:
self.log.debug('validProxy_a: %s validation fail' % raw_proxy)
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s validProxy_a complete' % time.ctime())
class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
def __validProxy(self):
"""
验证代理
:return:
"""
while True:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
if validUsefulProxy(each_proxy):
# 成功计数器加1
self.db.inckey(each_proxy, 1)
self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
else:
# 失败计数器减一
self.db.inckey(each_proxy, -1)
# self.db.delete(each_proxy)
self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
value = self.db.getvalue(each_proxy)
if value and value < -5:
# 计数器小于-5删除该代理
self.db.delete(each_proxy)
self.log.info('validProxy_a running normal')
def main(self):
self.__validProxy()
class ProxyRefreshSchedule(ProxyManager):
"""
代理定时刷新
"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('refresh_schedule')
def valid_proxy(self):
"""
valid_proxy
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s start valid proxy' % time.ctime())
while raw_proxy:
if validUsefulProxy(raw_proxy):
self.db.changeTable(self.useful_proxy_queue)
self.db.put(raw_proxy)
self.log.debug('proxy: %s validation passes' % raw_proxy)
else:
self.log.debug('proxy: %s validation fail' % raw_proxy)
pass
self.db.changeTable(self.raw_proxy_queue)
raw_proxy = self.db.pop()
self.log.info('%s valid proxy complete' % time.ctime())
class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
def __validProxy__(self):
"""
验证代理
:return:
"""
while 1:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if validUsefulProxy(each_proxy):
self.log.debug(
'proxy: {} validation pass'.format(each_proxy))
if validTelnetProxy(each_proxy):
self.log.debug(
'proxy: {} telnet pass'.format(each_proxy))
else:
self.db.delete(each_proxy)
self.log.info(
'proxy: {} telnet fail'.format(each_proxy))
else:
self.db.delete(each_proxy)
self.log.info(
'proxy: {} validation fail'.format(each_proxy))
self.log.info(u'代理验证程序运行正常')
def main(self):
self.__validProxy__()
class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
def validProxy(self):
"""
验证代理
:return:
"""
while True:
# for num in range(5):
self.db.changeTable(self.useful_proxy_queue)
each_proxys = self.db.getAll()
print "验证所有ip", each_proxys
if not each_proxys:
time.sleep(100)
for each_proxy in each_proxys:
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
print "验证ip：", each_proxy
if validUsefulProxy(each_proxy):
# 成功计数器加1
self.db.inckey(each_proxy, 1)
self.log.debug(
'validProxy_b: {} validation pass'.format(each_proxy))
else:
# print "删除:",each_proxy
# self.db.delete(each_proxy)
# 失败计数器减一
self.db.inckey(each_proxy, -1)
# self.db.delete(each_proxy)
self.log.info(
'validProxy_b: {} validation fail'.format(each_proxy))
value = self.db.getvalue(each_proxy)
if value and int(value) < -1:
# 计数器小于-5删除该代理
self.db.delete(each_proxy)
self.log.info('validProxy_a running normal')
class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.db = DbClient()
self.log = LogHandler('valid_schedule')
def __validProxy(self):
"""
验证代理
:return:
"""
time.sleep(60 * 0 * random.random())
while True:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if isinstance(each_proxy, bytes):
each_proxy = each_proxy.decode('utf-8')
if validUsefulProxy(each_proxy) == True:
# 成功计数器加1
self.db.inckey(each_proxy, 1)
self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
else:
# 失败计数器减一
print "原有value " + str(self.db.getvalue(each_proxy))
if self.db.getvalue(each_proxy) >= 0:
self.db.inckey(each_proxy, -1*int(self.db.getvalue(each_proxy)))
else:
self.db.inckey(each_proxy, -1)
# self.db.delete(each_proxy)
self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
value = self.db.getvalue(each_proxy)
print value
if None != value and int(value) < 0:
# 计数器小于-5删除该代理
print "删除" + each_proxy
self.db.delete(each_proxy)
self.log.info('validProxy_a running normal')
def main(self):
self.__validProxy()
class ProxyValidSchedule(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('valid_schedule')
def __validProxy__(self):
"""
验证代理
:return:
"""
while 1:
self.db.changeTable(self.useful_proxy_queue)
for each_proxy in self.db.getAll():
if validUsefulProxy(each_proxy):
self.log.debug('proxy: {} validation pass'.format(each_proxy))
else:
self.db.delete(each_proxy)
self.log.info('proxy: {} validation fail'.format(each_proxy))
self.log.info(u'代理验证程序运行正常')
def main(self):
self.__validProxy__()
