class ProxyRefreshSchedule(ProxyManager): """ 代理定时刷新 """ def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('refresh_schedule') def validProxy(self): """ 验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue :return: """ self.db.changeTable(self.raw_proxy_queue) raw_proxy = self.db.pop() self.log.info('%s start validProxy_a' % time.ctime()) # exist_proxy = self.db.getAll() while raw_proxy: self.db.changeTable(self.useful_proxy_queue) exist_proxy = self.db.getAll() if validUsefulProxy(raw_proxy) and (raw_proxy not in exist_proxy): # self.db.changeTable(self.useful_proxy_queue) self.db.put(raw_proxy) self.log.info('validProxy_a: %s validation pass' % raw_proxy) else: self.log.debug('validProxy_a: %s validation fail' % raw_proxy) self.db.changeTable(self.raw_proxy_queue) raw_proxy = self.db.pop() self.log.info('%s validProxy_a complete' % time.ctime())
class ProxyValidSchedule(ProxyManager): def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('valid_schedule') def __validProxy(self): """ 验证代理 :return: """ while True: self.db.changeTable(self.useful_proxy_queue) for each_proxy in self.db.getAll(): if isinstance(each_proxy, bytes): each_proxy = each_proxy.decode('utf-8') if validUsefulProxy(each_proxy): # 成功计数器加1 self.db.inckey(each_proxy, 1) self.log.debug('validProxy_b: {} validation pass'.format(each_proxy)) else: # 失败计数器减一 self.db.inckey(each_proxy, -1) # self.db.delete(each_proxy) self.log.info('validProxy_b: {} validation fail'.format(each_proxy)) value = self.db.getvalue(each_proxy) if value and value < -5: # 计数器小于-5删除该代理 self.db.delete(each_proxy) self.log.info('validProxy_a running normal') def main(self): self.__validProxy()
class ProxyRefreshSchedule(ProxyManager): """ 代理定时刷新 """ def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('refresh_schedule') def valid_proxy(self): """ valid_proxy :return: """ self.db.changeTable(self.raw_proxy_queue) raw_proxy = self.db.pop() self.log.info('%s start valid proxy' % time.ctime()) while raw_proxy: if validUsefulProxy(raw_proxy): self.db.changeTable(self.useful_proxy_queue) self.db.put(raw_proxy) self.log.debug('proxy: %s validation passes' % raw_proxy) else: self.log.debug('proxy: %s validation fail' % raw_proxy) pass self.db.changeTable(self.raw_proxy_queue) raw_proxy = self.db.pop() self.log.info('%s valid proxy complete' % time.ctime())
class ProxyValidSchedule(ProxyManager): def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('valid_schedule') def __validProxy__(self): """ 验证代理 :return: """ while 1: self.db.changeTable(self.useful_proxy_queue) for each_proxy in self.db.getAll(): if validUsefulProxy(each_proxy): self.log.debug( 'proxy: {} validation pass'.format(each_proxy)) if validTelnetProxy(each_proxy): self.log.debug( 'proxy: {} telnet pass'.format(each_proxy)) else: self.db.delete(each_proxy) self.log.info( 'proxy: {} telnet fail'.format(each_proxy)) else: self.db.delete(each_proxy) self.log.info( 'proxy: {} validation fail'.format(each_proxy)) self.log.info(u'代理验证程序运行正常') def main(self): self.__validProxy__()
class ProxyValidSchedule(ProxyManager): def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('valid_schedule') def validProxy(self): """ 验证代理 :return: """ while True: # for num in range(5): self.db.changeTable(self.useful_proxy_queue) each_proxys = self.db.getAll() print "验证所有ip", each_proxys if not each_proxys: time.sleep(100) for each_proxy in each_proxys: if isinstance(each_proxy, bytes): each_proxy = each_proxy.decode('utf-8') print "验证ip:", each_proxy if validUsefulProxy(each_proxy): # 成功计数器加1 self.db.inckey(each_proxy, 1) self.log.debug( 'validProxy_b: {} validation pass'.format(each_proxy)) else: # print "删除:",each_proxy # self.db.delete(each_proxy) # 失败计数器减一 self.db.inckey(each_proxy, -1) # self.db.delete(each_proxy) self.log.info( 'validProxy_b: {} validation fail'.format(each_proxy)) value = self.db.getvalue(each_proxy) if value and int(value) < -1: # 计数器小于-5删除该代理 self.db.delete(each_proxy) self.log.info('validProxy_a running normal')
class ProxyValidSchedule(ProxyManager): def __init__(self): ProxyManager.__init__(self) self.db = DbClient() self.log = LogHandler('valid_schedule') def __validProxy(self): """ 验证代理 :return: """ time.sleep(60 * 0 * random.random()) while True: self.db.changeTable(self.useful_proxy_queue) for each_proxy in self.db.getAll(): if isinstance(each_proxy, bytes): each_proxy = each_proxy.decode('utf-8') if validUsefulProxy(each_proxy) == True: # 成功计数器加1 self.db.inckey(each_proxy, 1) self.log.debug('validProxy_b: {} validation pass'.format(each_proxy)) else: # 失败计数器减一 print "原有value " + str(self.db.getvalue(each_proxy)) if self.db.getvalue(each_proxy) >= 0: self.db.inckey(each_proxy, -1*int(self.db.getvalue(each_proxy))) else: self.db.inckey(each_proxy, -1) # self.db.delete(each_proxy) self.log.info('validProxy_b: {} validation fail'.format(each_proxy)) value = self.db.getvalue(each_proxy) print value if None != value and int(value) < 0: # 计数器小于-5删除该代理 print "删除" + each_proxy self.db.delete(each_proxy) self.log.info('validProxy_a running normal') def main(self): self.__validProxy()
class ProxyValidSchedule(ProxyManager): def __init__(self): ProxyManager.__init__(self) self.log = LogHandler('valid_schedule') def __validProxy__(self): """ 验证代理 :return: """ while 1: self.db.changeTable(self.useful_proxy_queue) for each_proxy in self.db.getAll(): if validUsefulProxy(each_proxy): self.log.debug('proxy: {} validation pass'.format(each_proxy)) else: self.db.delete(each_proxy) self.log.info('proxy: {} validation fail'.format(each_proxy)) self.log.info(u'代理验证程序运行正常') def main(self): self.__validProxy__()