def monitor_proxies(): redis_0 = RedisDB() config = os.path.join(os.path.dirname(__file__) + '/../config.conf') redis_key = tools.get_conf_value(config, 'redis', 'redis_key') redis_key2 = tools.get_conf_value(config, 'redis', 'redis_key2') sum = redis_0.count(redis_key) sum2 = redis_0.count(redis_key2) log.debug("douban当前redis库中剩余ip总数:%d" % sum) log.debug("weibo当前redis库中剩余ip总数:%d" % sum2)
# proxies = { # "https": "https:" + ip_addr, # "http": "http:" + ip_addr # } try: if ping(ip) == True: #if check_ip(ips) == True: redis_0.sadd(redis_key, ip_addr) redis_0.sadd(redis_key2, ip_addr) log.debug("%s成功添加到redis" % ip) else: log.info("%s代理无效" % ip_addr) except Exception as e: log.info("出现异常终止") sys.exit(0) if __name__ == "__main__": # pid = os.getpid() # write(pid) while True: if redis_0.count(redis_key) >= MAX_POOL: time.sleep(15) continue else: ips = get_proxies() for ip in ips: save_proxies(ip[0], ip[1])