def getIpstate():
ipstate = RedisUtils.get("ipstate")
if (ipstate):
return "%s" % ipstate
else:
RedisUtils.set("ipstate", 1)
return "1"
def process_proxy():
currentKey = getIpstate();
key = "ip%s-" % currentKey
keys = RedisUtils.getKeys(key)
keyNum = len(keys)
if (keyNum < 1):
logger.info("代理池异常数量<1 sleepping 120s")
RedisUtils.set("ipstate", int(currentKey) + 1)
time.sleep(1200)
key = "ip%s-" % getIpstate()
keys = RedisUtils.getKeys(key)
keyNum = len(keys)
randNum = random.randint(0, keyNum - 1)
current = keys[randNum]
value = RedisUtils.get(current)
proxies = {}
proxies["http"] = r'http://%s' % value
removeIp(proxies)
return proxies