#!/usr/bin/env python import redis import IpBroker import pickle import time import gzip from cStringIO import StringIO logger = IpBroker.get_logger("IP_CLIENT") class IpClient(object): """IpClient for get proxy ip, port and type """ def __init__(self): self.broker = redis.StrictRedis(**IpBroker.IP_BROKER) self.pointer = 0 self.ip_cache = {} self.ip_len = 0 self.ips = [] self.expire_time = 1 * 60 * 60 # One hour def getIps(self, length=1): """Get length ips from pool,return ip dict ip -> type&port http == port + 100000 https == port + 1000000 sock5 == port + 10000000 """ self.pointer = (
#!/usr/bin/python import redis import pickle import time import IpBroker import requests import threading import gzip from cStringIO import StringIO from Queue import Queue requests.HTTPErrorDEFAULT_RETRIES = 2 logger = IpBroker.get_logger("IP_PRODUCER") THREAD_COUNT = 300 class IpProducer(object): """IpProducer to put proxy ip into pool, port and type """ def __init__(self): self.broker = redis.StrictRedis(**IpBroker.IP_BROKER) def putIps(self, ips): """Get length ips from pool,return ip dict ip -> type&port http == port + 100000 https == port + 1000000 sock5 == port + 10000000 """