#!/usr/bin/env python
import redis
import IpBroker
import pickle
import time
import gzip
from cStringIO import StringIO
logger = IpBroker.get_logger("IP_CLIENT")
class IpClient(object):
"""IpClient for get proxy ip, port and type
"""
def __init__(self):
self.broker = redis.StrictRedis(**IpBroker.IP_BROKER)
self.pointer = 0
self.ip_cache = {}
self.ip_len = 0
self.ips = []
self.expire_time = 1 * 60 * 60 # One hour
def getIps(self, length=1):
"""Get length ips from pool,return ip dict
ip -> type&port
http == port + 100000
https == port + 1000000
sock5 == port + 10000000
"""
self.pointer = (
#!/usr/bin/python
import redis
import pickle
import time
import IpBroker
import requests
import threading
import gzip
from cStringIO import StringIO
from Queue import Queue
requests.HTTPErrorDEFAULT_RETRIES = 2
logger = IpBroker.get_logger("IP_PRODUCER")
THREAD_COUNT = 300
class IpProducer(object):
"""IpProducer to put proxy ip into pool, port and type
"""
def __init__(self):
self.broker = redis.StrictRedis(**IpBroker.IP_BROKER)
def putIps(self, ips):
"""Get length ips from pool,return ip dict
ip -> type&port
http == port + 100000
https == port + 1000000
sock5 == port + 10000000
"""
