class ConfigHandler(withMetaclass(Singleton)):
def __init__(self):
pass
@LazyProperty
def serverHost(self):
return os.environ.get("HOST", setting.HOST)
@LazyProperty
def serverPort(self):
return os.environ.get("PORT", setting.PORT)
@LazyProperty
def dbConn(self):
return os.getenv("DB_CONN", setting.DB_CONN)
@LazyProperty
def tableName(self):
return os.getenv("TABLE_NAME", setting.TABLE_NAME)
@property
def fetchers(self):
reload_six(setting)
return setting.PROXY_FETCHER
@LazyProperty
def httpUrl(self):
return os.getenv("HTTP_URL", setting.HTTP_URL)
@LazyProperty
def httpsUrl(self):
return os.getenv("HTTPS_URL", setting.HTTPS_URL)
@LazyProperty
def verifyTimeout(self):
return int(os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT))
# @LazyProperty
# def proxyCheckCount(self):
# return int(os.getenv("PROXY_CHECK_COUNT", setting.PROXY_CHECK_COUNT))
@LazyProperty
def maxFailCount(self):
return int(os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT))
# @LazyProperty
# def maxFailRate(self):
# return int(os.getenv("MAX_FAIL_RATE", setting.MAX_FAIL_RATE))
@LazyProperty
def poolSizeMin(self):
return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN))
@LazyProperty
def proxyRegion(self):
return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION))
@LazyProperty
def timezone(self):
return os.getenv("TIMEZONE", setting.TIMEZONE)
class ConfigHandler(withMetaclass(Singleton)):
def __init__(self):
pass
@LazyProperty
def serverHost(self):
return os.environ.get("HOST", setting.HOST)
@LazyProperty
def serverHostIp(self):
return os.environ.get("HOSTIP", setting.HOSTIP)
@LazyProperty
def serverPort(self):
return os.environ.get("PORT", setting.PORT)
@LazyProperty
def dbConn(self):
return os.getenv("DB_CONN", setting.DB_CONN)
@LazyProperty
def tableName(self):
return os.getenv("TABLE_NAME", setting.TABLE_NAME)
@property
def fetchers(self):
reload_six(setting)
return setting.PROXY_FETCHER
@LazyProperty
def verifyUrl(self):
return os.getenv("VERIFY_URL", setting.VERIFY_URL)
@LazyProperty
def verifyTimeout(self):
return os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)
@LazyProperty
def maxFailCount(self):
return os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT)
@LazyProperty
def timezone(self):
return os.getenv("TIMEZONE", getattr(setting, 'TIMEZONE', None))
class ProxyValidator(withMetaclass(Singleton)):
pre_validator = []
http_validator = []
https_validator = []
@classmethod
def addPreValidator(cls, func):
cls.pre_validator.append(func)
return func
@classmethod
def addHttpValidator(cls, func):
cls.http_validator.append(func)
return func
@classmethod
def addHttpsValidator(cls, func):
cls.https_validator.append(func)
return func
class DbClient(withMetaclass(Singleton)):
"""
DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getCount/changeTable方法
抽象方法定义：
get(): 随机返回一个proxy;
put(proxy): 存入一个proxy;
pop(): 顺序返回并删除一个proxy;
update(proxy): 更新指定proxy信息;
delete(proxy): 删除指定proxy;
exists(proxy): 判断指定proxy是否存在;
getAll(): 返回所有代理;
clean(): 清除所有proxy信息;
getCount(): 返回proxy统计信息;
changeTable(name): 切换操作对象
所有方法需要相应类去具体实现：
ssdb: ssdbClient.py
redis: redisClient.py
mongodb: mongodbClient.py
"""
def __init__(self, db_conn):
"""
init
:return:
"""
self.db_conn = db_conn
self.parseDbConn(db_conn)
self.__printConfig()
self.__initDbClient()
@classmethod
def parseDbConn(cls, db_conn):
db_conf = urlparse(db_conn)
cls.db_type = db_conf.scheme.upper().strip()
cls.db_host = db_conf.hostname
cls.db_port = db_conf.port
cls.db_user = db_conf.username
cls.db_pwd = db_conf.password
cls.db_name = db_conf.path[1:]
return cls
def __initDbClient(self):
"""
init DB Client
:return:
"""
__type = None
if "SSDB" == self.db_type:
__type = "ssdbClient"
elif "REDIS" == self.db_type:
__type = "redisClient"
elif "MONGODB" == self.db_type:
__type = "mongodbClient"
else:
pass
assert __type, 'type error, Not support DB type: {}'.format(self.db_type)
self.client = getattr(__import__(__type), "%sClient" % self.db_type.title())(host=self.db_host,
port=self.db_port,
username=self.db_user,
password=self.db_pwd,
db=self.db_name)
def __printConfig(self):
print("============ DATABASE CONFIGURE =========================")
print("DB_TYPE: %s" % self.db_type)
print("DB_HOST: %s" % self.db_host)
print("DB_PORT: %s" % self.db_port)
print("DB_NAME: %s" % self.db_name)
print("DB_USER: %s" % self.db_user)
print("=========================================================")
def get(self, **kwargs):
return self.client.get(**kwargs)
def put(self, key, **kwargs):
return self.client.put(key, **kwargs)
def update(self, key, value, **kwargs):
return self.client.update(key, value, **kwargs)
def delete(self, key, **kwargs):
return self.client.delete(key, **kwargs)
def exists(self, key, **kwargs):
return self.client.exists(key, **kwargs)
def pop(self, **kwargs):
return self.client.pop(**kwargs)
def getAll(self):
return self.client.getAll()
def clear(self):
return self.client.clear()
def changeTable(self, name):
self.client.changeTable(name)
def getCount(self):
return self.client.getCount()