class ConfigHandler(withMetaclass(Singleton)): def __init__(self): pass @LazyProperty def serverHost(self): return os.environ.get("HOST", setting.HOST) @LazyProperty def serverPort(self): return os.environ.get("PORT", setting.PORT) @LazyProperty def dbConn(self): return os.getenv("DB_CONN", setting.DB_CONN) @LazyProperty def tableName(self): return os.getenv("TABLE_NAME", setting.TABLE_NAME) @property def fetchers(self): reload_six(setting) return setting.PROXY_FETCHER @LazyProperty def httpUrl(self): return os.getenv("HTTP_URL", setting.HTTP_URL) @LazyProperty def httpsUrl(self): return os.getenv("HTTPS_URL", setting.HTTPS_URL) @LazyProperty def verifyTimeout(self): return int(os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)) # @LazyProperty # def proxyCheckCount(self): # return int(os.getenv("PROXY_CHECK_COUNT", setting.PROXY_CHECK_COUNT)) @LazyProperty def maxFailCount(self): return int(os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT)) # @LazyProperty # def maxFailRate(self): # return int(os.getenv("MAX_FAIL_RATE", setting.MAX_FAIL_RATE)) @LazyProperty def poolSizeMin(self): return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN)) @LazyProperty def proxyRegion(self): return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION)) @LazyProperty def timezone(self): return os.getenv("TIMEZONE", setting.TIMEZONE)
class ConfigHandler(withMetaclass(Singleton)): def __init__(self): pass @LazyProperty def serverHost(self): return os.environ.get("HOST", setting.HOST) @LazyProperty def serverHostIp(self): return os.environ.get("HOSTIP", setting.HOSTIP) @LazyProperty def serverPort(self): return os.environ.get("PORT", setting.PORT) @LazyProperty def dbConn(self): return os.getenv("DB_CONN", setting.DB_CONN) @LazyProperty def tableName(self): return os.getenv("TABLE_NAME", setting.TABLE_NAME) @property def fetchers(self): reload_six(setting) return setting.PROXY_FETCHER @LazyProperty def verifyUrl(self): return os.getenv("VERIFY_URL", setting.VERIFY_URL) @LazyProperty def verifyTimeout(self): return os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT) @LazyProperty def maxFailCount(self): return os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT) @LazyProperty def timezone(self): return os.getenv("TIMEZONE", getattr(setting, 'TIMEZONE', None))
class ProxyValidator(withMetaclass(Singleton)): pre_validator = [] http_validator = [] https_validator = [] @classmethod def addPreValidator(cls, func): cls.pre_validator.append(func) return func @classmethod def addHttpValidator(cls, func): cls.http_validator.append(func) return func @classmethod def addHttpsValidator(cls, func): cls.https_validator.append(func) return func
class DbClient(withMetaclass(Singleton)): """ DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getCount/changeTable方法 抽象方法定义: get(): 随机返回一个proxy; put(proxy): 存入一个proxy; pop(): 顺序返回并删除一个proxy; update(proxy): 更新指定proxy信息; delete(proxy): 删除指定proxy; exists(proxy): 判断指定proxy是否存在; getAll(): 返回所有代理; clean(): 清除所有proxy信息; getCount(): 返回proxy统计信息; changeTable(name): 切换操作对象 所有方法需要相应类去具体实现: ssdb: ssdbClient.py redis: redisClient.py mongodb: mongodbClient.py """ def __init__(self, db_conn): """ init :return: """ self.db_conn = db_conn self.parseDbConn(db_conn) self.__printConfig() self.__initDbClient() @classmethod def parseDbConn(cls, db_conn): db_conf = urlparse(db_conn) cls.db_type = db_conf.scheme.upper().strip() cls.db_host = db_conf.hostname cls.db_port = db_conf.port cls.db_user = db_conf.username cls.db_pwd = db_conf.password cls.db_name = db_conf.path[1:] return cls def __initDbClient(self): """ init DB Client :return: """ __type = None if "SSDB" == self.db_type: __type = "ssdbClient" elif "REDIS" == self.db_type: __type = "redisClient" elif "MONGODB" == self.db_type: __type = "mongodbClient" else: pass assert __type, 'type error, Not support DB type: {}'.format(self.db_type) self.client = getattr(__import__(__type), "%sClient" % self.db_type.title())(host=self.db_host, port=self.db_port, username=self.db_user, password=self.db_pwd, db=self.db_name) def __printConfig(self): print("============ DATABASE CONFIGURE =========================") print("DB_TYPE: %s" % self.db_type) print("DB_HOST: %s" % self.db_host) print("DB_PORT: %s" % self.db_port) print("DB_NAME: %s" % self.db_name) print("DB_USER: %s" % self.db_user) print("=========================================================") def get(self, **kwargs): return self.client.get(**kwargs) def put(self, key, **kwargs): return self.client.put(key, **kwargs) def update(self, key, value, **kwargs): return self.client.update(key, value, **kwargs) def delete(self, key, **kwargs): return self.client.delete(key, **kwargs) def exists(self, key, **kwargs): return self.client.exists(key, **kwargs) def pop(self, **kwargs): return self.client.pop(**kwargs) def getAll(self): return self.client.getAll() def clear(self): return self.client.clear() def changeTable(self, name): self.client.changeTable(name) def getCount(self): return self.client.getCount()