from proxypool.db import RedisClient conn = RedisClient() def set(proxy): result = conn.add(proxy) print(proxy) print('录入成功' if result else '录入失败') def scan(): print('请输入代理, 输入exit退出读入') while True: proxy = input() if proxy == 'exit': break set(proxy) if __name__ == '__main__': scan()
def __init__(self, upper_threshold): self._upper_threshold = upper_threshold #150 self._conn = RedisClient() self._tester = ValidityTester() self._crawler = FreeProxyGetter()
def get_proxy(): g = RedisClient() return g.random()
def fetch(): client = RedisClient() if client.random() is None: return '代理池为空' return client.random()
def __init__(self): """ 初始化 Redis """ self.redis = RedisClient() self.loop = asyncio.get_event_loop()
def set_raw_proxies(self, proxies): self._raw_proxies = proxies self._conn = RedisClient()
def __init__(self): self.redis = RedisClient()
def __init__(self): self._conn = RedisClient() self._crawler = Crawler()
def __init__(self): self.redis = RedisClient() self.valid_proxy_num = 0
def get_conn(): if not hasattr(g, 'redis'): # 用于判断对象是否包含对应的属性 g.redis = RedisClient() return g.redis
def __init__(self, *args, **kwargs): super(TestRedisClient, self).__init__(*args, **kwargs) self.redis = RedisClient(proxy_key=TEST_REDIS_KEY)
def __init__(self): #创建对象,供该对象中其他方法使用 self.redis = RedisClient()
def __init__(self, data): self.redis = RedisClient(data) self.data = data
def count(): client = RedisClient() print('总数:', client.count()) return str(client.count())
def __init__(self): # 如果不声明以下代码 会报错:There is no current event loop in thread,参考https://blog.csdn.net/jusang486/article/details/82382358 asyncio.set_event_loop(asyncio.new_event_loop()) self.redis = RedisClient()
def get_conn(): '''打开一个redis连接''' if not hasattr(g, 'redis_client'): g.redis_client = RedisClient() return g.redis_client
def get_conn(): if not hasattr(g, 'redis'): g.redis = RedisClient() return g.redis
def __init__(self, threshold): self._threshold = threshold self._conn = RedisClient() self._tester = ValidityTester() self._crawler = CrawlProxiesFromWebsite() #从代理网站抓取的类
def __init__(self, threshold): self._threshold = threshold self._conn = RedisClient() self._tester = ValidityTester() self._crawler = FreeProxyGetter()
def __init__(self): self.client = RedisClient()
def __init__(self): self.redis = RedisClient() self.crawler = Crawler()
from proxypool.log import logger from proxypool.db import RedisClient redis = RedisClient() def image_pipeline(item): logger.info(f"image_pipeline ...") return item def mongo_pipeline(item): logger.info(f"mongo_pipeline ...") return item def mysql_pipeline(item): logger.info(f"mysql_pipeline ...") return item def redis_pipeline(item): logger.info("redis_pipeline ...") if item: redis.add(item) return item ITEM_PIPELINES = { image_pipeline: 300, mongo_pipeline: 301,