from proxypool.db import RedisClient
conn = RedisClient()
def set(proxy):
result = conn.add(proxy)
print(proxy)
print('录入成功' if result else '录入失败')
def scan():
print('请输入代理, 输入exit退出读入')
while True:
proxy = input()
if proxy == 'exit':
break
set(proxy)
if __name__ == '__main__':
scan()
def __init__(self, upper_threshold):
self._upper_threshold = upper_threshold #150
self._conn = RedisClient()
self._tester = ValidityTester()
self._crawler = FreeProxyGetter()
def get_proxy():
g = RedisClient()
return g.random()
def fetch():
client = RedisClient()
if client.random() is None:
return '代理池为空'
return client.random()
def __init__(self):
"""
初始化 Redis
"""
self.redis = RedisClient()
self.loop = asyncio.get_event_loop()
def set_raw_proxies(self, proxies):
self._raw_proxies = proxies
self._conn = RedisClient()
def __init__(self):
self.redis = RedisClient()
def __init__(self):
self._conn = RedisClient()
self._crawler = Crawler()
def __init__(self):
self.redis = RedisClient()
self.valid_proxy_num = 0
def get_conn():
if not hasattr(g, 'redis'): # 用于判断对象是否包含对应的属性
g.redis = RedisClient()
return g.redis
def __init__(self, *args, **kwargs):
super(TestRedisClient, self).__init__(*args, **kwargs)
self.redis = RedisClient(proxy_key=TEST_REDIS_KEY)
def __init__(self):
#创建对象，供该对象中其他方法使用
self.redis = RedisClient()
def __init__(self, data):
self.redis = RedisClient(data)
self.data = data
def count():
client = RedisClient()
print('总数：', client.count())
return str(client.count())
def __init__(self):
# 如果不声明以下代码 会报错：There is no current event loop in thread，参考https://blog.csdn.net/jusang486/article/details/82382358
asyncio.set_event_loop(asyncio.new_event_loop())
self.redis = RedisClient()
def get_conn():
'''打开一个redis连接'''
if not hasattr(g, 'redis_client'):
g.redis_client = RedisClient()
return g.redis_client
def get_conn():
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis
def __init__(self, threshold):
self._threshold = threshold
self._conn = RedisClient()
self._tester = ValidityTester()
self._crawler = CrawlProxiesFromWebsite() #从代理网站抓取的类
def __init__(self, threshold):
self._threshold = threshold
self._conn = RedisClient()
self._tester = ValidityTester()
self._crawler = FreeProxyGetter()
def __init__(self):
self.client = RedisClient()
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
from proxypool.log import logger
from proxypool.db import RedisClient
redis = RedisClient()
def image_pipeline(item):
logger.info(f"image_pipeline ...")
return item
def mongo_pipeline(item):
logger.info(f"mongo_pipeline ...")
return item
def mysql_pipeline(item):
logger.info(f"mysql_pipeline ...")
return item
def redis_pipeline(item):
logger.info("redis_pipeline ...")
if item:
redis.add(item)
return item
ITEM_PIPELINES = {
image_pipeline: 300,
mongo_pipeline: 301,
