示例#1
0
from proxypool.db import RedisClient

conn = RedisClient()


def set(proxy):
    result = conn.add(proxy)
    print(proxy)
    print('录入成功' if result else '录入失败')


def scan():
    print('请输入代理, 输入exit退出读入')
    while True:
        proxy = input()
        if proxy == 'exit':
            break
        set(proxy)


if __name__ == '__main__':
    scan()
示例#2
0
文件： scheduler.py 项目： xuqisong/IP- 
 def __init__(self, upper_threshold):
     self._upper_threshold = upper_threshold  #150
     self._conn = RedisClient()
     self._tester = ValidityTester()
     self._crawler = FreeProxyGetter()
示例#3
0
def get_proxy():
    g = RedisClient()
    return g.random()
示例#4
0
def fetch():
    client = RedisClient()
    if client.random() is None:
        return '代理池为空'
    return client.random()
示例#5
0
 def __init__(self):
     """
     初始化 Redis
     """
     self.redis = RedisClient()
     self.loop = asyncio.get_event_loop()
示例#6
0
 def set_raw_proxies(self, proxies):
     self._raw_proxies = proxies
     self._conn = RedisClient()
示例#7
0
 def __init__(self):
     self.redis = RedisClient()
示例#8
0
 def __init__(self):
     self._conn = RedisClient()
     self._crawler = Crawler()
示例#9
0
 def __init__(self):
     self.redis = RedisClient()
     self.valid_proxy_num = 0
示例#10
0
def get_conn():
    if not hasattr(g, 'redis'):  # 用于判断对象是否包含对应的属性
        g.redis = RedisClient()
    return g.redis
示例#11
0
 def __init__(self, *args, **kwargs):
     super(TestRedisClient, self).__init__(*args, **kwargs)
     self.redis = RedisClient(proxy_key=TEST_REDIS_KEY)
示例#12
0
 def __init__(self):
     #创建对象，供该对象中其他方法使用
     self.redis = RedisClient()
示例#13
0
文件： tester.py 项目： bigbigx/IP_Pool 
 def __init__(self, data):
     self.redis = RedisClient(data)
     self.data = data
示例#14
0
def count():
    client = RedisClient()
    print('总数：', client.count())
    return str(client.count())
示例#15
0
 def __init__(self):
     # 如果不声明以下代码 会报错：There is no current event loop in thread，参考https://blog.csdn.net/jusang486/article/details/82382358
     asyncio.set_event_loop(asyncio.new_event_loop())
     self.redis = RedisClient()
示例#16
0
def get_conn():
    '''打开一个redis连接'''
    if not hasattr(g, 'redis_client'):
        g.redis_client = RedisClient()
    return g.redis_client
示例#17
0
def get_conn():
    if not hasattr(g, 'redis'):
        g.redis = RedisClient()
        return g.redis
示例#18
0
 def __init__(self, threshold):
     self._threshold = threshold
     self._conn = RedisClient()
     self._tester = ValidityTester()
     self._crawler = CrawlProxiesFromWebsite()  #从代理网站抓取的类
示例#19
0
 def __init__(self, threshold):
     self._threshold = threshold
     self._conn = RedisClient()
     self._tester = ValidityTester()
     self._crawler = FreeProxyGetter()
示例#20
0
 def __init__(self):
     self.client = RedisClient()
示例#21
0
 def __init__(self):
     self.redis = RedisClient()
     self.crawler = Crawler()
示例#22
0
from proxypool.log import logger
from proxypool.db import RedisClient

redis = RedisClient()


def image_pipeline(item):
    logger.info(f"image_pipeline ...")
    return item


def mongo_pipeline(item):
    logger.info(f"mongo_pipeline ...")
    return item


def mysql_pipeline(item):
    logger.info(f"mysql_pipeline ...")
    return item


def redis_pipeline(item):
    logger.info("redis_pipeline ...")
    if item:
        redis.add(item)
    return item


ITEM_PIPELINES = {
    image_pipeline: 300,
    mongo_pipeline: 301,