def add_dup_detail(detail_url, spider_name, channel_id=0):
"""
把当前详细页加入集合
:param detail_url:
:param spider_name:
:param channel_id:
:return:
"""
detail_dup_key = 'dup:%s:%s' % (spider_name, channel_id)
detail_url_finger = get_request_finger(detail_url)
return redis_client.sadd(detail_dup_key, detail_url_finger)
def add_cookies(spider_name, cookies_obj):
"""
添加 cookies
:param spider_name:
:param cookies_obj:
:return:
"""
cookies_str = _get_cookies_str(cookies_obj)
cookies_id = _get_finger(cookies_str)
key_id = 'scrapy:cookies_id:%(cookies_id)s' % {'cookies_id': cookies_id}
key_set = 'scrapy:cookies_set:%(spider_name)s' % {
'spider_name': spider_name
}
if redis_client.sismember(key_set, cookies_id):
return False
redis_client.set(key_id, cookies_str)
redis_client.sadd(key_set, cookies_id)
return True
def put_task(spider_name, *task_ids):
key_set = 'scrapy:tasks_set:%(spider_name)s' % {'spider_name': spider_name}
redis_client.sadd(key_set, *task_ids)
def add_proxy(spider_name, *proxy):
key_set = 'scrapy:proxies_set:%(spider_name)s' % {
'spider_name': spider_name
}
return redis_client.sadd(key_set, *proxy)
