def set_anti_spider_task(spider_name, msg):
"""
设置任务队列
msg = {
'url': url,
'time': time.strftime("%Y-%m-%d %H:%M:%S")
}
:param spider_name:
:param msg:
:return:
"""
key = 'scrapy:anti_spider_task_weixin:%s' % spider_name
q_task = RedisQueue(key, redis_client=redis_client)
q_msg = json.dumps(msg) if isinstance(msg, dict) else msg
# 因为微信反爬策略是通过IP限制, 这里仅仅处理一个任务
if q_task.empty():
q_task.put(q_msg)
def _get_anti_spider_result(spider_name):
"""获取任务队列"""
key = 'scrapy:anti_spider_result_weixin:%s' % spider_name
q_result = RedisQueue(key, redis_client=redis_client)
result = q_result.get(timeout=60)
return json.loads(result) if result else {}
def _set_anti_spider_result(spider_name, msg):
"""设置结果队列"""
key = 'scrapy:anti_spider_result_weixin:%s' % spider_name
q_result = RedisQueue(key, redis_client=redis_client)
q_msg = json.dumps(msg) if isinstance(msg, dict) else msg
q_result.put(q_msg)
def _get_anti_spider_task(spider_name):
"""获取任务队列"""
key = 'anti_spider_task_weixin:%s' % spider_name
q_task = RedisQueue(key, redis_client=redis_client)
result = q_task.get(timeout=60)
return json.loads(result) if result else {}
def send_urls(brand_name, url_list):
queue = RedisQueue(brand_name)
for url in url_list:
queue.put(url)