def __parse_response(self, response, spider):
"""处理响应,将新的请求放入请求队列,提取到的item抛给输入管理器处理"""
try:
callback = getattr(spider, response.request.callback)
for each in callback(response):
if isinstance(each, req.Request): # 将新产生的请求添加至队列
RequestQueue.push(each, spider)
elif isinstance(each, dict): # 将提取到的item抛给op_manager进行处理
self.op_manager(each, spider)
except Exception as err:
self.__save_mode(response.request, err)
def __init_requests(self):
for spider in self.spiders:
for request in spider.start_requests():
RequestQueue.push(request, spider)
