def __parse_response(self, response, spider): """处理响应,将新的请求放入请求队列,提取到的item抛给输入管理器处理""" try: callback = getattr(spider, response.request.callback) for each in callback(response): if isinstance(each, req.Request): # 将新产生的请求添加至队列 RequestQueue.push(each, spider) elif isinstance(each, dict): # 将提取到的item抛给op_manager进行处理 self.op_manager(each, spider) except Exception as err: self.__save_mode(response.request, err)
def __init_requests(self): for spider in self.spiders: for request in spider.start_requests(): RequestQueue.push(request, spider)