def __init__(self):
self.spider = Spider()
self.scheduler = Scheduler()
self.pipeline = Pipeline()
self.downloader = Downloader()
self.spider_mid = SpiderMiddleware() # 初始化爬虫中间件对象
self.downloader_mid = DownloaderMiddleware() # 初始化下载器中间件对象
class Engine(object):
'''
a. 对外提供整个的程序的入口
b. 依次调用其他组件对外提供的接口，实现整个框架的运作(驱动)
'''
def __init__(self):
"""实例化其他各个组件"""
self.spider = Spider() # 接收爬虫对象
self.scheduler = Scheduler() # 初始化调度器对象
self.downloader = Downloader() # 初始化下载器对象
self.pipeline = Pipeline() # 初始化管道对象
self.spider_mid = SpiderMiddleware() # 初始化爬虫中间件对象
self.downloader_mid = DownloaderMiddleware() # 初始化下载器中间件对象
def start(self):
'''提供程序启动入口, 启动整个引擎'''
# 测试log功能
start_time = datetime.now()
logger.info('爬虫启动:{}'.format(start_time))
self._start_engine()
end_time = datetime.now()
print("爬虫结束:{}".format(end_time))
print('爬虫共运行:{}秒'.format(
(end_time - start_time).total_seconds())) # total_seconds
def _start_engine(self):
'''依次调用其他组件对外提供的接口，实现整个框架的运作(驱动)'''
# 1. 爬虫模块发出初始请求:调用爬虫的方法
start_request = self.spider.start_requests()
# 2. 把初始请求添加给调度器(队列)
# 利用爬虫中间件预处理请求对象
start_request = self.spider_mid.process_request(start_request)
self.scheduler.add_request(start_request)
# 3. 从调度器获取请求对象，准备交给下载器发起请求，获取一个响应对象
request = self.scheduler.get_request()
# 利用下载器中间件预处理请求对象
request = self.downloader_mid.process_request(request)
# 4. 利用下载器发起请求
response = self.downloader.get_response(request)
# 5. 利用爬虫的解析响应的方法，处理响应，得到结果
result = self.spider.parse(response)
# 6. 判断结果对象
# 6.1 如果是请求对象，那么就再交给调度器
if isinstance(result, Request):
# 利用爬虫中间件预处理请求对象
result = self.spider_mid.process_request(result)
self.scheduler.add_request(result)
# 6.2 否则，就交给管道处理
else:
self.pipeline.process_item(result)
def __init__(self):
"""实例化其他各个组件"""
self.spider = Spider() # 接收爬虫对象
self.scheduler = Scheduler() # 初始化调度器对象
self.downloader = Downloader() # 初始化下载器对象
self.pipeline = Pipeline() # 初始化管道对象
self.spider_mid = SpiderMiddleware() # 初始化爬虫中间件对象
self.downloader_mid = DownloaderMiddleware() # 初始化下载器中间件对象
class Engine():
def __init__(self):
self.spider = Spider()
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
def start(self):
# 被调用执行引擎逻辑的入口函数
self._start_engine()
def _start_engine(self):
# 框架运行的逻辑
# 1. 爬虫模块发出初始请求
start_request = self.spider.start_request()
# 利用爬虫中间件预处理请求对象
start_request = self.spider_mid.process_request(start_request)
# 2. 把初始请求添加给调度器
self.scheduler.add_request(start_request)
# 3. 从调度器获取请求对象
request = self.scheduler.get_request()
# 利用下载器中间件预处理请求对象
request = self.downloader_mid.process_request(request)
# 4. 利用下载器发起请求
response = self.downloader.get_response(request)
# 利用下载器中间件预处理响应对象
response = self.downloader_mid.process_response(response)
# 利用爬虫中间件预处理响应对象
response = self.spider_mid.process_response(response)
# 5. 利用爬虫的解析响应的方法，处理响应，得到结果
result = self.spider.parse(response)
# 6. 判断结果对象
if isinstance(result, Request):
# 利用爬虫中间件预处理请求对象
result = self.spider_mid.process_request(result)
# 6.1 如果是请求对象，那么就再交给调度器
self.scheduler.add_request(result)
else:
# 6.2 否则，就交给管道处理
self.pipeline.process_item(result)
def __init__(self, spider):
self.spider = spider
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
# 实例化中间件
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
def __init__(self,spider):
# self.url = url
self.spider_middleware = SpiderMiddleware()
self.downloader_middleware = DownloaderMiddleware()
self.spider = spider
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline= Pipeline()
self.total_request_num = 0
self.total_response_num = 0
def __init__(self, spiders):
self.spiders = spiders
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
self.total_response_nums = 0
self.total_request_nums = 0
class Engine(object):
def __init__(self, spider):
# self.url = url
self.spider_middleware = SpiderMiddleware()
self.downloader_middleware = DownloaderMiddleware()
self.spider = spider
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
self.total_request_num = 0
self.total_response_num = 0
def start(self):
start_time = datetime.now()
# print(3333)
self._start_engine()
end_time = datetime.now()
# print(1111)
logger.info('总时间{}'.format((end_time - start_time).total_seconds()))
# print(222)
def _request(self):
request = self.scheduler.get_request()
self.total_request_num += 1
if request is None:
return
request = self.downloader_middleware.process_request(request)
response = self.downloader.get_response(request)
self.downloader_middleware.process_response(response)
# response = self.spider_middleware.process_response(response)
self.spider_middleware.process_response(response)
results = self.spider.parse(response)
for result in results:
if isinstance(result, Request):
self.spider_middleware.process_request(request)
self.scheduler.add_request(result)
else:
self.pipeline.process_item(result)
self.total_response_num += 1
def _start_engine(self):
requests = self.spider.start_requests()
for request in requests:
request = self.spider_middleware.process_request(request)
self.scheduler.add_request(request)
while True:
self._request()
if self.total_response_num >= self.total_request_num:
return
class Engine(object):
def __init__(self):
self.spider = Spider()
self.scheduler = Scheduler()
self.pipeline = Pipeline()
self.downloader = Downloader()
self.spider_mid = SpiderMiddleware() # 初始化爬虫中间件对象
self.downloader_mid = DownloaderMiddleware() # 初始化下载器中间件对象
def start(self):
"""启动整个引擎,主要调用逻辑代码写在_start_engine中"""
start = datetime.now() # 获取当前时间
logger.info('开始运行时间:%s' % start)
self._start_engine()
stop = datetime.now()
logger.info('运行结束时间:%s' % stop)
# 运行总耗时时间
logger.info('耗时: %.2f' % (stop - start).total_seconds())
def _start_engine(self):
'''实现个业务之间对接'''
# 爬虫模块发出初始化请求
start_request = self.spider.start_request()
# 爬虫中间件
start_request = self.spider_mid.process_request(start_request)
# 把初始化请求添加给调度器
self.scheduler.add_request(start_request)
# 从调度器获取请求对象
request = self.scheduler.get_request()
# 利用下载中间件预处理请求对象
request = self.downloader_mid.process_request(request)
# 调用下载器发送请求
response = self.downloader.get_response(request)
# 下载中间件预处理对象
response = self.downloader_mid.process_response(response)
# 调用爬虫处理响应方法,处理响应,得到结果
result = self.spider.parse(response)
if isinstance(result, Request):
result = self.spider_mid.process_request(result)
self.scheduler.add_request(result)
else:
self.pipeline.process_item(result)
def __init__(self, pipelines={}, middlewares={}):
# self.spider = Spider()
# self.spider = spider
self.spider = self._auto_import_instances(SPIDERS)
self.scheduler = Scheduler()
self.pipelines = pipelines
# self.pipelines = self._auto_import_instances(PIPELINES)
self.downloader = Downloader()
self.spider_mid = SpiderMiddleware() # 初始化爬虫中间件对象
# self.downloader_mid = DownloaderMiddleware() # 初始化下载器中间件对象
self.downloader_mid = middlewares # 初始化下载器中间件对象
# 计算总响应数量
self.total_response_nums = 0
# 总请求数量
self.total_request_nums = 0
# 线程池
self.pool = Pool()
self.is_running = False
class Engine():
def __init__(self, spiders):
self.spiders = spiders
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
self.total_response_nums = 0
self.total_request_nums = 0
def start(self):
# 被调用执行引擎逻辑的入口函数
start = datetime.now() # 起始时间
logger.info("开始运行时间：%s" % start) # 使用日志记录起始运行时间
self._start_engine()
stop = datetime.now() # 结束时间
logger.info("结束运行时间：%s" % stop) # 使用日志记录结束运行时间
logger.info("耗时：%.2f" % (stop - start).total_seconds()) # 使用日志记录运行耗时
logger.info("总的请求数量:{}".format(self.total_request_nums))
logger.info("总的响应数量:{}".format(self.total_response_nums))
def _start_requests(self):
"""把爬虫中所有起始的url构造request并添加到请求队列中"""
for spider_name, spider in self.spiders.items():
for start_request in spider.start_requests():
# 1. 爬虫模块发出初始请求
# 利用爬虫中间件预处理请求对象
start_request = self.spider_mid.process_request(start_request)
# 给request对象增加spider_name的属性
start_request.spider_name = spider_name
# 2. 把初始请求添加给调度器
self.scheduler.add_request(start_request)
# 总请求数 + 1
self.total_request_nums += 1
def _execute_request_response_item(self):
"""队列中取出一个request,直到处理完毕"""
# 3. 从调度器获取请求对象
request = self.scheduler.get_request()
# 判断队列是否取空
if request is None:
return # 提前终止
# 利用下载器中间件预处理请求对象
request = self.downloader_mid.process_request(request)
# 4. 利用下载器发起请求
response = self.downloader.get_response(request)
# 传递meta
response.meta = request.meta
# 利用下载器中间件预处理响应对象
response = self.downloader_mid.process_response(response)
# 利用爬虫中间件预处理响应对象
response = self.spider_mid.process_response(response)
# 根据爬虫名获取爬虫类对象
spider = self.spiders[request.spider_name]
# 5. 利用爬虫的解析响应的方法，处理响应，得到结果
# request.parse指定的解析函数 = getattr(爬虫类对象, 指定的解析函数的字符串)
parse_func = getattr(spider, request.parse)
results = parse_func(response)
for result in results:
# 6. 判断结果对象
if isinstance(result, Request):
# 利用爬虫中间件预处理请求对象
result = self.spider_mid.process_request(result)
# 给request对象增加spider_name的属性
result.spider_name = request.spider_name
# 6.1 如果是请求对象，那么就再交给调度器
self.scheduler.add_request(result)
# 总请求数 + 1
self.total_request_nums += 1
else:
# 6.2 否则，就交给管道处理
self.pipeline.process_item(result)
# 总响应数 +1
self.total_response_nums += 1
def _start_engine(self):
# 框架运行的逻辑
self._start_requests() # 把所有初始request放入队列
while True:
time.sleep(0.1)
self._execute_request_response_item() # 处理一个从队列中取出的request
# 程序退出的条件
if self.scheduler.q.empty():
break # 判断队列为空
class Engine():
def __init__(self):
self.spider = Spider()
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
def start(self):
# 被调用执行引擎逻辑的入口函数
start = datetime.now() # 起始时间
logger.info("开始运行时间：%s" % start) # 使用日志记录起始运行时间
self._start_engine()
stop = datetime.now() # 结束时间
logger.info("结束运行时间：%s" % stop) # 使用日志记录结束运行时间
logger.info("耗时：%.2f" % (stop - start).total_seconds()) # 使用日志记录运行耗时
def _start_engine(self):
# 框架运行的逻辑
# 1. 爬虫模块发出初始请求
start_request = self.spider.start_request()
# 利用爬虫中间件预处理请求对象
start_request = self.spider_mid.process_request(start_request)
# 2. 把初始请求添加给调度器
self.scheduler.add_request(start_request)
# 3. 从调度器获取请求对象
request = self.scheduler.get_request()
# 利用下载器中间件预处理请求对象
request = self.downloader_mid.process_request(request)
# 4. 利用下载器发起请求
response = self.downloader.get_response(request)
# 利用下载器中间件预处理响应对象
response = self.downloader_mid.process_response(response)
# 利用爬虫中间件预处理响应对象
response = self.spider_mid.process_response(response)
# 5. 利用爬虫的解析响应的方法，处理响应，得到结果
result = self.spider.parse(response)
# 6. 判断结果对象
if isinstance(result, Request):
# 利用爬虫中间件预处理请求对象
result = self.spider_mid.process_request(result)
# 6.1 如果是请求对象，那么就再交给调度器
self.scheduler.add_request(result)
else:
# 6.2 否则，就交给管道处理
self.pipeline.process_item(result)
class Engine(object):
def __init__(self, spider):
self.spider = spider
self.scheduler = Scheduler()
self.downloader = Downloader()
self.pipeline = Pipeline()
# 实例化中间件
self.spider_mid = SpiderMiddleware()
self.downloader_mid = DownloaderMiddleware()
def _start_engine(self):
# 编写引擎调度流程
# 1调用spider模块的start_requests方法，获取起始的请求
start_request = self.spider.start_requests()
# ----1调用爬虫中间件的process_request方法处理请求
start_request = self.spider_mid.process_request(start_request)
# 2将起始的请求传递给调度器模块的add_request方法
self.scheduler.add_request(start_request)
# 3调用调度器模块的get_request方法，从带爬取队列中获取一个请求
request = self.scheduler.get_request()
# ----2调用下载器中间件类的process_request方法
request = self.downloader_mid.process_request(request)
# 4将获取到的请求交给下载器模块的get_response方法,获得请求对应的响应
response = self.downloader.get_response(request)
# ----3调用下载器中间件的process_response方法
response = self.downloader_mid.process_response(response)
# 5.将响应交给spider模块的parse方法用于解析，获取解析结果
result = self.spider.parse(response)
# 6判断解析结果类型
if isinstance(result, Request):
# ----1调用爬虫中间件的process_request方法处理请求
result = self.spider_mid.process_request(result)
# 如果是request对象，则调用调度器模块的add_request方法，建请求放入队列
self.scheduler.add_request(result)
if isinstance(result, Item):
result = self.spider_mid.process_item(result)
# 如果是item对象，则调用pipeline模块的process_item方法处理
self.pipeline.process_item(result)
else:
raise Exception('框架不支持的数据类型')
def start(self):
start = datetime.now()
logger.info("引擎启动时间{}".format(start))
self._start_engine()
stop = datetime.now()
logger.info("引擎停止时间{}".format(stop))
logger.info("爬虫运行时间{}".format((stop-start).total_seconds()))
