from scrapy.crawler import CrawlerProcess
process = CrawlerProcess(...)
process.crawl(spider_cls, *args, **kwargs)
process.start()

class CrawlerRunner:
    def __init__(self, settings=None):
        if isinstance(settings, dict) or settings is None:
            settings = Settings(settings)
        self.settings = settings
        # 'SPIDER_LOADER_CLASS'配置的'scrapy.spiderloader.SpiderLoader'，
        # 属于辅助方法，主要作用是将spider模块加载进入进程空间
        self.spider_loader = self._get_spider_loader(settings)
        # 维护crawler列表，意思就是字面的含义，每调用process.crawl（核心是CrawlerRunner.crawl）就生成一个crawler
        #对象放入_crawlers列表，对应对象的deferred放入 _active列表。
        self._crawlers = set()
        self._active = set()
        self.bootstrap_failed = False
        # 如果通过`TWISTED_REACTOR`配置项配置了指定reator实现，就加载指定的reactor实现
        # 例如在linux下指定使用"twisted.internet.pollreactor.PollReactor"
        self._handle_twisted_reactor()

class CrawlerRunner:
    def crawl(self, crawler_or_spidercls, *args, **kwargs):
        """
        Run a crawler with the provided arguments.
        Returns a deferred that is fired when the crawling is finished.
        """
        ...
        # 生成`scrapy.crawler.Crawler object`对象，最核心的是执行`Crawler.__init__`
        crawler = self.create_crawler(crawler_or_spidercls)
        # 这个_crawl method是重点
        return self._crawl(crawler, *args, **kwargs)

class CrawlerRunner:
    def _crawl(self, crawler, *args, **kwargs):
        # crawlers就是CrawlerRunner._crawlers的别名
        self.crawlers.add(crawler)
        #执行`Crawler.crawl`方法，该方法是逻辑重点,它生成 `crawler：spider, engine ` 组合
        #并返回该crawler对应的deferred
        d = crawler.crawl(*args, **kwargs)
        self._active.add(d)

        def _done(result):
            self.crawlers.discard(crawler)
            self._active.discard(d)
            self.bootstrap_failed |= not getattr(crawler, 'spider', None)
            return result
        # 
        return d.addBoth(_done)

class Crawler:
    @defer.inlineCallbacks
    def crawl(self, *args, **kwargs):
        if self.crawling:
            raise RuntimeError("Crawling already taking place")
        self.crawling = True

        try:
            # 生成spider object
            self.spider = self._create_spider(*args, **kwargs)
            # 生成engine 对象
            self.engine = self._create_engine()
            start_requests = iter(self.spider.start_requests())
            #
            yield self.engine.open_spider(self.spider, start_requests)
            yield defer.maybeDeferred(self.engine.start)
        except Exception:
            self.crawling = False
            if self.engine is not None:
                yield self.engine.close()
            raise

runner = CrawlerRunner()
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until all crawling jobs are finished

class ExecutionEngine:
    def __init__(self, crawler, spider_closed_callback):
        self.crawler = crawler
        self.settings = crawler.settings
        self.signals = crawler.signals
        self.logformatter = crawler.logformatter
        self.slot = None
        self.spider = None
        self.running = False
        self.paused = False
        self.scheduler_cls = load_object(self.settings['SCHEDULER'])
        downloader_cls = load_object(self.settings['DOWNLOADER'])
        # 生成downloader（包括downloadmiddleware）
        self.downloader = downloader_cls(crawler)
        #
        self.scraper = Scraper(crawler)
        self._spider_closed_callback = spider_closed_callback

class ExecutionEngine:
    @defer.inlineCallbacks
    def open_spider(self, spider, start_requests=(), close_if_idle=True):
        if not self.has_capacity():
            raise RuntimeError("No free spider slot when opening %r" % spider.name)
        logger.info("Spider opened", extra={'spider': spider})
        #生成callback nextcall，当_next_request被调用时，会去执行ExecutionEngine.crawl
        nextcall = CallLaterOnce(self._next_request, spider)
        scheduler = self.scheduler_cls.from_crawler(self.crawler)
        # 让各个middleware的process_start_requests方法对原始start_requst处理得到最终的start_requests(Request对象)
        start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
        #构建slot engine
        slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
        self.slot = slot
        self.spider = spider
        yield scheduler.open(spider)
        yield self.scraper.open_spider(spider)
        self.crawler.stats.open_spider(spider)
        yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
        # 将callback nextcall放入reactor的执行队列中
        slot.nextcall.schedule()
        #设置heartbeat deferred每5秒执行一次
        slot.heartbeat.start(5)

class CrawlerProcess(CrawlerRunner):
    def start(self, stop_after_crawl=True):
        """
        This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
        size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
        based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.

        If ``stop_after_crawl`` is True, the reactor will be stopped after all
        crawlers have finished, using :meth:`join`.

        :param boolean stop_after_crawl: stop or not the reactor when all
            crawlers have finished
        """
        from twisted.internet import reactor
        if stop_after_crawl:
            d = self.join()
            # Don't start the reactor if the deferreds are already fired
            if d.called:
                return
            d.addBoth(self._stop_reactor)
        #使用twisted内置的域名解析
        resolver_class = load_object(self.settings["DNS_RESOLVER"])
        resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
        resolver.install_on_reactor()
        tp = reactor.getThreadPool()
        tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
        # 启动
        reactor.run(installSignalHandlers=False)  # blocking call

class ExecutionEngine:
    def crawl(self, request, spider):
        if spider not in self.open_spiders:
            raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
        #将requst放入scheduler，并信号发送
        self.schedule(request, spider)
        #即ExecutionEngine._next_request defered封装再次放入callback队列
        self.slot.nextcall.schedule()

    def schedule(self, request, spider):
        #以sender是Anonymous身份，向所有receivers发送signals.request_scheduled信号
        self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
        #将request放入scheduler
        if not self.slot.scheduler.enqueue_request(request):
            self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)

    def _handle_downloader_output(self, response, request, spider):
        if not isinstance(response, (Request, Response, Failure)):
            raise TypeError(
                "Incorrect type: expected Request, Response or Failure, got %s: %r"
                % (type(response), response)
            )
        # downloader middleware can return requests (for example, redirects)
        if isinstance(response, Request):
            self.crawl(response, spider)
            return
        # response is a Response or Failure
        d = self.scraper.enqueue_scrape(response, request, spider)
        d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': spider}))
        return d

class Downloader:
    def __init__(self, crawler):
        self.settings = crawler.settings
        self.signals = crawler.signals
        self.slots = {}
        self.active = set()
        #加载不同协议对应的下载处理器
        self.handlers = DownloadHandlers(crawler)
        self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
        self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
        self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
        self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
        # 根据'DOWNLOADER_MIDDLEWARES'，'DOWNLOADER_MIDDLEWARES_BASE'配置，将所有相关dlmw 模块生成
        self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
        self._slot_gc_loop = task.LoopingCall(self._slot_gc)
        self._slot_gc_loop.start(60)

DOWNLOADER_MIDDLEWARES = {}

DOWNLOADER_MIDDLEWARES_BASE = {
    # Engine side
    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
    # Downloader side
}

class Downloader:
    def fetch(self, request, spider):
        def _deactivate(response):
            self.active.remove(request)
            return response

        self.active.add(request)
        # 执行下载，细节见下一小节的说明
        dfd = self.middleware.download(self._enqueue_request, request, spider)
        return dfd.addBoth(_deactivate)

class Downloader:
    def download(self, download_func, request, spider):
        @defer.inlineCallbacks
        def process_request(request):
            for method in self.methods['process_request']:
                response = yield deferred_from_coro(method(request=request, spider=spider))
                if response is not None and not isinstance(response, (Response, Request)):
                    raise _InvalidOutput(
                        "Middleware %s.process_request must return None, Response or Request, got %s"
                        % (method.__self__.__class__.__name__, response.__class__.__name__)
                    )
                #这意味着只要其中一个dlmw的process_request返回了响应（非None），就不会执行后续的
                if response:
                    return response
            #只要任意一个dlmw的process_request返回了响应（非None），就不会执行了，
            # download_func是Downloader._enqueue_request,它使用系统的各协议下载器（见Downloader.handlers）进行下载
            return (yield download_func(request=request, spider=spider))

        @defer.inlineCallbacks
        def process_response(response):
            ....
        ....

        deferred = mustbe_deferred(process_request, request)
        deferred.addErrback(process_exception)
        deferred.addCallback(process_response)
        return deferred

class Scraper:
    def __init__(self, crawler):
        self.slot = None
        self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
        itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
        self.itemproc = itemproc_cls.from_crawler(crawler)
        self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
        self.crawler = crawler
        self.signals = crawler.signals
        self.logformatter = crawler.logformatter