def _make_data(settings=None):
crawler = Crawler(Spider, settings=settings)
spider = Spider("dummy")
return {
"stats": crawler.stats.get_stats(),
"crawler": crawler,
"spider": spider,
"runner": SpiderMonitorRunner(spider=spider),
"job": None,
}
def make_data(settings):
crawler = get_crawler(settings_dict=settings)
spider = Spider.from_crawler(crawler, "example.com")
return {
"stats": crawler.stats.get_stats(),
"crawler": crawler,
"spider": spider,
"runner": SpiderMonitorRunner(spider=spider),
"job": None,
}
def make_data_for_monitor(settings=None, stats=None):
crawler = get_crawler(settings_dict=settings or {})
spider = Spider.from_crawler(crawler, "example.com")
return {
"stats": stats or {},
"crawler": crawler,
"spider": spider,
"runner": SpiderMonitorRunner(spider=spider),
"job": None,
}
def _run_suites(self, spider, suites):
data = self._generate_data_for_spider(spider)
for suite in suites:
runner = SpiderMonitorRunner(spider=spider)
runner.run(suite, **data)