def run(self, args, opts):
url = args[0] if args else None
if url:
# first argument may be a local file
url = guess_scheme(url)
spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif url:
spidercls = spidercls_for_request(spider_loader,
Request(url),
spidercls,
log_multiple=True)
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
crawler = self.crawler_process._create_crawler(spidercls)
# The Shell class needs a persistent engine in the crawler
crawler.engine = crawler._create_engine()
crawler.engine.start()
self._start_crawler_thread()
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
shell.start(url=url, redirect=not opts.no_redirect)
def run(self, args, opts):
url = args[0] if args else None
shell = Shell(self.crawler,
update_vars=self.update_vars,
code=opts.code)
self._start_crawler_thread()
shell.start(url=url)
def run(self, args, opts):
url = args[0] if args else None
spiders = self.crawler_process.spiders
spidercls = DefaultSpider
if opts.spider:
spidercls = spiders.load(opts.spider)
elif url:
spidercls = spidercls_for_request(spiders,
Request(url),
spidercls,
log_multiple=True)
# The crawler is created this way since the Shell manually handles the
# crawling engine, so the set up in the crawl method won't work
crawler = self.crawler_process._create_logged_crawler(spidercls)
# The Shell class needs a persistent engine in the crawler
crawler.engine = crawler._create_engine()
crawler.engine.start()
self.crawler_process.start(start_reactor=False)
self._start_crawler_thread()
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
shell.start(url=url)
def run(self, args, opts):
url = args[0] if args else None
spider = None
if opts.spider:
spider = self.crawler.spiders.create(opts.spider)
shell = Shell(self.crawler,
update_vars=self.update_vars,
code=opts.code)
self._start_crawler_thread()
shell.start(url=url, spider=spider)
def run(self, args, opts):
crawler = self.crawler_process.create_crawler()
url = args[0] if args else None
spider = crawler.spiders.create(opts.spider) if opts.spider else None
self.crawler_process.start_crawling()
self._start_crawler_thread()
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
shell.start(url=url, spider=spider)
def run(self, args, opts):
url = args[0] if args else None
shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True, \
code=opts.code)
def err(f):
log.err(f, "Shell error")
self.exitcode = 1
d = shell.start(url=url)
d.addErrback(err)
d.addBoth(lambda _: self.crawler.stop())
self.crawler.start()