def parse(self, response): yield None yield Request( "http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html?highlight=middleware", callback=self.dummy, ) yield subspider(TestSubSpider0.name, cmdargs={1: 2}, start_urls=["http://vk.com/feed"])
def test_remote_spider_start(self): os.environ['SCRAPY_SETTINGS_MODULE'] = 'test.test_subspiders.crawler.settings' self.enqueue(subspider('testspider0', cmdargs={1:2,}), None) # print self.pend(lambda:self.json('/listspiders.json?project=default')) wd = 100 while not "Process finished: project='default' spider='testspider0'" in self.scrapyd.stderr.readline(): wd -= 1 self.assertTrue(bool(wd), 'testspider0 unfinished!') wd = 100 while not "Process finished: project='default' spider='testsubspider0'" in self.scrapyd.stderr.readline(): wd -= 1 self.assertTrue(bool(wd), 'testspider0 unfinished!')