Example #1
0
 def parse(self, response):
     yield None
     yield Request(
         "http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html?highlight=middleware",
         callback=self.dummy,
     )
     yield subspider(TestSubSpider0.name, cmdargs={1: 2}, start_urls=["http://vk.com/feed"])
Example #2
0
    def test_remote_spider_start(self):
        os.environ['SCRAPY_SETTINGS_MODULE'] = 'test.test_subspiders.crawler.settings'
        self.enqueue(subspider('testspider0', cmdargs={1:2,}), None)
        # print self.pend(lambda:self.json('/listspiders.json?project=default'))
        wd = 100
        while not "Process finished: project='default' spider='testspider0'" in self.scrapyd.stderr.readline():
            wd -= 1
        self.assertTrue(bool(wd), 'testspider0 unfinished!')

        wd = 100
        while not "Process finished: project='default' spider='testsubspider0'" in self.scrapyd.stderr.readline():
            wd -= 1
        self.assertTrue(bool(wd), 'testspider0 unfinished!')