def test_start_already_running_exception(self):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
yield self.assertFailure(e.start(), RuntimeError).addBoth(
lambda exc: self.assertEqual(str(exc), "Engine already running"))
yield e.stop()
def test_close_engine_spiders_downloader(self):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
self.assertTrue(e.running)
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
def test_close_engine_spiders_downloader(self):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
self.assertTrue(e.running)
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
def test_deprecated_schedule(self):
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.schedule(Request("data:,"), spider)
yield e.close()
def test_download_deprecated_spider_arg(self):
with pytest.warns(ScrapyDeprecationWarning,
match="Passing a 'spider' argument to "
"ExecutionEngine.download is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.download(Request("data:,"), spider)
yield e.close()
def test_deprecated_has_capacity(self):
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.has_capacity is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
self.assertTrue(e.has_capacity())
spider = TestSpider()
yield e.open_spider(spider, [])
self.assertFalse(e.has_capacity())
e.start()
yield e.close()
self.assertTrue(e.has_capacity())
def test_close_engine_spiders_downloader(self):
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.open_spiders is deprecated, "
"please use ExecutionEngine.spider instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
self.assertTrue(e.running)
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
def test_deprecated_has_capacity(self):
with warnings.catch_warnings(record=True) as warning_list:
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
self.assertTrue(e.has_capacity())
spider = TestSpider()
yield e.open_spider(spider, [])
self.assertFalse(e.has_capacity())
e.start()
yield e.close()
self.assertTrue(e.has_capacity())
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
def test_download_deprecated_spider_arg(self):
with warnings.catch_warnings(record=True) as warning_list:
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.download(Request("data:,"), spider)
yield e.close()
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
)
def test_deprecated_schedule(self):
with warnings.catch_warnings(record=True) as warning_list:
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.schedule(Request("data:,"), spider)
yield e.close()
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
)
def test_close_engine_spiders_downloader(self):
with warnings.catch_warnings(record=True) as warning_list:
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
self.assertTrue(e.running)
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
)
