class SpiderManagerTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') self.tmpdir = self.mktemp() os.mkdir(self.tmpdir) self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = {'SPIDER_MODULES': ['test_spiders_xxx']} self.manager = SpiderManager(Settings(settings)) self.lw = LogWrapper() self.lw.setUp() def tearDown(self): sys.path.remove(self.tmpdir) self.lw.tearDown() def test_get_spiders(self): self.assertSetEqual( set(self.manager.get_spiders()), set(['spider1', 'spider2', 'spider3'])) def test_create_by_name(self): spider1 = self.manager.create_spider_by_name('spider1') self.assertEqual(spider1.__class__.__name__, 'Spider1') spider2 = self.manager.create_spider_by_name('spider2', {'p1': 1, 'p2': 2}) self.assertEqual(spider2.__class__.__name__, 'Spider2') self.assertEqual(spider2.p1, 1) self.assertEqual(spider2.p2, 2) def test_get_spiders_by_url(self): self.assertEqual(self.manager.get_spiders_by_url('http://crawlmi1.org/test'), ['spider1']) self.assertEqual(self.manager.get_spiders_by_url('http://crawlmi2.org/test'), ['spider2']) self.assertEqual(set(self.manager.get_spiders_by_url('http://crawlmi3.org/test')), set(['spider1', 'spider2'])) self.assertEqual(self.manager.get_spiders_by_url('http://crawlmi999.org/test'), []) self.assertEqual(self.manager.get_spiders_by_url('http://spider3.com'), ['spider3']) def test_create_spiders_by_url(self): spider = self.manager.create_spider_by_url('http://crawlmi1.org/test') self.assertEqual(spider.__class__.__name__, 'Spider1') spider = self.manager.create_spider_by_url('http://crawlmi3.org/test') self.assertIsNone(spider) self.assertTrue(self.lw.get_first_line().startswith('[crawlmi] ERROR: More than one spider can handle:')) spider = self.manager.create_spider_by_url('http://crawlmi999.org/test') self.assertIsNone(spider) self.assertTrue(self.lw.get_first_line().startswith('[crawlmi] ERROR: Unable to find spider that handles:')) def test_load_spider_module(self): settings = {'SPIDER_MODULES': ['crawlmi.tests.test_spider_manager.test_spiders.spider1']} self.manager = SpiderManager(Settings(settings)) self.assertEqual(len(self.manager._spiders), 1) def test_load_base_spider(self): settings = {'SPIDER_MODULES': ['crawlmi.tests.test_spider_manager.test_spiders.spider0']} self.manager = SpiderManager(Settings(settings)) self.assertEqual(len(self.manager._spiders), 0)