def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
self.tmpdir = self.mktemp()
os.mkdir(self.tmpdir)
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
self.spiderman = TwistedPluginSpiderManager()
assert not self.spiderman.loaded
self.spiderman.load(['test_spiders_xxx'])
assert self.spiderman.loaded
class TwistedPluginSpiderManagerTest(unittest.TestCase):
def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
self.tmpdir = self.mktemp()
os.mkdir(self.tmpdir)
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
self.spiderman = TwistedPluginSpiderManager()
assert not self.spiderman.loaded
self.spiderman.load(['test_spiders_xxx'])
assert self.spiderman.loaded
def tearDown(self):
del self.spiderman
sys.path.remove(self.tmpdir)
def test_list(self):
self.assertEqual(set(self.spiderman.list()),
set(['spider1', 'spider2']))
def test_create(self):
spider1 = self.spiderman.create("spider1")
self.assertEqual(spider1.__class__.__name__, 'Spider1')
spider2 = self.spiderman.create("spider2", foo="bar")
self.assertEqual(spider2.__class__.__name__, 'Spider2')
self.assertEqual(spider2.foo, 'bar')
def test_create_uses_cache(self):
# TwistedPluginSpiderManager uses an internal cache which is
# invalidated in close_spider() but this isn't necessarily the best
# thing to do in all cases.
spider1 = self.spiderman.create("spider1")
spider2 = self.spiderman.create("spider1")
assert spider1 is spider2
def test_find_by_request(self):
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
['spider1'])
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')),
['spider2'])
self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))),
set(['spider1', 'spider2']))
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
[])
def test_close_spider_remove_refs(self):
spider = self.spiderman.create("spider1")
wref = weakref.ref(spider)
assert wref()
self.spiderman.close_spider(spider)
del spider
assert not wref()
def test_close_spider_invalidates_cache(self):
spider1 = self.spiderman.create("spider1")
self.spiderman.close_spider(spider1)
spider2 = self.spiderman.create("spider1")
assert spider1 is not spider2
