def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
self.tmpdir = self.mktemp()
os.mkdir(self.tmpdir)
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
settings = {'SPIDER_MODULES': ['test_spiders_xxx']}
self.manager = SpiderManager(Settings(settings))
self.lw = LogWrapper()
self.lw.setUp()
def __init__(self, settings, project, command_invoked='', clock=None):
'''Constructor of Engine should be very lightweight, so that things
can be easily unittested. For any more complicated initialization
use `setup()`.
'''
self.settings = settings
self.project = project
self.spiders = SpiderManager(settings)
self.stop_if_idle = True
self.initialized = False # True, when `setup()` has been called
# name of the command invoking the engine. E.g. `crawl`, `shell`, etc.
self.command_invoked = command_invoked
self.spider = None
self.pending_requests = 0
self.running = False
self.paused = False
# clock is used in unittests
self.clock = clock or reactor
self.processing = ScheduledCall(self._process_queue, clock=self.clock)
def test_load_base_spider(self):
settings = {'SPIDER_MODULES': ['crawlmi.tests.test_spider_manager.test_spiders.spider0']}
self.manager = SpiderManager(Settings(settings))
self.assertEqual(len(self.manager._spiders), 0)
