def install(self, options):
self.event = Event()
sm = SpiderManager()
self.spider_class = sm.create(options.spider)
spider_qk = "spider:q:%s" % self.spider_class.name
schedule_qk = "schedule:q:%s" % self.spider_class.name
self.log = Logger().getlog(level=logging.getLevelName(options.level))
self.spider_queue = Queue(spider_qk, self.redis)
self.schedule_queue = Queue(schedule_qk, self.redis)
self.pipeline_queue = queue.Queue()
self.fp = Dupefilter.from_crawler(self)
self.fp.open(self.spider_class.name)
self.forever = options.forever
self.running = False
if options.urls:
self.addition_urls = options.urls.split(',')
else:
self.addition_urls = []
self.schedule = Schedule(self)
