def install(self, options): self.event = Event() sm = SpiderManager() self.spider_class = sm.create(options.spider) spider_qk = "spider:q:%s" % self.spider_class.name schedule_qk = "schedule:q:%s" % self.spider_class.name self.log = Logger().getlog(level=logging.getLevelName(options.level)) self.spider_queue = Queue(spider_qk, self.redis) self.schedule_queue = Queue(schedule_qk, self.redis) self.pipeline_queue = queue.Queue() self.fp = Dupefilter.from_crawler(self) self.fp.open(self.spider_class.name) self.forever = options.forever self.running = False if options.urls: self.addition_urls = options.urls.split(',') else: self.addition_urls = [] self.schedule = Schedule(self)