def open(self, spider):
self.spider = spider
self.queue_key = self.get_queue_key(spider)
self.queue = SpiderQueue(self.redis_cli, spider, self.queue_key)
self.df = RFPDupeFilter(self.redis_cli,
self.get_dupefilter_key(spider))
if spider is not None:
spider.log("Queue key of redis (%s)" % self.get_queue_key(spider),
log.INFO)
spider.log(
"Dupefilter key of redis (%s)" %
self.get_dupefilter_key(spider), log.INFO)
# notice if there are requests already in the queue
if len(self.queue):
spider.log("Resuming crawl (%d requests scheduled)" %
len(self.queue))
