def _dq(self): activef = join(self.dqdir, 'active.json') if exists(activef): with open(activef) as f: prios = json.load(f) else: prios = () q = PriorityQueue(self._newdq, startprios=prios) if q: log.msg("Resuming crawl (%d requests scheduled)" % len(q), \ spider=self.spider) return q
def open(self, spider): self.spider = spider self.mqs = PriorityQueue(self._newmq) self.dqs = self._dq() if self.dqdir else None return self.df.open()
def setUp(self): self.q = PriorityQueue(self.qfactory)