def _dq(self):
activef = join(self.dqdir, 'active.json')
if exists(activef):
with open(activef) as f:
prios = json.load(f)
else:
prios = ()
q = PriorityQueue(self._newdq, startprios=prios)
if q:
log.msg("Resuming crawl (%d requests scheduled)" % len(q), \
spider=self.spider)
return q
def _loadinfo(self, chunksize):
infopath = self._infopath()
if os.path.exists(infopath):
with open(infopath) as f:
info = json.load(f)
else:
info = {
'chunksize': chunksize,
'size': 0,
'tail': [0, 0, 0],
'head': [0, 0],
}
return info
def _loadinfo(self, chunksize):
infopath = self._infopath()
if os.path.exists(infopath):
with open(infopath) as f:
info = json.load(f)
else:
info = {
'chunksize': chunksize,
'size': 0,
'tail': [0, 0, 0],
'head': [0, 0],
}
return info