class RedisSpiderQueue(object): implements(ISpiderQueue) def __init__(self, settings): redis_host = settings.get('REDIS_HOST') redis_port = settings.getint('REDIS_PORT', 6379) server = redis.StrictRedis(host=redis_host, port=redis_port) self.queue = RedisPriorityQueue(server, 'scraperd:spider:queue') def add(self, name, **spider_args): d = spider_args.copy() d['name'] = name priority = float(d.pop('priority', 0)) self.queue.push(d, priority) def pop(self): return self.queue.pop() def count(self): return len(self.queue)
def __init__(self, settings): redis_host = settings.get('REDIS_HOST') redis_port = settings.getint('REDIS_PORT', 6379) server = redis.StrictRedis(host=redis_host, port=redis_port) self.queue = RedisPriorityQueue(server, 'scraperd:spider:queue')