class QueueBackend(QueueInterface): def __init__(self, prefix='spider_task', **kwargs): super(QueueInterface, self).__init__(**kwargs) self.queue_object = PriorityQueue(prefix) def put(self, task, priority): # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue task._rnd = random.random() self.queue_object.push(task, priority) def get(self, timeout): task = self.queue_object.pop() if task is None: raise Queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): try: while True: self.get(0) except Queue.Empty: pass
class QueueBackend(QueueInterface): def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s', self.queue_name) def put(self, task, priority, schedule_time=None): if schedule_time is not None: raise SpiderMisuseError('Redis task queue does not support ' 'delayed task') # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue task.redis_qr_rnd = random.random() self.queue_object.push(task, priority) def get(self): task = self.queue_object.pop() if task is None: raise queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): self.queue_object.clear()
class QueueBackend(QueueInterface): def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s' % self.queue_name) def put(self, task, priority, schedule_time=None): # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue if schedule_time is not None: raise SpiderMisuseError('Redis task queue does not support ' 'delayed task') task._rnd = random.random() self.queue_object.push(task, priority) def get(self): task = self.queue_object.pop() if task is None: raise queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): self.queue_object.clear()