class QueueBackend(QueueInterface):
def __init__(self, prefix='spider_task', **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.queue_object = PriorityQueue(prefix)
def put(self, task, priority):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self, timeout):
task = self.queue_object.pop()
if task is None:
raise Queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
try:
while True:
self.get(0)
except Queue.Empty:
pass
class QueueBackend(QueueInterface):
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueBackend, self).__init__(spider_name, **kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name, **kwargs)
logging.debug('Redis queue key: %s' % self.queue_name)
def put(self, task, priority, schedule_time=None):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
if schedule_time is not None:
raise SpiderMisuseError('Redis task queue does not support '
'delayed task')
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self):
task = self.queue_object.pop()
if task is None:
raise queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
self.queue_object.clear()
class QueueBackend(QueueInterface):
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name)
logging.debug('Redis queue key: %s' % self.queue_name)
def put(self, task, priority, schedule_time=None):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
if schedule_time is not None:
raise SpiderMisuseError(
'Redis task queue does not support delayed task')
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self):
task = self.queue_object.pop()
if task is None:
raise queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
con = StrictRedis()
con.delete(self.queue_name)
