class QueueBackend(QueueInterface): def __init__(self, prefix='spider_task', **kwargs): super(QueueInterface, self).__init__(**kwargs) self.queue_object = PriorityQueue(prefix) def put(self, task, priority): # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue task._rnd = random.random() self.queue_object.push(task, priority) def get(self, timeout): task = self.queue_object.pop() if task is None: raise Queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): try: while True: self.get(0) except Queue.Empty: pass
class QueueBackend(QueueInterface): def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s' % self.queue_name) def put(self, task, priority, schedule_time=None): # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue if schedule_time is not None: raise SpiderMisuseError('Redis task queue does not support ' 'delayed task') task._rnd = random.random() self.queue_object.push(task, priority) def get(self): task = self.queue_object.pop() if task is None: raise queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): self.queue_object.clear()
class QueueBackend(QueueInterface): def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s', self.queue_name) def put(self, task, priority, schedule_time=None): if schedule_time is not None: raise SpiderMisuseError('Redis task queue does not support ' 'delayed task') # Add attribute with random value # This is required because qr library # does not allow to store multiple values with same hash # in the PriorityQueue task.redis_qr_rnd = random.random() self.queue_object.push(task, priority) def get(self): task = self.queue_object.pop() if task is None: raise queue.Empty() else: return task def size(self): return len(self.queue_object) def clear(self): self.queue_object.clear()
def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueInterface, self).__init__(**kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name)
def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s', self.queue_name)
def __init__(self, spider_name, queue_name=None, **kwargs): super(QueueBackend, self).__init__(spider_name, **kwargs) self.spider_name = spider_name if queue_name is None: queue_name = 'task_queue_%s' % spider_name self.queue_name = queue_name self.queue_object = PriorityQueue(queue_name, **kwargs) logging.debug('Redis queue key: %s' % self.queue_name)
def __init__(self, prefix='spider_task', **kwargs): super(QueueInterface, self).__init__(**kwargs) self.queue_object = PriorityQueue(prefix)
# from keras.models import Sequential # from keras.layers import Dense import numpy as np import math import redis from redis import Redis from rq import Queue , Connection , Worker from flask import jsonify from qr import PriorityQueue import json os.system('sudo service redis start') PORT = 6379 HOST = 'localhost' prq =PriorityQueue('q',host=HOST,port=PORT) """ 1. Model Creator 2. Calculation Distributor 3. Graph Creator """ def dumper(obj): try: return obj.toJSON() except: return obj.__dict__