class QueueBackend(QueueInterface):
def __init__(self, prefix='spider_task', **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.queue_object = PriorityQueue(prefix)
def put(self, task, priority):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self, timeout):
task = self.queue_object.pop()
if task is None:
raise Queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
try:
while True:
self.get(0)
except Queue.Empty:
pass
class QueueBackend(QueueInterface):
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueBackend, self).__init__(spider_name, **kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name, **kwargs)
logging.debug('Redis queue key: %s' % self.queue_name)
def put(self, task, priority, schedule_time=None):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
if schedule_time is not None:
raise SpiderMisuseError('Redis task queue does not support '
'delayed task')
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self):
task = self.queue_object.pop()
if task is None:
raise queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
self.queue_object.clear()
class QueueBackend(QueueInterface):
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueBackend, self).__init__(spider_name, **kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name, **kwargs)
logging.debug('Redis queue key: %s', self.queue_name)
def put(self, task, priority, schedule_time=None):
if schedule_time is not None:
raise SpiderMisuseError('Redis task queue does not support '
'delayed task')
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
task.redis_qr_rnd = random.random()
self.queue_object.push(task, priority)
def get(self):
task = self.queue_object.pop()
if task is None:
raise queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
self.queue_object.clear()
class QueueBackend(QueueInterface):
def __init__(self, prefix='spider_task', **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.queue_object = PriorityQueue(prefix)
def put(self, task, priority):
# Add attribute with random value
# This is required because qr library
# does not allow to store multiple values with same hash
# in the PriorityQueue
task._rnd = random.random()
self.queue_object.push(task, priority)
def get(self, timeout):
task = self.queue_object.pop()
if task is None:
raise Queue.Empty()
else:
return task
def size(self):
return len(self.queue_object)
def clear(self):
try:
while True:
self.get(0)
except Queue.Empty:
pass
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name)
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueBackend, self).__init__(spider_name, **kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name, **kwargs)
logging.debug('Redis queue key: %s', self.queue_name)
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name)
def __init__(self, spider_name, queue_name=None, **kwargs):
super(QueueBackend, self).__init__(spider_name, **kwargs)
self.spider_name = spider_name
if queue_name is None:
queue_name = 'task_queue_%s' % spider_name
self.queue_name = queue_name
self.queue_object = PriorityQueue(queue_name, **kwargs)
logging.debug('Redis queue key: %s' % self.queue_name)
def __init__(self, prefix='spider_task', **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.queue_object = PriorityQueue(prefix)
def __init__(self, prefix='spider_task', **kwargs):
super(QueueInterface, self).__init__(**kwargs)
self.queue_object = PriorityQueue(prefix)
# from keras.models import Sequential
# from keras.layers import Dense
import numpy as np
import math
import redis
from redis import Redis
from rq import Queue , Connection , Worker
from flask import jsonify
from qr import PriorityQueue
import json
os.system('sudo service redis start')
PORT = 6379
HOST = 'localhost'
prq =PriorityQueue('q',host=HOST,port=PORT)
"""
1. Model Creator
2. Calculation Distributor
3. Graph Creator
"""
def dumper(obj):
try:
return obj.toJSON()
except:
return obj.__dict__
