def test_dequeue_deleted_jobs(self):
"""Dequeueing deleted jobs from queues don't blow the stack."""
q = Queue()
for _ in range(1, 1000):
job = q.enqueue(say_hello)
job.delete()
q.dequeue()
def test_skip_queue(self):
"""Ensure the skip_queue option functions"""
q = Queue('foo')
job1 = q.enqueue(say_hello)
job2 = q.enqueue(say_hello)
assert q.dequeue() == job1
skip_job = q.enqueue(say_hello, at_front=True)
assert q.dequeue() == skip_job
assert q.dequeue() == job2
def test_dequeue_ignores_nonexisting_jobs(self):
"""Dequeuing silently ignores non-existing jobs."""
q = Queue()
uuid = '49f205ab-8ea3-47dd-a1b5-bfa186870fc8'
q.push_job_id(uuid)
q.push_job_id(uuid)
result = q.enqueue(say_hello, 'Nick', foo='bar')
q.push_job_id(uuid)
# Dequeue simply ignores the missing job and returns None
self.assertEquals(q.count, 4)
self.assertEquals(q.dequeue().id, result.id)
self.assertIsNone(q.dequeue())
self.assertEquals(q.count, 0)
def test_dequeue_class_method(self):
"""Dequeueing class method jobs from queues."""
q = Queue()
q.enqueue(Number.divide, 3, 4)
job = q.dequeue()
self.assertEquals(job.instance.__dict__, Number.__dict__)
self.assertEquals(job.func.__name__, 'divide')
self.assertEquals(job.args, (3, 4))
def test_dequeue_instance_method(self):
"""Dequeueing instance method jobs from queues."""
q = Queue()
c = Calculator(2)
result = q.enqueue(c.calculate, 3, 4)
job = q.dequeue()
# The instance has been pickled and unpickled, so it is now a separate
# object. Test for equality using each object's __dict__ instead.
self.assertEquals(job.instance.__dict__, c.__dict__)
self.assertEquals(job.func.__name__, 'calculate')
self.assertEquals(job.args, (3, 4))
def test_dequeue_instance_method(self):
"""Dequeueing instance method jobs from queues."""
q = Queue()
c = Calculator(2)
result = q.enqueue(c.calculate, 3, 4)
job = q.dequeue()
# The instance has been pickled and unpickled, so it is now a separate
# object. Test for equality using each object's __dict__ instead.
self.assertEquals(job.instance.__dict__, c.__dict__)
self.assertEquals(job.func.__name__, 'calculate')
self.assertEquals(job.args, (3, 4))
def test_dequeue_instance_method(self):
"""Dequeueing instance method jobs from queues."""
q = Queue()
n = Number(2)
q.enqueue(n.div, 4)
job = q.dequeue()
# The instance has been pickled and unpickled, so it is now a separate
# object. Test for equality using each object's __dict__ instead.
self.assertEquals(job.instance.__dict__, n.__dict__)
self.assertEquals(job.func.__name__, 'div')
self.assertEquals(job.args, (4,))
def test_dequeue_instance_method(self):
"""Dequeueing instance method jobs from queues."""
q = Queue()
n = Number(2)
q.enqueue(n.div, 4)
job = q.dequeue()
# The instance has been pickled and unpickled, so it is now a separate
# object. Test for equality using each object's __dict__ instead.
self.assertEquals(job.instance.__dict__, n.__dict__)
self.assertEquals(job.func.__name__, 'div')
self.assertEquals(job.args, (4, ))
def test_dequeue(self):
"""Dequeueing jobs from queues."""
# Set up
q = Queue()
result = q.enqueue(say_hello, 'Rick', foo='bar')
# Dequeue a job (not a job ID) off the queue
self.assertEquals(q.count, 1)
job = q.dequeue()
self.assertEquals(job.id, result.id)
self.assertEquals(job.func, say_hello)
self.assertEquals(job.origin, q.name)
self.assertEquals(job.args[0], 'Rick')
self.assertEquals(job.kwargs['foo'], 'bar')
# ...and assert the queue count when down
self.assertEquals(q.count, 0)
def test_dequeue(self):
"""Dequeueing jobs from queues."""
# Set up
q = Queue()
result = q.enqueue(say_hello, 'Rick', foo='bar')
# Dequeue a job (not a job ID) off the queue
self.assertEquals(q.count, 1)
job = q.dequeue()
self.assertEquals(job.id, result.id)
self.assertEquals(job.func, say_hello)
self.assertEquals(job.origin, q.name)
self.assertEquals(job.args[0], 'Rick')
self.assertEquals(job.kwargs['foo'], 'bar')
# ...and assert the queue count when down
self.assertEquals(q.count, 0)
class WikiCiteServer(ApplicationSession):
@inlineCallbacks
def onJoin(self, details):
logging.info("session ready")
counter = 0
while True:
for change in queue.jobs:
self.publish(u'com.cocytus.onchange', change)
yield sleep(1)
#from autobahn.twisted.wamp import ApplicationRunner
#runner = ApplicationRunner(url = "ws://0.0.0.0:12345/citeserver", realm = "realm1")
#runner.run(WikiCiteServer) # this doesn't return
while True:
job = queue.dequeue()
if job is None:
print('no job ffound yet, sleeping')
logging.debug(u'No job found yet, sleeping')
time.sleep(1)
continue
if job.result is None:
print('job unexecuted: '+ str(job))
logging.debug(u'Job not executed yet; executing: '+str(job))
job.perform()
print("Job result is "+str(job.result))
if 'doi' in job.result and isinstance(job.result['doi'], dict) and (job.result['doi']['added'] or job.result['doi']['deleted']): # one is not empty
logging.info(u'change detected: '+str(job.result))
#and we have something intriquing to push to crossref
print("pushing to crossref")
@inlineCallbacks
def onJoin(self, details):
logging.info("session ready")
counter = 0
while True:
for change in queue.jobs:
self.publish(u'com.cocytus.onchange', change)
yield sleep(1)
#from autobahn.twisted.wamp import ApplicationRunner
#runner = ApplicationRunner(url = "ws://0.0.0.0:12345/citeserver", realm = "realm1")
#runner.run(WikiCiteServer) # this doesn't return
while True:
job = queue.dequeue()
if job is None:
print('no job ffound yet, sleeping')
logging.debug(u'No job found yet, sleeping')
time.sleep(1)
continue
if job.result is None:
print('job unexecuted: ' + str(job))
logging.debug(u'Job not executed yet; executing: ' + str(job))
job.perform()
print("Job result is " + str(job.result))
if 'doi' in job.result and isinstance(
job.result['doi'],
dict) and (job.result['doi']['added']
or job.result['doi']['deleted']): # one is not empty
class RedisStorage(Singleton):
"""
This is the wrapper class object that wraps the
base redis interface to a more comprehensive interface.
This is also a Singleton design pattern.
"""
# The connection
_redis = None
# The count
_count = 0
def __init__(self):
# Init the Redis
os.system("redis-server &")
# Connect to the server
self._redis = Queue(connection=Redis())
# Override the singleton base class
Singleton.__init__(self)
return
def append(self, type, request):
"""
This is the append method that will insert a new row.
:param type: The request type
:param request: The request itself
:return: count The item count
"""
# Generate a tag
tag = ":" + str(uuid.uuid4())
# Add it
self._redis.enqueue(type + tag, request)
# Get the new count to validate
self._count = self._redis.count
return self._count
def consume(self):
"""
This gets an entry from the redis database.
:return: The request
"""
# Get an entry to process
obj = self._redis.dequeue()
# Update the count
self._count = self._redis.count
# Return the object
return obj
def count(self):
"""
This is the count generator
:return:
"""
return self._count
class RedisStorage(Singleton):
"""
This is the wrapper class object that wraps the
base redis interface to a more comprehensive interface.
This is also a Singleton design pattern.
"""
# The connection
_redis = None
# The count
_count = 0
def __init__(self):
# Init the Redis
os.system("redis-server &")
# Connect to the server
self._redis = Queue(connection=Redis())
# Override the singleton base class
Singleton.__init__(self)
return
def append(self, type, request):
"""
This is the append method that will insert a new row.
:param type: The request type
:param request: The request itself
:return: count The item count
"""
# Generate a tag
tag = ":" + str(uuid.uuid4())
# Add it
self._redis.enqueue(type + tag, request)
# Get the new count to validate
self._count = self._redis.count
return self._count
def consume(self):
"""
This gets an entry from the redis database.
:return: The request
"""
# Get an entry to process
obj = self._redis.dequeue()
# Update the count
self._count = self._redis.count
# Return the object
return obj
def count(self):
"""
This is the count generator
:return:
"""
return self._count
def _get_job(self):
queue = Queue(self.task_name, connection=self.redis_conn)
return queue.dequeue()
def upload_file():
def create_queue(url):
resp = requests.get(url)
return len(resp.text.split())
def perfromElasticSearchPost(data2, myuuid):
#data = get_function(data2)
# call get function(uniqueid)
data2['uuid'] = myuuid
es.index(index="test-index", doc_type='plan', body=data2)
return None
#
for header in request.headers:
print(header)
data = request.get_json()
with open(r'''usecase_schema.json''') as json_schema:
schema = json.load(json_schema)
#initializaing the queue
redis_conn = Redis()
q = Queue(connection=redis_conn)
q.enqueue_call(func=create_queue, args=('data', ), timeout=0)
q.dequeue()
uniqueId = uuid.uuid4()
get_data = perfromElasticSearchPost(data, uniqueId)
myJSONValidation = Draft3Validator(schema).is_valid(data)
if (myJSONValidation == True):
for key, value in data.items():
if type(value) is dict:
dUId = uuid.uuid4()
dUId = str(dUId)
value['uuid'] = dUId
value2 = json.dumps(value)
conn.set(dUId, value2)
print('Dict', key)
print(conn.get(dUId))
print()
#conn.set(key, dUId)
data[key] = dUId
# print(key)
# print(dUId)
elif type(value) is list:
print("Inside list", key)
for i, x in enumerate(value):
if (type(x) is dict):
sUId = uuid.uuid4()
sUId = str(sUId)
x['uuid'] = sUId
x2 = json.dumps(x)
conn.set(sUId, x2)
print('Dict', i)
print(conn.get(sUId))
print()
value[i] = sUId
else:
conn.set(key, value)
#
#
#
encoded = jwt.encode(
{
'exp':
datetime.datetime.utcnow() + datetime.timedelta(seconds=36000)
}, 'secret')
#encoded=str(encoded)
token = encoded.decode('utf-8')
print(type(token))
# response = make_response(jsonify(data), 200)
# response.headers["ETag"] = str(hashlib.sha256("data".encode('utf-8')).hexdigest())
# response.headers["Cache-Control"] = "private, max-age=300"
# print('etag', response.headers["ETag"])
#return response
uniqueId = str(uniqueId)
data['token'] = token
data['uuid'] = uniqueId
#data['etag'] = response.headers["ETag"]
data2 = json.dumps(data)
#es.index(index="test-index", doc_type='plan', body=data2)
conn.set(uniqueId, data2)
print()
print(uniqueId, ":")
print('token', '\n', token)
print(conn.get(uniqueId))
return jsonify({"product": data2})
else:
return "JSON was not validated by the schema"
