def test_dequeue_deleted_jobs(self): """Dequeueing deleted jobs from queues don't blow the stack.""" q = Queue() for _ in range(1, 1000): job = q.enqueue(say_hello) job.delete() q.dequeue()
def test_skip_queue(self): """Ensure the skip_queue option functions""" q = Queue('foo') job1 = q.enqueue(say_hello) job2 = q.enqueue(say_hello) assert q.dequeue() == job1 skip_job = q.enqueue(say_hello, at_front=True) assert q.dequeue() == skip_job assert q.dequeue() == job2
def test_dequeue_ignores_nonexisting_jobs(self): """Dequeuing silently ignores non-existing jobs.""" q = Queue() uuid = '49f205ab-8ea3-47dd-a1b5-bfa186870fc8' q.push_job_id(uuid) q.push_job_id(uuid) result = q.enqueue(say_hello, 'Nick', foo='bar') q.push_job_id(uuid) # Dequeue simply ignores the missing job and returns None self.assertEquals(q.count, 4) self.assertEquals(q.dequeue().id, result.id) self.assertIsNone(q.dequeue()) self.assertEquals(q.count, 0)
def test_dequeue_class_method(self): """Dequeueing class method jobs from queues.""" q = Queue() q.enqueue(Number.divide, 3, 4) job = q.dequeue() self.assertEquals(job.instance.__dict__, Number.__dict__) self.assertEquals(job.func.__name__, 'divide') self.assertEquals(job.args, (3, 4))
def test_dequeue_instance_method(self): """Dequeueing instance method jobs from queues.""" q = Queue() c = Calculator(2) result = q.enqueue(c.calculate, 3, 4) job = q.dequeue() # The instance has been pickled and unpickled, so it is now a separate # object. Test for equality using each object's __dict__ instead. self.assertEquals(job.instance.__dict__, c.__dict__) self.assertEquals(job.func.__name__, 'calculate') self.assertEquals(job.args, (3, 4))
def test_dequeue_instance_method(self): """Dequeueing instance method jobs from queues.""" q = Queue() n = Number(2) q.enqueue(n.div, 4) job = q.dequeue() # The instance has been pickled and unpickled, so it is now a separate # object. Test for equality using each object's __dict__ instead. self.assertEquals(job.instance.__dict__, n.__dict__) self.assertEquals(job.func.__name__, 'div') self.assertEquals(job.args, (4,))
def test_dequeue_instance_method(self): """Dequeueing instance method jobs from queues.""" q = Queue() n = Number(2) q.enqueue(n.div, 4) job = q.dequeue() # The instance has been pickled and unpickled, so it is now a separate # object. Test for equality using each object's __dict__ instead. self.assertEquals(job.instance.__dict__, n.__dict__) self.assertEquals(job.func.__name__, 'div') self.assertEquals(job.args, (4, ))
def test_dequeue(self): """Dequeueing jobs from queues.""" # Set up q = Queue() result = q.enqueue(say_hello, 'Rick', foo='bar') # Dequeue a job (not a job ID) off the queue self.assertEquals(q.count, 1) job = q.dequeue() self.assertEquals(job.id, result.id) self.assertEquals(job.func, say_hello) self.assertEquals(job.origin, q.name) self.assertEquals(job.args[0], 'Rick') self.assertEquals(job.kwargs['foo'], 'bar') # ...and assert the queue count when down self.assertEquals(q.count, 0)
class WikiCiteServer(ApplicationSession): @inlineCallbacks def onJoin(self, details): logging.info("session ready") counter = 0 while True: for change in queue.jobs: self.publish(u'com.cocytus.onchange', change) yield sleep(1) #from autobahn.twisted.wamp import ApplicationRunner #runner = ApplicationRunner(url = "ws://0.0.0.0:12345/citeserver", realm = "realm1") #runner.run(WikiCiteServer) # this doesn't return while True: job = queue.dequeue() if job is None: print('no job ffound yet, sleeping') logging.debug(u'No job found yet, sleeping') time.sleep(1) continue if job.result is None: print('job unexecuted: '+ str(job)) logging.debug(u'Job not executed yet; executing: '+str(job)) job.perform() print("Job result is "+str(job.result)) if 'doi' in job.result and isinstance(job.result['doi'], dict) and (job.result['doi']['added'] or job.result['doi']['deleted']): # one is not empty logging.info(u'change detected: '+str(job.result)) #and we have something intriquing to push to crossref print("pushing to crossref")
@inlineCallbacks def onJoin(self, details): logging.info("session ready") counter = 0 while True: for change in queue.jobs: self.publish(u'com.cocytus.onchange', change) yield sleep(1) #from autobahn.twisted.wamp import ApplicationRunner #runner = ApplicationRunner(url = "ws://0.0.0.0:12345/citeserver", realm = "realm1") #runner.run(WikiCiteServer) # this doesn't return while True: job = queue.dequeue() if job is None: print('no job ffound yet, sleeping') logging.debug(u'No job found yet, sleeping') time.sleep(1) continue if job.result is None: print('job unexecuted: ' + str(job)) logging.debug(u'Job not executed yet; executing: ' + str(job)) job.perform() print("Job result is " + str(job.result)) if 'doi' in job.result and isinstance( job.result['doi'], dict) and (job.result['doi']['added'] or job.result['doi']['deleted']): # one is not empty
class RedisStorage(Singleton): """ This is the wrapper class object that wraps the base redis interface to a more comprehensive interface. This is also a Singleton design pattern. """ # The connection _redis = None # The count _count = 0 def __init__(self): # Init the Redis os.system("redis-server &") # Connect to the server self._redis = Queue(connection=Redis()) # Override the singleton base class Singleton.__init__(self) return def append(self, type, request): """ This is the append method that will insert a new row. :param type: The request type :param request: The request itself :return: count The item count """ # Generate a tag tag = ":" + str(uuid.uuid4()) # Add it self._redis.enqueue(type + tag, request) # Get the new count to validate self._count = self._redis.count return self._count def consume(self): """ This gets an entry from the redis database. :return: The request """ # Get an entry to process obj = self._redis.dequeue() # Update the count self._count = self._redis.count # Return the object return obj def count(self): """ This is the count generator :return: """ return self._count
def _get_job(self): queue = Queue(self.task_name, connection=self.redis_conn) return queue.dequeue()
def upload_file(): def create_queue(url): resp = requests.get(url) return len(resp.text.split()) def perfromElasticSearchPost(data2, myuuid): #data = get_function(data2) # call get function(uniqueid) data2['uuid'] = myuuid es.index(index="test-index", doc_type='plan', body=data2) return None # for header in request.headers: print(header) data = request.get_json() with open(r'''usecase_schema.json''') as json_schema: schema = json.load(json_schema) #initializaing the queue redis_conn = Redis() q = Queue(connection=redis_conn) q.enqueue_call(func=create_queue, args=('data', ), timeout=0) q.dequeue() uniqueId = uuid.uuid4() get_data = perfromElasticSearchPost(data, uniqueId) myJSONValidation = Draft3Validator(schema).is_valid(data) if (myJSONValidation == True): for key, value in data.items(): if type(value) is dict: dUId = uuid.uuid4() dUId = str(dUId) value['uuid'] = dUId value2 = json.dumps(value) conn.set(dUId, value2) print('Dict', key) print(conn.get(dUId)) print() #conn.set(key, dUId) data[key] = dUId # print(key) # print(dUId) elif type(value) is list: print("Inside list", key) for i, x in enumerate(value): if (type(x) is dict): sUId = uuid.uuid4() sUId = str(sUId) x['uuid'] = sUId x2 = json.dumps(x) conn.set(sUId, x2) print('Dict', i) print(conn.get(sUId)) print() value[i] = sUId else: conn.set(key, value) # # # encoded = jwt.encode( { 'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=36000) }, 'secret') #encoded=str(encoded) token = encoded.decode('utf-8') print(type(token)) # response = make_response(jsonify(data), 200) # response.headers["ETag"] = str(hashlib.sha256("data".encode('utf-8')).hexdigest()) # response.headers["Cache-Control"] = "private, max-age=300" # print('etag', response.headers["ETag"]) #return response uniqueId = str(uniqueId) data['token'] = token data['uuid'] = uniqueId #data['etag'] = response.headers["ETag"] data2 = json.dumps(data) #es.index(index="test-index", doc_type='plan', body=data2) conn.set(uniqueId, data2) print() print(uniqueId, ":") print('token', '\n', token) print(conn.get(uniqueId)) return jsonify({"product": data2}) else: return "JSON was not validated by the schema"