def execute(tag, queue, server, qurl): tq = TaskQueue(queue_name=queue, queue_server=server, n_threads=0, qurl=qurl) print("Pulling from {}://{}".format(server, queue)) tries = 0 with tq: while LOOP: task = 'unknown' try: task = tq.lease(tag=tag, seconds=int(LEASE_SECONDS)) tries += 1 print(task) task.execute() tq.delete(task) logger.log('INFO', task, "succesfully executed") tries = 0 except TaskQueue.QueueEmpty: time.sleep(random_exponential_window_backoff(tries)) continue except EmptyVolumeException: logger.log('WARNING', task, "raised an EmptyVolumeException") tq.delete(task) except Exception as e: logger.log('ERROR', task, "raised {}\n {}".format(e, traceback.format_exc())) raise #this will restart the container in kubernetes
def mv(src, dest): """ Moves the contents of a queue to another service or location. Do not run this process while a queue is being worked. Moving an sqs queue to a file queue may result in duplicated tasks. """ src = normalize_path(src) dest = normalize_path(dest) tqd = TaskQueue(dest, progress=False) tqs = TaskQueue(src, progress=False) total = tqs.enqueued with tqdm(total=total, desc="Moving") as pbar: while True: try: tasks = tqs.lease(num_tasks=10, seconds=10) except QueueEmptyError: break tqd.insert(tasks) tqs.delete(tasks) pbar.update(len(tasks))
def test_get(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) n_inserts = 5 tq.purge() tq.insert((PrintTask() for _ in range(n_inserts))) for i in range(n_inserts): t = tq.lease() tq.delete(t)
def test_is_empty(): tq = TaskQueue(FILE_QURL) tq.purge() assert tq.is_empty() == True tq.insert(PrintTask("hello")) assert tq.is_empty() == False task = tq.lease() tq.delete(task) assert tq.is_empty() == True
def test_lease(sqs): path = getpath("sqs") tq = TaskQueue(path, n_threads=0) n_inserts = 20 tq.purge() tq.insert(( PrintTask(str(x)) for x in range(n_inserts) )) tasks = tq.lease(num_tasks=10, wait_sec=0) assert len(tasks) == 10 tq.delete(tasks) tasks = tq.lease(num_tasks=10, wait_sec=0) assert len(tasks) == 10 tq.delete(tasks) try: tasks = tq.lease(num_tasks=10, wait_sec=0) assert False except QueueEmptyError: pass
def testall(self): queue = TaskQueue() id = queue.add (channel='C', name='N', content=dict(xxx=123)) self.assertIsNotNone(id) lease = queue.lease (channel='C') self.assertIsNotNone(lease) self.assertEqual(lease['id'], id) self.assertEqual(lease['name'], 'N') self.assertEqual(lease['channel'], 'C') lease = queue.lease (channel='C') self.assertIsNone(lease) lease = queue.lease (channel='NOT FOUND') self.assertIsNone(lease) queue.add (channel='C', name='N1', content=dict(xxx=123)) queue.add (channel='C', name='N2', content=dict(xxx=123)) lease = queue.lease (channel='C') self.assertTrue(queue.delete (lease, 'DONE')) self.assertFalse(queue.delete (lease, 'DONE')) lease = queue.lease (channel='C') self.assertTrue(queue.delete (lease, 'DONE')) lease = queue.lease (channel='C') self.assertIsNone(lease)
def delete(self, request): """Remove a task from the queue Args: request: A DeleteTaskRequest containing: a LeaseTaskRequest that was acquired form a prior call to taskqueue.lease a task_result that indicates whether the task was completed successfully If the task execution failed and you want it to be re-tried automatically, then do NOT call delete, just come back later after the lease times out and the task will be availble for lease again (unless the retry limit is reached). Returns: A DeleteTaskResponse. status == 'OK' on success, or 'NOT FOUND' if the task_id does not exist. Any other value indicates failure. """ response = DeleteTaskResponse(status='NOT FOUND') try: EndpointsHelper.authenticate() lease = {'id': request.lease.task_id, 'channel': request.lease.task.channel, 'name': request.lease.task.name} if TaskQueue.delete (lease, request.task_result): response.status='OK' except Exception, err: response.status=str(err)
def test_400_errors(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) tq.delete('nonexistent')