def execute(tag, queue, server, qurl):
tq = TaskQueue(queue_name=queue,
queue_server=server,
n_threads=0,
qurl=qurl)
print("Pulling from {}://{}".format(server, queue))
tries = 0
with tq:
while LOOP:
task = 'unknown'
try:
task = tq.lease(tag=tag, seconds=int(LEASE_SECONDS))
tries += 1
print(task)
task.execute()
tq.delete(task)
logger.log('INFO', task, "succesfully executed")
tries = 0
except TaskQueue.QueueEmpty:
time.sleep(random_exponential_window_backoff(tries))
continue
except EmptyVolumeException:
logger.log('WARNING', task, "raised an EmptyVolumeException")
tq.delete(task)
except Exception as e:
logger.log('ERROR', task,
"raised {}\n {}".format(e, traceback.format_exc()))
raise #this will restart the container in kubernetes
def mv(src, dest):
"""
Moves the contents of a queue to another
service or location. Do not run this
process while a queue is being worked.
Moving an sqs queue to a file queue
may result in duplicated tasks.
"""
src = normalize_path(src)
dest = normalize_path(dest)
tqd = TaskQueue(dest, progress=False)
tqs = TaskQueue(src, progress=False)
total = tqs.enqueued
with tqdm(total=total, desc="Moving") as pbar:
while True:
try:
tasks = tqs.lease(num_tasks=10, seconds=10)
except QueueEmptyError:
break
tqd.insert(tasks)
tqs.delete(tasks)
pbar.update(len(tasks))
def test_get(sqs, protocol):
path = getpath(protocol)
tq = TaskQueue(path, n_threads=0)
n_inserts = 5
tq.purge()
tq.insert((PrintTask() for _ in range(n_inserts)))
for i in range(n_inserts):
t = tq.lease()
tq.delete(t)
def test_is_empty():
tq = TaskQueue(FILE_QURL)
tq.purge()
assert tq.is_empty() == True
tq.insert(PrintTask("hello"))
assert tq.is_empty() == False
task = tq.lease()
tq.delete(task)
assert tq.is_empty() == True
def test_lease(sqs):
path = getpath("sqs")
tq = TaskQueue(path, n_threads=0)
n_inserts = 20
tq.purge()
tq.insert(( PrintTask(str(x)) for x in range(n_inserts) ))
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert len(tasks) == 10
tq.delete(tasks)
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert len(tasks) == 10
tq.delete(tasks)
try:
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert False
except QueueEmptyError:
pass
def testall(self):
queue = TaskQueue()
id = queue.add (channel='C', name='N', content=dict(xxx=123))
self.assertIsNotNone(id)
lease = queue.lease (channel='C')
self.assertIsNotNone(lease)
self.assertEqual(lease['id'], id)
self.assertEqual(lease['name'], 'N')
self.assertEqual(lease['channel'], 'C')
lease = queue.lease (channel='C')
self.assertIsNone(lease)
lease = queue.lease (channel='NOT FOUND')
self.assertIsNone(lease)
queue.add (channel='C', name='N1', content=dict(xxx=123))
queue.add (channel='C', name='N2', content=dict(xxx=123))
lease = queue.lease (channel='C')
self.assertTrue(queue.delete (lease, 'DONE'))
self.assertFalse(queue.delete (lease, 'DONE'))
lease = queue.lease (channel='C')
self.assertTrue(queue.delete (lease, 'DONE'))
lease = queue.lease (channel='C')
self.assertIsNone(lease)
def delete(self, request):
"""Remove a task from the queue
Args:
request: A DeleteTaskRequest containing:
a LeaseTaskRequest that was acquired form a prior call to taskqueue.lease
a task_result that indicates whether the task was completed successfully
If the task execution failed and you want it to be re-tried automatically, then do NOT call delete,
just come back later after the lease times out and the task will be availble for lease again
(unless the retry limit is reached).
Returns:
A DeleteTaskResponse. status == 'OK' on success, or 'NOT FOUND' if the task_id does not exist. Any other value indicates failure.
"""
response = DeleteTaskResponse(status='NOT FOUND')
try:
EndpointsHelper.authenticate()
lease = {'id': request.lease.task_id, 'channel': request.lease.task.channel, 'name': request.lease.task.name}
if TaskQueue.delete (lease, request.task_result):
response.status='OK'
except Exception, err:
response.status=str(err)
def test_400_errors(sqs, protocol):
path = getpath(protocol)
tq = TaskQueue(path, n_threads=0)
tq.delete('nonexistent')
