def add_tasks(self):
""" The only action performed by producer.
Sends BulkAdd request to TaskQueue protobuffer API.
Every added task is reported to validation log.
"""
bulk_add = taskqueue_service_pb2.TaskQueueBulkAddRequest()
tasks = {}
for _ in range(BULK_SIZES):
add_task = bulk_add.add_request.add()
add_task.app_id = bytes(TEST_PROJECT, 'utf8')
add_task.queue_name = bytes(PULL_QUEUE, 'utf8')
add_task.mode = taskqueue_service_pb2.TaskQueueMode.PULL
add_task.task_name = bytes(str(uuid.uuid4()), 'utf8')
work_times = self.generate_work_times()
add_task.body = bytes(work_times, 'utf8') + b':' + next(BIG_PAYLOADS)
add_task.eta_usec = 0
tasks[add_task.task_name] = work_times
# Send request to TaskQueue
with self.client.protobuf('BulkAdd', bulk_add) as response:
# Report what tasks were added
for task in response.taskresult:
if task.result == taskqueue_service_pb2.TaskQueueServiceError.OK:
self.locust.log_action(
'ADDED', str(task.chosen_task_name, 'utf8'),
tasks[task.chosen_task_name]
)
else:
self.locust.log_action(task.result, '?', '?')
async def test_add_lease_prolong_delete(taskqueue):
# Initialize tasks
queue_str = 'pull-queue-b'
queue_bytes = bytes(queue_str, 'utf8')
tasks = []
for n in range(10):
add_task = taskqueue_service_pb2.TaskQueueAddRequest()
add_task.app_id = bytes(TEST_PROJECT, 'utf8')
add_task.queue_name = queue_bytes
add_task.mode = taskqueue_service_pb2.TaskQueueMode.PULL
add_task.task_name = b'task-%d' % n
add_task.body = b'some-payload-%d' % n
add_task.eta_usec = 0
tasks.append(add_task)
bulk_add = taskqueue_service_pb2.TaskQueueBulkAddRequest()
bulk_add.add_request.extend(tasks[:5])
# Add tasks using single and bulk add
await taskqueue.protobuf('BulkAdd', bulk_add)
asyncio_tasks = []
for task in tasks[5:]:
asyncio_tasks.append(taskqueue.protobuf('Add', task))
await asyncio.wait(asyncio_tasks)
remote_time = await taskqueue.remote_time_usec()
# Lease 10 tasks for 5 second
lease_req = taskqueue_service_pb2.TaskQueueQueryAndOwnTasksRequest()
lease_req.queue_name = queue_bytes
lease_req.lease_seconds = 5
lease_req.max_tasks = 10
leased = await taskqueue.protobuf('QueryAndOwnTasks', lease_req)
# Make sure eta_usec is ~5s greater than current remote_time
assert all(
task.eta_usec == pytest.approx(remote_time + 5_000_000, abs=600_000)
for task in leased.task)
async def test_add_lease_retry_retry_delete_pg(taskqueue, pull_queues_backend):
if pull_queues_backend == CASSANDRA:
pytest.skip('Skipped for Cassandra backend')
# Initialize tasks
queue_str = 'pull-queue-5-retry'
queue_bytes = bytes(queue_str, 'utf8')
add_tasks = []
for n in range(4):
add_task = taskqueue_service_pb2.TaskQueueAddRequest()
add_task.app_id = bytes(TEST_PROJECT, 'utf8')
add_task.queue_name = queue_bytes
add_task.mode = taskqueue_service_pb2.TaskQueueMode.PULL
add_task.task_name = b'task-%d' % n
add_task.body = b'some-payload-%d' % n
add_task.eta_usec = 0
add_tasks.append(add_task)
bulk_add = taskqueue_service_pb2.TaskQueueBulkAddRequest()
bulk_add.add_request.extend(add_tasks)
start_time = await taskqueue.remote_time_usec()
total_delay = 0
# Add tasks using bulk add
_, delay = await taskqueue.timed_protobuf('BulkAdd', bulk_add)
total_delay += delay
# Lease 4 tasks for 2 seconds
lease_req = taskqueue_service_pb2.TaskQueueQueryAndOwnTasksRequest()
lease_req.queue_name = queue_bytes
lease_req.lease_seconds = 2
lease_req.max_tasks = 4
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == [1, 1, 1, 1]
# Try to lease 4 tasks for 2 seconds
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == []
# Give 3 seconds for lease to expire
time.sleep(3)
# Lease 2 tasks for 2 seconds (retry)
lease_req.max_tasks = 2
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == [2, 2]
# Lease 2 tasks for 2 seconds (retry)
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == [2, 2]
# Try to lease 2 tasks for 2 seconds
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == []
# Give 3 seconds for lease to expire
time.sleep(3)
# Try to lease 3 tasks for 2 seconds
lease_req.max_tasks = 3
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == [3, 3, 3]
# Give 3 seconds for lease to expire
time.sleep(3)
# Try to lease 3 tasks for 2 seconds
leased, delay = await taskqueue.timed_protobuf('QueryAndOwnTasks',
lease_req)
total_delay += delay
assert [task.retry_count for task in leased.task] == [3, 4, 4]
# Verify listed tasks
listed = await taskqueue.rest('GET', path_suffix=f'/{queue_str}/tasks')
sorting_key = lambda item: (item['retry_count'], int(item['leaseTimestamp']
))
tasks = sorted(listed.json['items'], key=sorting_key)
actual = [(task['retry_count'], int(task['leaseTimestamp']))
for task in tasks]
expected = [
(3, pytest.approx(start_time + 8_000_000, abs=300_000 + total_delay)),
(3, pytest.approx(start_time + 11_000_000, abs=300_000 + total_delay)),
(4, pytest.approx(start_time + 11_000_000, abs=300_000 + total_delay)),
(4, pytest.approx(start_time + 11_000_000, abs=300_000 + total_delay)),
]
assert actual == expected
# Delete tasks
req = taskqueue_service_pb2.TaskQueueDeleteRequest()
req.queue_name = queue_bytes
req.task_name.extend([task.task_name for task in add_tasks])
await taskqueue.protobuf('Delete', req)
# Verify that queue is empty
listed = await taskqueue.rest('GET', path_suffix=f'/{queue_str}/tasks')
assert listed.json == {
'kind': 'taskqueues#tasks'
} # items should be missing