async def test_task_lifecycle_in_pull_queue(pull_queue_context):
tq = pull_queue_context['queue']
payload = {'this': {'is': {'a': {'test': uuid.uuid4().hex}}}}
tag = 'smoke-test'
# DRAIN
await tq.drain()
# INSERT
inserted = await tq.insert(encode(json.dumps(payload)), tag=encode(tag))
assert inserted
# add created task(and the queue to delete it from) to the tasks list,
# so that regardless of what happens, the teardown will clean it up.
pull_queue_context['tasks_to_cleanup'].append(inserted)
# GET
assert inserted == await tq.get(inserted['name'], full=True)
# LIST
listed = await tq.list(full=True)
assert listed.get('tasks')
assert inserted in listed['tasks']
# LEASE
leased = await tq.lease(num_tasks=1,
lease_seconds=10,
task_filter=f'tag={encode(tag)}')
assert leased.get('tasks') and len(leased['tasks']) == 1
leased_message = leased['tasks'][0]['pullMessage']
assert payload == json.loads(decode(leased_message['payload']))
assert tag == decode(leased_message['tag'])
# RENEW
renewed = await tq.renew(leased['tasks'][0], lease_seconds=10)
for k, v in renewed.items():
if k == 'scheduleTime':
assert v != leased['tasks'][0][k]
else:
assert v == leased['tasks'][0][k]
# ack?
# cancel?
# DELETE
assert not await tq.delete(renewed['name'])
# inserted task has been deleted successfully,
# so remove it from the list to avoid unnecessary delete attempt
pull_queue_context['tasks_to_cleanup'].remove(inserted)
async def test_task_lifecycle():
project = os.environ['GCLOUD_PROJECT']
creds = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
task_queue = 'test-pull'
async with aiohttp.ClientSession() as session:
tq = TaskQueue(project, creds, task_queue, session=session)
payload = {'this': {'is': {'a': {'test': uuid.uuid4().hex}}}}
tag = 'smoke-test'
# DRAIN
await tq.drain()
# INSERT
inserted = await tq.insert(encode(json.dumps(payload)),
tag=encode(tag))
assert inserted
# GET
assert inserted == await tq.get(inserted['name'], full=True)
# LIST
listed = await tq.list(full=True)
assert listed.get('tasks')
assert inserted in listed['tasks']
# LEASE
leased = await tq.lease(num_tasks=1, lease_seconds=10,
task_filter=f'tag={encode(tag)}')
assert leased.get('tasks') and len(leased['tasks']) == 1
leased_message = leased['tasks'][0]['pullMessage']
assert payload == json.loads(decode(leased_message['payload']))
assert tag == decode(leased_message['tag'])
# RENEW
renewed = await tq.renew(leased['tasks'][0], lease_seconds=10)
for k, v in renewed.items():
if k == 'scheduleTime':
assert v != leased['tasks'][0][k]
else:
assert v == leased['tasks'][0][k]
# ack?
# cancel?
# DELETE
assert not await tq.delete(renewed['name'])
async def test_task_multiple_leases(caplog, mocker, creds, project,
pull_queue_name, tm_session):
def get_mock_coro(return_value):
@asyncio.coroutine
def mock_coro(*args, **kwargs):
# pylint: disable=unused-argument
yield # ensure all tasks are processed at once
time.sleep(9)
return return_value
return mocker.Mock(wraps=mock_coro)
tasks = [
'{"test_idx": 1}',
'{"test_idx": 2}',
]
worker = get_mock_coro('ok')
tm = TaskManager(project,
pull_queue_name,
worker,
service_file=creds,
batch_size=len(tasks),
lease_seconds=4,
session=tm_session)
tm.session = tm_session
# drain old tasks
await tm.tq.drain()
# insert new ones
for task in tasks:
await tm.tq.insert(encode(task),
tag=encode('gcloud-aio-manager-test-multilease'))
tm.start()
await asyncio.sleep(10)
tm.stop()
await asyncio.sleep(0) # allow tm.poll() to terminate
assert worker.mock_calls == [mocker.call(t) for t in tasks]
for record in caplog.records:
assert record.levelname != 'ERROR'
async def test_task_multiple_leases(caplog, mocker):
project = os.environ['GCLOUD_PROJECT']
creds = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
task_queue = 'test-pull'
def get_mock_coro(return_value):
@asyncio.coroutine
def mock_coro(*args, **kwargs):
# pylint: disable=unused-argument
yield # ensure all tasks are processed at once
time.sleep(9)
return return_value
return mocker.Mock(wraps=mock_coro)
tasks = [
'{"test_idx": 1}',
'{"test_idx": 2}',
]
worker = get_mock_coro('ok')
async with aiohttp.ClientSession() as session:
tm = TaskManager(project, creds, task_queue, worker,
batch_size=len(tasks), lease_seconds=4,
session=session)
# drain old tasks
await tm.tq.drain()
# insert new ones
for task in tasks:
await tm.tq.insert(
encode(task), tag=encode('gcloud-aio-manager-test-multilease'))
tm.start()
await asyncio.sleep(10)
tm.stop()
assert worker.mock_calls == [mocker.call(t) for t in tasks]
for record in caplog.records:
assert record.levelname != 'ERROR'
async def test_task_lifecycle(mocker):
project = os.environ['GCLOUD_PROJECT']
creds = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
task_queue = 'test-pull'
def get_mock_coro(return_value):
@asyncio.coroutine
def mock_coro(*args, **kwargs):
# pylint: disable=unused-argument
return return_value
return mocker.Mock(wraps=mock_coro)
tasks = [
'{"test_idx": 1}',
'{"test_idx": 2}',
'{"test_idx": 3}',
'{"test_idx": 4}',
'not-a-json-task',
]
worker = get_mock_coro('ok')
async with aiohttp.ClientSession() as session:
tm = TaskManager(project, creds, task_queue, worker,
batch_size=len(tasks), session=session)
# DRAIN
await tm.tq.drain()
# START
tm.start()
# INSERT
for task in tasks:
await tm.tq.insert(
encode(task), tag=encode('gcloud-aio-manager-test-lifecycle'))
await asyncio.sleep(3)
tm.stop()
assert worker.mock_calls == [mocker.call(t) for t in tasks]
async def test_task_lifecycle(mocker, creds, project, pull_queue_name,
tm_session):
def get_mock_coro(return_value):
@asyncio.coroutine
def mock_coro(*args, **kwargs):
# pylint: disable=unused-argument
return return_value
return mocker.Mock(wraps=mock_coro)
tasks = [
'{"test_idx": 1}',
'{"test_idx": 2}',
'{"test_idx": 3}',
'{"test_idx": 4}',
'not-a-json-task',
]
worker = get_mock_coro('ok')
tm = TaskManager(project,
pull_queue_name,
worker,
service_file=creds,
batch_size=len(tasks),
session=tm_session)
# DRAIN
await tm.tq.drain()
# START
tm.start()
# INSERT
for task in tasks:
await tm.tq.insert(encode(task),
tag=encode('gcloud-aio-manager-test-lifecycle'))
await asyncio.sleep(3)
tm.stop()
await asyncio.sleep(0) # allow tm.poll() to terminate
assert worker.mock_calls == [mocker.call(t) for t in tasks]
