async def test_basic_commit_after_timeout(
committer: Committer,
default_connection,
initial_request,
asyncio_sleep,
sleep_queues,
):
sleep_called = sleep_queues[FLUSH_SECONDS].called
sleep_results = sleep_queues[FLUSH_SECONDS].results
cursor1 = Cursor(offset=321)
cursor2 = Cursor(offset=1)
write_called_queue = asyncio.Queue()
write_result_queue = asyncio.Queue()
default_connection.write.side_effect = make_queue_waiter(
write_called_queue, write_result_queue
)
read_called_queue = asyncio.Queue()
read_result_queue = asyncio.Queue()
default_connection.read.side_effect = make_queue_waiter(
read_called_queue, read_result_queue
)
read_result_queue.put_nowait(StreamingCommitCursorResponse(initial={}))
write_result_queue.put_nowait(None)
async with committer:
# Set up connection
await write_called_queue.get()
await read_called_queue.get()
default_connection.write.assert_has_calls([call(initial_request)])
# Commit cursors
commit_fut1 = asyncio.ensure_future(committer.commit(cursor1))
commit_fut2 = asyncio.ensure_future(committer.commit(cursor2))
empty_fut = asyncio.ensure_future(committer.wait_until_empty())
assert not commit_fut1.done()
assert not commit_fut2.done()
assert not empty_fut.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_called_with(FLUSH_SECONDS)
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
# Called with second cursor
default_connection.write.assert_has_calls(
[call(initial_request), call(as_request(cursor2))]
)
assert not commit_fut1.done()
assert not commit_fut2.done()
assert not empty_fut.done()
# Send the connection response with 1 ack since only one request was sent.
await read_result_queue.put(as_response(count=1))
await commit_fut1
await commit_fut2
await empty_fut
async def test_publishes_retried_on_restart(
committer: Committer,
default_connection,
initial_request,
asyncio_sleep,
sleep_queues,
):
sleep_called = sleep_queues[FLUSH_SECONDS].called
sleep_results = sleep_queues[FLUSH_SECONDS].results
cursor1 = Cursor(offset=321)
cursor2 = Cursor(offset=1)
write_called_queue = asyncio.Queue()
write_result_queue = asyncio.Queue()
default_connection.write.side_effect = make_queue_waiter(
write_called_queue, write_result_queue)
read_called_queue = asyncio.Queue()
read_result_queue = asyncio.Queue()
default_connection.read.side_effect = make_queue_waiter(
read_called_queue, read_result_queue)
read_result_queue.put_nowait(StreamingCommitCursorResponse(initial={}))
write_result_queue.put_nowait(None)
async with committer:
# Set up connection
await write_called_queue.get()
await read_called_queue.get()
default_connection.write.assert_has_calls([call(initial_request)])
# Write message 1
commit_fut1 = asyncio.ensure_future(committer.commit(cursor1))
assert not commit_fut1.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_called_with(FLUSH_SECONDS)
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
default_connection.write.assert_has_calls(
[call(initial_request),
call(as_request(cursor1))])
assert not commit_fut1.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_has_calls(
[call(FLUSH_SECONDS), call(FLUSH_SECONDS)])
# Write message 2
commit_fut2 = asyncio.ensure_future(committer.commit(cursor2))
assert not commit_fut2.done()
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
default_connection.write.assert_has_calls([
call(initial_request),
call(as_request(cursor1)),
call(as_request(cursor2)),
])
assert not commit_fut1.done()
assert not commit_fut2.done()
# Fail the connection with a retryable error
await read_called_queue.get()
await read_result_queue.put(InternalServerError("retryable"))
await sleep_queues[_MIN_BACKOFF_SECS].called.get()
await sleep_queues[_MIN_BACKOFF_SECS].results.put(None)
# Reinitialization
await write_called_queue.get()
await write_result_queue.put(None)
await read_called_queue.get()
await read_result_queue.put(StreamingCommitCursorResponse(initial={}))
# Re-sending messages on the new stream
await write_called_queue.get()
await write_result_queue.put(None)
asyncio_sleep.assert_has_calls([
call(FLUSH_SECONDS),
call(FLUSH_SECONDS),
call(FLUSH_SECONDS),
call(_MIN_BACKOFF_SECS),
])
default_connection.write.assert_has_calls([
# Aggregates response calls on second pass
call(initial_request),
call(as_request(cursor2)),
])
# Sending the response for the one commit finishes both
await read_called_queue.get()
await read_result_queue.put(as_response(count=1))
await commit_fut1
await commit_fut2
async def test_commits_multi_cycle(
committer: Committer,
default_connection,
initial_request,
asyncio_sleep,
sleep_queues,
):
sleep_called = sleep_queues[FLUSH_SECONDS].called
sleep_results = sleep_queues[FLUSH_SECONDS].results
cursor1 = Cursor(offset=321)
cursor2 = Cursor(offset=1)
write_called_queue = asyncio.Queue()
write_result_queue = asyncio.Queue()
default_connection.write.side_effect = make_queue_waiter(
write_called_queue, write_result_queue)
read_called_queue = asyncio.Queue()
read_result_queue = asyncio.Queue()
default_connection.read.side_effect = make_queue_waiter(
read_called_queue, read_result_queue)
read_result_queue.put_nowait(StreamingCommitCursorResponse(initial={}))
write_result_queue.put_nowait(None)
async with committer:
# Set up connection
await write_called_queue.get()
await read_called_queue.get()
default_connection.write.assert_has_calls([call(initial_request)])
# Write message 1
commit_fut1 = asyncio.ensure_future(committer.commit(cursor1))
assert not commit_fut1.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_called_with(FLUSH_SECONDS)
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
default_connection.write.assert_has_calls(
[call(initial_request),
call(as_request(cursor1))])
assert not commit_fut1.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_has_calls(
[call(FLUSH_SECONDS), call(FLUSH_SECONDS)])
# Write message 2
commit_fut2 = asyncio.ensure_future(committer.commit(cursor2))
assert not commit_fut2.done()
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
default_connection.write.assert_has_calls([
call(initial_request),
call(as_request(cursor1)),
call(as_request(cursor2)),
])
assert not commit_fut1.done()
assert not commit_fut2.done()
# Send the connection responses
await read_result_queue.put(as_response(count=2))
await commit_fut1
await commit_fut2
async def test_wait_until_empty_completes_on_failure(
committer: Committer,
default_connection,
initial_request,
asyncio_sleep,
sleep_queues,
):
sleep_called = sleep_queues[FLUSH_SECONDS].called
sleep_results = sleep_queues[FLUSH_SECONDS].results
cursor1 = Cursor(offset=1)
write_called_queue = asyncio.Queue()
write_result_queue = asyncio.Queue()
default_connection.write.side_effect = make_queue_waiter(
write_called_queue, write_result_queue
)
read_called_queue = asyncio.Queue()
read_result_queue = asyncio.Queue()
default_connection.read.side_effect = make_queue_waiter(
read_called_queue, read_result_queue
)
read_result_queue.put_nowait(StreamingCommitCursorResponse(initial={}))
write_result_queue.put_nowait(None)
async with committer:
# Set up connection
await write_called_queue.get()
await read_called_queue.get()
default_connection.write.assert_has_calls([call(initial_request)])
# New committer is empty.
await committer.wait_until_empty()
# Write message 1
commit_fut1 = asyncio.ensure_future(committer.commit(cursor1))
empty_fut = asyncio.ensure_future(committer.wait_until_empty())
assert not commit_fut1.done()
assert not empty_fut.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_called_with(FLUSH_SECONDS)
# Handle the connection write
await sleep_results.put(None)
await write_called_queue.get()
await write_result_queue.put(None)
default_connection.write.assert_has_calls(
[call(initial_request), call(as_request(cursor1))]
)
assert not commit_fut1.done()
assert not empty_fut.done()
# Wait for writes to be waiting
await sleep_called.get()
asyncio_sleep.assert_has_calls([call(FLUSH_SECONDS), call(FLUSH_SECONDS)])
# Fail the connection with a permanent error
await read_called_queue.get()
await read_result_queue.put(InvalidArgument("permanent"))
with pytest.raises(InvalidArgument):
await empty_fut