async def test_restart_from_historical(): """ If we ask the conversation to start again while we're reading historical events we should re-send the most recent page request. In this scenario, we start reading the stream at event 10, we receive a page with 2 events, we request the next page starting at 12. When we restart the conversation, we should again request the page starting at 12. """ conversation_id = uuid.uuid4() output = TeeQueue() convo = CatchupSubscription("my-stream", start_from=10, conversation_id=conversation_id) await convo.start(output) await reply_to( convo, (ReadStreamEventsResponseBuilder(stream="stream-123").with_event( event_number=10).with_event( event_number=11).with_next_event_number(12).build()), output, ) await convo.start(output) [first_page, second_page, second_page_again ] = [read_as(proto.ReadStreamEvents, m) for m in output.items] assert second_page.from_event_number == second_page_again.from_event_number
async def test_should_perform_a_catchup_when_subscription_is_confirmed(): """ When we have read all the events in the stream, we should send a request to subscribe for new events. We should start reading catchup events from the `next_event_number` returned by the historical event read. """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) await reply_to( convo, ReadStreamEventsResponseBuilder().with_next_event_number( 17).at_end_of_stream().build(), output, ) await confirm_subscription(convo, output, event_number=42, commit_pos=40) [read_historial, subscribe, catch_up] = await output.next_event(3) assert read_historial.command == msg.TcpCommand.ReadStreamEventsForward assert subscribe.command == msg.TcpCommand.SubscribeToStream assert catch_up.command == msg.TcpCommand.ReadStreamEventsForward payload = proto.ReadStreamEvents() payload.ParseFromString(catch_up.payload) assert payload.event_stream_id == "my-stream" assert payload.from_event_number == 17
async def test_subscribe_with_context_manager(): conversation_id = uuid.uuid4() output = TeeQueue() convo = CatchupSubscription("my-stream", conversation_id=conversation_id) await convo.start(output) # Create a subscription with three events in it await reply_to(convo, EMPTY_STREAM_PAGE, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, EMPTY_STREAM_PAGE, output) for i in range(0, 3): await reply_to(convo, event_appeared(event_id=uuid.uuid4(), event_number=i), output) async with (await convo.result) as subscription: events_seen = 0 async for _ in subscription.events: events_seen += 1 if events_seen == 3: break # Having exited the context manager it should send # an unsubscribe message [read_historical, subscribe, catch_up, unsubscribe] = output.items assert unsubscribe.command == msg.TcpCommand.UnsubscribeFromStream assert unsubscribe.conversation_id == conversation_id
async def test_start_read_phase(): """ A "catchup" subscription starts by iterating the events in the stream until it reaches the most recent event. This is the "Read" phase. """ output = TeeQueue() conversation_id = uuid.uuid4() convo = CatchupSubscription("my-stream", start_from=0, conversation_id=conversation_id) await convo.start(output) [request] = output.items body = proto.ReadStreamEvents() body.ParseFromString(request.payload) assert request.command is msg.TcpCommand.ReadStreamEventsForward assert body.event_stream_id == "my-stream" assert body.from_event_number == 0 assert body.resolve_link_tos is True assert body.require_master is False assert body.max_count == 100
async def test_subscription_dropped_mid_stream(): convo = CatchupSubscription("my-stream") output = TeeQueue() empty_page = (ReadStreamEventsResponseBuilder( stream="stream-123").at_end_of_stream().build()) await reply_to(convo, empty_page, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, empty_page, output) subscription = convo.result.result() await reply_to(convo, event_appeared(), output) await drop_subscription(convo, output) events = [e async for e in subscription.events] assert len(events) == 1
async def test_restart_from_catchup(): """ If the connection drops during the catchup phase, we need to unsubscribe from the stream and then go back to reading historical events starting from the last page. => Request historical events <= Receive 1 event, next_event = 1 => Subscribe <= Confirmed => Catch up from 1 ** Restart ** => Unsubscribe <= Confirmed => Read historical from 1 <= Empty page => Subscribe """ conversation_id = uuid.uuid4() output = TeeQueue() convo = CatchupSubscription("my-stream", conversation_id=conversation_id) await convo.start(output) await output.get() page_one = (ReadStreamEventsResponseBuilder().with_event( event_number=1).with_next_event_number(1).at_end_of_stream().build()) await reply_to(convo, page_one, output) await output.get() await confirm_subscription(convo, output, event_number=10, commit_pos=10) first_catch_up = read_as(proto.ReadStreamEvents, await output.get()) await reply_to(convo, page_one, output) # Restart await convo.start(output) unsubscribe = await output.get() assert first_catch_up.from_event_number == 1 assert unsubscribe.command == msg.TcpCommand.UnsubscribeFromStream await drop_subscription(convo, output) second_catchup = read_as(proto.ReadStreamEvents, await output.get()) assert second_catchup.from_event_number == 1
async def test_unsubscription(): correlation_id = uuid.uuid4() output = TeeQueue() convo = CatchupSubscription("my-stream", conversation_id=correlation_id) await convo.start(output) await reply_to(convo, EMPTY_STREAM_PAGE, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, EMPTY_STREAM_PAGE, output) sub = convo.result.result() await sub.unsubscribe() [read_historical, subscribe, catch_up, unsubscribe] = output.items assert unsubscribe.command == msg.TcpCommand.UnsubscribeFromStream assert unsubscribe.conversation_id == correlation_id
async def test_paging(): """ During the read phase, we expect to page through multiple batches of events. In this scenario we have two batches, each of two events. """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) await output.get() event_1_id = uuid.uuid4() event_2_id = uuid.uuid4() event_3_id = uuid.uuid4() event_4_id = uuid.uuid4() first_response = (ReadStreamEventsResponseBuilder().with_event( event_id=event_1_id, event_number=32).with_event( event_id=event_2_id, event_number=33).with_next_event_number(34)).build() second_response = (ReadStreamEventsResponseBuilder().with_event( event_id=event_3_id, event_number=34).with_event(event_id=event_4_id, event_number=35)).build() await reply_to(convo, first_response, output) subscription = await convo.result event_1 = await anext(subscription.events) event_2 = await anext(subscription.events) assert event_1.id == event_1_id assert event_2.id == event_2_id reply = await output.get() body = proto.ReadStreamEvents() body.ParseFromString(reply.payload) assert body.from_event_number == 34 await reply_to(convo, second_response, output) event_3 = await anext(subscription.events) event_4 = await anext(subscription.events) assert event_3.id == event_3_id assert event_4.id == event_4_id
async def test_paging_projection(): """ """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) await output.get() event_1_id = uuid.uuid4() event_2_id = uuid.uuid4() event_3_id = uuid.uuid4() event_4_id = uuid.uuid4() first_response = (ReadStreamEventsResponseBuilder().with_event( event_id=event_1_id, event_number=0, link_event_number=32).with_event( event_id=event_2_id, event_number=0, link_event_number=33).with_next_event_number(34)).build() second_response = (ReadStreamEventsResponseBuilder().with_event( event_id=event_3_id, event_number=0, link_event_number=34).with_event(event_id=event_4_id, event_number=0, link_event_number=35)).build() await reply_to(convo, first_response, output) subscription = await convo.result event_1 = await anext(subscription.events) event_2 = await anext(subscription.events) assert event_1.id == event_1_id assert event_2.id == event_2_id reply = await output.get() body = proto.ReadStreamEvents() body.ParseFromString(reply.payload) assert body.from_event_number == 34 await reply_to(convo, second_response, output) event_3 = await anext(subscription.events) event_4 = await anext(subscription.events) assert event_3.id == event_3_id assert event_4.id == event_4_id
async def test_subscription_failure_mid_stream(): output = TeeQueue() convo = CatchupSubscription("my-stream") event_id = uuid.uuid4() await reply_to(convo, EMPTY_STREAM_PAGE, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, EMPTY_STREAM_PAGE, output) subscription = convo.result.result() await reply_to(convo, event_appeared(event_id=event_id), output) await drop_subscription( convo, output, msg.SubscriptionDropReason.SubscriberMaxCountReached) with pytest.raises(exn.SubscriptionFailed): event = await anext(subscription.events) assert event.id == event_id await anext(subscription.events)
async def test_live_restart(): """ If we reset the conversation while we are live, we should first unsubscribe then start a historical read from the last read event. => Read historial <= empty => subscribe <= confirmed => catchup <= empty <= event 1 appeared <= event 2 appeared RESTART => unsubscribe <= dropped => Read historical from 2 """ output = TeeQueue() convo = CatchupSubscription("my-stream") await convo.start(output) await reply_to(convo, EMPTY_STREAM_PAGE, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, EMPTY_STREAM_PAGE, output) await reply_to(convo, event_appeared(event_number=1), output) await reply_to(convo, event_appeared(event_number=2), output) output.items.clear() await convo.start(output) await drop_subscription(convo, output) [unsubscribe, read_historical] = output.items read_historical = read_as(proto.ReadStreamEvents, read_historical) assert unsubscribe.command == msg.TcpCommand.UnsubscribeFromStream assert read_historical.from_event_number == 2
async def test_subscribes_at_end_of_stream(): """ When we have read all the events in the stream, we should send a request to subscribe for new events. """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) await output.get() await reply_to( convo, ReadStreamEventsResponseBuilder().at_end_of_stream().build(), output) reply = await output.get() payload = proto.SubscribeToStream() payload.ParseFromString(reply.payload) assert reply.command == msg.TcpCommand.SubscribeToStream assert payload.event_stream_id == "my-stream" assert payload.resolve_link_tos is True
async def test_historical_duplicates(): """ It's possible that we receive the reply to a ReadStreamEvents request after we've resent the request. This will result in our receiving a duplicate page. In this instance, we should not raise duplicate events. => Request historical RESTART => Request historical <= 2 events <= 3 events Should only see the 3 unique events """ two_events = (ReadStreamEventsResponseBuilder().with_event( event_number=1).with_event(event_number=2).with_next_event_number( 2).at_end_of_stream().build()) three_events = (ReadStreamEventsResponseBuilder().with_event( event_number=1).with_event(event_number=2).with_event(event_number=3). with_next_event_number(3).at_end_of_stream().build()) output = TeeQueue() convo = CatchupSubscription("my-stream") await convo.start(output) await convo.start(output) await reply_to(convo, two_events, output) await reply_to(convo, three_events, output) [event_1, event_2, event_3] = await anext(convo.subscription.events, 3) assert event_1.event_number == 1 assert event_2.event_number == 2 assert event_3.event_number == 3
async def test_end_of_stream(): """ During the Read phase, we yield the events to the subscription so that the user is unaware of the chicanery in the background. When we reach the end of the stream, we should send a subscribe message to start the volatile subscription. """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) event_1_id = uuid.uuid4() event_2_id = uuid.uuid4() response = (ReadStreamEventsResponseBuilder( stream="stream-123").at_end_of_stream().with_event( event_id=event_1_id, event_number=32).with_event(event_id=event_2_id, event_number=33)).build() await reply_to(convo, response, output) subscription = await convo.result event_1 = await anext(subscription.events) event_2 = await anext(subscription.events) assert event_1.stream == "stream-123" assert event_1.id == event_1_id assert event_1.event_number == 32 assert event_2.stream == "stream-123" assert event_2.id == event_2_id assert event_2.event_number == 33
async def test_subscription_duplicates(): """ If we receive subscription events while catching up, we buffer them internally. If we restart the conversation at that point we need to make sure we clear our buffer and do not raise duplicate events. => Request historical <= Empty => Subscribe to stream <= Confirmed => Request catchup <= Subscribed event 2 appeared <= Event 1, not end of stream RESTART => Drop subscription <= Dropped => Request historical from_event = 1 <= Receive event 2 at end of stream => Subscribe <= Confirmed => Catchup <= Subscribed event 3 appeared <= Empty Should yield [event 1, event 2, event 3] """ event_1_not_end_of_stream = (ReadStreamEventsResponseBuilder().with_event( event_number=1).with_next_event_number(2).build()) event_2_at_end_of_stream = (ReadStreamEventsResponseBuilder().with_event( event_number=2).with_next_event_number(2).at_end_of_stream().build()) output = TeeQueue() convo = CatchupSubscription("my-stream") await convo.start(output) await reply_to(convo, EMPTY_STREAM_PAGE, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, event_appeared(event_number=2), output) await reply_to(convo, event_1_not_end_of_stream, output) # RESTART await convo.start(output) output.items.clear() await drop_subscription(convo, output) second_read_historical = read_as(proto.ReadStreamEvents, output.items[0]) await reply_to(convo, event_2_at_end_of_stream, output) await confirm_subscription(convo, output, event_number=10, commit_pos=10) await reply_to(convo, event_appeared(event_number=3), output) await reply_to(convo, EMPTY_STREAM_PAGE, output) [event_1, event_2, event_3] = await anext(convo.subscription.events, 3) assert event_1.event_number == 1 assert event_2.event_number == 2 assert event_3.event_number == 3 assert second_read_historical.from_event_number == 2
async def test_should_return_catchup_events_before_subscribed_events(): """ It's possible that the following sequence of events occurs: * The client reads the last batch of events from a stream containing 50 events. * The client sends SubscribeToStream * Event 51 is written to the stream * The server creates a subscription starting at event 51 and responds with SubscriptionConfirmed * Event 52 is written to the stream * The client receives event 52. To solve this problem, the client needs to perform an additional read to fetch any missing events created between the last batch and the subscription confirmation. -------------- In this scenario, we read a single event (1) from the end of the stream and expect to create a subscription. We receive event 4 immediately on the subscription. We expect that the client requests missing events. We receive two pages, of one event each: 3, and 4, and then drop the subscription. Lastly, we expect that the events are yielded in the correct order despite being received out of order and that we have no duplicates. """ convo = CatchupSubscription("my-stream") output = TeeQueue() await convo.start(output) await output.get() last_page = ( ReadStreamEventsResponseBuilder().at_end_of_stream().with_event( event_number=1, type="a").build()) subscribed_event = event_appeared(event_number=4, type="d") first_catchup = (ReadStreamEventsResponseBuilder().with_event( event_number=2, type="b").build()) second_catchup = (ReadStreamEventsResponseBuilder().with_event( event_number=3, type="c").with_event(event_number=4, type="d")).build() await reply_to(convo, last_page, output) assert (await output.get()).command == msg.TcpCommand.SubscribeToStream await confirm_subscription(convo, output, event_number=3) await reply_to(convo, subscribed_event, output) assert (await output.get()).command == msg.TcpCommand.ReadStreamEventsForward await reply_to(convo, first_catchup, output) assert (await output.get()).command == msg.TcpCommand.ReadStreamEventsForward await reply_to(convo, second_catchup, output) await drop_subscription(convo, output) events = [] subscription = await convo.result async for e in subscription.events: events.append(e) assert len(events) == 4 [a, b, c, d] = events assert a.event_number == 1 assert b.event_number == 2 assert c.event_number == 3 assert d.event_number == 4