def _setup_error_after_data(self): subscriptions = SubscriptionState(loop=self.loop) client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp1 = TopicPartition('some_topic', 0) tp2 = TopicPartition('some_topic', 1) subscriptions.subscribe(set(["some_topic"])) subscriptions.assign_from_subscribed({tp1, tp2}) assignment = subscriptions.subscription.assignment subscriptions.seek(tp1, 0) subscriptions.seek(tp2, 0) # Add some data messages = [ConsumerRecord( topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4, headers=[])] partition_records = PartitionRecords( tp2, mock.Mock(), [], 0, None, None, False, READ_UNCOMMITTED) partition_records._records_iterator = iter(messages) fetcher._records[tp2] = FetchResult( tp2, assignment=assignment, loop=self.loop, partition_records=partition_records, backoff=0) # Add some error fetcher._records[tp1] = FetchError( loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0) return fetcher, tp1, tp2, messages
def _setup_error_after_data(self): subscriptions = SubscriptionState(loop=self.loop) client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) fetcher = Fetcher(client, subscriptions, loop=self.loop) tp1 = TopicPartition('some_topic', 0) tp2 = TopicPartition('some_topic', 1) subscriptions.subscribe(set(["some_topic"])) subscriptions.assign_from_subscribed({tp1, tp2}) assignment = subscriptions.subscription.assignment subscriptions.seek(tp1, 0) subscriptions.seek(tp2, 0) # Add some data messages = [ConsumerRecord( topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4)] fetcher._records[tp2] = FetchResult( tp2, assignment=assignment, loop=self.loop, message_iterator=iter(messages), backoff=0, fetch_offset=0) # Add some error fetcher._records[tp1] = FetchError( loop=self.loop, error=OffsetOutOfRangeError({}), backoff=0) return fetcher, tp1, tp2, messages
def _setup_error_after_data(self): subscriptions = SubscriptionState() client = AIOKafkaClient(bootstrap_servers=[]) fetcher = Fetcher(client, subscriptions) tp1 = TopicPartition('some_topic', 0) tp2 = TopicPartition('some_topic', 1) subscriptions.subscribe(set(["some_topic"])) subscriptions.assign_from_subscribed({tp1, tp2}) assignment = subscriptions.subscription.assignment subscriptions.seek(tp1, 0) subscriptions.seek(tp2, 0) # Add some data messages = [ ConsumerRecord(topic="some_topic", partition=1, offset=0, timestamp=0, timestamp_type=0, key=None, value=b"some", checksum=None, serialized_key_size=0, serialized_value_size=4, headers=[]) ] partition_records = PartitionRecords(tp2, mock.Mock(), [], 0, None, None, False, READ_UNCOMMITTED) partition_records._records_iterator = iter(messages) fetcher._records[tp2] = FetchResult( tp2, assignment=assignment, partition_records=partition_records, backoff=0) # Add some error fetcher._records[tp1] = FetchError(error=OffsetOutOfRangeError({}), backoff=0) return fetcher, tp1, tp2, messages
async def test_proc_fetch_request(self): client = AIOKafkaClient(bootstrap_servers=[]) subscriptions = SubscriptionState() fetcher = Fetcher(client, subscriptions, auto_offset_reset="latest") tp = TopicPartition('test', 0) tp_info = (tp.topic, [(tp.partition, 4, 100000)]) req = FetchRequest( -1, # replica_id 100, 100, [tp_info]) async def ready(conn): return True def force_metadata_update(): fut = create_future() fut.set_result(False) return fut client.ready = mock.MagicMock() client.ready.side_effect = ready client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = force_metadata_update client.send = mock.MagicMock() builder = LegacyRecordBatchBuilder(magic=1, compression_type=0, batch_size=99999999) builder.append(offset=4, value=b"test msg", key=None, timestamp=None) raw_batch = bytes(builder.build()) fetch_response = FetchResponse([('test', [(0, 0, 9, raw_batch)])]) async def send(node, request): nonlocal fetch_response return fetch_response client.send.side_effect = send subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # The partition has no active position, so will ignore result needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) # The partition's position does not match request's fetch offset subscriptions.seek(tp, 0) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] self.assertEqual(buf.getone().value, b"test msg") # If position changed after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) subscriptions.seek(tp, 10) self.assertIsNone(buf.getone()) # If assignment is lost after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) subscriptions.unsubscribe() self.assertIsNone(buf.getone()) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # error -> no partition found (UnknownTopicOrPartitionError) subscriptions.seek(tp, 4) fetcher._records.clear() fetch_response = FetchResponse([('test', [(0, 3, 9, raw_batch)])]) cc = client.force_metadata_update.call_count needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(client.force_metadata_update.call_count, cc + 1) # error -> topic auth failed (TopicAuthorizationFailedError) fetch_response = FetchResponse([('test', [(0, 29, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(TopicAuthorizationFailedError): await fetcher.next_record([]) # error -> unknown fetch_response = FetchResponse([('test', [(0, -1, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) # error -> offset out of range with offset strategy fetch_response = FetchResponse([('test', [(0, 1, 9, raw_batch)])]) needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(tp_state.has_valid_position, False) self.assertEqual(tp_state.awaiting_reset, True) self.assertEqual(tp_state.reset_strategy, OffsetResetStrategy.LATEST) # error -> offset out of range without offset strategy subscriptions.seek(tp, 4) fetcher._default_reset_strategy = OffsetResetStrategy.NONE needs_wake_up = await fetcher._proc_fetch_request(assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(OffsetOutOfRangeError): await fetcher.next_record([]) await fetcher.close()
def test_proc_fetch_request(self): client = AIOKafkaClient( loop=self.loop, bootstrap_servers=[]) subscriptions = SubscriptionState(loop=self.loop) fetcher = Fetcher( client, subscriptions, auto_offset_reset="latest", loop=self.loop) tp = TopicPartition('test', 0) tp_info = (tp.topic, [(tp.partition, 4, 100000)]) req = FetchRequest( -1, # replica_id 100, 100, [tp_info]) client.ready = mock.MagicMock() client.ready.side_effect = asyncio.coroutine(lambda a: True) client.force_metadata_update = mock.MagicMock() client.force_metadata_update.side_effect = asyncio.coroutine( lambda: False) client.send = mock.MagicMock() builder = LegacyRecordBatchBuilder( magic=1, compression_type=0, batch_size=99999999) builder.append(offset=4, value=b"test msg", key=None, timestamp=None) raw_batch = bytes(builder.build()) client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 0, 9, raw_batch)])])) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # The partition has no active position, so will ignore result needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) # The partition's position does not match request's fetch offset subscriptions.seek(tp, 0) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(fetcher._records, {}) subscriptions.seek(tp, 4) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, True) buf = fetcher._records[tp] self.assertEqual(buf.getone().value, b"test msg") # If position changed after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) subscriptions.seek(tp, 10) self.assertIsNone(buf.getone()) # If assignment is lost after fetch request passed subscriptions.seek(tp, 4) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) subscriptions.unsubscribe() self.assertIsNone(buf.getone()) subscriptions.assign_from_user({tp}) assignment = subscriptions.subscription.assignment tp_state = assignment.state_value(tp) # error -> no partition found (UnknownTopicOrPartitionError) subscriptions.seek(tp, 4) fetcher._records.clear() client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 3, 9, raw_batch)])])) cc = client.force_metadata_update.call_count needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(client.force_metadata_update.call_count, cc + 1) # error -> topic auth failed (TopicAuthorizationFailedError) client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 29, 9, raw_batch)])])) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(TopicAuthorizationFailedError): yield from fetcher.next_record([]) # error -> unknown client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, -1, 9, raw_batch)])])) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, False) # error -> offset out of range with offset strategy client.send.side_effect = asyncio.coroutine( lambda n, r: FetchResponse( [('test', [(0, 1, 9, raw_batch)])])) needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, False) self.assertEqual(tp_state.has_valid_position, False) self.assertEqual(tp_state.awaiting_reset, True) self.assertEqual(tp_state.reset_strategy, OffsetResetStrategy.LATEST) # error -> offset out of range without offset strategy subscriptions.seek(tp, 4) fetcher._default_reset_strategy = OffsetResetStrategy.NONE needs_wake_up = yield from fetcher._proc_fetch_request( assignment, 0, req) self.assertEqual(needs_wake_up, True) with self.assertRaises(OffsetOutOfRangeError): yield from fetcher.next_record([]) yield from fetcher.close()