def _check_specified_offsets_initializer(
self,
source: KafkaSource,
offsets: Dict[KafkaTopicPartition, int],
reset_strategy: KafkaOffsetResetStrategy,
is_start: bool = True):
if is_start:
field_name = 'startingOffsetsInitializer'
else:
field_name = 'stoppingOffsetsInitializer'
offsets_initializer = get_field_value(source.get_java_function(),
field_name)
self.assertEqual(
offsets_initializer.getClass().getCanonicalName(),
'org.apache.flink.connector.kafka.source.enumerator.initializer'
'.SpecifiedOffsetsInitializer')
initial_offsets = get_field_value(offsets_initializer,
'initialOffsets')
self.assertTrue(
is_instance_of(initial_offsets,
get_gateway().jvm.java.util.Map))
self.assertEqual(initial_offsets.size(), len(offsets))
for j_topic_partition in initial_offsets:
topic_partition = KafkaTopicPartition(
j_topic_partition.topic(), j_topic_partition.partition())
self.assertIsNotNone(offsets.get(topic_partition))
self.assertEqual(initial_offsets[j_topic_partition],
offsets[topic_partition])
offset_reset_strategy = get_field_value(offsets_initializer,
'offsetResetStrategy')
self.assertTrue(
offset_reset_strategy.equals(
reset_strategy._to_j_offset_reset_strategy()))
def _check_reader_handled_offsets_initializer(
self,
source: KafkaSource,
offset: int,
reset_strategy: KafkaOffsetResetStrategy,
is_start: bool = True):
if is_start:
field_name = 'startingOffsetsInitializer'
else:
field_name = 'stoppingOffsetsInitializer'
offsets_initializer = get_field_value(source.get_java_function(),
field_name)
self.assertEqual(
offsets_initializer.getClass().getCanonicalName(),
'org.apache.flink.connector.kafka.source.enumerator.initializer'
'.ReaderHandledOffsetsInitializer')
starting_offset = get_field_value(offsets_initializer,
'startingOffset')
self.assertEqual(starting_offset, offset)
offset_reset_strategy = get_field_value(offsets_initializer,
'offsetResetStrategy')
self.assertTrue(
offset_reset_strategy.equals(
reset_strategy._to_j_offset_reset_strategy()))
def _get_kafka_source_configuration(source: KafkaSource):
jvm = get_gateway().jvm
j_source = source.get_java_function()
j_to_configuration = j_source.getClass().getDeclaredMethod(
'getConfiguration', to_jarray(jvm.java.lang.Class, []))
j_to_configuration.setAccessible(True)
j_configuration = j_to_configuration.invoke(
j_source, to_jarray(jvm.java.lang.Object, []))
return Configuration(j_configuration=j_configuration)
def _check_timestamp_offsets_initializer(self,
source: KafkaSource,
timestamp: int,
is_start: bool = True):
if is_start:
field_name = 'startingOffsetsInitializer'
else:
field_name = 'stoppingOffsetsInitializer'
offsets_initializer = get_field_value(source.get_java_function(),
field_name)
self.assertEqual(
offsets_initializer.getClass().getCanonicalName(),
'org.apache.flink.connector.kafka.source.enumerator.initializer'
'.TimestampOffsetsInitializer')
starting_timestamp = get_field_value(offsets_initializer,
'startingTimestamp')
self.assertEqual(starting_timestamp, timestamp)
def _check_bounded(source: KafkaSource):
self.assertEqual(
get_field_value(source.get_java_function(),
'boundedness').toString(),
'CONTINUOUS_UNBOUNDED')
def _check_bounded(source: KafkaSource):
self.assertEqual(
self._get_java_field(source.get_java_function(),
'boundedness').toString(), 'BOUNDED')
