def kafka_connector_assertion(self, flink_kafka_consumer_clz,
flink_kafka_producer_clz):
source_topic = 'test_source_topic'
sink_topic = 'test_sink_topic'
props = {
'bootstrap.servers': 'localhost:9092',
'group.id': 'test_group'
}
type_info = Types.ROW([Types.INT(), Types.STRING()])
# Test for kafka consumer
deserialization_schema = JsonRowDeserializationSchema.builder() \
.type_info(type_info=type_info).build()
flink_kafka_consumer = flink_kafka_consumer_clz(
source_topic, deserialization_schema, props)
flink_kafka_consumer.set_start_from_earliest()
flink_kafka_consumer.set_commit_offsets_on_checkpoints(True)
j_properties = get_field_value(
flink_kafka_consumer.get_java_function(), 'properties')
self.assertEqual('localhost:9092',
j_properties.getProperty('bootstrap.servers'))
self.assertEqual('test_group', j_properties.getProperty('group.id'))
self.assertTrue(
get_field_value(flink_kafka_consumer.get_java_function(),
'enableCommitOnCheckpoints'))
j_start_up_mode = get_field_value(
flink_kafka_consumer.get_java_function(), 'startupMode')
j_deserializer = get_field_value(
flink_kafka_consumer.get_java_function(), 'deserializer')
j_deserialize_type_info = invoke_java_object_method(
j_deserializer, "getProducedType")
deserialize_type_info = typeinfo._from_java_type(
j_deserialize_type_info)
self.assertTrue(deserialize_type_info == type_info)
self.assertTrue(
j_start_up_mode.equals(
get_gateway().jvm.org.apache.flink.streaming.connectors.kafka.
config.StartupMode.EARLIEST))
j_topic_desc = get_field_value(
flink_kafka_consumer.get_java_function(), 'topicsDescriptor')
j_topics = invoke_java_object_method(j_topic_desc, 'getFixedTopics')
self.assertEqual(['test_source_topic'], list(j_topics))
# Test for kafka producer
serialization_schema = JsonRowSerializationSchema.builder().with_type_info(type_info) \
.build()
flink_kafka_producer = flink_kafka_producer_clz(
sink_topic, serialization_schema, props)
flink_kafka_producer.set_write_timestamp_to_kafka(False)
j_producer_config = get_field_value(
flink_kafka_producer.get_java_function(), 'producerConfig')
self.assertEqual('localhost:9092',
j_producer_config.getProperty('bootstrap.servers'))
self.assertEqual('test_group',
j_producer_config.getProperty('group.id'))
self.assertFalse(
get_field_value(flink_kafka_producer.get_java_function(),
'writeTimestampToKafka'))
def setUp(self) -> None:
self.env = StreamExecutionEnvironment.get_execution_environment()
getConfigurationMethod = invoke_java_object_method(
self.env._j_stream_execution_environment, "getConfiguration")
getConfigurationMethod.setString("akka.ask.timeout", "20 s")
self.test_sink = DataStreamTestSinkFunction()