def test_planner_selection(self):
gateway = get_gateway()
CLASS_NAME = gateway.jvm.EnvironmentSettings.CLASS_NAME
builder = EnvironmentSettings.new_instance()
OLD_PLANNER_FACTORY = get_private_field(builder._j_builder,
"OLD_PLANNER_FACTORY")
OLD_EXECUTOR_FACTORY = get_private_field(builder._j_builder,
"OLD_EXECUTOR_FACTORY")
BLINK_PLANNER_FACTORY = get_private_field(builder._j_builder,
"BLINK_PLANNER_FACTORY")
BLINK_EXECUTOR_FACTORY = get_private_field(builder._j_builder,
"BLINK_EXECUTOR_FACTORY")
# test the default behaviour to make sure it is consistent with the python doc
envrionment_settings = builder.build()
self.assertEqual(
envrionment_settings._j_environment_settings.toPlannerProperties()
[CLASS_NAME], OLD_PLANNER_FACTORY)
self.assertEqual(
envrionment_settings._j_environment_settings.toExecutorProperties(
)[CLASS_NAME], OLD_EXECUTOR_FACTORY)
# test use_old_planner
envrionment_settings = builder.use_old_planner().build()
self.assertEqual(
envrionment_settings._j_environment_settings.toPlannerProperties()
[CLASS_NAME], OLD_PLANNER_FACTORY)
self.assertEqual(
envrionment_settings._j_environment_settings.toExecutorProperties(
)[CLASS_NAME], OLD_EXECUTOR_FACTORY)
# test use_blink_planner
envrionment_settings = builder.use_blink_planner().build()
self.assertEqual(
envrionment_settings._j_environment_settings.toPlannerProperties()
[CLASS_NAME], BLINK_PLANNER_FACTORY)
self.assertEqual(
envrionment_settings._j_environment_settings.toExecutorProperties(
)[CLASS_NAME], BLINK_EXECUTOR_FACTORY)
# test use_any_planner
envrionment_settings = builder.use_any_planner().build()
self.assertTrue(CLASS_NAME not in envrionment_settings.
_j_environment_settings.toPlannerProperties())
self.assertTrue(CLASS_NAME not in envrionment_settings.
_j_environment_settings.toExecutorProperties())
def check_blink_planner(self, settings: EnvironmentSettings):
gateway = get_gateway()
CLASS_NAME = gateway.jvm.EnvironmentSettings.CLASS_NAME
builder = EnvironmentSettings.new_instance()
BLINK_PLANNER_FACTORY = get_private_field(builder._j_builder,
"BLINK_PLANNER_FACTORY")
BLINK_EXECUTOR_FACTORY = get_private_field(builder._j_builder,
"BLINK_EXECUTOR_FACTORY")
self.assertEqual(
settings._j_environment_settings.toPlannerProperties()[CLASS_NAME],
BLINK_PLANNER_FACTORY)
self.assertEqual(
settings._j_environment_settings.toExecutorProperties()
[CLASS_NAME], BLINK_EXECUTOR_FACTORY)
def test_jdbc_sink(self):
ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3),
('deeefg', 4)],
type_info=Types.ROW(
[Types.STRING(),
Types.INT()]))
jdbc_connection_options = JdbcConnectionOptions.JdbcConnectionOptionsBuilder()\
.with_driver_name('com.mysql.jdbc.Driver')\
.with_user_name('root')\
.with_password('password')\
.with_url('jdbc:mysql://server-name:server-port/database-name').build()
jdbc_execution_options = JdbcExecutionOptions.builder().with_batch_interval_ms(2000)\
.with_batch_size(100).with_max_retries(5).build()
jdbc_sink = JdbcSink.sink("insert into test table", ds.get_type(),
jdbc_connection_options,
jdbc_execution_options)
ds.add_sink(jdbc_sink).name('jdbc sink')
plan = eval(self.env.get_execution_plan())
self.assertEqual('Sink: jdbc sink', plan['nodes'][1]['type'])
j_output_format = get_private_field(jdbc_sink.get_java_function(),
'outputFormat')
connection_options = JdbcConnectionOptions(
get_private_field(
get_private_field(j_output_format, 'connectionProvider'),
'jdbcOptions'))
self.assertEqual(jdbc_connection_options.get_db_url(),
connection_options.get_db_url())
self.assertEqual(jdbc_connection_options.get_driver_name(),
connection_options.get_driver_name())
self.assertEqual(jdbc_connection_options.get_password(),
connection_options.get_password())
self.assertEqual(jdbc_connection_options.get_user_name(),
connection_options.get_user_name())
exec_options = JdbcExecutionOptions(
get_private_field(j_output_format, 'executionOptions'))
self.assertEqual(jdbc_execution_options.get_batch_interval_ms(),
exec_options.get_batch_interval_ms())
self.assertEqual(jdbc_execution_options.get_batch_size(),
exec_options.get_batch_size())
self.assertEqual(jdbc_execution_options.get_max_retries(),
exec_options.get_max_retries())
def kafka_connector_assertion(self, flink_kafka_consumer_clz, flink_kafka_producer_clz):
source_topic = 'test_source_topic'
sink_topic = 'test_sink_topic'
props = {'bootstrap.servers': 'localhost:9092', 'group.id': 'test_group'}
type_info = Types.ROW([Types.INT(), Types.STRING()])
# Test for kafka consumer
deserialization_schema = JsonRowDeserializationSchema.builder() \
.type_info(type_info=type_info).build()
flink_kafka_consumer = flink_kafka_consumer_clz(source_topic, deserialization_schema, props)
flink_kafka_consumer.set_start_from_earliest()
flink_kafka_consumer.set_commit_offsets_on_checkpoints(True)
j_properties = get_private_field(flink_kafka_consumer.get_java_function(), 'properties')
self.assertEqual('localhost:9092', j_properties.getProperty('bootstrap.servers'))
self.assertEqual('test_group', j_properties.getProperty('group.id'))
self.assertTrue(get_private_field(flink_kafka_consumer.get_java_function(),
'enableCommitOnCheckpoints'))
j_start_up_mode = get_private_field(flink_kafka_consumer.get_java_function(), 'startupMode')
j_deserializer = get_private_field(flink_kafka_consumer.get_java_function(), 'deserializer')
j_deserialize_type_info = invoke_java_object_method(j_deserializer, "getProducedType")
deserialize_type_info = typeinfo._from_java_type(j_deserialize_type_info)
self.assertTrue(deserialize_type_info == type_info)
self.assertTrue(j_start_up_mode.equals(get_gateway().jvm
.org.apache.flink.streaming.connectors
.kafka.config.StartupMode.EARLIEST))
j_topic_desc = get_private_field(flink_kafka_consumer.get_java_function(),
'topicsDescriptor')
j_topics = invoke_java_object_method(j_topic_desc, 'getFixedTopics')
self.assertEqual(['test_source_topic'], list(j_topics))
# Test for kafka producer
serialization_schema = JsonRowSerializationSchema.builder().with_type_info(type_info) \
.build()
flink_kafka_producer = flink_kafka_producer_clz(sink_topic, serialization_schema, props)
flink_kafka_producer.set_write_timestamp_to_kafka(False)
j_producer_config = get_private_field(flink_kafka_producer.get_java_function(),
'producerConfig')
self.assertEqual('localhost:9092', j_producer_config.getProperty('bootstrap.servers'))
self.assertEqual('test_group', j_producer_config.getProperty('group.id'))
self.assertFalse(get_private_field(flink_kafka_producer.get_java_function(),
'writeTimestampToKafka'))