def test_planner_selection(self): gateway = get_gateway() CLASS_NAME = gateway.jvm.EnvironmentSettings.CLASS_NAME builder = EnvironmentSettings.new_instance() OLD_PLANNER_FACTORY = get_private_field(builder._j_builder, "OLD_PLANNER_FACTORY") OLD_EXECUTOR_FACTORY = get_private_field(builder._j_builder, "OLD_EXECUTOR_FACTORY") BLINK_PLANNER_FACTORY = get_private_field(builder._j_builder, "BLINK_PLANNER_FACTORY") BLINK_EXECUTOR_FACTORY = get_private_field(builder._j_builder, "BLINK_EXECUTOR_FACTORY") # test the default behaviour to make sure it is consistent with the python doc envrionment_settings = builder.build() self.assertEqual( envrionment_settings._j_environment_settings.toPlannerProperties() [CLASS_NAME], OLD_PLANNER_FACTORY) self.assertEqual( envrionment_settings._j_environment_settings.toExecutorProperties( )[CLASS_NAME], OLD_EXECUTOR_FACTORY) # test use_old_planner envrionment_settings = builder.use_old_planner().build() self.assertEqual( envrionment_settings._j_environment_settings.toPlannerProperties() [CLASS_NAME], OLD_PLANNER_FACTORY) self.assertEqual( envrionment_settings._j_environment_settings.toExecutorProperties( )[CLASS_NAME], OLD_EXECUTOR_FACTORY) # test use_blink_planner envrionment_settings = builder.use_blink_planner().build() self.assertEqual( envrionment_settings._j_environment_settings.toPlannerProperties() [CLASS_NAME], BLINK_PLANNER_FACTORY) self.assertEqual( envrionment_settings._j_environment_settings.toExecutorProperties( )[CLASS_NAME], BLINK_EXECUTOR_FACTORY) # test use_any_planner envrionment_settings = builder.use_any_planner().build() self.assertTrue(CLASS_NAME not in envrionment_settings. _j_environment_settings.toPlannerProperties()) self.assertTrue(CLASS_NAME not in envrionment_settings. _j_environment_settings.toExecutorProperties())
def check_blink_planner(self, settings: EnvironmentSettings): gateway = get_gateway() CLASS_NAME = gateway.jvm.EnvironmentSettings.CLASS_NAME builder = EnvironmentSettings.new_instance() BLINK_PLANNER_FACTORY = get_private_field(builder._j_builder, "BLINK_PLANNER_FACTORY") BLINK_EXECUTOR_FACTORY = get_private_field(builder._j_builder, "BLINK_EXECUTOR_FACTORY") self.assertEqual( settings._j_environment_settings.toPlannerProperties()[CLASS_NAME], BLINK_PLANNER_FACTORY) self.assertEqual( settings._j_environment_settings.toExecutorProperties() [CLASS_NAME], BLINK_EXECUTOR_FACTORY)
def test_jdbc_sink(self): ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3), ('deeefg', 4)], type_info=Types.ROW( [Types.STRING(), Types.INT()])) jdbc_connection_options = JdbcConnectionOptions.JdbcConnectionOptionsBuilder()\ .with_driver_name('com.mysql.jdbc.Driver')\ .with_user_name('root')\ .with_password('password')\ .with_url('jdbc:mysql://server-name:server-port/database-name').build() jdbc_execution_options = JdbcExecutionOptions.builder().with_batch_interval_ms(2000)\ .with_batch_size(100).with_max_retries(5).build() jdbc_sink = JdbcSink.sink("insert into test table", ds.get_type(), jdbc_connection_options, jdbc_execution_options) ds.add_sink(jdbc_sink).name('jdbc sink') plan = eval(self.env.get_execution_plan()) self.assertEqual('Sink: jdbc sink', plan['nodes'][1]['type']) j_output_format = get_private_field(jdbc_sink.get_java_function(), 'outputFormat') connection_options = JdbcConnectionOptions( get_private_field( get_private_field(j_output_format, 'connectionProvider'), 'jdbcOptions')) self.assertEqual(jdbc_connection_options.get_db_url(), connection_options.get_db_url()) self.assertEqual(jdbc_connection_options.get_driver_name(), connection_options.get_driver_name()) self.assertEqual(jdbc_connection_options.get_password(), connection_options.get_password()) self.assertEqual(jdbc_connection_options.get_user_name(), connection_options.get_user_name()) exec_options = JdbcExecutionOptions( get_private_field(j_output_format, 'executionOptions')) self.assertEqual(jdbc_execution_options.get_batch_interval_ms(), exec_options.get_batch_interval_ms()) self.assertEqual(jdbc_execution_options.get_batch_size(), exec_options.get_batch_size()) self.assertEqual(jdbc_execution_options.get_max_retries(), exec_options.get_max_retries())
def kafka_connector_assertion(self, flink_kafka_consumer_clz, flink_kafka_producer_clz): source_topic = 'test_source_topic' sink_topic = 'test_sink_topic' props = {'bootstrap.servers': 'localhost:9092', 'group.id': 'test_group'} type_info = Types.ROW([Types.INT(), Types.STRING()]) # Test for kafka consumer deserialization_schema = JsonRowDeserializationSchema.builder() \ .type_info(type_info=type_info).build() flink_kafka_consumer = flink_kafka_consumer_clz(source_topic, deserialization_schema, props) flink_kafka_consumer.set_start_from_earliest() flink_kafka_consumer.set_commit_offsets_on_checkpoints(True) j_properties = get_private_field(flink_kafka_consumer.get_java_function(), 'properties') self.assertEqual('localhost:9092', j_properties.getProperty('bootstrap.servers')) self.assertEqual('test_group', j_properties.getProperty('group.id')) self.assertTrue(get_private_field(flink_kafka_consumer.get_java_function(), 'enableCommitOnCheckpoints')) j_start_up_mode = get_private_field(flink_kafka_consumer.get_java_function(), 'startupMode') j_deserializer = get_private_field(flink_kafka_consumer.get_java_function(), 'deserializer') j_deserialize_type_info = invoke_java_object_method(j_deserializer, "getProducedType") deserialize_type_info = typeinfo._from_java_type(j_deserialize_type_info) self.assertTrue(deserialize_type_info == type_info) self.assertTrue(j_start_up_mode.equals(get_gateway().jvm .org.apache.flink.streaming.connectors .kafka.config.StartupMode.EARLIEST)) j_topic_desc = get_private_field(flink_kafka_consumer.get_java_function(), 'topicsDescriptor') j_topics = invoke_java_object_method(j_topic_desc, 'getFixedTopics') self.assertEqual(['test_source_topic'], list(j_topics)) # Test for kafka producer serialization_schema = JsonRowSerializationSchema.builder().with_type_info(type_info) \ .build() flink_kafka_producer = flink_kafka_producer_clz(sink_topic, serialization_schema, props) flink_kafka_producer.set_write_timestamp_to_kafka(False) j_producer_config = get_private_field(flink_kafka_producer.get_java_function(), 'producerConfig') self.assertEqual('localhost:9092', j_producer_config.getProperty('bootstrap.servers')) self.assertEqual('test_group', j_producer_config.getProperty('group.id')) self.assertFalse(get_private_field(flink_kafka_producer.get_java_function(), 'writeTimestampToKafka'))