def test_kinesis_firehose_sink(self): _load_specific_flink_module_jars('/flink-connectors/' 'flink-sql-connector-aws-kinesis-firehose') sink_properties = { 'aws.region': 'eu-west-1', 'aws.credentials.provider.basic.accesskeyid': 'aws_access_key_id', 'aws.credentials.provider.basic.secretkey': 'aws_secret_access_key' } ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3), ('deeefg', 4)], type_info=Types.ROW([Types.STRING(), Types.INT()])) kinesis_firehose_sink = KinesisFirehoseSink.builder() \ .set_firehose_client_properties(sink_properties) \ .set_serialization_schema(SimpleStringSchema()) \ .set_delivery_stream_name('stream-1') \ .set_fail_on_error(False) \ .set_max_batch_size(500) \ .set_max_in_flight_requests(50) \ .set_max_buffered_requests(10000) \ .set_max_batch_size_in_bytes(5 * 1024 * 1024) \ .set_max_time_in_buffer_ms(5000) \ .set_max_record_size_in_bytes(1 * 1024 * 1024) \ .build() ds.sink_to(kinesis_firehose_sink).name('kinesis firehose sink') plan = eval(self.env.get_execution_plan()) self.assertEqual('kinesis firehose sink: Writer', plan['nodes'][1]['type']) self.assertEqual(get_field_value(kinesis_firehose_sink.get_java_function(), 'failOnError'), False) self.assertEqual( get_field_value(kinesis_firehose_sink.get_java_function(), 'deliveryStreamName'), 'stream-1')
def setUp(self) -> None: self.env = StreamExecutionEnvironment.get_execution_environment() self.test_sink = DataStreamTestSinkFunction() _load_specific_flink_module_jars( '/flink-connectors/flink-connector-files') _load_specific_flink_module_jars( '/flink-connectors/flink-connector-sink-common')
def setUp(self) -> None: self.env = StreamExecutionEnvironment.get_execution_environment() # Cache current ContextClassLoader, we will replace it with a temporary URLClassLoader to # load specific connector jars with given module path to do dependency isolation. And We # will change the ClassLoader back to the cached ContextClassLoader after the test case # finished. self._cxt_clz_loader = get_gateway().jvm.Thread.currentThread().getContextClassLoader() _load_specific_flink_module_jars(self._get_jars_relative_path())
def test_use_modules(self): # please do not change this order since ModuleMock depends on FunctionDefinitionMock _load_specific_flink_module_jars('/flink-table/flink-table-common') _load_specific_flink_module_jars('/flink-table/flink-table-api-java') self.t_env.load_module('x', Module( get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("x") )) self.t_env.load_module('y', Module( get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("y") )) self.check_list_modules('core', 'x', 'y') self.check_list_full_modules(3, 'core', 'x', 'y') self.t_env.use_modules('y', 'core') self.check_list_modules('y', 'core') self.check_list_full_modules(2, 'y', 'core', 'x')
def test_kafka_connector_universal(self): _load_specific_flink_module_jars('/flink-connectors/flink-sql-connector-kafka') self.kafka_connector_assertion(FlinkKafkaConsumer, FlinkKafkaProducer)
def setUp(self) -> None: self.env = StreamExecutionEnvironment.get_execution_environment() self._cxt_clz_loader = get_gateway().jvm.Thread.currentThread().getContextClassLoader() _load_specific_flink_module_jars('/flink-connectors/flink-connector-jdbc')
def setUpClass(cls): super(KafkaDescriptorTests, cls).setUpClass() cls._cxt_clz_loader = get_gateway().jvm.Thread.currentThread().getContextClassLoader() _load_specific_flink_module_jars('/flink-connectors/flink-connector-kafka')
def setUpClass(cls): super(ElasticsearchDescriptorTest, cls).setUpClass() cls._cxt_clz_loader = get_gateway().jvm.Thread.currentThread().getContextClassLoader() _load_specific_flink_module_jars('/flink-connectors/flink-connector-elasticsearch-base')
def test_kafka_connector_011(self): _load_specific_flink_module_jars( '/flink-connectors/flink-sql-connector-kafka-0.11') self.kafka_connector_assertion(FlinkKafkaConsumer011, FlinkKafkaProducer011)
def setUpClass(cls): cls._cxt_clz_loader = get_gateway().jvm.Thread.currentThread( ).getContextClassLoader() _load_specific_flink_module_jars( '/flink-connectors/flink-connector-kafka-base')
def setUp(self): self._cxt_clz_loader = get_gateway().jvm.Thread.currentThread( ).getContextClassLoader() _load_specific_flink_module_jars( '/flink-connectors/flink-sql-connector-rabbitmq')