def test_csv_row_serialization_schema(self):
jvm = get_gateway().jvm
JRow = jvm.org.apache.flink.types.Row
j_row = JRow(3)
j_row.setField(0, "BEGIN")
j_row.setField(2, "END")
def field_assertion(field_info, csv_value, value, field_delimiter):
row_info = Types.ROW([Types.STRING(), field_info, Types.STRING()])
expected_csv = "BEGIN" + field_delimiter + csv_value + field_delimiter + "END\n"
j_row.setField(1, value)
csv_row_serialization_schema = CsvRowSerializationSchema.Builder(row_info)\
.set_escape_character('*').set_quote_character('\'')\
.set_array_element_delimiter(':').set_field_delimiter(';').build()
csv_row_deserialization_schema = CsvRowDeserializationSchema.Builder(row_info)\
.set_escape_character('*').set_quote_character('\'')\
.set_array_element_delimiter(':').set_field_delimiter(';').build()
csv_row_serialization_schema._j_serialization_schema.open(
jvm.org.apache.flink.connector.testutils.formats.
DummyInitializationContext())
csv_row_deserialization_schema._j_deserialization_schema.open(
jvm.org.apache.flink.connector.testutils.formats.
DummyInitializationContext())
serialized_bytes = csv_row_serialization_schema._j_serialization_schema.serialize(
j_row)
self.assertEqual(expected_csv,
str(serialized_bytes, encoding='utf-8'))
j_deserialized_row = csv_row_deserialization_schema._j_deserialization_schema\
.deserialize(expected_csv.encode("utf-8"))
self.assertTrue(j_row.equals(j_deserialized_row))
field_assertion(Types.STRING(), "'123''4**'", "123'4*", ";")
field_assertion(Types.STRING(), "'a;b''c'", "a;b'c", ";")
field_assertion(Types.INT(), "12", 12, ";")
test_j_row = JRow(2)
test_j_row.setField(0, "1")
test_j_row.setField(1, "hello")
field_assertion(Types.ROW([Types.STRING(),
Types.STRING()]), "'1:hello'", test_j_row,
";")
test_j_row.setField(1, "hello world")
field_assertion(Types.ROW([Types.STRING(),
Types.STRING()]), "'1:hello world'",
test_j_row, ";")
field_assertion(Types.STRING(), "null", "null", ";")
def write_to_kafka(env):
ds = env.from_collection([(1, 'hi'), (2, 'hello'), (3, 'hi'), (4, 'hello'),
(5, 'hi'), (6, 'hello'), (6, 'hello')],
type_info=Types.ROW([Types.INT(),
Types.STRING()]))
serialization_schema = AvroRowSerializationSchema(avro_schema_string="""
{
"type": "record",
"name": "TestRecord",
"fields": [
{"name": "id", "type": "int"},
{"name": "name", "type": "string"}
]
}""")
kafka_producer = FlinkKafkaProducer(
topic='test_avro_topic',
serialization_schema=serialization_schema,
producer_config={
'bootstrap.servers': 'localhost:9092',
'group.id': 'test_group'
})
# note that the output type of ds must be RowTypeInfo
ds.add_sink(kafka_producer)
env.execute()
def test_cassandra_sink(self):
type_info = Types.ROW([Types.STRING(), Types.INT()])
ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3), ('deeefg', 4)],
type_info=type_info)
cassandra_sink_builder = CassandraSink.add_sink(ds)
cassandra_sink = cassandra_sink_builder\
.set_host('localhost', 9876) \
.set_query('query') \
.enable_ignore_null_fields() \
.set_mapper_options(MapperOptions()
.ttl(1)
.timestamp(100)
.tracing(True)
.if_not_exists(False)
.consistency_level(ConsistencyLevel.ANY)
.save_null_fields(True)) \
.set_max_concurrent_requests(1000) \
.build()
cassandra_sink.name('cassandra_sink').set_parallelism(3)
plan = eval(self.env.get_execution_plan())
self.assertEqual("Sink: cassandra_sink", plan['nodes'][1]['type'])
self.assertEqual(3, plan['nodes'][1]['parallelism'])
def field_assertion(field_info, csv_value, value, field_delimiter):
row_info = Types.ROW([Types.STRING(), field_info, Types.STRING()])
expected_csv = "BEGIN" + field_delimiter + csv_value + field_delimiter + "END\n"
j_row.setField(1, value)
csv_row_serialization_schema = CsvRowSerializationSchema.Builder(row_info)\
.set_escape_character('*').set_quote_character('\'')\
.set_array_element_delimiter(':').set_field_delimiter(';').build()
csv_row_deserialization_schema = CsvRowDeserializationSchema.Builder(row_info)\
.set_escape_character('*').set_quote_character('\'')\
.set_array_element_delimiter(':').set_field_delimiter(';').build()
csv_row_serialization_schema._j_serialization_schema.open(
jvm.org.apache.flink.connector.testutils.formats.
DummyInitializationContext())
csv_row_deserialization_schema._j_deserialization_schema.open(
jvm.org.apache.flink.connector.testutils.formats.
DummyInitializationContext())
serialized_bytes = csv_row_serialization_schema._j_serialization_schema.serialize(
j_row)
self.assertEqual(expected_csv,
str(serialized_bytes, encoding='utf-8'))
j_deserialized_row = csv_row_deserialization_schema._j_deserialization_schema\
.deserialize(expected_csv.encode("utf-8"))
self.assertTrue(j_row.equals(j_deserialized_row))
def pandas_udaf():
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
t_env = StreamTableEnvironment.create(stream_execution_environment=env)
# define the source with watermark definition
ds = env.from_collection(
collection=[
(Instant.of_epoch_milli(1000), 'Alice', 110.1),
(Instant.of_epoch_milli(4000), 'Bob', 30.2),
(Instant.of_epoch_milli(3000), 'Alice', 20.0),
(Instant.of_epoch_milli(2000), 'Bob', 53.1),
(Instant.of_epoch_milli(5000), 'Alice', 13.1),
(Instant.of_epoch_milli(3000), 'Bob', 3.1),
(Instant.of_epoch_milli(7000), 'Bob', 16.1),
(Instant.of_epoch_milli(10000), 'Alice', 20.1)
],
type_info=Types.ROW([Types.INSTANT(), Types.STRING(), Types.FLOAT()]))
table = t_env.from_data_stream(
ds,
Schema.new_builder()
.column_by_expression("ts", "CAST(f0 AS TIMESTAMP_LTZ(3))")
.column("f1", DataTypes.STRING())
.column("f2", DataTypes.FLOAT())
.watermark("ts", "ts - INTERVAL '3' SECOND")
.build()
).alias("ts, name, price")
# define the sink
t_env.create_temporary_table(
'sink',
TableDescriptor.for_connector('print')
.schema(Schema.new_builder()
.column('name', DataTypes.STRING())
.column('total_price', DataTypes.FLOAT())
.column('w_start', DataTypes.TIMESTAMP_LTZ())
.column('w_end', DataTypes.TIMESTAMP_LTZ())
.build())
.build())
@udaf(result_type=DataTypes.FLOAT(), func_type="pandas")
def mean_udaf(v):
return v.mean()
# define the tumble window operation
table = table.window(Tumble.over(lit(5).seconds).on(col("ts")).alias("w")) \
.group_by(table.name, col('w')) \
.select(table.name, mean_udaf(table.price), col("w").start, col("w").end)
# submit for execution
table.execute_insert('sink') \
.wait()
def tumble_window_demo():
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
t_env = StreamTableEnvironment.create(stream_execution_environment=env)
# define the source with watermark definition
ds = env.from_collection(
collection=[
(Instant.of_epoch_milli(1000), 'Alice', 110.1),
(Instant.of_epoch_milli(4000), 'Bob', 30.2),
(Instant.of_epoch_milli(3000), 'Alice', 20.0),
(Instant.of_epoch_milli(2000), 'Bob', 53.1),
(Instant.of_epoch_milli(5000), 'Alice', 13.1),
(Instant.of_epoch_milli(3000), 'Bob', 3.1),
(Instant.of_epoch_milli(7000), 'Bob', 16.1),
(Instant.of_epoch_milli(10000), 'Alice', 20.1)
],
type_info=Types.ROW([Types.INSTANT(), Types.STRING(), Types.FLOAT()]))
table = t_env.from_data_stream(
ds,
Schema.new_builder()
.column_by_expression("ts", "CAST(f0 AS TIMESTAMP(3))")
.column("f1", DataTypes.STRING())
.column("f2", DataTypes.FLOAT())
.watermark("ts", "ts - INTERVAL '3' SECOND")
.build()
).alias("ts", "name", "price")
# define the sink
t_env.create_temporary_table(
'sink',
TableDescriptor.for_connector('print')
.schema(Schema.new_builder()
.column('name', DataTypes.STRING())
.column('total_price', DataTypes.FLOAT())
.build())
.build())
# define the over window operation
table = table.over_window(
Over.partition_by(col("name"))
.order_by(col("ts"))
.preceding(row_interval(2))
.following(CURRENT_ROW)
.alias('w')) \
.select(table.name, table.price.max.over(col('w')))
# submit for execution
table.execute_insert('sink') \
.wait()
def read_from_kafka(env):
deserialization_schema = JsonRowDeserializationSchema.Builder() \
.type_info(Types.ROW([Types.INT(), Types.STRING()])) \
.build()
kafka_consumer = FlinkKafkaConsumer(
topics='test_csv_topic',
deserialization_schema=deserialization_schema,
properties={'bootstrap.servers': 'localhost:9092', 'group.id': 'test_group_1'}
)
kafka_consumer.set_start_from_earliest()
env.add_source(kafka_consumer).print()
env.execute()
def write_to_kafka(env):
type_info = Types.ROW([Types.INT(), Types.STRING()])
ds = env.from_collection([
(1, 'hi'), (2, 'hello'), (3, 'hi'), (4, 'hello'), (5, 'hi'), (6, 'hello'), (6, 'hello')],
type_info=type_info)
serialization_schema = CsvRowSerializationSchema.Builder(type_info).build()
kafka_producer = FlinkKafkaProducer(
topic='test_csv_topic',
serialization_schema=serialization_schema,
producer_config={'bootstrap.servers': 'localhost:9092', 'group.id': 'test_group'}
)
# note that the output type of ds must be RowTypeInfo
ds.add_sink(kafka_producer)
env.execute()
def test_jdbc_sink(self):
ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3),
('deeefg', 4)],
type_info=Types.ROW(
[Types.STRING(),
Types.INT()]))
jdbc_connection_options = JdbcConnectionOptions.JdbcConnectionOptionsBuilder()\
.with_driver_name('com.mysql.jdbc.Driver')\
.with_user_name('root')\
.with_password('password')\
.with_url('jdbc:mysql://server-name:server-port/database-name').build()
jdbc_execution_options = JdbcExecutionOptions.builder().with_batch_interval_ms(2000)\
.with_batch_size(100).with_max_retries(5).build()
jdbc_sink = JdbcSink.sink("insert into test table", ds.get_type(),
jdbc_connection_options,
jdbc_execution_options)
ds.add_sink(jdbc_sink).name('jdbc sink')
plan = eval(self.env.get_execution_plan())
self.assertEqual('Sink: jdbc sink', plan['nodes'][1]['type'])
j_output_format = get_field_value(jdbc_sink.get_java_function(),
'outputFormat')
connection_options = JdbcConnectionOptions(
get_field_value(
get_field_value(j_output_format, 'connectionProvider'),
'jdbcOptions'))
self.assertEqual(jdbc_connection_options.get_db_url(),
connection_options.get_db_url())
self.assertEqual(jdbc_connection_options.get_driver_name(),
connection_options.get_driver_name())
self.assertEqual(jdbc_connection_options.get_password(),
connection_options.get_password())
self.assertEqual(jdbc_connection_options.get_user_name(),
connection_options.get_user_name())
exec_options = JdbcExecutionOptions(
get_field_value(j_output_format, 'executionOptions'))
self.assertEqual(jdbc_execution_options.get_batch_interval_ms(),
exec_options.get_batch_interval_ms())
self.assertEqual(jdbc_execution_options.get_batch_size(),
exec_options.get_batch_size())
self.assertEqual(jdbc_execution_options.get_max_retries(),
exec_options.get_max_retries())
def test_rabbitmq_connectors(self):
connection_config = RMQConnectionConfig.Builder() \
.set_host('localhost') \
.set_port(5672) \
.set_virtual_host('/') \
.set_user_name('guest') \
.set_password('guest') \
.build()
type_info = Types.ROW([Types.INT(), Types.STRING()])
deserialization_schema = JsonRowDeserializationSchema.builder() \
.type_info(type_info=type_info).build()
rmq_source = RMQSource(
connection_config, 'source_queue', True, deserialization_schema)
self.assertEqual(
get_field_value(rmq_source.get_java_function(), 'queueName'), 'source_queue')
self.assertTrue(get_field_value(rmq_source.get_java_function(), 'usesCorrelationId'))
serialization_schema = JsonRowSerializationSchema.builder().with_type_info(type_info) \
.build()
rmq_sink = RMQSink(connection_config, 'sink_queue', serialization_schema)
self.assertEqual(
get_field_value(rmq_sink.get_java_function(), 'queueName'), 'sink_queue')
def test_pulsar_sink(self):
ds = self.env.from_collection([('ab', 1), ('bdc', 2), ('cfgs', 3),
('deeefg', 4)],
type_info=Types.ROW(
[Types.STRING(),
Types.INT()]))
TEST_OPTION_NAME = 'pulsar.producer.chunkingEnabled'
pulsar_sink = PulsarSink.builder() \
.set_service_url('pulsar://localhost:6650') \
.set_admin_url('http://localhost:8080') \
.set_producer_name('fo') \
.set_topics('ada') \
.set_serialization_schema(
PulsarSerializationSchema.flink_schema(SimpleStringSchema())) \
.set_delivery_guarantee(DeliveryGuarantee.AT_LEAST_ONCE) \
.set_topic_routing_mode(TopicRoutingMode.ROUND_ROBIN) \
.delay_sending_message(MessageDelayer.fixed(Duration.of_seconds(12))) \
.set_config(TEST_OPTION_NAME, True) \
.set_properties({'pulsar.producer.batchingMaxMessages': '100'}) \
.build()
ds.sink_to(pulsar_sink).name('pulsar sink')
plan = eval(self.env.get_execution_plan())
self.assertEqual('pulsar sink: Writer', plan['nodes'][1]['type'])
configuration = get_field_value(pulsar_sink.get_java_function(),
"sinkConfiguration")
self.assertEqual(
configuration.getString(
ConfigOptions.key('pulsar.client.serviceUrl').string_type().
no_default_value()._j_config_option),
'pulsar://localhost:6650')
self.assertEqual(
configuration.getString(
ConfigOptions.key('pulsar.admin.adminUrl').string_type().
no_default_value()._j_config_option), 'http://localhost:8080')
self.assertEqual(
configuration.getString(
ConfigOptions.key('pulsar.producer.producerName').string_type(
).no_default_value()._j_config_option), 'fo - %s')
j_pulsar_serialization_schema = get_field_value(
pulsar_sink.get_java_function(), 'serializationSchema')
j_serialization_schema = get_field_value(j_pulsar_serialization_schema,
'serializationSchema')
self.assertTrue(
is_instance_of(
j_serialization_schema,
'org.apache.flink.api.common.serialization.SimpleStringSchema')
)
self.assertEqual(
configuration.getString(
ConfigOptions.key('pulsar.sink.deliveryGuarantee').string_type(
).no_default_value()._j_config_option), 'at-least-once')
j_topic_router = get_field_value(pulsar_sink.get_java_function(),
"topicRouter")
self.assertTrue(
is_instance_of(
j_topic_router,
'org.apache.flink.connector.pulsar.sink.writer.router.RoundRobinTopicRouter'
))
j_message_delayer = get_field_value(pulsar_sink.get_java_function(),
'messageDelayer')
delay_duration = get_field_value(j_message_delayer, 'delayDuration')
self.assertEqual(delay_duration, 12000)
test_option = ConfigOptions.key(
TEST_OPTION_NAME).boolean_type().no_default_value()
self.assertEqual(
configuration.getBoolean(test_option._j_config_option), True)
self.assertEqual(
configuration.getLong(
ConfigOptions.key('pulsar.producer.batchingMaxMessages').
long_type().no_default_value()._j_config_option), 100)