def python_data_stream_example():
env = StreamExecutionEnvironment.get_execution_environment()
# Set the parallelism to be one to make sure that all data including fired timer and normal data
# are processed by the same worker and the collected result would be in order which is good for
# assertion.
env.set_parallelism(1)
env.set_stream_time_characteristic(TimeCharacteristic.EventTime)
type_info = Types.ROW_NAMED(['createTime', 'orderId', 'payAmount', 'payPlatform', 'provinceId'],
[Types.LONG(), Types.LONG(), Types.DOUBLE(), Types.INT(),
Types.INT()])
json_row_schema = JsonRowDeserializationSchema.builder().type_info(type_info).build()
kafka_props = {'bootstrap.servers': 'localhost:9092', 'group.id': 'pyflink-e2e-source'}
kafka_consumer = FlinkKafkaConsumer("timer-stream-source", json_row_schema, kafka_props)
kafka_producer = FlinkKafkaProducer("timer-stream-sink", SimpleStringSchema(), kafka_props)
watermark_strategy = WatermarkStrategy.for_bounded_out_of_orderness(Duration.of_seconds(5))\
.with_timestamp_assigner(KafkaRowTimestampAssigner())
kafka_consumer.set_start_from_earliest()
ds = env.add_source(kafka_consumer).assign_timestamps_and_watermarks(watermark_strategy)
ds.key_by(MyKeySelector(), key_type_info=Types.LONG()) \
.process(MyProcessFunction(), output_type=Types.STRING()) \
.add_sink(kafka_producer)
env.execute_async("test data stream timer")
def _create_orc_basic_row_and_data() -> Tuple[RowType, RowTypeInfo, List[Row]]:
row_type = DataTypes.ROW([
DataTypes.FIELD('char', DataTypes.CHAR(10)),
DataTypes.FIELD('varchar', DataTypes.VARCHAR(10)),
DataTypes.FIELD('bytes', DataTypes.BYTES()),
DataTypes.FIELD('boolean', DataTypes.BOOLEAN()),
DataTypes.FIELD('decimal', DataTypes.DECIMAL(2, 0)),
DataTypes.FIELD('int', DataTypes.INT()),
DataTypes.FIELD('bigint', DataTypes.BIGINT()),
DataTypes.FIELD('double', DataTypes.DOUBLE()),
DataTypes.FIELD('date', DataTypes.DATE().bridged_to('java.sql.Date')),
DataTypes.FIELD('timestamp', DataTypes.TIMESTAMP(3).bridged_to('java.sql.Timestamp')),
])
row_type_info = Types.ROW_NAMED(
['char', 'varchar', 'bytes', 'boolean', 'decimal', 'int', 'bigint', 'double',
'date', 'timestamp'],
[Types.STRING(), Types.STRING(), Types.PRIMITIVE_ARRAY(Types.BYTE()), Types.BOOLEAN(),
Types.BIG_DEC(), Types.INT(), Types.LONG(), Types.DOUBLE(), Types.SQL_DATE(),
Types.SQL_TIMESTAMP()]
)
data = [Row(
char='char',
varchar='varchar',
bytes=b'varbinary',
boolean=True,
decimal=Decimal(1.5),
int=2147483647,
bigint=-9223372036854775808,
double=2e-308,
date=date(1970, 1, 1),
timestamp=datetime(1970, 1, 2, 3, 4, 5, 600000),
)]
return row_type, row_type_info, data
def test_from_collection_with_data_types(self):
# verify from_collection for the collection with single object.
ds = self.env.from_collection(['Hi', 'Hello'], type_info=Types.STRING())
ds.add_sink(self.test_sink)
self.env.execute("test from collection with single object")
results = self.test_sink.get_results(False)
expected = ['Hello', 'Hi']
results.sort()
expected.sort()
self.assertEqual(expected, results)
# verify from_collection for the collection with multiple objects like tuple.
ds = self.env.from_collection([(1, None, 1, True, 32767, -2147483648, 1.23, 1.98932,
bytearray(b'flink'), 'pyflink', datetime.date(2014, 9, 13),
datetime.time(hour=12, minute=0, second=0,
microsecond=123000),
datetime.datetime(2018, 3, 11, 3, 0, 0, 123000), [1, 2, 3],
decimal.Decimal('1000000000000000000.05'),
decimal.Decimal('1000000000000000000.0599999999999'
'9999899999999999')),
(2, None, 2, True, 43878, 9147483648, 9.87, 2.98936,
bytearray(b'flink'), 'pyflink', datetime.date(2015, 10, 14),
datetime.time(hour=11, minute=2, second=2,
microsecond=234500),
datetime.datetime(2020, 4, 15, 8, 2, 6, 235000), [2, 4, 6],
decimal.Decimal('2000000000000000000.74'),
decimal.Decimal('2000000000000000000.061111111111111'
'11111111111111'))],
type_info=Types.ROW(
[Types.LONG(), Types.LONG(), Types.SHORT(),
Types.BOOLEAN(), Types.SHORT(), Types.INT(),
Types.FLOAT(), Types.DOUBLE(),
Types.PICKLED_BYTE_ARRAY(),
Types.STRING(), Types.SQL_DATE(), Types.SQL_TIME(),
Types.SQL_TIMESTAMP(),
Types.BASIC_ARRAY(Types.LONG()), Types.BIG_DEC(),
Types.BIG_DEC()]))
ds.add_sink(self.test_sink)
self.env.execute("test from collection with tuple object")
results = self.test_sink.get_results(False)
# if user specifies data types of input data, the collected result should be in row format.
expected = [
'+I[1, null, 1, true, 32767, -2147483648, 1.23, 1.98932, [102, 108, 105, 110, 107], '
'pyflink, 2014-09-13, 12:00:00, 2018-03-11 03:00:00.123, [1, 2, 3], '
'1000000000000000000.05, 1000000000000000000.05999999999999999899999999999]',
'+I[2, null, 2, true, -21658, 557549056, 9.87, 2.98936, [102, 108, 105, 110, 107], '
'pyflink, 2015-10-14, 11:02:02, 2020-04-15 08:02:06.235, [2, 4, 6], '
'2000000000000000000.74, 2000000000000000000.06111111111111111111111111111]']
results.sort()
expected.sort()
self.assertEqual(expected, results)
def _create_parquet_basic_row_and_data() -> Tuple[RowType, RowTypeInfo, List[Row]]:
row_type = DataTypes.ROW([
DataTypes.FIELD('char', DataTypes.CHAR(10)),
DataTypes.FIELD('varchar', DataTypes.VARCHAR(10)),
DataTypes.FIELD('binary', DataTypes.BINARY(10)),
DataTypes.FIELD('varbinary', DataTypes.VARBINARY(10)),
DataTypes.FIELD('boolean', DataTypes.BOOLEAN()),
DataTypes.FIELD('decimal', DataTypes.DECIMAL(2, 0)),
DataTypes.FIELD('int', DataTypes.INT()),
DataTypes.FIELD('bigint', DataTypes.BIGINT()),
DataTypes.FIELD('double', DataTypes.DOUBLE()),
DataTypes.FIELD('date', DataTypes.DATE().bridged_to('java.sql.Date')),
DataTypes.FIELD('time', DataTypes.TIME().bridged_to('java.sql.Time')),
DataTypes.FIELD('timestamp', DataTypes.TIMESTAMP(3).bridged_to('java.sql.Timestamp')),
DataTypes.FIELD('timestamp_ltz', DataTypes.TIMESTAMP_LTZ(3)),
])
row_type_info = Types.ROW_NAMED(
['char', 'varchar', 'binary', 'varbinary', 'boolean', 'decimal', 'int', 'bigint', 'double',
'date', 'time', 'timestamp', 'timestamp_ltz'],
[Types.STRING(), Types.STRING(), Types.PRIMITIVE_ARRAY(Types.BYTE()),
Types.PRIMITIVE_ARRAY(Types.BYTE()), Types.BOOLEAN(), Types.BIG_DEC(), Types.INT(),
Types.LONG(), Types.DOUBLE(), Types.SQL_DATE(), Types.SQL_TIME(), Types.SQL_TIMESTAMP(),
Types.INSTANT()]
)
datetime_ltz = datetime.datetime(1970, 2, 3, 4, 5, 6, 700000, tzinfo=pytz.timezone('UTC'))
timestamp_ltz = Instant.of_epoch_milli(
(
calendar.timegm(datetime_ltz.utctimetuple()) +
calendar.timegm(time.localtime(0))
) * 1000 + datetime_ltz.microsecond // 1000
)
data = [Row(
char='char',
varchar='varchar',
binary=b'binary',
varbinary=b'varbinary',
boolean=True,
decimal=Decimal(1.5),
int=2147483647,
bigint=-9223372036854775808,
double=2e-308,
date=datetime.date(1970, 1, 1),
time=datetime.time(1, 1, 1),
timestamp=datetime.datetime(1970, 1, 2, 3, 4, 5, 600000),
timestamp_ltz=timestamp_ltz
)]
return row_type, row_type_info, data
def python_data_stream_example():
env = StreamExecutionEnvironment.get_execution_environment()
# Set the parallelism to be one to make sure that all data including fired timer and normal data
# are processed by the same worker and the collected result would be in order which is good for
# assertion.
env.set_parallelism(1)
env.set_stream_time_characteristic(TimeCharacteristic.EventTime)
t_env = StreamTableEnvironment.create(stream_execution_environment=env)
create_kafka_source_ddl = """
CREATE TABLE payment_msg(
createTime VARCHAR,
rt as TO_TIMESTAMP(createTime),
orderId BIGINT,
payAmount DOUBLE,
payPlatform INT,
provinceId INT,
WATERMARK FOR rt as rt - INTERVAL '2' SECOND
) WITH (
'connector.type' = 'kafka',
'connector.version' = 'universal',
'connector.topic' = 'timer-stream-source',
'connector.properties.bootstrap.servers' = 'localhost:9092',
'connector.properties.group.id' = 'test_3',
'connector.startup-mode' = 'earliest-offset',
'format.type' = 'json'
)
"""
t_env.execute_sql(create_kafka_source_ddl)
t = t_env.from_path("payment_msg").select("createTime, orderId, payAmount, payPlatform,"
" provinceId")
source_type_info = Types.ROW([
Types.STRING(),
Types.LONG(),
Types.DOUBLE(),
Types.INT(),
Types.INT()])
ds = t_env.to_append_stream(table=t, type_info=source_type_info)
producer_props = {'bootstrap.servers': 'localhost:9092', 'group.id': 'pyflink-e2e-source'}
kafka_producer = FlinkKafkaProducer("timer-stream-sink", SimpleStringSchema(),
producer_props)
ds.key_by(MyKeySelector(), key_type_info=Types.LONG()) \
.process(MyProcessFunction(), output_type=Types.STRING()) \
.add_sink(kafka_producer)
env.execute_async("test data stream timer")
def test_from_java_type(self):
basic_int_type_info = Types.INT()
self.assertEqual(basic_int_type_info,
_from_java_type(basic_int_type_info.get_java_type_info()))
basic_short_type_info = Types.SHORT()
self.assertEqual(basic_short_type_info,
_from_java_type(basic_short_type_info.get_java_type_info()))
basic_long_type_info = Types.LONG()
self.assertEqual(basic_long_type_info,
_from_java_type(basic_long_type_info.get_java_type_info()))
basic_float_type_info = Types.FLOAT()
self.assertEqual(basic_float_type_info,
_from_java_type(basic_float_type_info.get_java_type_info()))
basic_double_type_info = Types.DOUBLE()
self.assertEqual(basic_double_type_info,
_from_java_type(basic_double_type_info.get_java_type_info()))
basic_char_type_info = Types.CHAR()
self.assertEqual(basic_char_type_info,
_from_java_type(basic_char_type_info.get_java_type_info()))
basic_byte_type_info = Types.BYTE()
self.assertEqual(basic_byte_type_info,
_from_java_type(basic_byte_type_info.get_java_type_info()))
basic_big_int_type_info = Types.BIG_INT()
self.assertEqual(basic_big_int_type_info,
_from_java_type(basic_big_int_type_info.get_java_type_info()))
basic_big_dec_type_info = Types.BIG_DEC()
self.assertEqual(basic_big_dec_type_info,
_from_java_type(basic_big_dec_type_info.get_java_type_info()))
basic_sql_date_type_info = Types.SQL_DATE()
self.assertEqual(basic_sql_date_type_info,
_from_java_type(basic_sql_date_type_info.get_java_type_info()))
basic_sql_time_type_info = Types.SQL_TIME()
self.assertEqual(basic_sql_time_type_info,
_from_java_type(basic_sql_time_type_info.get_java_type_info()))
basic_sql_timestamp_type_info = Types.SQL_TIMESTAMP()
self.assertEqual(basic_sql_timestamp_type_info,
_from_java_type(basic_sql_timestamp_type_info.get_java_type_info()))
row_type_info = Types.ROW([Types.INT(), Types.STRING()])
self.assertEqual(row_type_info, _from_java_type(row_type_info.get_java_type_info()))
tuple_type_info = Types.TUPLE([Types.CHAR(), Types.INT()])
self.assertEqual(tuple_type_info, _from_java_type(tuple_type_info.get_java_type_info()))
primitive_int_array_type_info = Types.PRIMITIVE_ARRAY(Types.INT())
self.assertEqual(primitive_int_array_type_info,
_from_java_type(primitive_int_array_type_info.get_java_type_info()))
object_array_type_info = Types.OBJECT_ARRAY(Types.SQL_DATE())
self.assertEqual(object_array_type_info,
_from_java_type(object_array_type_info.get_java_type_info()))
pickled_byte_array_type_info = Types.PICKLED_BYTE_ARRAY()
self.assertEqual(pickled_byte_array_type_info,
_from_java_type(pickled_byte_array_type_info.get_java_type_info()))
sql_date_type_info = Types.SQL_DATE()
self.assertEqual(sql_date_type_info,
_from_java_type(sql_date_type_info.get_java_type_info()))
map_type_info = Types.MAP(Types.INT(), Types.STRING())
self.assertEqual(map_type_info,
_from_java_type(map_type_info.get_java_type_info()))
list_type_info = Types.LIST(Types.INT())
self.assertEqual(list_type_info,
_from_java_type(list_type_info.get_java_type_info()))
def to_java_typeinfo(type_info: TypeInformation):
if isinstance(type_info, BasicTypeInfo):
basic_type = type_info._basic_type
if basic_type == BasicType.STRING:
j_typeinfo = JTypes.STRING
elif basic_type == BasicType.BYTE:
j_typeinfo = JTypes.LONG
elif basic_type == BasicType.BOOLEAN:
j_typeinfo = JTypes.BOOLEAN
elif basic_type == BasicType.SHORT:
j_typeinfo = JTypes.LONG
elif basic_type == BasicType.INT:
j_typeinfo = JTypes.LONG
elif basic_type == BasicType.LONG:
j_typeinfo = JTypes.LONG
elif basic_type == BasicType.FLOAT:
j_typeinfo = JTypes.DOUBLE
elif basic_type == BasicType.DOUBLE:
j_typeinfo = JTypes.DOUBLE
elif basic_type == BasicType.CHAR:
j_typeinfo = JTypes.STRING
elif basic_type == BasicType.BIG_INT:
j_typeinfo = JTypes.BIG_INT
elif basic_type == BasicType.BIG_DEC:
j_typeinfo = JTypes.BIG_DEC
elif basic_type == BasicType.INSTANT:
j_typeinfo = JTypes.INSTANT
else:
raise TypeError("Invalid BasicType %s." % basic_type)
elif isinstance(type_info, PrimitiveArrayTypeInfo):
element_type = type_info._element_type
if element_type == Types.BOOLEAN():
j_typeinfo = JPrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.BYTE():
j_typeinfo = JPrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.SHORT():
j_typeinfo = JPrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.INT():
j_typeinfo = JPrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.LONG():
j_typeinfo = JPrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.FLOAT():
j_typeinfo = JPrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.DOUBLE():
j_typeinfo = JPrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO
elif element_type == Types.CHAR():
j_typeinfo = JPrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO
else:
raise TypeError("Invalid element type for a primitive array.")
elif isinstance(type_info, BasicArrayTypeInfo):
element_type = type_info._element_type
if element_type == Types.BOOLEAN():
j_typeinfo = JBasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO
elif element_type == Types.BYTE():
j_typeinfo = JBasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO
elif element_type == Types.SHORT():
j_typeinfo = JBasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO
elif element_type == Types.INT():
j_typeinfo = JBasicArrayTypeInfo.INT_ARRAY_TYPE_INFO
elif element_type == Types.LONG():
j_typeinfo = JBasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO
elif element_type == Types.FLOAT():
j_typeinfo = JBasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO
elif element_type == Types.DOUBLE():
j_typeinfo = JBasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO
elif element_type == Types.CHAR():
j_typeinfo = JBasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO
elif element_type == Types.STRING():
j_typeinfo = JBasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO
else:
raise TypeError("Invalid element type for a basic array.")
elif isinstance(type_info, ObjectArrayTypeInfo):
element_type = type_info._element_type
j_typeinfo = JTypes.OBJECT_ARRAY(to_java_typeinfo(element_type))
elif isinstance(type_info, MapTypeInfo):
j_key_typeinfo = to_java_typeinfo(type_info._key_type_info)
j_value_typeinfo = to_java_typeinfo(type_info._value_type_info)
j_typeinfo = JMapTypeInfo(j_key_typeinfo, j_value_typeinfo)
else:
j_typeinfo = JPickledByteArrayTypeInfo.PICKLED_BYTE_ARRAY_TYPE_INFO
return j_typeinfo
