def register_rides_source(st_env):
st_env \
.connect( # declare the external system to connect to
Kafka()
.version("universal")
.topic("Rides")
.start_from_earliest()
.property("zookeeper.connect", "zookeeper:2181")
.property("bootstrap.servers", "kafka:9092")) \
.with_format( # declare a format for this system
Json()
.fail_on_missing_field(True)
.schema(DataTypes.ROW([
DataTypes.FIELD("rideId", DataTypes.BIGINT()),
DataTypes.FIELD("isStart", DataTypes.BOOLEAN()),
DataTypes.FIELD("eventTime", DataTypes.STRING()),
DataTypes.FIELD("lon", DataTypes.FLOAT()),
DataTypes.FIELD("lat", DataTypes.FLOAT()),
DataTypes.FIELD("psgCnt", DataTypes.INT()),
DataTypes.FIELD("taxiId", DataTypes.BIGINT())]))) \
.with_schema( # declare the schema of the table
Schema()
.field("rideId", DataTypes.BIGINT())
.field("taxiId", DataTypes.BIGINT())
.field("isStart", DataTypes.BOOLEAN())
.field("lon", DataTypes.FLOAT())
.field("lat", DataTypes.FLOAT())
.field("psgCnt", DataTypes.INT())
.field("eventTime", DataTypes.STRING())) \
.in_append_mode() \
.create_temporary_table("source")
def group_by_agg_streaming():
s_env = StreamExecutionEnvironment.get_execution_environment()
s_env.set_parallelism(1)
# use blink table planner
st_env = StreamTableEnvironment.create(
s_env,
environment_settings=EnvironmentSettings.new_instance(
).in_streaming_mode().use_blink_planner().build())
# use flink table planner
# st_env = StreamTableEnvironment.create(s_env)
source_file = os.getcwd() + "/../resources/table_orders.csv"
st_env.register_table_source(
"Orders",
CsvTableSource(source_file, ["a", "b", "c", "rowtime"], [
DataTypes.STRING(),
DataTypes.INT(),
DataTypes.INT(),
DataTypes.TIMESTAMP()
]))
st_env.connect(
Elasticsearch()
.version("6")
.host("localhost", 9200, "http")
.index("group_by_agg_streaming")
.document_type('pyflink')
.key_delimiter("_")
.key_null_literal("null")
.failure_handler_ignore()
.disable_flush_on_checkpoint()
.bulk_flush_max_actions(2)
.bulk_flush_max_size("1 mb")
.bulk_flush_interval(5000)
) \
.with_schema(
Schema()
.field("a", DataTypes.STRING())
.field("b", DataTypes.STRING())
) \
.with_format(
Json()
.derive_schema()
) \
.in_upsert_mode() \
.register_table_sink("result")
orders = st_env.scan("Orders")
groub_by_table = orders.group_by("a").select("a, b.sum as d")
# Because the schema of index user in elasticsearch is
# {"a":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
# "b":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}
# so we need to cast the type in our demo.
st_env.register_table("group_table", groub_by_table)
result = st_env.sql_query("SELECT a, CAST(d AS VARCHAR) from group_table")
result.insert_into("result")
st_env.execute("group by agg streaming")
Kafka()
.version("0.11")
.topic("input")
.start_from_earliest()
.property("zookeeper.connect", "zookeeper:2181")
.property("bootstrap.servers", "kafka:9092")
) \
.with_format( # declare a format for this system
Json()
.fail_on_missing_field(True)
.json_schema(
"{"
" type: 'object',"
" properties: {"
" timestamp: {"
" type: 'string'"
" },"
" page: {"
" type: 'string'"
" }"
" }"
"}"
)
) \
.with_schema( # declare the schema of the table
Schema()
.field("timestamp", DataTypes.TIMESTAMP()).proctime()
.field("page", DataTypes.STRING())
) \
.in_append_mode() \
.register_table_source("ClickEvent Source")
.start_from_specific_offset(0,496)
.property("zookeeper.connect", "6.86.2.170:2181")
.property("bootstrap.servers", "6.86.2.170:9092")
) \
.with_format( # declare a format for this system
Json()
.fail_on_missing_field(True)
.json_schema(
"{"
" type: 'object',"
" properties: {"
" a: {"
" type: 'string'"
" },"
" b: {"
" type: 'number'"
" },"
" c: {"
" type: 'string'"
" },"
" time: {"
" type: 'string',"
" format: 'date-time'"
" }"
" }"
"}"
)
) \
.with_schema( # declare the schema of the table
Schema()
.field("rowtime", DataTypes.TIMESTAMP())
.rowtime(
.use_blink_planner().build())
st_env \
.connect( # declare the external system to connect to
Kafka()
.version("0.11")
.topic("Rides")
.start_from_earliest()
.property("zookeeper.connect", "zookeeper:2181")
.property("bootstrap.servers", "kafka:9092")) \
.with_format( # declare a format for this system
Json()
.fail_on_missing_field(True)
.schema(DataTypes.ROW([
DataTypes.FIELD("rideId", DataTypes.BIGINT()),
DataTypes.FIELD("isStart", DataTypes.BOOLEAN()),
DataTypes.FIELD("eventTime", DataTypes.TIMESTAMP()),
DataTypes.FIELD("lon", DataTypes.FLOAT()),
DataTypes.FIELD("lat", DataTypes.FLOAT()),
DataTypes.FIELD("psgCnt", DataTypes.INT()),
DataTypes.FIELD("taxiId", DataTypes.BIGINT())]))) \
.with_schema( # declare the schema of the table
Schema()
.field("rideId", DataTypes.BIGINT())
.field("taxiId", DataTypes.BIGINT())
.field("isStart", DataTypes.BOOLEAN())
.field("lon", DataTypes.FLOAT())
.field("lat", DataTypes.FLOAT())
.field("psgCnt", DataTypes.INT())
.field("rideTime", DataTypes.TIMESTAMP())
.rowtime(
Rowtime()
from pyflink.table.descriptors import Kafka, Json, OldCsv, Schema, FileSystem
directories = ['/flink/lib']
for directory in directories:
for jar in glob.glob(os.path.join(directory, '*.jar')):
sys.path.append(jar)
# from org.apache.flink.streaming.connectors.kafka import FlinkKafkaConsumer11
# from org.apache.flink.streaming.connectors.kafka import FlinkKafkaConsumer09
OldCsv()
print("debug 010")
Kafka()
print("debug 020")
Json()
print("debug 030")
sourcetable = table_env \
.connect(Kafka()
.properties({'update-mode': 'append', 'connector.topic': 'machine.data',
'connector.properties.zookeeper.connect': 'localhost:2181',
'connector.properties.bootstrap.servers.': 'localhost:9092'})) \
.with_format(Json().
json_schema(
"{type:'object',properties:{thing: {type: 'string'},quantity:{type:'string'},phenomenonTime:{type:'integer'},result:{type:'number'}}}") \
.fail_on_missing_field(False)) \
.with_schema(Schema()
.field("thing", DataTypes.STRING())
.field("quantity", DataTypes.STRING())
