def test_ksql_create_stream_w_properties(self):
""" Test GET requests """
topic = self.exist_topic
stream_name = self.test_prefix + "test_ksql_create_stream"
stream_name = "test_ksql_create_stream"
ksql_string = "CREATE STREAM {} (ORDER_ID INT, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
WITH (kafka_topic='{}', value_format='JSON');".format(
stream_name, topic)
streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
if 'TEST_KSQL_CREATE_STREAM' not in utils.get_all_streams(
self.api_client):
r = self.api_client.ksql(ksql_string,
stream_properties=streamProperties)
self.assertEqual(r[0]['commandStatus']['status'], 'SUCCESS')
producer = Producer({'bootstrap.servers': self.bootstrap_servers})
producer.produce(
self.exist_topic,
'''{"order_id":3,"total_amount":43,"customer_name":"Palo Alto"}''')
producer.flush()
print()
chunks = self.api_client.query("select * from {}".format(stream_name),
stream_properties=streamProperties,
idle_timeout=10)
for chunk in chunks:
pass
assert json.loads(chunk)['row']['columns'][-1] == 'Palo Alto'
def test_ksql_parse_query_final_message(self):
topic = "TEST_KSQL_PARSE_QUERY_FINAL_MESSAGE_TOPIC"
stream_name = "TEST_KSQL_PARSE_QUERY_FINAL_MESSAGE_STREAM"
producer = Producer({"bootstrap.servers": self.bootstrap_servers})
producer.produce(topic, """{"order_id":3,"my_struct":{"a":1,"b":"bbb"}, "my_map":{"x":3, "y":4}, "my_array":[1,2,3], "total_amount":43,"customer_name":"Palo Alto"}""")
producer.flush()
ksql_string = "CREATE STREAM {} (ORDER_ID INT, MY_STRUCT STRUCT<A INT, B VARCHAR>, MY_MAP MAP<VARCHAR, INT>, MY_ARRAY ARRAY<INT>, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
WITH (kafka_topic='{}', value_format='JSON');".format(
stream_name, topic
)
streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
if stream_name not in utils.get_all_streams(self.api_client):
r = self.api_client.ksql(ksql_string, stream_properties=streamProperties)
self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")
chunks = self.api_client.query(
"select * from {} EMIT CHANGES LIMIT 1".format(stream_name), stream_properties=streamProperties, return_objects=True
)
for row_obj in chunks:
self.assertEqual(row_obj["ORDER_ID"], 3)
self.assertEqual(row_obj["MY_STRUCT"], {"A": 1, "B": "bbb"})
self.assertEqual(row_obj["MY_MAP"], {"x": 3, "y": 4})
self.assertEqual(row_obj["MY_ARRAY"], [1, 2, 3])
self.assertEqual(row_obj["TOTAL_AMOUNT"], 43)
self.assertEqual(row_obj["CUSTOMER_NAME"], "Palo Alto")
def test_ksql_create_stream_w_properties(self):
""" Test GET requests """
topic = self.exist_topic
stream_name = "TEST_KSQL_CREATE_STREAM"
ksql_string = "CREATE STREAM {} (ORDER_ID INT, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
WITH (kafka_topic='{}', value_format='JSON');".format(
stream_name, topic)
streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
if "TEST_KSQL_CREATE_STREAM" not in utils.get_all_streams(
self.api_client):
r = self.api_client.ksql(ksql_string,
stream_properties=streamProperties)
self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")
producer = Producer({"bootstrap.servers": self.bootstrap_servers})
producer.produce(
self.exist_topic,
"""{"order_id":3,"total_amount":43,"customer_name":"Palo Alto"}""")
producer.flush()
chunks = self.api_client.query(
"select * from {} EMIT CHANGES".format(stream_name),
stream_properties=streamProperties)
for chunk in chunks:
self.assertTrue(chunk)
break
def test_get_all_streams(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_get_all_streams"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
filtered_streams = utils.get_all_streams(self.api_client,
prefix=self.test_prefix)
self.assertEqual(filtered_streams, [stream_name.upper()])
def test_ksql_create_stream_w_properties(self):
""" Test GET requests """
topic = self.exist_topic
stream_name = "TEST_KSQL_CREATE_STREAM"
ksql_string = "CREATE STREAM {} (ORDER_ID INT, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
WITH (kafka_topic='{}', value_format='JSON');".format(
stream_name, topic
)
streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
if "TEST_KSQL_CREATE_STREAM" not in utils.get_all_streams(self.api_client):
r = self.api_client.ksql(ksql_string, stream_properties=streamProperties)
self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")
producer = Producer({"bootstrap.servers": self.bootstrap_servers})
producer.produce(self.exist_topic, """{"order_id":3,"total_amount":43,"customer_name":"Palo Alto"}""")
producer.flush()
# test legacy HTTP/1.1 request
chunks = self.api_client.query(
"select * from {} EMIT CHANGES".format(stream_name), stream_properties=streamProperties
)
header = next(chunks)
self.assertEqual(header, """[{"header":{"queryId":"none","schema":"`ORDER_ID` INTEGER, `TOTAL_AMOUNT` DOUBLE, `CUSTOMER_NAME` STRING"}},\n""")
for chunk in chunks:
self.assertEqual(chunk, """{"row":{"columns":[3,43.0,"Palo Alto"]}},\n""")
break
# test new HTTP/2 request
chunks = self.api_client.query(
"select * from {} EMIT CHANGES".format(stream_name), stream_properties=streamProperties, use_http2=True
)
header = next(chunks)
header_obj = json.loads(header)
self.assertEqual(header_obj["columnNames"], ['ORDER_ID', 'TOTAL_AMOUNT', 'CUSTOMER_NAME'])
self.assertEqual(header_obj["columnTypes"], ['INTEGER', 'DOUBLE', 'STRING'])
for chunk in chunks:
chunk_obj = json.loads(chunk)
self.assertEqual(chunk_obj, [3,43.0, "Palo Alto"])
break
def test_inserts_stream(self):
topic = self.exist_topic
stream_name = "TEST_INSERTS_STREAM_STREAM"
ksql_string = "CREATE STREAM {} (ORDER_ID INT, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
WITH (kafka_topic='{}', value_format='JSON');".format(
stream_name, topic
)
streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
if "TEST_KSQL_CREATE_STREAM" not in utils.get_all_streams(self.api_client):
r = self.api_client.ksql(ksql_string, stream_properties=streamProperties)
self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")
rows = [
{"ORDER_ID": 1, "TOTAL_AMOUNT": 23.5, "CUSTOMER_NAME": "abc"},
{"ORDER_ID": 2, "TOTAL_AMOUNT": 3.7, "CUSTOMER_NAME": "xyz"}
]
results = self.api_client.inserts_stream(stream_name, rows)
for result in results:
self.assertEqual(result["status"], "ok")