def test_create_stream_as_with_conditions_with_startwith(self):
src_table = 'pageviews_original'
columns_type = [
'name string', 'age bigint', 'userid string', 'pageid bigint'
]
topic = self.exist_topic
table_name = 'create_stream_as_with_conditions_with_startwith'
kafka_topic = 'create_stream_as_with_conditions_with_startwith'
value_format = 'DELIMITED'
select_columns = ['rowtime as logtime', '*']
conditions = "userid = 'foo_%'"
utils.drop_stream(self.api_client, src_table)
utils.drop_stream(self.api_client, table_name)
try:
r = self.api_client.create_stream(table_name=src_table,
columns_type=columns_type,
topic=topic,
value_format=value_format)
except KSQLError as e:
pass
r = self.api_client.create_stream_as(table_name=table_name,
src_table=src_table,
kafka_topic=kafka_topic,
select_columns=select_columns,
timestamp='logtime',
value_format=value_format,
conditions=conditions)
self.assertTrue(r)
def test_ksql_create_stream_as_with_wrong_timestamp(self):
src_table = 'prebid_traffic_log_total_stream'
columns_type = [
'name string', 'age bigint', 'userid string', 'pageid bigint'
]
topic = self.exist_topic
table_name = 'prebid_traffic_log_valid_stream'
kafka_topic = 'prebid_traffic_log_valid_topic'
value_format = 'DELIMITED'
select_columns = ['*']
timestamp = 'foo'
utils.drop_stream(self.api_client, src_table)
utils.drop_stream(self.api_client, table_name)
try:
r = self.api_client.create_stream(table_name=src_table,
columns_type=columns_type,
topic=topic,
value_format=value_format)
except KSQLError as e:
raise
with self.assertRaises(KSQLError):
r = self.api_client.create_stream_as(table_name=table_name,
src_table=src_table,
kafka_topic=kafka_topic,
select_columns=select_columns,
timestamp=timestamp,
value_format=value_format)
def test_ksql_create_stream_as_with_wrong_timestamp(self):
src_table = "prebid_traffic_log_total_stream"
columns_type = ["name string", "age bigint", "userid string", "pageid bigint"]
topic = self.exist_topic
table_name = "prebid_traffic_log_valid_stream"
kafka_topic = "prebid_traffic_log_valid_topic"
value_format = "DELIMITED"
select_columns = ["*"]
timestamp = "foo"
utils.drop_stream(self.api_client, src_table)
utils.drop_stream(self.api_client, table_name)
try:
self.api_client.create_stream(
table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format
)
except KSQLError:
pass
with self.assertRaises(KSQLError):
self.api_client.create_stream_as(
table_name=table_name,
src_table=src_table,
kafka_topic=kafka_topic,
select_columns=select_columns,
timestamp=timestamp,
value_format=value_format,
)
def test_create_stream_as_with_conditions_with_startwith(self):
src_table = "pageviews_original"
columns_type = ["name string", "age bigint", "userid string", "pageid bigint"]
topic = self.exist_topic
table_name = "create_stream_as_with_conditions_with_startwith"
kafka_topic = "create_stream_as_with_conditions_with_startwith"
value_format = "DELIMITED"
select_columns = ["rowtime as logtime", "*"]
conditions = "userid = 'foo_%'"
utils.drop_stream(self.api_client, src_table)
utils.drop_stream(self.api_client, table_name)
try:
r = self.api_client.create_stream(
table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format
)
except KSQLError:
pass
r = self.api_client.create_stream_as(
table_name=table_name,
src_table=src_table,
kafka_topic=kafka_topic,
select_columns=select_columns,
timestamp="logtime",
value_format=value_format,
conditions=conditions,
)
self.assertTrue(r)
def test_drop_all_streams(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_drop_all_streams"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
utils.drop_all_streams(self.api_client, prefix=self.test_prefix)
self.assertFalse(utils.get_stream_info(self.api_client, stream_name))
def test_get_stream_info(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_get_stream_info"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
stream_info = utils.get_stream_info(self.api_client, stream_name)
# print(stream_info['topic'])
self.assertEqual(stream_info["topic"], self.exist_topic)
def test_get_all_streams(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_get_all_streams"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
filtered_streams = utils.get_all_streams(self.api_client,
prefix=self.test_prefix)
self.assertEqual(filtered_streams, [stream_name.upper()])
def test_ksql_create_stream_by_builder_api(self):
table_name = "test_table"
columns_type = ["viewtime bigint", "userid varchar", "pageid varchar"]
topic = self.exist_topic
value_format = "DELIMITED"
utils.drop_stream(self.api_client, table_name)
r = self.api_client.create_stream(
table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format
)
self.assertTrue(r)
def test_ksql_create_stream_by_builder_api(self):
table_name = 'test_table'
columns_type = ['viewtime bigint', 'userid varchar', 'pageid varchar']
topic = self.exist_topic
value_format = 'DELIMITED'
utils.drop_stream(self.api_client, table_name)
r = self.api_client.create_stream(table_name=table_name,
columns_type=columns_type,
topic=topic,
value_format=value_format)
self.assertTrue(r)
def test_raise_create_error_topic_already_registered(self):
table_name = "foo_table"
columns_type = ["name string", "age bigint"]
topic = self.exist_topic
value_format = "DELIMITED"
utils.drop_stream(self.api_client, table_name)
self.api_client.create_stream(
table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format
)
with self.assertRaises(KSQLError):
self.api_client.create_stream(
table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format
)
def test_raise_create_error_topic_already_registered(self):
table_name = 'foo_table'
columns_type = ['name string', 'age bigint']
topic = self.exist_topic
value_format = 'DELIMITED'
utils.drop_stream(self.api_client, table_name)
r = self.api_client.create_stream(table_name=table_name,
columns_type=columns_type,
topic=topic,
value_format=value_format)
with self.assertRaises(KSQLError):
r = self.api_client.create_stream(table_name=table_name,
columns_type=columns_type,
topic=topic,
value_format=value_format)
def test_get_dependent_queries(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_get_dependent_queries"
stream_name_as = stream_name + "_as"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
ksql_string = "CREATE STREAM {} as select * from {};".format(
stream_name_as, stream_name)
self.api_client.ksql(ksql_string)
read_queries, write_queries = utils.get_dependent_queries(
self.api_client, stream_name_as)
self.assertEqual(read_queries, [])
self.assertTrue(write_queries[0].startswith(
"CSAS_KSQL_PYTHON_TEST_TEST_GET_DEPENDENT_QUERIES_AS"))
def test_drop_stream_create_as_stream(self):
topic = self.exist_topic
stream_name = self.test_prefix + "_test_drop_stream"
stream_name_as = stream_name + "_as"
utils.drop_stream(self.api_client, stream_name)
ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \
WITH (kafka_topic='{}', value_format='DELIMITED');".format(
stream_name, topic)
self.api_client.ksql(ksql_string)
ksql_string = "CREATE STREAM {} as select * from {};".format(
stream_name_as, stream_name)
self.api_client.ksql(ksql_string)
self.assertTrue(utils.get_stream_info(self.api_client, stream_name_as))
utils.drop_stream(self.api_client, stream_name_as)
self.assertFalse(utils.get_stream_info(self.api_client,
stream_name_as))
def test_ksql_create_stream_by_builder(self):
sql_type = 'create'
table_type = 'stream'
table_name = 'test_table'
columns_type = ['viewtime bigint', 'userid varchar', 'pageid varchar']
topic = self.exist_topic
value_format = 'DELIMITED'
utils.drop_stream(self.api_client, table_name)
ksql_string = SQLBuilder.build(sql_type=sql_type,
table_type=table_type,
table_name=table_name,
columns_type=columns_type,
topic=topic,
value_format=value_format)
r = self.api_client.ksql(ksql_string)
self.assertEqual(r[0]['commandStatus']['status'], 'SUCCESS')
def test_ksql_create_stream_by_builder(self):
sql_type = "create"
table_type = "stream"
table_name = "test_table"
columns_type = ["viewtime bigint", "userid varchar", "pageid varchar"]
topic = self.exist_topic
value_format = "DELIMITED"
utils.drop_stream(self.api_client, table_name)
ksql_string = SQLBuilder.build(
sql_type=sql_type,
table_type=table_type,
table_name=table_name,
columns_type=columns_type,
topic=topic,
value_format=value_format,
)
r = self.api_client.ksql(ksql_string)
self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")
