def test_create_stream_as_with_conditions_with_startwith(self): src_table = 'pageviews_original' columns_type = [ 'name string', 'age bigint', 'userid string', 'pageid bigint' ] topic = self.exist_topic table_name = 'create_stream_as_with_conditions_with_startwith' kafka_topic = 'create_stream_as_with_conditions_with_startwith' value_format = 'DELIMITED' select_columns = ['rowtime as logtime', '*'] conditions = "userid = 'foo_%'" utils.drop_stream(self.api_client, src_table) utils.drop_stream(self.api_client, table_name) try: r = self.api_client.create_stream(table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format) except KSQLError as e: pass r = self.api_client.create_stream_as(table_name=table_name, src_table=src_table, kafka_topic=kafka_topic, select_columns=select_columns, timestamp='logtime', value_format=value_format, conditions=conditions) self.assertTrue(r)
def test_ksql_create_stream_as_with_wrong_timestamp(self): src_table = 'prebid_traffic_log_total_stream' columns_type = [ 'name string', 'age bigint', 'userid string', 'pageid bigint' ] topic = self.exist_topic table_name = 'prebid_traffic_log_valid_stream' kafka_topic = 'prebid_traffic_log_valid_topic' value_format = 'DELIMITED' select_columns = ['*'] timestamp = 'foo' utils.drop_stream(self.api_client, src_table) utils.drop_stream(self.api_client, table_name) try: r = self.api_client.create_stream(table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format) except KSQLError as e: raise with self.assertRaises(KSQLError): r = self.api_client.create_stream_as(table_name=table_name, src_table=src_table, kafka_topic=kafka_topic, select_columns=select_columns, timestamp=timestamp, value_format=value_format)
def test_ksql_create_stream_as_with_wrong_timestamp(self): src_table = "prebid_traffic_log_total_stream" columns_type = ["name string", "age bigint", "userid string", "pageid bigint"] topic = self.exist_topic table_name = "prebid_traffic_log_valid_stream" kafka_topic = "prebid_traffic_log_valid_topic" value_format = "DELIMITED" select_columns = ["*"] timestamp = "foo" utils.drop_stream(self.api_client, src_table) utils.drop_stream(self.api_client, table_name) try: self.api_client.create_stream( table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format ) except KSQLError: pass with self.assertRaises(KSQLError): self.api_client.create_stream_as( table_name=table_name, src_table=src_table, kafka_topic=kafka_topic, select_columns=select_columns, timestamp=timestamp, value_format=value_format, )
def test_create_stream_as_with_conditions_with_startwith(self): src_table = "pageviews_original" columns_type = ["name string", "age bigint", "userid string", "pageid bigint"] topic = self.exist_topic table_name = "create_stream_as_with_conditions_with_startwith" kafka_topic = "create_stream_as_with_conditions_with_startwith" value_format = "DELIMITED" select_columns = ["rowtime as logtime", "*"] conditions = "userid = 'foo_%'" utils.drop_stream(self.api_client, src_table) utils.drop_stream(self.api_client, table_name) try: r = self.api_client.create_stream( table_name=src_table, columns_type=columns_type, topic=topic, value_format=value_format ) except KSQLError: pass r = self.api_client.create_stream_as( table_name=table_name, src_table=src_table, kafka_topic=kafka_topic, select_columns=select_columns, timestamp="logtime", value_format=value_format, conditions=conditions, ) self.assertTrue(r)
def test_drop_all_streams(self): topic = self.exist_topic stream_name = self.test_prefix + "_test_drop_all_streams" utils.drop_stream(self.api_client, stream_name) ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \ WITH (kafka_topic='{}', value_format='DELIMITED');".format( stream_name, topic) self.api_client.ksql(ksql_string) utils.drop_all_streams(self.api_client, prefix=self.test_prefix) self.assertFalse(utils.get_stream_info(self.api_client, stream_name))
def test_get_stream_info(self): topic = self.exist_topic stream_name = self.test_prefix + "_test_get_stream_info" utils.drop_stream(self.api_client, stream_name) ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \ WITH (kafka_topic='{}', value_format='DELIMITED');".format( stream_name, topic) self.api_client.ksql(ksql_string) stream_info = utils.get_stream_info(self.api_client, stream_name) # print(stream_info['topic']) self.assertEqual(stream_info["topic"], self.exist_topic)
def test_get_all_streams(self): topic = self.exist_topic stream_name = self.test_prefix + "_test_get_all_streams" utils.drop_stream(self.api_client, stream_name) ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \ WITH (kafka_topic='{}', value_format='DELIMITED');".format( stream_name, topic) self.api_client.ksql(ksql_string) filtered_streams = utils.get_all_streams(self.api_client, prefix=self.test_prefix) self.assertEqual(filtered_streams, [stream_name.upper()])
def test_ksql_create_stream_by_builder_api(self): table_name = "test_table" columns_type = ["viewtime bigint", "userid varchar", "pageid varchar"] topic = self.exist_topic value_format = "DELIMITED" utils.drop_stream(self.api_client, table_name) r = self.api_client.create_stream( table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format ) self.assertTrue(r)
def test_ksql_create_stream_by_builder_api(self): table_name = 'test_table' columns_type = ['viewtime bigint', 'userid varchar', 'pageid varchar'] topic = self.exist_topic value_format = 'DELIMITED' utils.drop_stream(self.api_client, table_name) r = self.api_client.create_stream(table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format) self.assertTrue(r)
def test_raise_create_error_topic_already_registered(self): table_name = "foo_table" columns_type = ["name string", "age bigint"] topic = self.exist_topic value_format = "DELIMITED" utils.drop_stream(self.api_client, table_name) self.api_client.create_stream( table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format ) with self.assertRaises(KSQLError): self.api_client.create_stream( table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format )
def test_raise_create_error_topic_already_registered(self): table_name = 'foo_table' columns_type = ['name string', 'age bigint'] topic = self.exist_topic value_format = 'DELIMITED' utils.drop_stream(self.api_client, table_name) r = self.api_client.create_stream(table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format) with self.assertRaises(KSQLError): r = self.api_client.create_stream(table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format)
def test_get_dependent_queries(self): topic = self.exist_topic stream_name = self.test_prefix + "_test_get_dependent_queries" stream_name_as = stream_name + "_as" utils.drop_stream(self.api_client, stream_name) ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \ WITH (kafka_topic='{}', value_format='DELIMITED');".format( stream_name, topic) self.api_client.ksql(ksql_string) ksql_string = "CREATE STREAM {} as select * from {};".format( stream_name_as, stream_name) self.api_client.ksql(ksql_string) read_queries, write_queries = utils.get_dependent_queries( self.api_client, stream_name_as) self.assertEqual(read_queries, []) self.assertTrue(write_queries[0].startswith( "CSAS_KSQL_PYTHON_TEST_TEST_GET_DEPENDENT_QUERIES_AS"))
def test_drop_stream_create_as_stream(self): topic = self.exist_topic stream_name = self.test_prefix + "_test_drop_stream" stream_name_as = stream_name + "_as" utils.drop_stream(self.api_client, stream_name) ksql_string = "CREATE STREAM {} (viewtime bigint, userid varchar, pageid varchar) \ WITH (kafka_topic='{}', value_format='DELIMITED');".format( stream_name, topic) self.api_client.ksql(ksql_string) ksql_string = "CREATE STREAM {} as select * from {};".format( stream_name_as, stream_name) self.api_client.ksql(ksql_string) self.assertTrue(utils.get_stream_info(self.api_client, stream_name_as)) utils.drop_stream(self.api_client, stream_name_as) self.assertFalse(utils.get_stream_info(self.api_client, stream_name_as))
def test_ksql_create_stream_by_builder(self): sql_type = 'create' table_type = 'stream' table_name = 'test_table' columns_type = ['viewtime bigint', 'userid varchar', 'pageid varchar'] topic = self.exist_topic value_format = 'DELIMITED' utils.drop_stream(self.api_client, table_name) ksql_string = SQLBuilder.build(sql_type=sql_type, table_type=table_type, table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format) r = self.api_client.ksql(ksql_string) self.assertEqual(r[0]['commandStatus']['status'], 'SUCCESS')
def test_ksql_create_stream_by_builder(self): sql_type = "create" table_type = "stream" table_name = "test_table" columns_type = ["viewtime bigint", "userid varchar", "pageid varchar"] topic = self.exist_topic value_format = "DELIMITED" utils.drop_stream(self.api_client, table_name) ksql_string = SQLBuilder.build( sql_type=sql_type, table_type=table_type, table_name=table_name, columns_type=columns_type, topic=topic, value_format=value_format, ) r = self.api_client.ksql(ksql_string) self.assertEqual(r[0]["commandStatus"]["status"], "SUCCESS")