def setUp(self): self.url = "http://ksql-server:8088" self.api_client = KSQLAPI(url=self.url, check_version=False) self.exist_topic = 'exist_topic' bootstrap_servers = 'kafka:29092' if check_kafka_available(bootstrap_servers): producer = Producer({'bootstrap.servers': bootstrap_servers}) producer.produce(self.exist_topic, "test_message") producer.flush()
def setUp(self): self.url = "http://localhost:8088" self.api_client = KSQLAPI(url=self.url, check_version=False) self.test_prefix = "ksql_python_test" self.exist_topic = 'exist_topic' self.bootstrap_servers = 'localhost:29092' if utils.check_kafka_available(self.bootstrap_servers): producer = Producer({'bootstrap.servers': self.bootstrap_servers}) producer.produce(self.exist_topic, "test_message") producer.flush()
def __init__(self, user=None, security_enabled=False, ssl_cert_ca_verify=False): try: from ksql import KSQLAPI except ImportError: raise KSqlApiException('Module missing: pip install ksql') self._api_url = KAFKA.KSQL_API_URL.get().strip('/') if KAFKA.KSQL_API_URL.get() else '' self.user = user self.client = client = KSQLAPI(self._api_url)
def test_ksql_show_tables_with_api_key(self): api_client = KSQLAPI(url=self.url, check_version=False, api_key='foo', secret='bar') ksql_string = "show tables;" r = api_client.ksql(ksql_string) self.assertEqual(r, [{ "@type": "tables", "statementText": "show tables;", "tables": [], "warnings": [] }])
def __init__(self): url = "http://localhost:8088" self.api_client = KSQLAPI(url) self.topic = "test08" self.bootstrap_servers = "localhost:9092" if utils.check_kafka_available(self.bootstrap_servers): value_schema_str = """ { "type": "record", "namespace": "com.example", "name": "value", "fields": [ {"name":"LOCATION", "type":"string"}, {"name":"DATETIME", "type":"string"}, {"name":"SENTIMENT", "type":"string"}, {"name":"TEXT", "type":"string"} ] } """ key_schema_str = """ { "type": "record", "namespace": "com.example", "name": "key", "fields": [ {"name":"LOCATION", "type":"string"}, {"name":"DATETIME", "type":"string"}, {"name":"SENTIMENT", "type":"string"}, {"name":"TEXT", "type":"string"} ] } """ value_schema = avro.loads(value_schema_str) key_schema = avro.loads(key_schema_str) self.key = { "LOCATION": "LOCATION", "DATETIME": "DATETIME", "SENTIMENT": "SENTIMENT", "TEXT": "TEXT" } self.producer = AvroProducer( { 'bootstrap.servers': self.bootstrap_servers, 'on_delivery': delivery_report, 'schema.registry.url': 'http://localhost:8081' }, default_key_schema=None, default_value_schema=value_schema) else: print("Could not connect to Kafka") exit(-1)
from ksql import KSQLAPI # Refer to https://pypi.org/project/ksql/ client = KSQLAPI('http://localhost:8088') query = client.query('select * from table1') for item in query: print(item)
def test_with_timeout(self): api_client = KSQLAPI(url=self.url, timeout=10, check_version=False) self.assertEquals(api_client.timeout, 10)
"""Load configuration from .yaml file.""" import confuse from faker import Faker from time import sleep from json import dumps from kafka import KafkaProducer from ksql import KSQLAPI client = KSQLAPI('http://ksqldb-server:8088') producer = KafkaProducer(bootstrap_servers=['broker:29092'], value_serializer=lambda x: dumps(x).encode('utf-8')) fake = Faker() config = confuse.Configuration('mocker') config.set_file('/app/config.yaml') rate = config['transactions']['rate'].get(int) topic = config['kafka']['topic'].get() print("Start sending transactions") while True: producer.send(topic, value={ 'transaction_id': "RF" + str(fake.pyint(5)), 'transaction_type': "transaction_" + str(fake.pyint(1)), 'from_account': fake.iban(), 'to_account': fake.iban(), 'amount_cents': fake.pyint(), 'created_at': fake.date_time().strftime("%Y/%m/%d, %H:%M:%S") })
def __init__(self, url): self.client = KSQLAPI(url)
def get_ksql_client(): client = KSQLAPI(KSQL_URL) return client
def test_with_timeout(self): api_client = KSQLAPI(url='http://foo', timeout=10) self.assertEquals(api_client.timeout, 10)
def setUp(self): self.url = "http://ksql-server:8080" self.api_client = KSQLAPI(url=self.url) self.exist_topic = 'exist_topic'
from ksql import KSQLAPI client = KSQLAPI('http://192.168.99.100:8088') s = client.ksql('show streams') print(s)
# print(cmd) insert_table_cmd = (""" CREATE TABLE IF NOT EXISTS updates ( update varchar(255) ); """) try: conn = pgdb.Connection(database='postgres', host='localhost', user='******', password='******') cur = conn.cursor() print('Connection successful!') client = KSQLAPI('http://127.0.0.1:59090/browser/') except: print('Connection unsuccessful!') print( "Type a SQL command below. \nWhen you are done typing your command, press ENTER on a blank line or type 'stop'.\nTo completely quite out of this interface, type 'quit all'." ) raw_input_lines = [] outer_loop = True while (outer_loop): print("> ") inner_loop = True while (inner_loop): raw_input = input() if (raw_input) and (raw_input != 'stop'): raw_input_lines.append(raw_input)
import logging from ksql import KSQLAPI logging.basicConfig(level=logging.DEBUG) #client = KSQLAPI('http://ec2-52-41-32-196.us-west-2.compute.amazonaws.com:8088') client = KSQLAPI('http://10.0.0.13:8088') #client.create_stream() client.ksql('show tables') #client = KSQLAPI('http://ec2-52-41-32-196.us-west-2.compute.amazonaws.com:8088')
from ksql import KSQLAPI import sys import string import random import json endpoint = str(sys.argv[1]) topic = str(sys.argv[2]) client = KSQLAPI(endpoint) ## Create a stream from topic stream_name_random = ''.join( random.choice(string.ascii_uppercase) for _ in range(10)) client.create_stream(table_name=stream_name_random, columns_type=["number bigint", "category varchar"], topic=topic, value_format="json") query_res = client.ksql("LIST STREAMS;") streams_names = [stream['name'] for stream in query_res[0]['streams']] if stream_name_random in streams_names: print("STREAM CREATED") else: print("Stream not created") exit(1) # Create a materialized table from stream for the average number over categories materialized_table_random_name = ''.join( random.choice(string.ascii_uppercase) for _ in range(10))