def setUp(self):
self.url = "http://ksql-server:8088"
self.api_client = KSQLAPI(url=self.url, check_version=False)
self.exist_topic = 'exist_topic'
bootstrap_servers = 'kafka:29092'
if check_kafka_available(bootstrap_servers):
producer = Producer({'bootstrap.servers': bootstrap_servers})
producer.produce(self.exist_topic, "test_message")
producer.flush()
def setUp(self):
self.url = "http://localhost:8088"
self.api_client = KSQLAPI(url=self.url, check_version=False)
self.test_prefix = "ksql_python_test"
self.exist_topic = 'exist_topic'
self.bootstrap_servers = 'localhost:29092'
if utils.check_kafka_available(self.bootstrap_servers):
producer = Producer({'bootstrap.servers': self.bootstrap_servers})
producer.produce(self.exist_topic, "test_message")
producer.flush()
def __init__(self, user=None, security_enabled=False, ssl_cert_ca_verify=False):
try:
from ksql import KSQLAPI
except ImportError:
raise KSqlApiException('Module missing: pip install ksql')
self._api_url = KAFKA.KSQL_API_URL.get().strip('/') if KAFKA.KSQL_API_URL.get() else ''
self.user = user
self.client = client = KSQLAPI(self._api_url)
def test_ksql_show_tables_with_api_key(self):
api_client = KSQLAPI(url=self.url,
check_version=False,
api_key='foo',
secret='bar')
ksql_string = "show tables;"
r = api_client.ksql(ksql_string)
self.assertEqual(r, [{
"@type": "tables",
"statementText": "show tables;",
"tables": [],
"warnings": []
}])
def __init__(self):
url = "http://localhost:8088"
self.api_client = KSQLAPI(url)
self.topic = "test08"
self.bootstrap_servers = "localhost:9092"
if utils.check_kafka_available(self.bootstrap_servers):
value_schema_str = """
{
"type": "record",
"namespace": "com.example",
"name": "value",
"fields": [
{"name":"LOCATION", "type":"string"},
{"name":"DATETIME", "type":"string"},
{"name":"SENTIMENT", "type":"string"},
{"name":"TEXT", "type":"string"}
]
}
"""
key_schema_str = """
{
"type": "record",
"namespace": "com.example",
"name": "key",
"fields": [
{"name":"LOCATION", "type":"string"},
{"name":"DATETIME", "type":"string"},
{"name":"SENTIMENT", "type":"string"},
{"name":"TEXT", "type":"string"}
]
}
"""
value_schema = avro.loads(value_schema_str)
key_schema = avro.loads(key_schema_str)
self.key = {
"LOCATION": "LOCATION",
"DATETIME": "DATETIME",
"SENTIMENT": "SENTIMENT",
"TEXT": "TEXT"
}
self.producer = AvroProducer(
{
'bootstrap.servers': self.bootstrap_servers,
'on_delivery': delivery_report,
'schema.registry.url': 'http://localhost:8081'
},
default_key_schema=None,
default_value_schema=value_schema)
else:
print("Could not connect to Kafka")
exit(-1)
from ksql import KSQLAPI
# Refer to https://pypi.org/project/ksql/
client = KSQLAPI('http://localhost:8088')
query = client.query('select * from table1')
for item in query:
print(item)
def test_with_timeout(self):
api_client = KSQLAPI(url=self.url, timeout=10, check_version=False)
self.assertEquals(api_client.timeout, 10)
"""Load configuration from .yaml file."""
import confuse
from faker import Faker
from time import sleep
from json import dumps
from kafka import KafkaProducer
from ksql import KSQLAPI
client = KSQLAPI('http://ksqldb-server:8088')
producer = KafkaProducer(bootstrap_servers=['broker:29092'],
value_serializer=lambda x: dumps(x).encode('utf-8'))
fake = Faker()
config = confuse.Configuration('mocker')
config.set_file('/app/config.yaml')
rate = config['transactions']['rate'].get(int)
topic = config['kafka']['topic'].get()
print("Start sending transactions")
while True:
producer.send(topic,
value={
'transaction_id': "RF" + str(fake.pyint(5)),
'transaction_type': "transaction_" + str(fake.pyint(1)),
'from_account': fake.iban(),
'to_account': fake.iban(),
'amount_cents': fake.pyint(),
'created_at':
fake.date_time().strftime("%Y/%m/%d, %H:%M:%S")
})
def __init__(self, url):
self.client = KSQLAPI(url)
def get_ksql_client():
client = KSQLAPI(KSQL_URL)
return client
def test_with_timeout(self):
api_client = KSQLAPI(url='http://foo', timeout=10)
self.assertEquals(api_client.timeout, 10)
def setUp(self):
self.url = "http://ksql-server:8080"
self.api_client = KSQLAPI(url=self.url)
self.exist_topic = 'exist_topic'
from ksql import KSQLAPI
client = KSQLAPI('http://192.168.99.100:8088')
s = client.ksql('show streams')
print(s)
# print(cmd)
insert_table_cmd = ("""
CREATE TABLE IF NOT EXISTS updates (
update varchar(255)
);
""")
try:
conn = pgdb.Connection(database='postgres',
host='localhost',
user='******',
password='******')
cur = conn.cursor()
print('Connection successful!')
client = KSQLAPI('http://127.0.0.1:59090/browser/')
except:
print('Connection unsuccessful!')
print(
"Type a SQL command below. \nWhen you are done typing your command, press ENTER on a blank line or type 'stop'.\nTo completely quite out of this interface, type 'quit all'."
)
raw_input_lines = []
outer_loop = True
while (outer_loop):
print("> ")
inner_loop = True
while (inner_loop):
raw_input = input()
if (raw_input) and (raw_input != 'stop'):
raw_input_lines.append(raw_input)
import logging
from ksql import KSQLAPI
logging.basicConfig(level=logging.DEBUG)
#client = KSQLAPI('http://ec2-52-41-32-196.us-west-2.compute.amazonaws.com:8088')
client = KSQLAPI('http://10.0.0.13:8088')
#client.create_stream()
client.ksql('show tables')
#client = KSQLAPI('http://ec2-52-41-32-196.us-west-2.compute.amazonaws.com:8088')
from ksql import KSQLAPI
import sys
import string
import random
import json
endpoint = str(sys.argv[1])
topic = str(sys.argv[2])
client = KSQLAPI(endpoint)
## Create a stream from topic
stream_name_random = ''.join(
random.choice(string.ascii_uppercase) for _ in range(10))
client.create_stream(table_name=stream_name_random,
columns_type=["number bigint", "category varchar"],
topic=topic,
value_format="json")
query_res = client.ksql("LIST STREAMS;")
streams_names = [stream['name'] for stream in query_res[0]['streams']]
if stream_name_random in streams_names:
print("STREAM CREATED")
else:
print("Stream not created")
exit(1)
# Create a materialized table from stream for the average number over categories
materialized_table_random_name = ''.join(
random.choice(string.ascii_uppercase) for _ in range(10))
