def __init__(self, config): # pragma: no cover
"""
Streaming client implementation based on Kafka.
Configuration keys:
KAFKA_ADDRESS
KAFKA_CONSUMER_GROUP
KAFKA_TOPIC
TIMEOUT
EVENTHUB_KAFKA_CONNECTION_STRING
"""
self.logger = Logger()
self.topic = config.get("KAFKA_TOPIC")
if config.get("TIMEOUT"):
try:
self.timeout = int(config.get("TIMEOUT"))
except ValueError:
self.timeout = None
else:
self.timeout = None
kafka_config = self.create_kafka_config(config)
self.admin = admin.AdminClient(kafka_config)
if config.get("KAFKA_CONSUMER_GROUP") is None:
self.logger.info('Creating Producer')
self.producer = Producer(kafka_config)
else:
self.logger.info('Creating Consumer')
self.consumer = Consumer(kafka_config)
def delete_kafka_topics(kafka_broker, topic_regex):
# test valid broker, it will throw in case of invalid broker
registered_topics = get_topics(kafka_broker)
kba = admin.AdminClient({'bootstrap.servers': kafka_broker})
""" delete topics """
# Call delete_topics to asynchronously delete topics, a future is returned.
# By default this operation on the broker returns immediately while
# topics are deleted in the background. But here we give it some time (30s)
# to propagate in the cluster before returning.
#
# Returns a dict of <topic,future>.
topics = []
regtopic = re.compile(topic_regex)
for artop in registered_topics[1]:
if regtopic.match(artop):
topics.append(artop)
if not topics:
return []
fs = kba.delete_topics(topics)
# Wait for operation to finish.
for topic, f in fs.items():
# it will throw if topic can not be deleted
f.result() # The result itself is None
print("Topic {} deleted".format(topic))
return topics
def __init__(self, config):
"""
Class to create a KafkaStreamingClient instance.
:param config: Dictionary file with all the relevant parameters.
:param topic: A string kafka topic.
"""
self.topic = config.get("KAFKA_TOPIC")
if config.get("TIMEOUT"):
try:
self.timeout = int(config.get("TIMEOUT"))
except ValueError:
self.timeout = None
else:
self.timeout = None
kafka_config = self.create_kafka_config(config)
self.admin = admin.AdminClient(kafka_config)
if config.get("KAFKA_CONSUMER_GROUP") is None:
self.producer = Producer(kafka_config)
else:
self.app_host = config.get("APP_HOST")
self.app_port = config.get("APP_PORT")
self.consumer = Consumer(kafka_config)
def initialize_admin(self):
"""
Creates an kafka admin object
:return: kafka admin object
"""
adm = admin.AdminClient(self.__config)
return adm
def create_kafka_admin(self):
try:
print("Connecting to Kafka Server....")
admin_client = admin.AdminClient(
{'bootstrap.servers': self.broker})
print("Successfully Connected!")
return admin_client
except Exception as ex:
print('Exception while connecting Kafka')
raise
def __init__(self, app):
self.app = app
self.system_topics = None
self.bootstrap_server = app.config["KAFKA_BOOTSTRAP_SERVER"]
# Create kafka admin client if there is a connection
if not self.bootstrap_server:
self.k_admin_client = None
self.k_admin_client = kafka_admin.AdminClient(
{'bootstrap.servers': self.bootstrap_server})
# the system_topics are updated in the get_connection method
self.get_connection()
def create_topic(cfg: config.Config):
"""Ensure that the required Kafka topic exists."""
# based on code in https://github.com/confluentinc/confluent-kafka-python
client = admin.AdminClient(cfg.get_kafka_config())
topic = admin.NewTopic(cfg.kafka_topic,
num_partitions=3,
replication_factor=1)
topic_map = client.create_topics([topic])
for (topic, future) in topic_map.items():
try:
future.result()
logger.info('Topic %s created', topic)
except Exception as err:
if err.args[0].code() != kafka.KafkaError.TOPIC_ALREADY_EXISTS:
raise
logger.debug('Topic %s already exists', topic)
def setConnection(self,
connectParamsFile='Vinnig/KafkaConnectionSettings.json'):
try:
home = os.path.expanduser("~")
filenameC = os.path.join(home, connectParamsFile)
with open(filenameC) as fc:
jsC = json.load(fc)
self.connectParams = jsC['connectParams']
self.topicBasename = jsC['topicBasename']
self.producer = Producer(
self.connectParams) # producer only used for log
self.producer.poll(0.1)
self.kafka_admin = admin.AdminClient(self.connectParams)
self.topics = self.kafka_admin.list_topics().topics
except Exception as e:
print('Problem connecting to Kafka: {}'.format(e))
def check_kafka(timeout=30):
end_time = time.time() + timeout
client = admin.AdminClient({"bootstrap.servers": KAFKA_BROKER_URL})
print(f"Waiting for kafka")
while time.time() < end_time:
try:
cluster_meta_data = client.list_topics(timeout=1)
if KAFKA_TOPIC in cluster_meta_data.topics.keys():
print("Kafka is up!")
return
time.sleep(0.3)
except KafkaException:
time.sleep(0.3)
continue
else:
raise Exception(f"Could not fin {KAFKA_TOPIC} in topics")
def __init__(self,
connectParamsFile='Vinnig/KafkaConnectionSettings.json',
connectParams=None,
topicBasename=''):
if (
connectParams == None
): # then read the configuration file. Location is relative to home directory
home = os.path.expanduser("~")
filename = os.path.join(home, connectParamsFile)
with open(filename) as f:
js = json.load(f)
connectParams = js['connectParams']
topicBasename = js['topicBasename']
self.kafka_admin = admin.AdminClient(connectParams)
self.topics = self.kafka_admin.list_topics().topics
self.connectParams = connectParams
self.topicBasename = topicBasename
self.device = None
self.deviceName = 'Unknown'
def __init__(self, config): # pragma: no cover
"""
Streaming client implementation based on Kafka.
Configuration keys:
KAFKA_ADDRESS
KAFKA_CONSUMER_GROUP
KAFKA_TOPIC
TIMEOUT
EVENT_HUB_KAFKA_CONNECTION_STRING
"""
self.logger = Logger()
self.topic = config.get("KAFKA_TOPIC")
if not self.topic:
raise ValueError("KAFKA_TOPIC is not set in the config object.")
if not config.get("KAFKA_ADDRESS"):
raise ValueError("KAFKA_ADDRESS is not set in the config object.")
if config.get("TIMEOUT"):
try:
self.timeout = int(config.get("TIMEOUT"))
except ValueError:
self.timeout = None
else:
self.timeout = None
kafka_config = self.create_kafka_config(config)
self.admin = admin.AdminClient(kafka_config)
if config.get("KAFKA_CONSUMER_GROUP") is None:
self.logger.info('Creating Producer')
self.producer = Producer(kafka_config)
self.run = False
else:
self.logger.info('Creating Consumer')
self.consumer = Consumer(kafka_config)
self.run = True
signal.signal(signal.SIGTERM, self.exit_gracefully)
def verify_admin():
""" Verify Admin API """
a = admin.AdminClient({'bootstrap.servers': bootstrap_servers})
our_topic = topic + '_admin_' + str(uuid.uuid4())
num_partitions = 2
topic_config = {"compression.type": "gzip"}
#
# First iteration: validate our_topic creation.
# Second iteration: create topic.
#
for validate in (True, False):
fs = a.create_topics([
admin.NewTopic(our_topic,
num_partitions=num_partitions,
config=topic_config,
replication_factor=1)
],
validate_only=validate,
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Find the topic in list_topics
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Increase the partition count
#
num_partitions += 3
fs = a.create_partitions(
[admin.NewPartitions(our_topic, new_total_count=num_partitions)],
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Verify with list_topics.
#
verify_topic_metadata(a, {our_topic: num_partitions})
def verify_config(expconfig, configs):
"""
Verify that the config key,values in expconfig are found
and matches the ConfigEntry in configs.
"""
for key, expvalue in expconfig.items():
entry = configs.get(key, None)
assert entry is not None, "Config {} not found in returned configs".format(
key)
assert entry.value == str(expvalue), \
"Config {} with value {} does not match expected value {}".format(key, entry, expvalue)
#
# Get current topic config
#
resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic)
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Now change the config.
#
topic_config["file.delete.delay.ms"] = 12345
topic_config["compression.type"] = "snappy"
for key, value in topic_config.items():
resource.set_config(key, value)
fs = a.alter_configs([resource])
fs[resource].result() # will raise exception on failure
#
# Read the config back again and verify.
#
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Delete the topic
#
fs = a.delete_topics([our_topic])
fs[our_topic].result() # will raise exception on failure
print("Topic {} marked for deletion".format(our_topic))
def __init__(self, conf):
self.conf = conf
kafka_conf = {'bootstrap.servers': conf.KAFKA_BROKERS}
kafka_admin = kad.AdminClient(kafka_conf)
self.admin = kafka_admin
data = res.json()
return data
with open(os.path.join(WORKING_DIRECTORY, 'city.list.json')) as json_file:
cities = json.load(json_file)
chile = []
for index, city in enumerate(cities):
if city["country"] == "CL":
chile.append(city)
logging.info(f"Total cities {len(chile)}")
calls = 0
kafka_client = admin.AdminClient(KAFKA_CONFIG)
new_topic = admin.NewTopic("weather", 1, 1)
kafka_client.create_topics([new_topic])
producer = Producer(KAFKA_CONFIG)
while True:
for index, city in enumerate(chile):
calls += 1
query = current_city(city['id'], api_key)
if query["cod"] != 200:
logging.error(f"Error in {index} {city}")
break
else:
logging.info(f"calls: {calls} city: {city['name']}")
producer.produce("weather", pickle.dumps(query))
EVENT_FILE = "test_events.json" # file of the records, not a json itself, but each row is
QUANTITIES = ["actSpeed_C11", "vaTorque_C11"]
RES_QUANTITY = "vaPower_C11"
MAX_JOIN_CNT = None # maximum of 1500 rows
MAX_BATCH_SIZE = 100
TRANSACTION_TIME = 0.1
VERBOSE = False
if sys.platform.startswith("win"):
pytest.skip("skipping unix-only tests", allow_module_level=True)
# Create a kafka producer and consumer instance and subscribe to the topics
print("Create Kafka instances.")
# create a Kafka admin client
k_admin_client = kafka_admin.AdminClient({'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS})
kafka_consumer = Consumer({
'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS,
'group.id': f"kafka-eof_{str(uuid.uuid4())}",
'auto.offset.reset': 'earliest',
'enable.auto.commit': False,
'enable.auto.offset.store': False,
'enable.partition.eof': False
})
kafka_consumer.subscribe([KAFKA_TOPIC_IN_0, KAFKA_TOPIC_IN_1])
# kafka_consumer.assign([TopicPartition(KAFKA_TOPIC_IN_0), TopicPartition(KAFKA_TOPIC_IN_1)])
# create a Kafka producer
kafka_producer = Producer({'bootstrap.servers': KAFKA_BOOTSTRAP_SERVERS,
def enqueue(self, filename):
"""
This method tries to post a message to the pre-defined kafka topic.
:param filename:
:return status False or True:
"""
status = False
if filename is None or len(filename) == 0:
logging_to_console_and_syslog(
"ConfluentKafkaMsgQAPI: filename is None or invalid")
return status
if self.producer_instance is None:
logging_to_console_and_syslog(
"KafkaMsgQAPI: Producer instance is None. Trying to create one.."
)
if not self.__producer_connect():
logging_to_console_and_syslog(
"Unable to create producer instance.")
return status
if not self.is_topic_created:
try:
if self.producer_instance.list_topics(self.topic, timeout=1.0):
logging_to_console_and_syslog(
"Found topic name = {} in the zookeeper.".format(
self.topic))
self.is_topic_created = True
except KafkaException:
kafka_admin_client = admin.AdminClient(self.producer_conf)
logging_to_console_and_syslog("Creating topic {}.".format(
self.topic))
ret = kafka_admin_client.create_topics(new_topics=[
admin.NewTopic(topic=self.topic, num_partitions=1)
],
operation_timeout=1.0)
logging_to_console_and_syslog("ret = {}".format(ret))
# Asynchronously produce a message, the delivery report callback
# will be triggered from poll() above, or flush() below, when the message has
# been successfully delivered or failed permanently.
logging_to_console_and_syslog(
"ConfluentKafkaMsgQAPI: Posting filename={} into "
"kafka broker={}, topic={}".format(filename, self.broker_name,
self.topic))
value = filename.encode('utf-8')
try:
# Produce line (without newline)
self.producer_instance.produce(
self.topic,
value,
callback=ConfluentKafkaMsgQAPI.delivery_callback)
status = True
except BufferError:
sys.stderr.write('%% Local producer queue is full '
'(%d messages awaiting delivery): try again\n' %
len(self.producer_instance))
status = False
except:
print("ConfluentKafkaMsgQAPI: Exception in user code:")
print("-" * 60)
traceback.print_exc(file=sys.stdout)
print("-" * 60)
status = False
else:
event = "ConfluentKafkaMsgQAPI: Posting filename={} into " \
"kafka broker={}, topic={}." \
.format(filename,
self.broker_name,
self.topic)
logging_to_console_and_syslog(event)
# Wait for any outstanding messages to be delivered and delivery report
# callbacks to be triggered.
# Serve delivery callback queue.
# NOTE: Since produce() is an asynchronous API this poll() call
# will most likely not serve the delivery callback for the
# last produce()d message.
self.producer_instance.poll(timeout=0.1)
# Wait until all messages have been delivered
# sys.stderr.write('%% Waiting for %d deliveries\n' % len(self.producer_instance))
self.producer_instance.flush(timeout=0.1)
return status
def create_admin_client(config):
global adminClient
adminClient = ckAdmin.AdminClient(config)
def verify_admin():
""" Verify Admin API """
a = admin.AdminClient({'bootstrap.servers': bootstrap_servers})
our_topic = topic + '_admin_' + str(uuid.uuid4())
num_partitions = 2
topic_config = {"compression.type": "gzip"}
#
# First iteration: validate our_topic creation.
# Second iteration: create topic.
#
for validate in (True, False):
fs = a.create_topics([
admin.NewTopic(our_topic,
num_partitions=num_partitions,
config=topic_config,
replication_factor=1)
],
validate_only=validate,
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Find the topic in list_topics
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Increase the partition count
#
num_partitions += 3
fs = a.create_partitions(
[admin.NewPartitions(our_topic, new_total_count=num_partitions)],
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Verify with list_topics.
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Verify with list_groups.
#
# Produce some messages
p = confluent_kafka.Producer({"bootstrap.servers": bootstrap_servers})
p.produce(our_topic, 'Hello Python!', headers=produce_headers)
p.produce(our_topic, key='Just a key and headers', headers=produce_headers)
def consume_messages(group_id):
# Consume messages
conf = {
'bootstrap.servers': bootstrap_servers,
'group.id': group_id,
'session.timeout.ms': 6000,
'enable.auto.commit': False,
'on_commit': print_commit_result,
'error_cb': error_cb,
'auto.offset.reset': 'earliest',
'enable.partition.eof': True
}
c = confluent_kafka.Consumer(conf)
c.subscribe([our_topic])
eof_reached = dict()
while True:
msg = c.poll()
if msg is None:
raise Exception(
'Got timeout from poll() without a timeout set: %s' % msg)
if msg.error():
if msg.error().code(
) == confluent_kafka.KafkaError._PARTITION_EOF:
print('Reached end of %s [%d] at offset %d' %
(msg.topic(), msg.partition(), msg.offset()))
eof_reached[(msg.topic(), msg.partition())] = True
if len(eof_reached) == len(c.assignment()):
print(
'EOF reached for all assigned partitions: exiting')
break
else:
print('Consumer error: %s: ignoring' % msg.error())
break
# Commit offset
c.commit(msg, asynchronous=False)
group1 = 'test-group-1'
group2 = 'test-group-2'
consume_messages(group1)
consume_messages(group2)
# list_groups without group argument
groups = set(group.id for group in a.list_groups(timeout=10))
assert group1 in groups, "Consumer group {} not found".format(group1)
assert group2 in groups, "Consumer group {} not found".format(group2)
# list_groups with group argument
groups = set(group.id for group in a.list_groups(group1))
assert group1 in groups, "Consumer group {} not found".format(group1)
groups = set(group.id for group in a.list_groups(group2))
assert group2 in groups, "Consumer group {} not found".format(group2)
def verify_config(expconfig, configs):
"""
Verify that the config key,values in expconfig are found
and matches the ConfigEntry in configs.
"""
for key, expvalue in expconfig.items():
entry = configs.get(key, None)
assert entry is not None, "Config {} not found in returned configs".format(
key)
assert entry.value == str(expvalue), \
"Config {} with value {} does not match expected value {}".format(key, entry, expvalue)
#
# Get current topic config
#
resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic)
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Now change the config.
#
topic_config["file.delete.delay.ms"] = 12345
topic_config["compression.type"] = "snappy"
for key, value in topic_config.items():
resource.set_config(key, value)
fs = a.alter_configs([resource])
fs[resource].result() # will raise exception on failure
#
# Read the config back again and verify.
#
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Delete the topic
#
fs = a.delete_topics([our_topic])
fs[our_topic].result() # will raise exception on failure
print("Topic {} marked for deletion".format(our_topic))
# Kafka Details
kafkaBrokerServer = hostname
kafkaBrokerPort = 9092
kafkaBroker = kafkaBrokerServer + ":" + str(kafkaBrokerPort)
zookeeperServer = hostname
zookeeperPort = 2185
zookeeper = zookeeperServer + ":" + str(zookeeperPort)
topic = 'mytopic'
print """\nINFO: Kakfa Connection Details:
Kafka Broker : %s
Zookeeper : %s
Topic : %s """ % (kafkaBroker, zookeeper, topic)
print "\nINFO: Creating connection obj for Admin Task"
connection = admin.AdminClient({'bootstrap.servers': kafkaBroker})
print "\nINFO: Check if Topic %s exists. Else Create it" % topic
check_topic_existence(connection, topic)
conf = {'bootstrap.servers': kafkaBroker}
print "\nINFO: Create Client obj for Kafka Connection"
client = Producer(**conf)
produce_messages(client, int(num_mesg), topic)
def check_kafka():
client = admin.AdminClient({"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS})
cluster_meta_data = client.list_topics(timeout=1)
# fail if custom topic is not present
return WEATHER_STATUS in cluster_meta_data.topics.keys()
def check_ksql():
client = admin.AdminClient({"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS})
cluster_meta_data = client.list_topics(timeout=1)
# fail if custom topic is not present
return TURNSTILE_ENTRIES_TABLE in cluster_meta_data.topics.keys()
def example_delete_topics(a, topics):
""" delete topics """
# Call delete_topics to asynchronously delete topics, a future is returned.
# By default this operation on the broker returns immediately while
# topics are deleted in the background. But here we give it some time (30s)
# to propagate in the cluster before returning.
#
# Returns a dict of <topic,future>.
fs = a.delete_topics(topics, operation_timeout=30)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
print("Topic {} deleted".format(topic))
except Exception as e:
print("Failed to delete topic {}: {}".format(topic, e))
conf = {
'bootstrap.servers': 'localhost:9092',
'client.id': 'test',
'default.topic.config': {
'acks': 'all'
}
}
a = admin.AdminClient(conf)
example_delete_topics(a, ['test-topic', 'orders-topic'])
