def check_kafka_connection(host, port):
"""Check connectability of Kafka Broker."""
conn = BrokerConnection(host, int(port), socket.AF_UNSPEC)
connected = conn.connect_blocking(timeout=1)
if connected:
conn.close()
return connected
def check_kafka_connection():
"""Check connectability of Kafka Broker."""
conn = BrokerConnection(SourcesConfig.SOURCES_KAFKA_HOST, int(SourcesConfig.SOURCES_KAFKA_PORT), socket.AF_UNSPEC)
connected = conn.connect_blocking(timeout=1)
if connected:
conn.close()
return connected
def __init__(self,
host,
port,
is_sasl=False,
sasl_username=None,
sasl_password=None):
self.sasl_plain_username = ""
self.sasl_plain_password = ""
if sasl_username is not None and sasl_password is not None:
self.sasl_plain_username = sasl_username,
self.sasl_plain_password = sasl_password
self.brokers_host = host
self.brokers_port = port
self.is_sasl = is_sasl
self.kafka_groups_response = None
self.kafka_topics_response = None
self.consumer_groups = []
self.topic_offsets_for_groups = []
self.topic_offsets_for_rate = []
self.topic_nps = {}
self.topic_data_metric = []
self.topic_list = []
self.consumer_offset_metric = []
self.bc = None
if not self.is_sasl:
self.bc = BrokerConnection(self.brokers_host, self.brokers_port,
socket.AF_INET)
print(self.bc.__str__())
print("connection successful : %s" % str(self.bc.connected()))
else:
self.bc = BrokerConnection(self.brokers_host,
self.brokers_port,
socket.AF_INET,
security_protocol='SASL_PLAINTEXT',
sasl_mechanism='PLAIN',
sasl_plain_username='******',
sasl_plain_password='******')
print(self.bc.__str__())
print("connection with sasl successful : %s" %
str(self.bc.connected()))
def list_avail_destinations(args, broker_addr, port):
""" List the groups currenly using """
bc = BrokerConnection(broker_addr, int(port), socket.AF_INET)
bc.connect_blocking()
list_groups_request = ListGroupsRequest_v1()
future = bc.send(list_groups_request)
while not future.is_done:
for resp, f in bc.recv():
f.success(resp)
bc.close()
# Only show CRR groups - might want to extend to lifecycle
for group in future.value.groups:
if group[0].find("backbeat-replication-group-") == 0:
print(group[0].replace("backbeat-replication-group-", ""))
def list_groups(bootstrap_server):
brokers = bootstrap_server.split(':')[0]
port = bootstrap_server.split(':')[1]
bc = BrokerConnection(brokers, port, socket.AF_INET)
bc.connect_blocking()
list_groups_request = ListGroupsRequest_v1()
future = bc.send(list_groups_request)
while not future.is_done:
for resp, f in bc.recv():
f.success(resp)
for group in future.value.groups:
print(group)
import socket
import os
# sudoPassword = '******'
# command = 'mount -t vboxsf kafka /home/kafka/'
# p = os.system('echo %s|sudo -S %s' % (sudoPassword, command))
#
# os.system('sudo /home/kafka/bin/zookeeper-server-start.sh -daemon /home/kafka/config/zookeeper.properties')
# os.system('sudo /home/kafka/bin/kafka-server-start.sh -daemon /home/kafka/config/server.properties')
with open("../cervantes.txt", "r") as f:
cervantes_file = f.readlines()
cervantes_file = [x.strip() for x in cervantes_file]
broker = BrokerConnection('localhost', 9092, socket.AF_INET)
broker.connect_blocking()
print(broker.connecting())
# To consume latest messages and auto-commit offsets
producer = KafkaProducer(bootstrap_servers='localhost:9092')
while not broker.connected():
print(123)
assert broker.connected() is True
count = 0
for x in cervantes_file:
time.sleep(0.5)
y = x.encode()
class MetricsExporter:
def __init__(self,
host,
port,
is_sasl=False,
sasl_username=None,
sasl_password=None):
self.sasl_plain_username = ""
self.sasl_plain_password = ""
if sasl_username is not None and sasl_password is not None:
self.sasl_plain_username = sasl_username,
self.sasl_plain_password = sasl_password
self.brokers_host = host
self.brokers_port = port
self.is_sasl = is_sasl
self.kafka_groups_response = None
self.kafka_topics_response = None
self.consumer_groups = []
self.topic_offsets_for_groups = []
self.topic_offsets_for_rate = []
self.topic_nps = {}
self.topic_data_metric = []
self.topic_list = []
self.consumer_offset_metric = []
self.bc = None
if not self.is_sasl:
self.bc = BrokerConnection(self.brokers_host, self.brokers_port,
socket.AF_INET)
print(self.bc.__str__())
print("connection successful : %s" % str(self.bc.connected()))
else:
self.bc = BrokerConnection(self.brokers_host,
self.brokers_port,
socket.AF_INET,
security_protocol='SASL_PLAINTEXT',
sasl_mechanism='PLAIN',
sasl_plain_username='******',
sasl_plain_password='******')
print(self.bc.__str__())
print("connection with sasl successful : %s" %
str(self.bc.connected()))
def get_groups(self):
cons_groups = []
list_groups_request = ListGroupsRequest_v1()
self.kafka_groups_response = self.bc.send(list_groups_request)
while not self.kafka_groups_response.is_done:
for resp, f in self.bc.recv():
print('group resp:', resp)
f.success(resp)
for group in self.kafka_groups_response.value.groups:
cons_groups.append(group[0])
# print('group[0]', group[0])
return cons_groups
# invoke in each iteration of group list
def get_topic_offsets(self, consumer_group):
fetch_offset_request = OffsetFetchRequest_v3(consumer_group, None)
print("offset fetch request: ", fetch_offset_request)
self.kafka_topics_response = self.bc.send(fetch_offset_request)
while not self.kafka_topics_response.is_done:
for resp, f in self.bc.recv():
# print('resp: ', resp)
f.success(resp)
for topic in self.kafka_topics_response.value.topics:
self.topic_list.append(topic[0])
print("topic: ", topic)
print("topic[0]", topic[0])
print("topic[1]", topic[1])
str_broker_host = self.brokers_host + ':' + str(self.brokers_port)
con = self.invoke_kafka_consumer(str_broker_host, self.is_sasl)
print("connection : %s" % con.bootstrap_connected())
print("consumer config is: ", con.config)
# TODO uncomment if necessary
# ps = [TopicPartition(topic[0], p) for p in con.partitions_for_topic(topic[0])]
# print("ps: ", ps)
end_offset = con.end_offsets([TopicPartition(topic[0], 0)])
# print('end_offset: ', end_offset)
topic_size = [*end_offset.values()]
# here to make list
# print('topic size: ', topic_size[0])
#
# print('offsets for {0}'.format(topic[0]))
# topic[1] is like [(0, 76, '', 0)]
for partition in topic[1]:
print("partition in topic cycle: ", partition)
# print("topic[1]", topic[1])
# print('- partition {0}, offset: {1} lag: {2}'.format(partition[0], int(partition[1]), lag_diff))
# data for lag and offset metrics
topic_consumer_lag_model = SimpleTopicConsumerLagMetricModel(
str(consumer_group), str(topic[0]), str(partition[0]),
str(topic_size[0]), str(partition[1]))
# data for rate and top 5 metrics
topic_consumer_rate_model = SimpleRateLagMetricModel(
str(consumer_group), str(topic[0]), str(partition[0]),
str(topic_size[0]), str(partition[1]))
# add new item to metrics list
self.topic_offsets_for_groups.append(
topic_consumer_lag_model.__str__())
# extending list of consumer metrics by other list with rate metrics to combine the response
self.topic_offsets_for_groups.extend(
topic_consumer_rate_model.__str__())
con.close()
return self.topic_offsets_for_groups
def get_topic_inbound_rate(self, consumer_group, topic_name):
pass
def get_topic_outbound_rate(self):
pass
def close_connection(self):
self.bc.close()
def init_connection(self):
if self.bc is not None:
self.bc.connect_blocking()
print(self.bc.__str__())
print("connection __SASL__ successful : %s" %
str(self.bc.connected()))
def invoke_kafka_consumer(self, p_str_broker_host, p_is_sasl):
if p_is_sasl:
# consumer = Consumer({
# 'bootstrap.servers': config.BOOTSTRAP_SERVERS,
# 'group.id': config.CONSUMER_GROUP,
# 'enable.auto.commit': False,
# })
return KafkaConsumer(bootstrap_servers=p_str_broker_host,
security_protocol='SASL_PLAINTEXT',
sasl_mechanism='PLAIN',
sasl_plain_username='******',
sasl_plain_password='******')
else:
return KafkaConsumer(bootstrap_servers=p_str_broker_host)
from kafka import BrokerConnection
from kafka.protocol.admin import *
import socket
# import kafka
bc = BrokerConnection('10.173.210.68', 9092, socket.AF_INET)
bc.connect_blocking()
list_groups_request = ListGroupsRequest_v1()
future = bc.send(list_groups_request)
while not future.is_done:
for resp, f in bc.recv():
f.success(resp)
for group in future.value.groups:
print(group)
from rest_framework.decorators import renderer_classes
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.settings import api_settings
from masu.config import Config
from masu.external.date_accessor import DateAccessor
from sources.config import Config as SourcesConfig
from sources.sources_http_client import SourceNotFoundError
from sources.sources_http_client import SourcesHTTPClient
from sources.sources_http_client import SourcesHTTPClientError
LOG = logging.getLogger(__name__)
BROKER_CONNECTION = BrokerConnection(SourcesConfig.SOURCES_KAFKA_HOST,
int(SourcesConfig.SOURCES_KAFKA_PORT),
socket.AF_UNSPEC)
BROKER_CONNECTION_ERROR = "Unable to establish connection with broker."
CELERY_WORKER_NOT_FOUND = "No running Celery workers were found."
def check_kafka_connection():
"""Check connectability of Kafka Broker."""
connected = BROKER_CONNECTION.connect_blocking(timeout=1)
if connected:
BROKER_CONNECTION.close()
return connected
def check_sources_connection():
"""Check sources-backend connection."""