def ensure_topic(topic,
num_partitions,
replication_factor,
logger,
timeout_ms=3000,
brokers='localhost'):
adminclient = KafkaAdminClient(bootstrap_servers=brokers,
client_id='ansible-rm')
topic_list = []
topic_list.append(
NewTopic(name=topic, num_partitions=1, replication_factor=1))
try:
adminclient.create_topics(new_topics=topic_list, validate_only=False)
# adminclient.delete_topics(topic_list)
logger.info('kafka topic ' + topic + ' created')
except TopicAlreadyExistsError as e:
logger.info('kafka topic ' + topic + ' exists')
except Exception as e:
logger.error('error creating kafka topic ' + topic)
raise Exception(
'Unknown error code during creation of topic `{}`: {}'.format(
topic, str(e)))
class KafkaAdminGroupReader:
def __init__(self, kafka_config):
self.log = logging.getLogger(__name__)
self.admin_client = KafkaAdminClient(
bootstrap_servers=kafka_config.broker_list, )
def read_group(self, groupid):
topics = set()
group_offsets = self.admin_client.list_consumer_group_offsets(groupid)
for tp in six.iterkeys(group_offsets):
topics.add(tp.topic)
return list(topics)
def read_groups(self, groupids=None, list_only=False):
if groupids is None:
groupids = self._list_groups()
if list_only:
return {groupid: [] for groupid in groupids}
groups = {}
for groupid in groupids:
topics = self.read_group(groupid)
groups[groupid] = topics
return groups
def _list_groups(self):
groups_and_protocol_types = self.admin_client.list_consumer_groups()
return [gpt[0] for gpt in groups_and_protocol_types]
def __init__(self, id: int, name: str):
if (id <= 0): raise ValueError("ID must be a positive integer")
super().__init__()
self._daemon: bool = True
self._id: int = id
self._name: str = name
self._status_up_time: datetime = datetime.now()
self._actual_status: Status = self.__set_status()
self._consumer = KafkaConsumer(f"get_status_{self.id}",
bootstrap_servers=["localhost:9092"])
self._reg_consumer = KafkaConsumer(
f"reg_response_{self.id}",
bootstrap_servers=["localhost:9092"],
consumer_timeout_ms=5000)
try:
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
admin_client.create_topics([
NewTopic(f"reg_response_{self.id}", 1, 1),
NewTopic(f"get_status_{self.id}", 1, 1)
])
except:
pass
self.__make_reg(id)
def delete_topic(username, title):
# Deletes a kafak topic, based on username and title
admin_client = KafkaAdminClient(bootstrap_servers='localhost:9092')
topic_lst = []
topic_name = "{}-{}".format(username, title)
topic_lst.append(topic_name)
admin_client.delete_topics(topic_lst)
def alternative_main(input_path, bootstrap_server, topic_name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:7092",
client_id='test')
topic_list = []
topic_list.append(
NewTopic(name="test", num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
#Starting session
spark = SparkSession.builder.appName('BigData1').getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
joined_raw = spark.read.parquet(input_path)
#print(joined_raw.show())
# Write key-value data from a DataFrame to a specific Kafka topic specified in an option
query = joined_raw.select(
F.col('time').cast("string").alias("key"),
F.to_json(F.struct("duration", "visibility")).alias("value"))
print(query.show())
query = joined_raw \
.select(F.col('time').cast("string").alias("key"), F.to_json(F.struct("duration", "visibility")).alias("value")) \
.write \
.format("kafka") \
.option("kafka.bootstrap.servers", bootstrap_server) \
.option("topic", "test") \
.save()
def init_admin_client(self):
start_time = time.monotonic()
wait_time = constants.MINUTE
while True:
if time.monotonic() - start_time > wait_time:
raise Timeout(
f"Timeout ({wait_time}) on creating admin client")
try:
self.admin_client = KafkaAdminClient(
api_version_auto_timeout_ms=constants.
API_VERSION_AUTO_TIMEOUT_MS,
bootstrap_servers=self.config["bootstrap_uri"],
client_id=self.config["client_id"],
security_protocol=self.config["security_protocol"],
ssl_cafile=self.config["ssl_cafile"],
ssl_certfile=self.config["ssl_certfile"],
ssl_keyfile=self.config["ssl_keyfile"],
client_factory=KarapaceKafkaClient,
)
break
except (NodeNotReadyError, NoBrokersAvailable, AssertionError):
self.log.warning(
"No Brokers available yet, retrying init_admin_client()")
except: # pylint: disable=bare-except
self.log.exception(
"Failed to initialize admin client, retrying init_admin_client()"
)
time.sleep(2.0)
def connect(self):
"""Make a connection"""
logger.info(f"connect kafka: {self.connection_conf}")
if self.admin_client:
try:
self.admin_client.close()
except:
pass
self.admin_client = KafkaAdminClient(bootstrap_servers=self.connection_conf)
topic_list = []
topic_list.append(
NewTopic(name=self.topic,
num_partitions=self.num_partitions,
replication_factor=self.replication_factor))
try:
self.admin_client.create_topics(new_topics=topic_list, validate_only=False)
except Exception as e:
logger.error(e)
if self.producer:
self.producer.close()
self.producer = KafkaProducer(bootstrap_servers=self.connection_conf)
if self.consumer:
self.consumer.close()
self.consumer = KafkaConsumer(self.topic, group_id=f"{self.topic}.spider.consumer", bootstrap_servers=self.connection_conf)
def get(self, topicName):
"""
Get Topic Detail.
"""
app.logger.info(
"Request to get details for topic {0}.".format(topicName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
result = admin.describe_topics([topicName])
except UnknownTopicOrPartitionError as e:
api.abort(500, e.description)
except Exception as e:
api.abort(500, str(e.args))
finally:
admin.close()
app.logger.debug(result)
if result[0]['error_code'] == 0:
return {
'partitions': len(result[0]['partitions']),
'replicas': len(result[0]['partitions'][0]['replicas'])
}
else:
api.abort(400, "Bad Request(Wrong Topic Name)")
def __init__( self , kafka_endpoint , access_key , access_certificate , ca_certificate ):
self.kafka_endpoint = kafka_endpoint
self.kafka_consumer = KafkaConsumer(
bootstrap_servers = self.kafka_endpoint
, value_deserializer = lambda m: json.loads( m.decode('utf-8') )
, ssl_keyfile = access_key
, ssl_certfile = access_certificate
, ssl_cafile = ca_certificate
, security_protocol = "SSL"
, api_version_auto_timeout_ms = 10000
)
topics = self.kafka_consumer.topics()
if 'os_stats' not in topics:
print( "os_stats topic not found ... creating" )
admin_client = KafkaAdminClient(
bootstrap_servers = self.kafka_endpoint
, ssl_keyfile = access_key
, ssl_certfile = access_certificate
, ssl_cafile = ca_certificate
, security_protocol = "SSL"
, api_version_auto_timeout_ms = 10000
)
topic_list = []
topic_list.append(NewTopic( name = "os_stats" , num_partitions = 1 , replication_factor = 1 ) )
admin_client.create_topics( new_topics = topic_list , validate_only = False )
self.kafka_consumer.subscribe( 'os_stats' )
def get_consumer(self):
if self.err_var is not None:
self.err_var.set("")
self.ui.update()
set_content(self)
from kafka.admin import KafkaAdminClient
adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker)
data = adminClient.list_consumer_group_offsets(self.cur_consumer)
self.content_text.config(state=NORMAL)
self.content_text.delete(1.0, END)
self.content_text.insert(
END, u"{} {} :\n\n".format(self.cur_broker, self.cur_consumer))
self.content_text.insert(
END,
u"{:20s} {:10s} {:12s} {:12s} {}\n".format("topic", "partition",
"offset", "newest",
"lag"))
for key in data.keys():
num = self.get_topic_single_partiton_offset(
self.cur_broker, key.topic, key.partition)
lag = int(num) - int(data[key].offset)
self.content_text.insert(
END, u"{:20s} {:10s} {:12s} {:12s} {}\n".format(
key.topic, str(key.partition), str(data[key].offset),
str(num), lag))
msg = "offset:{} lag:{}".format(num, lag)
self.write_file(msg, self.cur_consumer)
if len(data.keys()) == 0:
self.content_text.insert(END, u"没有查到数据")
self.content_text.config(state=DISABLED)
def test_validate_number_in_interval_double(self):
prod, cons = app.run(Config.K_MONITOR_TEST_TOPIC,
Config.PS_DATABASE_NAME,
Config.PS_TEST_WEBSITE_TABLE_NAME,
"tests/t_monitor_heavy_test.yml")
interval = File.read_time_interval("tests/t_monitor_heavy_test.yml")
time.sleep(interval * 2)
app.stop_monitor(prod, cons)
admin_client = KafkaAdminClient(
bootstrap_servers=[Config.K_HOST + ':' + Config.K_PORT],
security_protocol=Config.K_SECURITY_PROTOCOL,
ssl_cafile=Config.K_SSL_CAT_FILE,
ssl_certfile=Config.K_SSL_CERT_FILE,
ssl_keyfile=Config.K_SSL_KEY_FILE)
admin_client.delete_topics([Config.K_MONITOR_TEST_TOPIC])
monitors = File.read_monitors("tests/t_monitor_heavy_test.yml")
#send messages equals total urls count in 2 cycle is double the urls size
self.assertEqual(prod.get_message_count(), len(monitors) * 2)
def get(self, topicName):
"""
Get Topic Configuration.
"""
app.logger.info(
"Request to get Configuration for topic {0}.".format(topicName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
config_list = []
config = ConfigResource(ConfigResourceType.TOPIC, topicName)
topic_configs = admin.describe_configs([config])
topic_config = topic_configs[0].resources[0]
for c in topic_config[4]:
config_list.append({'key': c[0], 'value': c[1]})
return config_list
except Exception as e:
ns_topic.abort(500, str(e.args))
finally:
admin.close()
def create_group(phone_no):
if get_jwt_identity() == phone_no:
try:
group_id = request.json['group_id']
group_name = request.json['group_name']
created_at = request.json['created_at']
participants = request.json['participants']
# Instantiate kafka admin client to create a topic
client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id=phone_no)
topics = []
topics.append(
NewTopic(name=group_id, num_partitions=1,
replication_factor=1))
client.create_topics(new_topics=topics, validate_only=False)
group = Groups(group_id=group_id,
group_name=group_name,
created_at=created_at,
participants=participants)
group.save()
return make_response(
jsonify({'success': 'Group created successfully'}), 200)
except KeyError:
abort(400)
else:
abort(401)
def get(self, serviceName):
"""
Get Service State.
"""
app.logger.info(
"Request to get details for service {0}.".format(serviceName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
state_list = []
if serviceName.lower() == 'kafka':
for n in admin.describe_cluster()['brokers']:
h = n['host']
s = remote_execute(h, serviceName.lower(), 'get')
state_list.append({'host': h, 'state': s})
return state_list
except Exception as e:
ns_service.abort(500, str(e.args))
finally:
admin.close()
def create_topic(self):
""" Connects to cloudharness Events and creates a new topic
Return:
True if topic was created correctly, False otherwise.
"""
## Connect to kafka
admin_client = KafkaAdminClient(
bootstrap_servers=self._get_bootstrap_servers(),
client_id=self._get_client_id())
# ## Create topic
new_topic = NewTopic(name=self.topic_id,
num_partitions=1,
replication_factor=1)
try:
result = admin_client.create_topics(new_topics=[new_topic],
validate_only=False)
log.info(f"Created new topic {self.topic_id}")
return result
except TopicAlreadyExistsError as e:
# topic already exists "no worries", proceed
return True
except Exception as e:
log.error(f"Error creating the new Topics --> {e}", exc_info=True)
raise EventGeneralException from e
def main():
topic_name= sys.argv[1]
try:
num_partitions= int(sys.argv[2])
except:
num_partitions= 1
try:
replication_factor= int(sys.argv[3])
except:
replication_factor= 1
try:
bootstrap_servers=sys.argv[4]
logging.info('bootstrap_servers: ' + bootstrap_servers)
except:
logging.error('bootstrap_servers not specified')
os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers, client_id='test')
topic_list = []
topic_list.append(NewTopic(name= topic_name , num_partitions=num_partitions, replication_factor=replication_factor))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
logging.info('Topic {}, with num_partitions {} and replication_factor {} has been created'.format(topic_name,str(num_partitions),str(replication_factor)))
def put(self, topicName):
"""
Update Topic Configuration.
"""
ckey = request.json['key']
cvalue = request.json['value']
app.logger.info(
"Request to update configuration for topic {0} for key {1} and value {2}."
.format(topicName, ckey, cvalue))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
new_config = ConfigResource(ConfigResourceType.TOPIC, topicName,
{ckey: cvalue})
result = admin.alter_configs([new_config])
except UnknownTopicOrPartitionError as e:
api.abort(400, e.description)
except Exception as e:
ns_topic.abort(500, str(e.args))
finally:
admin.close()
if result.resources[0][0] == 0:
return {"configured": topicName}
else:
api.abort(400, "Bad Request(" + result.resources[0][1] + ")")
def get(self, topicName):
"""
Get Topic ACL.
"""
app.logger.info("Request to get ACL for topic {0}.".format(topicName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
acl_filter = ACLFilter(principal=None,
host="*",
operation=ACLOperation.ANY,
permission_type=ACLPermissionType.ANY,
resource_pattern=ResourcePattern(
ResourceType.TOPIC, topicName))
acls, error = admin.describe_acls(acl_filter)
acl_list = []
for a in acls:
princpl = a.principal
oprtn = a.operation.name
acl_list.append({'user': princpl, 'type': oprtn})
return acl_list
except Exception as e:
ns_acl.abort(500, str(e.args))
finally:
admin.close()
def delete(self, topicName):
"""
Delete Topic.
"""
app.logger.info(
"Request to delete topic witn name {0}.".format(topicName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
result = admin.delete_topics([topicName])
except UnknownTopicOrPartitionError as e:
api.abort(400, e.description)
except Exception as e:
api.abort(500, str(e.args))
finally:
admin.close()
app.logger.debug(result)
if result.topic_error_codes[0][1] == 0:
return {"deleted": topicName}
else:
api.abort(400, "Bad Request(Topic Deletion Failed)")
def init_admin_client(self):
try:
self.admin_client = KafkaAdminClient(
api_version_auto_timeout_ms=constants.
API_VERSION_AUTO_TIMEOUT_MS,
bootstrap_servers=self.config["bootstrap_uri"],
client_id=self.config["client_id"],
security_protocol=self.config["security_protocol"],
ssl_cafile=self.config["ssl_cafile"],
ssl_certfile=self.config["ssl_certfile"],
ssl_keyfile=self.config["ssl_keyfile"],
sasl_mechanism=self.config["sasl_mechanism"],
sasl_plain_username=self.config["sasl_plain_username"],
sasl_plain_password=self.config["sasl_plain_password"],
)
return True
except (NodeNotReadyError, NoBrokersAvailable, AssertionError):
self.log.warning(
"No Brokers available yet, retrying init_admin_client()")
time.sleep(2.0)
except: # pylint: disable=bare-except
self.log.exception(
"Failed to initialize admin client, retrying init_admin_client()"
)
time.sleep(2.0)
return False
def create_topic(name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id='test')
topic_list = []
topic_list.append(
NewTopic(name=str(name), num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
class AdminClient:
def __init__(self, record):
'''
record: not null, a list of input needed to create kafka topic
'''
self.topic_name = record[0]
self.replication_factor = record[1]
self.partition = record[2]
self.zoo_conn = record[3]
self.client = KafkaAdminClient(bootstrap_servers=zoo_conn)
def create_kafka_topic(self):
try:
self.client.creat_topics(NewTopic(
name=self.topic_name,
num_partitions=self.partition,
replication_factor=self.replication_factor),
validate_only=False)
except Exception as ex:
print("Error creating Kafka Topic \n")
print(str(ex))
def delete_kafka_topic(self, topic_name):
try:
self.client.delete_topics(NewTopic(name=topic_name))
print(topic_name + ' is successfully deleted')
except Exception as ex:
print('error deleting topic')
print(str(ex))
def run2():
admin_client = KafkaAdminClient(
bootstrap_servers="kafka:9092",
)
topic_list = []
topic_list.append(NewTopic(name="api_topic", num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
def _connect(self):
"""Connect to kafka admin client"""
try:
self._client = KafkaAdminClient(**self.kwargs)
logger.info('Connected to Kafka Admin client!')
except Exception as ex:
logger.error('Unable to connect to Kafka Admin client. %s',
str(ex))
def get_consumers(self, brokers):
from kafka.admin import KafkaAdminClient
adminClient = KafkaAdminClient(bootstrap_servers=brokers)
consumer_data = adminClient.list_consumer_groups()
consumers = []
for consumer in consumer_data:
consumers.append(consumer[0])
return consumers
def add_topic(username, title):
# Creates a new kafka topic, based on username and title
admin_client = KafkaAdminClient(bootstrap_servers='localhost:9092')
topic_lst = []
topic_name = "{}-{}".format(username, title)
topic_lst.append(
NewTopic(name=topic_name, num_partitions=1, replication_factor=1))
admin_client.create_topics(topic_lst)
def CreateKafkaTopic(topic_name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id='test')
topic_list = [
NewTopic(name=topic_name, num_partitions=1, replication_factor=1)
]
admin_client.create_topics(new_topics=topic_list)
def createKafkaTopic(topicName) :
try:
admin_client = KafkaAdminClient(bootstrap_servers=["localhost:9092"])
admin_client.create_topics(new_topics=topic_list, validate_only=False)
topic_list = []
topic_list.append(NewTopic(name=topicName, num_partitions=1, replication_factor=1))
except:
pass
def create_topic(self):
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=[settings.KAFKA['SERVER']])
topic = NewTopic(name=topic,
num_partitions=3,
topic_configs={'retention.ms': RETENTION_TIME},
replication_factor=1)
kafka_admin_client.create_topics([topic])
def lambda_handler(event, context):
responseData = {}
responseStatus = cfnresponse.SUCCESS
print("Request body is:", event)
if event['RequestType'] == 'Create':
bootstrap_uri = "Bootstrap servers not provided in env"
if 'BOOTSTRAP_SERVERS' in os.environ:
bootstrap_uri = os.environ['BOOTSTRAP_SERVERS']
print(bootstrap_uri)
else:
print(bootstrap_uri)
responseData[
'cause'] = "Bootstrap servers not mentioned in env variables"
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
try:
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_uri,
client_id='lambda')
except Exception as e:
responseData[
'status'] = "Failed to make KafkaAdmin Client, posssible reasons:bootstrap server name not resolvable, \
bootsrap servers not reacheable, MSK cluster not running"
print(e)
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if 'KafkaTopic' in event['ResourceProperties']:
topic_list = []
topic_list.append(
NewTopic(
name=event['ResourceProperties']['KafkaTopic']['name'],
num_partitions=int(event['ResourceProperties']
['KafkaTopic']['num_partitions']),
replication_factor=int(
event['ResourceProperties']['KafkaTopic']
['replication_factor'])))
try:
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
responseData['status'] = "Topic created successfully"
except Exception as e:
print("Failed to create topic:", e)
responseData['status'] = 'FAILED'
responseData['cause'] = e
cfnresponse.send(event, context, cfnresponse.FAILED,
responseData)
else:
responseData[
'cause'] = 'Failed to create topics, no KafkaTopic provided'
responseStatus = cfnresponse.FAILED
else:
#if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update':
responseData[
'cause'] = 'CloudFormation Delete and Update method not implemented, just return cnfresponse=SUCCSESS without any job/modifications'
responseStatus = cfnresponse.SUCCESS
cfnresponse.send(event, context, responseStatus, responseData)
