def create_topic(self, topic):
""" Creates a topic at Kafka.
Arguments:
topic (str): Name of the topic to create
"""
try:
admin_client = KafkaAdminClient(
bootstrap_servers=self.bootstrap_servers,
security_protocol=self.security_protocol,
ssl_cafile=self.ssl_cafile,
ssl_certfile=self.ssl_certfile,
ssl_keyfile=self.ssl_keyfile,
)
topic_list = []
topic_list.append(
NewTopic(topic, num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
admin_client.close()
except TopicAlreadyExistsError:
# we are fine if the Topic already exists
logger.info("Topic already exists")
pass
except Exception as ex:
logger.error("Could not create topic %s: %s", topic, str(ex))
raise Exception(ex)
def alternative_main(input_path, bootstrap_server, topic_name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:7092",
client_id='test')
topic_list = []
topic_list.append(
NewTopic(name="test", num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
#Starting session
spark = SparkSession.builder.appName('BigData1').getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
joined_raw = spark.read.parquet(input_path)
#print(joined_raw.show())
# Write key-value data from a DataFrame to a specific Kafka topic specified in an option
query = joined_raw.select(
F.col('time').cast("string").alias("key"),
F.to_json(F.struct("duration", "visibility")).alias("value"))
print(query.show())
query = joined_raw \
.select(F.col('time').cast("string").alias("key"), F.to_json(F.struct("duration", "visibility")).alias("value")) \
.write \
.format("kafka") \
.option("kafka.bootstrap.servers", bootstrap_server) \
.option("topic", "test") \
.save()
def __init__( self , kafka_endpoint , access_key , access_certificate , ca_certificate ):
self.kafka_endpoint = kafka_endpoint
self.kafka_consumer = KafkaConsumer(
bootstrap_servers = self.kafka_endpoint
, value_deserializer = lambda m: json.loads( m.decode('utf-8') )
, ssl_keyfile = access_key
, ssl_certfile = access_certificate
, ssl_cafile = ca_certificate
, security_protocol = "SSL"
, api_version_auto_timeout_ms = 10000
)
topics = self.kafka_consumer.topics()
if 'os_stats' not in topics:
print( "os_stats topic not found ... creating" )
admin_client = KafkaAdminClient(
bootstrap_servers = self.kafka_endpoint
, ssl_keyfile = access_key
, ssl_certfile = access_certificate
, ssl_cafile = ca_certificate
, security_protocol = "SSL"
, api_version_auto_timeout_ms = 10000
)
topic_list = []
topic_list.append(NewTopic( name = "os_stats" , num_partitions = 1 , replication_factor = 1 ) )
admin_client.create_topics( new_topics = topic_list , validate_only = False )
self.kafka_consumer.subscribe( 'os_stats' )
def main():
topic_name= sys.argv[1]
try:
num_partitions= int(sys.argv[2])
except:
num_partitions= 1
try:
replication_factor= int(sys.argv[3])
except:
replication_factor= 1
try:
bootstrap_servers=sys.argv[4]
logging.info('bootstrap_servers: ' + bootstrap_servers)
except:
logging.error('bootstrap_servers not specified')
os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_servers, client_id='test')
topic_list = []
topic_list.append(NewTopic(name= topic_name , num_partitions=num_partitions, replication_factor=replication_factor))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
logging.info('Topic {}, with num_partitions {} and replication_factor {} has been created'.format(topic_name,str(num_partitions),str(replication_factor)))
def create_kafka_topic():
logger.info("Request received - POST /create_kafka_topic")
if not request.is_json:
logger.warning("Format not valid")
return 'Format not valid', 400
try:
admin_client = KafkaAdminClient(bootstrap_servers=kafka_ip_port,
client_id='create_kafka_topic')
# Parse JSON
data = request.get_json()
logger.info("Data received: %s", data)
topic = data["topic"]
logger.info("Creating topic %s in Kafka", topic)
topic_list = []
topic_list.append(
NewTopic(name=topic, num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
admin_client.close()
except Exception as e:
logger.error("Error while parsing request")
logger.exception(e)
return str(e), 400
return '', 201
def ensure_topic(topic,
num_partitions,
replication_factor,
logger,
timeout_ms=3000,
brokers='localhost'):
adminclient = KafkaAdminClient(bootstrap_servers=brokers,
client_id='ansible-rm')
topic_list = []
topic_list.append(
NewTopic(name=topic, num_partitions=1, replication_factor=1))
try:
adminclient.create_topics(new_topics=topic_list, validate_only=False)
# adminclient.delete_topics(topic_list)
logger.info('kafka topic ' + topic + ' created')
except TopicAlreadyExistsError as e:
logger.info('kafka topic ' + topic + ' exists')
except Exception as e:
logger.error('error creating kafka topic ' + topic)
raise Exception(
'Unknown error code during creation of topic `{}`: {}'.format(
topic, str(e)))
def __init__(self, topicName):
admin_client = KafkaAdminClient(
bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'],
client_id='disco_eventproducer_admin')
try:
topic_list = [
NewTopic(name=topicName,
num_partitions=1,
replication_factor=2,
topic_configs={'retention.ms': 30758400000})
]
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
except Exception as e:
logging.warning(str(e))
pass
finally:
admin_client.close()
self.producer = KafkaProducer(
bootstrap_servers='localhost:9092',
value_serializer=lambda v: msgpack.packb(v, use_bin_type=True),
compression_type='snappy')
self.topicName = topicName
def create_topic(name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id='test')
topic_list = []
topic_list.append(
NewTopic(name=str(name), num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
def __enter__(self):
logger.debug("Start class OutputData!")
self.content_format = self.args.op_format.upper()
self.output_type = self.args.op_type.upper()
if self.output_type == "FILE":
self.fp = open(self.args.op_file, "w")
self.csv_list = []
elif self.output_type == "KAFKA":
# check topic exists
self.topic = self.args.op_topic
kafka_topic = NewTopic(name=self.topic,
num_partitions=1,
replication_factor=1)
client = KafkaAdminClient(bootstrap_servers=self.args.op_bootstrap)
try:
client.delete_topics([kafka_topic])
client.create_topics([kafka_topic])
except KafkaError:
logger.warn(
"delete or create kafka topic raised error, ignore it!")
self.producer = KafkaProducer(
bootstrap_servers=self.args.op_bootstrap)
elif self.output_type == "ES" or self.output_type == "ElasticSearch".upper(
):
self.es = Elasticsearch(
hosts=self.args.op_es_hosts,
sniff_on_start=True,
# sniff_on_connection_fail=True,
sniffer_timeout=20,
# http_auth=('user', 'secret')
)
self.es_index = self.args.op_index
return self
def handle(event, context):
if event.method == 'POST':
data = json.loads(event.body)
if "topic" not in data:
return {
"statusCode": 400,
"body": "Format not valid"
}
try:
topic = data["topic"]
admin_client = KafkaAdminClient(bootstrap_servers="kafka.deployment8:9092", client_id='create_kafka_topic')
topic_list = []
topic_list.append(NewTopic(name=topic, num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
admin_client.close()
except Exception as e:
return {
"statusCode": 400,
"body": "".format(e)
}
return {
"statusCode": 200,
"body": "OK"
}
else:
return {
"statusCode": 200,
"body": "No action for this endpoint"
}
def setup(topic_name):
# First, check if the topic already exists in kafka
kafka_client = KafkaClient(bootstrap_servers=KAFKA_SERVER,
api_version=(2, 5, 0))
future = kafka_client.cluster.request_update()
kafka_client.poll(future=future)
metadata = kafka_client.cluster
current_topics = metadata.topics()
kafka_client.close()
print('Active topics:', current_topics)
if topic_name not in current_topics:
print(f'Creating topic {topic_name}...')
kafka_admin_client = KafkaAdminClient(bootstrap_servers=KAFKA_SERVER,
api_version=(2, 5, 0))
topic_list = [
NewTopic(name=topic_name, num_partitions=1, replication_factor=1)
]
kafka_admin_client.create_topics(new_topics=topic_list,
validate_only=False)
kafka_admin_client.close()
else:
print(f'Topic {topic_name} exists')
def __init__(self, id: int, name: str):
if (id <= 0): raise ValueError("ID must be a positive integer")
super().__init__()
self._daemon: bool = True
self._id: int = id
self._name: str = name
self._status_up_time: datetime = datetime.now()
self._actual_status: Status = self.__set_status()
self._consumer = KafkaConsumer(f"get_status_{self.id}",
bootstrap_servers=["localhost:9092"])
self._reg_consumer = KafkaConsumer(
f"reg_response_{self.id}",
bootstrap_servers=["localhost:9092"],
consumer_timeout_ms=5000)
try:
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
admin_client.create_topics([
NewTopic(f"reg_response_{self.id}", 1, 1),
NewTopic(f"get_status_{self.id}", 1, 1)
])
except:
pass
self.__make_reg(id)
def create_group(phone_no):
if get_jwt_identity() == phone_no:
try:
group_id = request.json['group_id']
group_name = request.json['group_name']
created_at = request.json['created_at']
participants = request.json['participants']
# Instantiate kafka admin client to create a topic
client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id=phone_no)
topics = []
topics.append(
NewTopic(name=group_id, num_partitions=1,
replication_factor=1))
client.create_topics(new_topics=topics, validate_only=False)
group = Groups(group_id=group_id,
group_name=group_name,
created_at=created_at,
participants=participants)
group.save()
return make_response(
jsonify({'success': 'Group created successfully'}), 200)
except KeyError:
abort(400)
else:
abort(401)
def __init__(self,
threshold,
startTime,
endTime,
timeWindow,
probeData,
topicIn,
topicOut,
slideStep=3600):
self.threshold = threshold
self.startTime = startTime
self.endTime = endTime
self.timeWindow = timeWindow
# Slide the window by this amount
self.slideWindow = slideStep
# Ignore disconnection events followed by a reconnect within
# discoProbesWindow seconds
# Also report only probes disconnected discoProbesWindow
# seconds before/after the burst starting time
self.discoProbesWindow = 300
self.probeData = probeData
self.eventData = defaultdict(list)
self.numTotalProbes = {}
self.initNumProbes()
self.disconnectedProbes = {}
admin_client = KafkaAdminClient(
bootstrap_servers=['kafka1:9092', 'kafka2:9092', 'kafka3:9092'],
client_id='disco_disco_admin')
try:
topic_list = [
NewTopic(name=topicOut,
num_partitions=1,
replication_factor=2,
topic_configs={'retention.ms': 30758400000})
]
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
except Exception as e:
logging.warning(str(e))
pass
finally:
admin_client.close()
self.producer = KafkaProducer(
bootstrap_servers='localhost:9092',
value_serializer=lambda v: msgpack.packb(v, use_bin_type=True),
compression_type='snappy')
self.topicIn = topicIn
self.topicOut = topicOut
self.executor = ProcessPoolExecutor(max_workers=10)
def CreateKafkaTopic(topic_name):
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092",
client_id='test')
topic_list = [
NewTopic(name=topic_name, num_partitions=1, replication_factor=1)
]
admin_client.create_topics(new_topics=topic_list)
def run2():
admin_client = KafkaAdminClient(
bootstrap_servers="kafka:9092",
)
topic_list = []
topic_list.append(NewTopic(name="api_topic", num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
def create_topic(self):
kafka_admin_client = KafkaAdminClient(
bootstrap_servers=[settings.KAFKA['SERVER']])
topic = NewTopic(name=topic,
num_partitions=3,
topic_configs={'retention.ms': RETENTION_TIME},
replication_factor=1)
kafka_admin_client.create_topics([topic])
def add_topic(username, title):
# Creates a new kafka topic, based on username and title
admin_client = KafkaAdminClient(bootstrap_servers='localhost:9092')
topic_lst = []
topic_name = "{}-{}".format(username, title)
topic_lst.append(
NewTopic(name=topic_name, num_partitions=1, replication_factor=1))
admin_client.create_topics(topic_lst)
def createKafkaTopic(topicName) :
try:
admin_client = KafkaAdminClient(bootstrap_servers=["localhost:9092"])
admin_client.create_topics(new_topics=topic_list, validate_only=False)
topic_list = []
topic_list.append(NewTopic(name=topicName, num_partitions=1, replication_factor=1))
except:
pass
def lambda_handler(event, context):
responseData = {}
responseStatus = cfnresponse.SUCCESS
print("Request body is:", event)
if event['RequestType'] == 'Create':
bootstrap_uri = "Bootstrap servers not provided in env"
if 'BOOTSTRAP_SERVERS' in os.environ:
bootstrap_uri = os.environ['BOOTSTRAP_SERVERS']
print(bootstrap_uri)
else:
print(bootstrap_uri)
responseData[
'cause'] = "Bootstrap servers not mentioned in env variables"
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
try:
admin_client = KafkaAdminClient(bootstrap_servers=bootstrap_uri,
client_id='lambda')
except Exception as e:
responseData[
'status'] = "Failed to make KafkaAdmin Client, posssible reasons:bootstrap server name not resolvable, \
bootsrap servers not reacheable, MSK cluster not running"
print(e)
cfnresponse.send(event, context, cfnresponse.FAILED, responseData)
if 'KafkaTopic' in event['ResourceProperties']:
topic_list = []
topic_list.append(
NewTopic(
name=event['ResourceProperties']['KafkaTopic']['name'],
num_partitions=int(event['ResourceProperties']
['KafkaTopic']['num_partitions']),
replication_factor=int(
event['ResourceProperties']['KafkaTopic']
['replication_factor'])))
try:
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
responseData['status'] = "Topic created successfully"
except Exception as e:
print("Failed to create topic:", e)
responseData['status'] = 'FAILED'
responseData['cause'] = e
cfnresponse.send(event, context, cfnresponse.FAILED,
responseData)
else:
responseData[
'cause'] = 'Failed to create topics, no KafkaTopic provided'
responseStatus = cfnresponse.FAILED
else:
#if event['RequestType'] == 'Delete' or event['RequestType'] == 'Update':
responseData[
'cause'] = 'CloudFormation Delete and Update method not implemented, just return cnfresponse=SUCCSESS without any job/modifications'
responseStatus = cfnresponse.SUCCESS
cfnresponse.send(event, context, responseStatus, responseData)
class Kafka:
def __init__(self, bootstrap_servers):
self.bootstrap_servers = bootstrap_servers
self.client = KafkaAdminClient(bootstrap_servers=bootstrap_servers)
self.producer = KafkaProducer(bootstrap_servers=bootstrap_servers)
def create_topics(self, new_topics):
'''
create a list of topics
:param topic_name: a list of tuple including 'topic_name', 'partition_number', 'replica_factor'
:param num_partitions:
:param replica_factor:
:return:
'''
topics = []
for item in new_topics:
topic = NewTopic(item[0], item[1], item[2])
topics.append(topic)
self.client.create_topics(topics)
def delete_topics(self, topic_names):
'''
delete a list of topics
:param topic_names: a list of topics
:type list
:return:
'''
self.client.delete_topics(topic_names)
def send(self, topic_name, msg):
'''
produce message to this topic
:param topic_name: topic name
:param msg: message string
:type str
:return:
'''
self.producer.send(topic_name, msg.encode('utf-8'))
def get_consumer(self, topic_names, group_id=None):
'''
a list of topics for subscription
:param topic_names:
:return: topic to list of records since the last fetch for
subscribed list of topics and partitions
:type dict
'''
consumer = KafkaConsumer(bootstrap_servers=self.bootstrap_servers,
group_id=group_id)
consumer.subscribe(topic_names)
return consumer
def pause(self, consumer, topic, partition):
consumer.pause(TopicPartition(topic=topic, partition=partition))
def resume(self, consumer, topic, partition):
consumer.resume(TopicPartition(topic=topic, partition=partition))
def _create_topic(self):
if self.name in self._cluster.kafka.consumer().topics():
logger.warning('Asked to create a topic that already exists: %s. Deleting the topic first.', self.name)
self.delete()
logger.debug('Creating new topic %s', self.name)
topic = NewTopic(name=self.name, num_partitions=self._dataset.num_partitions, replication_factor=1)
admin_client = KafkaAdminClient(bootstrap_servers=self._cluster.kafka.brokers, request_timeout_ms=30000)
admin_client.create_topics(new_topics=[topic], timeout_ms=60000)
self.topic = topic
def __init__(self, topic):
admin_client = KafkaAdminClient(
bootstrap_servers = g_KAFKA_HOST,
client_id = "test"
)
topic_list = []
topic_list.append( NewTopic(name = topic, num_partitions = 1, replication_factor = 1) )
admin_client.create_topics( new_topics = topic_list, validate_only = False )
def init_front_server():
GE_metaData.drop_front_services()
front_dic = {
'SERVICE_NAME': gDefine.FRONT_SERVER_SERVICE_NAME,
'IP': gDefine.FRONT_SERVER_ENDPOINT_IP,
'PORT': gDefine.FRONT_SERVER_ENDPOINT_PORT
}
redis_dic = {
'SERVICE_NAME': gDefine.REDIS_SERVICE_NAME,
'IP': gDefine.REDIS_ENDPOINT_IP,
'PORT': gDefine.REDIS_ENDPOINT_PORT
}
kafka_dic = {
'SERVICE_NAME': gDefine.KAFKA_SERVICE_NAME,
'IP': gDefine.KAFKA_ENDPOINT_IP,
'PORT': gDefine.KAFKA_ENDPOINT_PORT
}
# write front server information to mongo_db
GE_metaData.set_front_services(front_dic)
# write redis service information to mongo_db
GE_metaData.set_front_services(redis_dic)
# write kafka service information to mongo_db
GE_metaData.set_front_services(kafka_dic)
# create kafka topic(GLOBAL)
admin_client = KafkaAdminClient(bootstrap_servers=gDefine.KAFKA_SERVER_URL,
client_id='test')
# check topic exist
'''-----------------------
try :
topic_list = []
topic_list.append(gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME)
admin_client.delete_topics(topic_list, timeout_ms=3*1000)
print('topic is deleted :', gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME)
except:
print('topic is not exist:', gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME)
time.sleep(5)
----------------'''
try:
topic_list = []
print('1')
topic_list.append(
NewTopic(name=gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME,
num_partitions=1,
replication_factor=1))
print('2')
admin_client.create_topics(new_topics=topic_list, validate_only=False)
print('3')
print('topic is created:', gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME)
except:
print('topic is exist', gDefine.GLOBAL_SCHEDULER_GLOBAL_TOPIC_NAME)
def check_topics(self, topic):
try:
admin_client = KafkaAdminClient(bootstrap_servers="192.168.1.28:9092", client_id='test')
topic_list = []
topic_list.append(NewTopic(name=topic, num_partitions=1, replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
print("topic creato: {}".format(topic))
except:
print("topic esistente: {}".format(topic))
return
def __init__(self):
super().__init__()
self._daemon: bool = True
self._status_consumer = KafkaConsumer("status", bootstrap_servers=["localhost:9092"], consumer_timeout_ms=5000)
self._reg_consumer = KafkaConsumer("reg", bootstrap_servers=["localhost:9092"])
admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092")
try:
admin_client.create_topics([NewTopic(f"status", 1, 1), NewTopic(f"reg", 1, 1)])
except:
pass
def create_topic(self, topic):
try:
admin_client = KafkaAdminClient(
bootstrap_servers=self.bootstrap_servers, )
topic_list = [(NewTopic(name=topic,
num_partitions=1,
replication_factor=1))]
admin_client.create_topics(new_topics=topic_list,
validate_only=False)
except TopicAlreadyExistsError:
pass # ignore
def create_topic(id):
filepath = "configuration/kafka_config.json"
kafka_ip, kafka_port = read_json_kafka(filepath)
admin_client = KafkaAdminClient(bootstrap_servers="{}:{}".format(
kafka_ip, kafka_port),
client_id='Aviral')
topic_list = []
topic_list.append(NewTopic(name=id, num_partitions=1,
replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
def __create_topics_if_missing(self, topic_names):
admin_client = KafkaAdminClient(bootstrap_servers=self.kafka_brokers,
client_id='test')
for topic in topic_names:
try:
new_topic = NewTopic(name=topic,
num_partitions=1,
replication_factor=1)
admin_client.create_topics(new_topics=[new_topic],
validate_only=False)
except TopicAlreadyExistsError:
print('Topic: {} already exists!')
def create_topic(self):
client = SimpleClient(self.BOOTSTRAP_SERVER)
broker_topics = client.topic_partitions
admin_client = KafkaAdminClient(bootstrap_servers=self.BOOTSTRAP_SERVER, client_id='test')
if self.TOPIC_NAME and self.TOPIC_NAME not in broker_topics:
topic_list = [NewTopic(name=self.TOPIC_NAME, num_partitions=self.NUM_PARTITIONS,
replication_factor=self.REPLICATION_FACTOR)]
try:
admin_client.create_topics(new_topics=topic_list, validate_only=False)
except Exception:
raise Exception('Unable to create topic')
elif self.TOPIC_NAME and self.TOPIC_NAME in broker_topics:
print('Topic already created')