def close(self, timeout=None):
"""
All fonction documentation available on http://kafka-python.readthedocs.io.
"""
tpl = templates.kafka_ops(method=CLOSE, timeout=timeout)
self.logSentEvent(shortEvt="req close",
tplEvt=self.encapsule(self.producerTpl, tpl))
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(CLOSE),
'timeout': timeout
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
self.producer.close(timeout=timeout)
tpl = templates.kafka_ops(method=CLOSE, timeout=timeout)
self.logRecvEvent(shortEvt="closed",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=CLOSE))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def partitions_for(self, topic):
"""
All fonction documentation available on http://kafka-python.readthedocs.io.
"""
tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic)
self.logSentEvent(shortEvt="req partitions_for",
tplEvt=self.encapsule(self.producerTpl, tpl))
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(PARTITIONS_FOR),
'topic': topic
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
partitions = self.producer.partitions_for(topic)
tpl = templates.kafka_ops(method=PARTITIONS_FOR,
topic=topic,
partitions=partitions)
self.logRecvEvent(shortEvt="resp partitions_for",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=PARTITIONS_FOR))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def send(self, topic, **kargs):
"""
Publish a message to a topic.
@topic (str): topic where the message will be published
@value (optional): message value as bytes.
@partition (int, optional): optionally specify a partition. If not set, the partition will be selected using the configured 'partitioner'.
@key (optional): a key to associate with the message. Can be used to determine which partition to send the message to.
@timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp. Defaults to current time.
"""
tpl = templates.kafka_ops(method=SEND, **kargs)
self.logSentEvent(shortEvt="req send",
tplEvt=self.encapsule(self.producerTpl, tpl))
# Timeout for record metadata retreving
if "timeout" in kargs:
timeout = kargs.pop("timeout")
else:
timeout = 2
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(SEND),
'topic': topic,
'timeout': timeout,
'kargs': kargs
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
future = self.producer.send(topic, **kargs)
record_metadata = future.get(timeout=timeout)
rec = {
"Topic": record_metadata[0],
"Partition": record_metadata[1],
"Offset": record_metadata[3],
"Timestamp": record_metadata[4],
"Checksum": record_metadata[5],
"Serialized_key_size": record_metadata[6],
"Serialized_value_size": record_metadata[7]
}
tpl = templates.kafka_ops(method=SEND, more=rec)
self.logRecvEvent(shortEvt="resp send",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl, templates.response_err(msg=e,
method=SEND))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def receivedErrorFromAgent(self, data):
"""
Function to reimplement
"""
if "cmd" in data:
if data['cmd'] in [ CONNECT, CLOSE, SEND, FLUSH,PARTITIONS_FOR ]:
tpl = self.encapsule(self.producerTpl, templates.response_err(msg=data['err-msg'], method=data['cmd'] ))
self.logRecvEvent( shortEvt = "response error", tplEvt = tpl )
else:
self.error("unknown command received: %s" % data["cmd"])
else:
self.error( 'Generic error: %s' % data )
def connect(self, **kargs):
"""
Instantiate the KafkaProducer and Fetch Kafka Cluster Metadata
@param kargs: keyword arguments from KafkaProducer class:
@type kargs: keyword
"""
if 'bootstrap_servers' in kargs:
bootstrap_servers = kargs.pop('bootstrap_servers')
else:
bootstrap_servers = self.bootstrap_servers
# Log start connexion event
self.producerTpl = templates.kafka_connect(
api=PRODUCER, bootstrap_servers=bootstrap_servers, **kargs)
tpl = templates.kafka_ops(method=CONNECT,
bootstrap_servers=bootstrap_servers,
**kargs)
self.logSentEvent(shortEvt="connection",
tplEvt=self.encapsule(self.producerTpl, tpl))
self.__kafka_connected = False
# Agent mode
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(CONNECT),
'bootstrap_servers': bootstrap_servers,
'kargs': kargs
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
self.producer = KafkaProducer(
bootstrap_servers=bootstrap_servers, **kargs)
tpl = templates.kafka_ops(method=CONNECT,
bootstrap_servers=bootstrap_servers,
**kargs)
self.logRecvEvent(shortEvt="connected",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=CONNECT))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
