def close(self, timeout=None): """ All fonction documentation available on http://kafka-python.readthedocs.io. """ tpl = templates.kafka_ops(method=CLOSE, timeout=timeout) self.logSentEvent(shortEvt="req close", tplEvt=self.encapsule(self.producerTpl, tpl)) if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(CLOSE), 'timeout': timeout } self.sendNotifyToAgent(data=remote_cfg) else: try: self.producer.close(timeout=timeout) tpl = templates.kafka_ops(method=CLOSE, timeout=timeout) self.logRecvEvent(shortEvt="closed", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=CLOSE)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def partitions_for(self, topic): """ All fonction documentation available on http://kafka-python.readthedocs.io. """ tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic) self.logSentEvent(shortEvt="req partitions_for", tplEvt=self.encapsule(self.producerTpl, tpl)) if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(PARTITIONS_FOR), 'topic': topic } self.sendNotifyToAgent(data=remote_cfg) else: try: partitions = self.producer.partitions_for(topic) tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic, partitions=partitions) self.logRecvEvent(shortEvt="resp partitions_for", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=PARTITIONS_FOR)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def send(self, topic, **kargs): """ Publish a message to a topic. @topic (str): topic where the message will be published @value (optional): message value as bytes. @partition (int, optional): optionally specify a partition. If not set, the partition will be selected using the configured 'partitioner'. @key (optional): a key to associate with the message. Can be used to determine which partition to send the message to. @timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp. Defaults to current time. """ tpl = templates.kafka_ops(method=SEND, **kargs) self.logSentEvent(shortEvt="req send", tplEvt=self.encapsule(self.producerTpl, tpl)) # Timeout for record metadata retreving if "timeout" in kargs: timeout = kargs.pop("timeout") else: timeout = 2 if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(SEND), 'topic': topic, 'timeout': timeout, 'kargs': kargs } self.sendNotifyToAgent(data=remote_cfg) else: try: future = self.producer.send(topic, **kargs) record_metadata = future.get(timeout=timeout) rec = { "Topic": record_metadata[0], "Partition": record_metadata[1], "Offset": record_metadata[3], "Timestamp": record_metadata[4], "Checksum": record_metadata[5], "Serialized_key_size": record_metadata[6], "Serialized_value_size": record_metadata[7] } tpl = templates.kafka_ops(method=SEND, more=rec) self.logRecvEvent(shortEvt="resp send", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=SEND)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def receivedErrorFromAgent(self, data): """ Function to reimplement """ if "cmd" in data: if data['cmd'] in [ CONNECT, CLOSE, SEND, FLUSH,PARTITIONS_FOR ]: tpl = self.encapsule(self.producerTpl, templates.response_err(msg=data['err-msg'], method=data['cmd'] )) self.logRecvEvent( shortEvt = "response error", tplEvt = tpl ) else: self.error("unknown command received: %s" % data["cmd"]) else: self.error( 'Generic error: %s' % data )
def connect(self, **kargs): """ Instantiate the KafkaProducer and Fetch Kafka Cluster Metadata @param kargs: keyword arguments from KafkaProducer class: @type kargs: keyword """ if 'bootstrap_servers' in kargs: bootstrap_servers = kargs.pop('bootstrap_servers') else: bootstrap_servers = self.bootstrap_servers # Log start connexion event self.producerTpl = templates.kafka_connect( api=PRODUCER, bootstrap_servers=bootstrap_servers, **kargs) tpl = templates.kafka_ops(method=CONNECT, bootstrap_servers=bootstrap_servers, **kargs) self.logSentEvent(shortEvt="connection", tplEvt=self.encapsule(self.producerTpl, tpl)) self.__kafka_connected = False # Agent mode if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(CONNECT), 'bootstrap_servers': bootstrap_servers, 'kargs': kargs } self.sendNotifyToAgent(data=remote_cfg) else: try: self.producer = KafkaProducer( bootstrap_servers=bootstrap_servers, **kargs) tpl = templates.kafka_ops(method=CONNECT, bootstrap_servers=bootstrap_servers, **kargs) self.logRecvEvent(shortEvt="connected", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=CONNECT)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)