def partitions_for(self, topic): """ All fonction documentation available on http://kafka-python.readthedocs.io. """ tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic) self.logSentEvent(shortEvt="req partitions_for", tplEvt=self.encapsule(self.producerTpl, tpl)) if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(PARTITIONS_FOR), 'topic': topic } self.sendNotifyToAgent(data=remote_cfg) else: try: partitions = self.producer.partitions_for(topic) tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic, partitions=partitions) self.logRecvEvent(shortEvt="resp partitions_for", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=PARTITIONS_FOR)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def close(self, timeout=None): """ All fonction documentation available on http://kafka-python.readthedocs.io. """ tpl = templates.kafka_ops(method=CLOSE, timeout=timeout) self.logSentEvent(shortEvt="req close", tplEvt=self.encapsule(self.producerTpl, tpl)) if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(CLOSE), 'timeout': timeout } self.sendNotifyToAgent(data=remote_cfg) else: try: self.producer.close(timeout=timeout) tpl = templates.kafka_ops(method=CLOSE, timeout=timeout) self.logRecvEvent(shortEvt="closed", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=CLOSE)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def receivedNotifyFromAgent(self, data): """ Function to reimplement """ if 'cmd' in data: if data['cmd'] == AGENT_INITIALIZED: tpl = TestTemplatesLib.TemplateMessage() layer = TestTemplatesLib.TemplateLayer('AGENT') layer.addKey("ready", True) layer.addKey(name='name', data=self.cfg['agent']['name']) layer.addKey(name='type', data=self.cfg['agent']['type']) tpl.addLayer(layer=layer) self.logRecvEvent(shortEvt="Agent Is Ready", tplEvt=tpl) elif data['cmd'] == "producer_{0}".format(CONNECT): self.__kafka_connected = True tpl = templates.kafka_ops( method=CONNECT, bootstrap_servers=self.bootstrap_servers) self.logRecvEvent(shortEvt="connected", tplEvt=self.encapsule(self.producerTpl, tpl)) elif data['cmd'] == "producer_{0}".format(SEND): record_metadata = data['result'] self.__kafka_send = True rec = { "Topic": record_metadata[0], "Partition": record_metadata[1], "Offset": record_metadata[3], "Timestamp": record_metadata[4], "Checksum": record_metadata[5], "Serialized_key_size": record_metadata[6], "Serialized_value_size": record_metadata[7] } tpl = templates.kafka_ops(method=SEND, more=rec) self.logRecvEvent(shortEvt="sended", tplEvt=self.encapsule(self.producerTpl, tpl)) elif data['cmd'] == "producer_{0}".format(FLUSH): tpl = templates.kafka_ops(method=FLUSH) self.logRecvEvent(shortEvt="flushed", tplEvt=self.encapsule(self.producerTpl, tpl)) elif data['cmd'] == "producer_{0}".format(PARTITIONS_FOR): partitions = data['result'] tpl = templates.kafka_ops(method=PARTITIONS_FOR, partitions=partitions) self.logRecvEvent(shortEvt="partitions_for", tplEvt=self.encapsule(self.producerTpl, tpl)) elif data['cmd'] == "producer_{0}".format(CLOSE): tpl = templates.kafka_ops(method=CLOSE) self.logRecvEvent(shortEvt="closed", tplEvt=self.encapsule(self.producerTpl, tpl)) else: self.warning('Notify received from agent: %s' % data)
def send(self, topic, **kargs): """ Publish a message to a topic. @topic (str): topic where the message will be published @value (optional): message value as bytes. @partition (int, optional): optionally specify a partition. If not set, the partition will be selected using the configured 'partitioner'. @key (optional): a key to associate with the message. Can be used to determine which partition to send the message to. @timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp. Defaults to current time. """ tpl = templates.kafka_ops(method=SEND, **kargs) self.logSentEvent(shortEvt="req send", tplEvt=self.encapsule(self.producerTpl, tpl)) # Timeout for record metadata retreving if "timeout" in kargs: timeout = kargs.pop("timeout") else: timeout = 2 if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(SEND), 'topic': topic, 'timeout': timeout, 'kargs': kargs } self.sendNotifyToAgent(data=remote_cfg) else: try: future = self.producer.send(topic, **kargs) record_metadata = future.get(timeout=timeout) rec = { "Topic": record_metadata[0], "Partition": record_metadata[1], "Offset": record_metadata[3], "Timestamp": record_metadata[4], "Checksum": record_metadata[5], "Serialized_key_size": record_metadata[6], "Serialized_value_size": record_metadata[7] } tpl = templates.kafka_ops(method=SEND, more=rec) self.logRecvEvent(shortEvt="resp send", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=SEND)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def isSend(self, timeout=2, record=None): """ Wait to receive response from "send" request and match returned RecordMetadata until the end of the timeout. @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float @param offset: Optional RecordMetadata that we expect to be assigned to consumer @type offset: RecordMetadata """ TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout) if record == None: record = { "Topic":TestOperatorsLib.Any(), "Partition": TestOperatorsLib.Any(), "Offset":TestOperatorsLib.Any() , "Timestamp":TestOperatorsLib.Any() , "Checksum": TestOperatorsLib.Any(), "Serialized_key_size":TestOperatorsLib.Any(), "Serialized_value_size": TestOperatorsLib.Any()} expected = templates.kafka_ops(method=SEND, more=record) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected ), timeout=timeout ) return evt
def connect(self, **kargs): """ Instantiate the KafkaProducer and Fetch Kafka Cluster Metadata @param kargs: keyword arguments from KafkaProducer class: @type kargs: keyword """ if 'bootstrap_servers' in kargs: bootstrap_servers = kargs.pop('bootstrap_servers') else: bootstrap_servers = self.bootstrap_servers # Log start connexion event self.producerTpl = templates.kafka_connect( api=PRODUCER, bootstrap_servers=bootstrap_servers, **kargs) tpl = templates.kafka_ops(method=CONNECT, bootstrap_servers=bootstrap_servers, **kargs) self.logSentEvent(shortEvt="connection", tplEvt=self.encapsule(self.producerTpl, tpl)) self.__kafka_connected = False # Agent mode if self.cfg['agent-support']: remote_cfg = { 'cmd': "producer_{0}".format(CONNECT), 'bootstrap_servers': bootstrap_servers, 'kargs': kargs } self.sendNotifyToAgent(data=remote_cfg) else: try: self.producer = KafkaProducer( bootstrap_servers=bootstrap_servers, **kargs) tpl = templates.kafka_ops(method=CONNECT, bootstrap_servers=bootstrap_servers, **kargs) self.logRecvEvent(shortEvt="connected", tplEvt=self.encapsule(self.producerTpl, tpl)) except KafkaError as e: tpl = self.encapsule( self.producerTpl, templates.response_err(msg=e, method=CONNECT)) self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def isClose(self, timeout=2): """ Wait to receive response from "close" request until the end of the timeout @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float """ TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout) # construct the expected template expected = templates.kafka_ops(method=CLOSE) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt
def isClose(self, timeout=2): """ Wait to receive response from "close" request until the end of the timeout @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float """ if not ( isinstance(timeout, int) or isinstance(timeout, float) ) or isinstance(timeout,bool): raise TestAdapterLib.ValueException(TestAdapterLib.caller(), "timeout argument is not a float or integer (%s)" % type(timeout) ) # construct the expected template expected = templates.kafka_ops(method=CLOSE) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt
def isPartitions_for(self, timeout=2,partitions=None): """ Wait to receive response from "partitions_for" request and match returned Topics until the end of the timeout. @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float @param offset: Optional list that we expect to be view by producer @type offset: list of of Topics """ if not ( isinstance(timeout, int) or isinstance(timeout, float) ) or isinstance(timeout,bool): raise TestAdapterLib.ValueException(TestAdapterLib.caller(), "timeout argument is not a float or integer (%s)" % type(timeout) ) if partitions == None: partitions= { "partitions":TestOperatorsLib.Any()} expected = templates.kafka_ops(method=PARTITIONS_FOR,more=partitions) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt
def isPartitions_for(self, timeout=2,partitions=None): """ Wait to receive response from "partitions_for" request and match returned Topics until the end of the timeout. @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float @param offset: Optional list that we expect to be view by producer @type offset: list of of Topics """ TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout) if partitions == None: partitions= { "partitions":TestOperatorsLib.Any()} expected = templates.kafka_ops(method=PARTITIONS_FOR,more=partitions) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt