def partitions_for(self, topic):
"""
All fonction documentation available on http://kafka-python.readthedocs.io.
"""
tpl = templates.kafka_ops(method=PARTITIONS_FOR, topic=topic)
self.logSentEvent(shortEvt="req partitions_for",
tplEvt=self.encapsule(self.producerTpl, tpl))
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(PARTITIONS_FOR),
'topic': topic
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
partitions = self.producer.partitions_for(topic)
tpl = templates.kafka_ops(method=PARTITIONS_FOR,
topic=topic,
partitions=partitions)
self.logRecvEvent(shortEvt="resp partitions_for",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=PARTITIONS_FOR))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def close(self, timeout=None):
"""
All fonction documentation available on http://kafka-python.readthedocs.io.
"""
tpl = templates.kafka_ops(method=CLOSE, timeout=timeout)
self.logSentEvent(shortEvt="req close",
tplEvt=self.encapsule(self.producerTpl, tpl))
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(CLOSE),
'timeout': timeout
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
self.producer.close(timeout=timeout)
tpl = templates.kafka_ops(method=CLOSE, timeout=timeout)
self.logRecvEvent(shortEvt="closed",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=CLOSE))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def receivedNotifyFromAgent(self, data):
"""
Function to reimplement
"""
if 'cmd' in data:
if data['cmd'] == AGENT_INITIALIZED:
tpl = TestTemplatesLib.TemplateMessage()
layer = TestTemplatesLib.TemplateLayer('AGENT')
layer.addKey("ready", True)
layer.addKey(name='name', data=self.cfg['agent']['name'])
layer.addKey(name='type', data=self.cfg['agent']['type'])
tpl.addLayer(layer=layer)
self.logRecvEvent(shortEvt="Agent Is Ready", tplEvt=tpl)
elif data['cmd'] == "producer_{0}".format(CONNECT):
self.__kafka_connected = True
tpl = templates.kafka_ops(
method=CONNECT, bootstrap_servers=self.bootstrap_servers)
self.logRecvEvent(shortEvt="connected",
tplEvt=self.encapsule(self.producerTpl, tpl))
elif data['cmd'] == "producer_{0}".format(SEND):
record_metadata = data['result']
self.__kafka_send = True
rec = {
"Topic": record_metadata[0],
"Partition": record_metadata[1],
"Offset": record_metadata[3],
"Timestamp": record_metadata[4],
"Checksum": record_metadata[5],
"Serialized_key_size": record_metadata[6],
"Serialized_value_size": record_metadata[7]
}
tpl = templates.kafka_ops(method=SEND, more=rec)
self.logRecvEvent(shortEvt="sended",
tplEvt=self.encapsule(self.producerTpl, tpl))
elif data['cmd'] == "producer_{0}".format(FLUSH):
tpl = templates.kafka_ops(method=FLUSH)
self.logRecvEvent(shortEvt="flushed",
tplEvt=self.encapsule(self.producerTpl, tpl))
elif data['cmd'] == "producer_{0}".format(PARTITIONS_FOR):
partitions = data['result']
tpl = templates.kafka_ops(method=PARTITIONS_FOR,
partitions=partitions)
self.logRecvEvent(shortEvt="partitions_for",
tplEvt=self.encapsule(self.producerTpl, tpl))
elif data['cmd'] == "producer_{0}".format(CLOSE):
tpl = templates.kafka_ops(method=CLOSE)
self.logRecvEvent(shortEvt="closed",
tplEvt=self.encapsule(self.producerTpl, tpl))
else:
self.warning('Notify received from agent: %s' % data)
def send(self, topic, **kargs):
"""
Publish a message to a topic.
@topic (str): topic where the message will be published
@value (optional): message value as bytes.
@partition (int, optional): optionally specify a partition. If not set, the partition will be selected using the configured 'partitioner'.
@key (optional): a key to associate with the message. Can be used to determine which partition to send the message to.
@timestamp_ms (int, optional): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp. Defaults to current time.
"""
tpl = templates.kafka_ops(method=SEND, **kargs)
self.logSentEvent(shortEvt="req send",
tplEvt=self.encapsule(self.producerTpl, tpl))
# Timeout for record metadata retreving
if "timeout" in kargs:
timeout = kargs.pop("timeout")
else:
timeout = 2
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(SEND),
'topic': topic,
'timeout': timeout,
'kargs': kargs
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
future = self.producer.send(topic, **kargs)
record_metadata = future.get(timeout=timeout)
rec = {
"Topic": record_metadata[0],
"Partition": record_metadata[1],
"Offset": record_metadata[3],
"Timestamp": record_metadata[4],
"Checksum": record_metadata[5],
"Serialized_key_size": record_metadata[6],
"Serialized_value_size": record_metadata[7]
}
tpl = templates.kafka_ops(method=SEND, more=rec)
self.logRecvEvent(shortEvt="resp send",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl, templates.response_err(msg=e,
method=SEND))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def isSend(self, timeout=2, record=None):
"""
Wait to receive response from "send" request and match returned RecordMetadata until the end of the timeout.
@param timeout: time max to wait to receive event in second (default=2s)
@type timeout: float
@param offset: Optional RecordMetadata that we expect to be assigned to consumer
@type offset: RecordMetadata
"""
TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout)
if record == None:
record = { "Topic":TestOperatorsLib.Any(),
"Partition": TestOperatorsLib.Any(),
"Offset":TestOperatorsLib.Any() ,
"Timestamp":TestOperatorsLib.Any() ,
"Checksum": TestOperatorsLib.Any(),
"Serialized_key_size":TestOperatorsLib.Any(),
"Serialized_value_size": TestOperatorsLib.Any()}
expected = templates.kafka_ops(method=SEND, more=record)
# try to match the template
evt = self.received( expected=self.encapsule( self.producerTpl ,expected ), timeout=timeout )
return evt
def connect(self, **kargs):
"""
Instantiate the KafkaProducer and Fetch Kafka Cluster Metadata
@param kargs: keyword arguments from KafkaProducer class:
@type kargs: keyword
"""
if 'bootstrap_servers' in kargs:
bootstrap_servers = kargs.pop('bootstrap_servers')
else:
bootstrap_servers = self.bootstrap_servers
# Log start connexion event
self.producerTpl = templates.kafka_connect(
api=PRODUCER, bootstrap_servers=bootstrap_servers, **kargs)
tpl = templates.kafka_ops(method=CONNECT,
bootstrap_servers=bootstrap_servers,
**kargs)
self.logSentEvent(shortEvt="connection",
tplEvt=self.encapsule(self.producerTpl, tpl))
self.__kafka_connected = False
# Agent mode
if self.cfg['agent-support']:
remote_cfg = {
'cmd': "producer_{0}".format(CONNECT),
'bootstrap_servers': bootstrap_servers,
'kargs': kargs
}
self.sendNotifyToAgent(data=remote_cfg)
else:
try:
self.producer = KafkaProducer(
bootstrap_servers=bootstrap_servers, **kargs)
tpl = templates.kafka_ops(method=CONNECT,
bootstrap_servers=bootstrap_servers,
**kargs)
self.logRecvEvent(shortEvt="connected",
tplEvt=self.encapsule(self.producerTpl, tpl))
except KafkaError as e:
tpl = self.encapsule(
self.producerTpl,
templates.response_err(msg=e, method=CONNECT))
self.logRecvEvent(shortEvt="response error", tplEvt=tpl)
def isClose(self, timeout=2): """ Wait to receive response from "close" request until the end of the timeout @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float """ TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout) # construct the expected template expected = templates.kafka_ops(method=CLOSE) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt
def isClose(self, timeout=2): """ Wait to receive response from "close" request until the end of the timeout @param timeout: time max to wait to receive event in second (default=2s) @type timeout: float """ if not ( isinstance(timeout, int) or isinstance(timeout, float) ) or isinstance(timeout,bool): raise TestAdapterLib.ValueException(TestAdapterLib.caller(), "timeout argument is not a float or integer (%s)" % type(timeout) ) # construct the expected template expected = templates.kafka_ops(method=CLOSE) # try to match the template evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout ) return evt
def isPartitions_for(self, timeout=2,partitions=None):
"""
Wait to receive response from "partitions_for" request and match returned Topics until the end of the timeout.
@param timeout: time max to wait to receive event in second (default=2s)
@type timeout: float
@param offset: Optional list that we expect to be view by producer
@type offset: list of of Topics
"""
if not ( isinstance(timeout, int) or isinstance(timeout, float) ) or isinstance(timeout,bool):
raise TestAdapterLib.ValueException(TestAdapterLib.caller(), "timeout argument is not a float or integer (%s)" % type(timeout) )
if partitions == None:
partitions= { "partitions":TestOperatorsLib.Any()}
expected = templates.kafka_ops(method=PARTITIONS_FOR,more=partitions)
# try to match the template
evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout )
return evt
def isPartitions_for(self, timeout=2,partitions=None):
"""
Wait to receive response from "partitions_for" request and match returned Topics until the end of the timeout.
@param timeout: time max to wait to receive event in second (default=2s)
@type timeout: float
@param offset: Optional list that we expect to be view by producer
@type offset: list of of Topics
"""
TestAdapterLib.check_timeout(caller=TestAdapterLib.caller(), timeout=timeout)
if partitions == None:
partitions= { "partitions":TestOperatorsLib.Any()}
expected = templates.kafka_ops(method=PARTITIONS_FOR,more=partitions)
# try to match the template
evt = self.received( expected=self.encapsule( self.producerTpl ,expected), timeout=timeout )
return evt
