def _register_consumer(self):
from pycompss.streams.components.objects.kafka_connectors import ODSConsumer # noqa: E501
if self.consumer is None:
if self.bootstrap_server is None:
self.bootstrap_server = ObjectDistroStream._request_bootstrap_server_info() # noqa: E501
logger.info("Creating internal consumer...")
self.consumer = ODSConsumer(self.bootstrap_server,
self.kafka_topic_name,
self.access_mode)
class ObjectDistroStream(DistroStreamImpl):
"""
Object Distributed Stream implementation.
Attributes:
- kafka_topic_name: Registered topic name on the Kafka backend
+ type: string
- bootstrap_server: Bootstrap server information
+ type: string
- publisher: Internal Kafka connector for publish
+ type: ODSPublisher
- consumer: Internal Kafka connector for consume
+ type: ODSConsumer
"""
TOPIC_REGULAR_MESSAGES_PREFIX = "regular-messages"
TOPIC_SYSTEM_MESSAGES = "system-messages"
DEFAULT_KAFKA_TIMEOUT = 200 # ms
def __init__(self, alias=None, access_mode=AT_MOST_ONCE):
"""
Creates a new ObjectDistroStream instance.
:param alias: Stream alias.
+ type: string
:param access_mode: Stream access mode.
+ type: ConsumerMode
:raise RegistrationException: When client cannot register the stream
into the server.
"""
super(ObjectDistroStream, self).__init__(alias=alias,
stream_type=OBJECT,
access_mode=access_mode,
internal_stream_info=[])
if alias is None:
self.kafka_topic_name = ObjectDistroStream.TOPIC_REGULAR_MESSAGES_PREFIX + "-" + self.id # noqa: E501
else:
self.kafka_topic_name = alias
self.bootstrap_server = None
self.publisher = None
self.consumer = None
def _register_publisher(self):
from pycompss.streams.components.objects.kafka_connectors import ODSPublisher # noqa: E501
if self.publisher is None:
if self.bootstrap_server is None:
self.bootstrap_server = ObjectDistroStream._request_bootstrap_server_info(
) # noqa: E501
logger.info("Creating internal producer...")
self.publisher = ODSPublisher(self.bootstrap_server)
def _register_consumer(self):
from pycompss.streams.components.objects.kafka_connectors import ODSConsumer # noqa: E501
if self.consumer is None:
if self.bootstrap_server is None:
self.bootstrap_server = ObjectDistroStream._request_bootstrap_server_info(
) # noqa: E501
logger.info("Creating internal consumer...")
self.consumer = ODSConsumer(self.bootstrap_server,
self.kafka_topic_name,
self.access_mode)
@staticmethod
def _request_bootstrap_server_info():
logger.info("Requesting bootstrap server...")
req = BootstrapServerRequest()
DistroStreamClientHandler.request(req)
# Retrieve answer
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise BackendException(error, req.get_error_msg())
# Parse answer
answer = req.get_response_msg()
if __debug__:
logger.debug("Retrieved bootstrap server information: " + answer)
return answer
def publish(self, message):
logger.info("Publishing new object...")
self._register_publisher()
self.publisher.publish(self.kafka_topic_name, message)
logger.info("Publishing new object")
def publish_list(self, messages):
logger.info("Publishing new List of objects...")
self._register_publisher()
for msg in messages:
self.publisher.publish(self.kafka_topic_name, msg)
logger.info("Published new List of objects")
def poll(self, timeout=DEFAULT_KAFKA_TIMEOUT):
logger.info("Polling new stream items...")
self._register_consumer()
return self.consumer.poll(timeout)