def __init__(self, alias=None,
consumer=None,
buffer_size=0,
number_of_consumer=1,
skip_on_error=False,
inbound_counter=None,
outbound_counter=None,
consumer_exception_handler=None,
**kwargs):
self._alias = alias
self._logger = _get_logger(__name__)
self._buffer_size = buffer_size
self._consumer = consumer
self._number_of_consumer = number_of_consumer
self._active_consumer_counter = AtomicCounter()
self._skip_on_error = skip_on_error
self._inbound_counter = inbound_counter if inbound_counter is not None else AtomicCounter()
self._outbound_counter = outbound_counter if outbound_counter is not None else AtomicCounter()
self._inbound = Queue(self._buffer_size)
self._outbound = None
self._consumer_exception_handler = consumer_exception_handler
self._additional_properties = kwargs
class Pipe(object):
def __init__(self, alias=None,
consumer=None,
buffer_size=0,
number_of_consumer=1,
skip_on_error=False,
inbound_counter=None,
outbound_counter=None,
consumer_exception_handler=None,
**kwargs):
self._alias = alias
self._logger = _get_logger(__name__)
self._buffer_size = buffer_size
self._consumer = consumer
self._number_of_consumer = number_of_consumer
self._active_consumer_counter = AtomicCounter()
self._skip_on_error = skip_on_error
self._inbound_counter = inbound_counter if inbound_counter is not None else AtomicCounter()
self._outbound_counter = outbound_counter if outbound_counter is not None else AtomicCounter()
self._inbound = Queue(self._buffer_size)
self._outbound = None
self._consumer_exception_handler = consumer_exception_handler
self._additional_properties = kwargs
def open(self, pipeline_running_status=None):
with self._active_consumer_counter.lock:
self._active_consumer_counter.increase()
self.info("open consumer, %s of %s consumer(s)", self._active_consumer_counter.value,
self._number_of_consumer)
try:
map(
lambda message: self._downstream(message),
self._read_consume_yield(self._read_from_stream)
)
except Exception as e:
self._handle_exception(exception=e, pipeline_running_status=pipeline_running_status)
with self._active_consumer_counter.lock:
self._active_consumer_counter.decrease()
self.info("close consumer, %s consumer(s) remaining", self._active_consumer_counter.value)
def _read_from_stream(self):
message = self._inbound.get()
self.debug("<< %s", message)
if Pipeline.is_end_of_stream(message):
self.info("<< %s", message)
with self._active_consumer_counter.lock:
if self._active_consumer_counter.value > 1:
# re-product end of stream signal for other sibling pipe processes
self._inbound.put(message)
else:
self._inbound_counter.increase()
return message
def _downstream(self, message=None):
"""
pass messages to the next pipe,
notice that if and only if when this is the last consumer of a pipe, it streams end of stream signal to next pipe.
:param message: data processed in this pipe
"""
if not Pipeline.is_end_of_stream(message):
self._outbound_counter.increase()
if self._outbound is None:
return
if Pipeline.is_end_of_stream(message):
# if and only if current pipe process is the last one remaining, send end-of-stream signal downstream.
with self._active_consumer_counter.lock:
if self._active_consumer_counter.value <= 1:
self._outbound.put(message)
self.info(">> %s", message)
else:
self._outbound.put(message)
self.debug(">> %s", message)
def _read_consume_yield(self, func_read_from_upstream):
return []
def _handle_consumer_exception(self, consumer_exception, message):
if self._consumer_exception_handler is None:
return False
try:
self._consumer_exception_handler(consumer_exception, message)
return True
except Exception as e:
self.warn("failed to invoke a consumer exception handler with a consumer exception. see cause -> %s", e.message)
return False
def _handle_exception(self, exception=None, pipeline_running_status=None):
with pipeline_running_status.get_lock():
if pipeline_running_status.value == Pipeline.RUNNING_STATUS_INTERRUPTED:
return
else:
pipeline_running_status.value = Pipeline.RUNNING_STATUS_INTERRUPTED
self.error("when processing data stream on pipeline, an unexpected exception has occurred. "
"This will cause this pipeline to stop. see cause -> %s\n%s",
exception,
traceback.format_exc())
def debug(self, message, *args, **kwargs):
self._log(logging.DEBUG, message, *args, **kwargs)
def info(self, message, *args, **kwargs):
self._log(logging.INFO, message, *args, **kwargs)
def warn(self, message, *args, **kwargs):
self._log(logging.WARNING, message, *args, **kwargs)
def error(self, message, *args, **kwargs):
self._log(logging.ERROR, message, *args, **kwargs)
def _log(self, level, message, *args, **kwargs):
self._logger.log(level, message, *args, **kwargs)
@property
def alias(self):
return self._alias
@property
def inbound(self):
return self._inbound
@property
def outbound(self):
return self._outbound
@outbound.setter
def outbound(self, outbound):
self._outbound = outbound
@property
def number_of_consumer(self):
return self._number_of_consumer
@property
def skip_on_error(self):
return self._skip_on_error
@property
def additional_properties(self):
return self._additional_properties
def inbound_count(self):
return self._inbound_counter.value
def outbound_count(self):
return self._outbound_counter.value
