def __init__(self, alias=None, consumer=None, buffer_size=0, number_of_consumer=1, skip_on_error=False, inbound_counter=None, outbound_counter=None, consumer_exception_handler=None, **kwargs): self._alias = alias self._logger = _get_logger(__name__) self._buffer_size = buffer_size self._consumer = consumer self._number_of_consumer = number_of_consumer self._active_consumer_counter = AtomicCounter() self._skip_on_error = skip_on_error self._inbound_counter = inbound_counter if inbound_counter is not None else AtomicCounter() self._outbound_counter = outbound_counter if outbound_counter is not None else AtomicCounter() self._inbound = Queue(self._buffer_size) self._outbound = None self._consumer_exception_handler = consumer_exception_handler self._additional_properties = kwargs
class Pipe(object): def __init__(self, alias=None, consumer=None, buffer_size=0, number_of_consumer=1, skip_on_error=False, inbound_counter=None, outbound_counter=None, consumer_exception_handler=None, **kwargs): self._alias = alias self._logger = _get_logger(__name__) self._buffer_size = buffer_size self._consumer = consumer self._number_of_consumer = number_of_consumer self._active_consumer_counter = AtomicCounter() self._skip_on_error = skip_on_error self._inbound_counter = inbound_counter if inbound_counter is not None else AtomicCounter() self._outbound_counter = outbound_counter if outbound_counter is not None else AtomicCounter() self._inbound = Queue(self._buffer_size) self._outbound = None self._consumer_exception_handler = consumer_exception_handler self._additional_properties = kwargs def open(self, pipeline_running_status=None): with self._active_consumer_counter.lock: self._active_consumer_counter.increase() self.info("open consumer, %s of %s consumer(s)", self._active_consumer_counter.value, self._number_of_consumer) try: map( lambda message: self._downstream(message), self._read_consume_yield(self._read_from_stream) ) except Exception as e: self._handle_exception(exception=e, pipeline_running_status=pipeline_running_status) with self._active_consumer_counter.lock: self._active_consumer_counter.decrease() self.info("close consumer, %s consumer(s) remaining", self._active_consumer_counter.value) def _read_from_stream(self): message = self._inbound.get() self.debug("<< %s", message) if Pipeline.is_end_of_stream(message): self.info("<< %s", message) with self._active_consumer_counter.lock: if self._active_consumer_counter.value > 1: # re-product end of stream signal for other sibling pipe processes self._inbound.put(message) else: self._inbound_counter.increase() return message def _downstream(self, message=None): """ pass messages to the next pipe, notice that if and only if when this is the last consumer of a pipe, it streams end of stream signal to next pipe. :param message: data processed in this pipe """ if not Pipeline.is_end_of_stream(message): self._outbound_counter.increase() if self._outbound is None: return if Pipeline.is_end_of_stream(message): # if and only if current pipe process is the last one remaining, send end-of-stream signal downstream. with self._active_consumer_counter.lock: if self._active_consumer_counter.value <= 1: self._outbound.put(message) self.info(">> %s", message) else: self._outbound.put(message) self.debug(">> %s", message) def _read_consume_yield(self, func_read_from_upstream): return [] def _handle_consumer_exception(self, consumer_exception, message): if self._consumer_exception_handler is None: return False try: self._consumer_exception_handler(consumer_exception, message) return True except Exception as e: self.warn("failed to invoke a consumer exception handler with a consumer exception. see cause -> %s", e.message) return False def _handle_exception(self, exception=None, pipeline_running_status=None): with pipeline_running_status.get_lock(): if pipeline_running_status.value == Pipeline.RUNNING_STATUS_INTERRUPTED: return else: pipeline_running_status.value = Pipeline.RUNNING_STATUS_INTERRUPTED self.error("when processing data stream on pipeline, an unexpected exception has occurred. " "This will cause this pipeline to stop. see cause -> %s\n%s", exception, traceback.format_exc()) def debug(self, message, *args, **kwargs): self._log(logging.DEBUG, message, *args, **kwargs) def info(self, message, *args, **kwargs): self._log(logging.INFO, message, *args, **kwargs) def warn(self, message, *args, **kwargs): self._log(logging.WARNING, message, *args, **kwargs) def error(self, message, *args, **kwargs): self._log(logging.ERROR, message, *args, **kwargs) def _log(self, level, message, *args, **kwargs): self._logger.log(level, message, *args, **kwargs) @property def alias(self): return self._alias @property def inbound(self): return self._inbound @property def outbound(self): return self._outbound @outbound.setter def outbound(self, outbound): self._outbound = outbound @property def number_of_consumer(self): return self._number_of_consumer @property def skip_on_error(self): return self._skip_on_error @property def additional_properties(self): return self._additional_properties def inbound_count(self): return self._inbound_counter.value def outbound_count(self): return self._outbound_counter.value