def _run_process(self, retries_limit=3, sleep_timeout=1): # run _send_upstream process with the queue stop_event = threading.Event() retry_options = RetryOptions(limit=retries_limit, backoff_ms=50, retry_on_timeouts=False) self.thread = threading.Thread( target=_send_upstream, args=(self.queue, self.client, CODEC_NONE, 0.3, # batch time (seconds) 3, # batch length Producer.ACK_AFTER_LOCAL_WRITE, Producer.DEFAULT_ACK_TIMEOUT, retry_options, stop_event)) self.thread.daemon = True self.thread.start() time.sleep(sleep_timeout) stop_event.set()
def __init__( self, client, req_acks=ACK_AFTER_LOCAL_WRITE, ack_timeout=DEFAULT_ACK_TIMEOUT, codec=None, codec_compresslevel=None, sync_fail_on_error=SYNC_FAIL_ON_ERROR_DEFAULT, async_send=False, batch_send=False, # deprecated, use async_send batch_send_every_n=BATCH_SEND_MSG_COUNT, batch_send_every_t=BATCH_SEND_DEFAULT_INTERVAL, async_retry_limit=ASYNC_RETRY_LIMIT, async_retry_backoff_ms=ASYNC_RETRY_BACKOFF_MS, async_retry_on_timeouts=ASYNC_RETRY_ON_TIMEOUTS, async_queue_maxsize=ASYNC_QUEUE_MAXSIZE, async_queue_put_timeout=ASYNC_QUEUE_PUT_TIMEOUT, async_log_messages_on_error=ASYNC_LOG_MESSAGES_ON_ERROR, async_stop_timeout=ASYNC_STOP_TIMEOUT_SECS, **kwargs): # async renamed async_send for python3.7 support if 'async' in kwargs: log.warning('Deprecated async option found -- use async_send') async_send = kwargs['async'] if async_send: assert batch_send_every_n > 0 assert batch_send_every_t > 0 assert async_queue_maxsize >= 0 self.client = client self.async_send = async_send self.req_acks = req_acks self.ack_timeout = ack_timeout self.stopped = False if codec is None: codec = CODEC_NONE elif codec not in ALL_CODECS: raise UnsupportedCodecError("Codec 0x%02x unsupported" % (codec, )) self.codec = codec self.codec_compresslevel = codec_compresslevel if self.async_send: # Messages are sent through this queue self.queue = Queue(async_queue_maxsize) self.async_queue_put_timeout = async_queue_put_timeout async_retry_options = RetryOptions( limit=async_retry_limit, backoff_ms=async_retry_backoff_ms, retry_on_timeouts=async_retry_on_timeouts) self.thread_stop_event = Event() self.thread = Thread(target=_send_upstream, args=(self.queue, self.client.copy(), self.codec, batch_send_every_t, batch_send_every_n, self.req_acks, self.ack_timeout, async_retry_options, self.thread_stop_event), kwargs={ 'log_messages_on_error': async_log_messages_on_error, 'stop_timeout': async_stop_timeout, 'codec_compresslevel': self.codec_compresslevel }) # Thread will die if main thread exits self.thread.daemon = True self.thread.start() def cleanup(obj): if not obj.stopped: obj.stop() self._cleanup_func = cleanup atexit.register(cleanup, self) else: self.sync_fail_on_error = sync_fail_on_error