def __init__(self, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
self._closed = False
self._flushes_in_progress = AtomicInteger()
self._appends_in_progress = AtomicInteger()
self._batches = collections.defaultdict(
collections.deque) # TopicPartition: [ProducerBatch]
self._tp_locks = {
None: threading.Lock()
} # TopicPartition: Lock, plus a lock to add entries
self._free = SimpleBufferPool(
self.config["buffer_memory"],
self.config["batch_size"],
metrics=self.config["metrics"],
metric_group_prefix=self.config["metric_group_prefix"],
)
self._incomplete = IncompleteProducerBatches()
# The following variables should only be accessed by the sender thread,
# so we don't need to protect them w/ locking.
self.muted = set()
self._drain_index = 0
def test_buffer_pool():
pool = SimpleBufferPool(1000, 1000)
buf1 = pool.allocate(1000, 1000)
message = ''.join(map(str, range(100)))
buf1.write(message.encode('utf-8'))
pool.deallocate(buf1)
buf2 = pool.allocate(1000, 1000)
assert buf2.read() == b''