class BaseExporter(object):
def __init__(self, **options):
options = Options(**options)
self.export_interval = options.export_interval
self.max_batch_size = options.max_batch_size
# TODO: queue should be moved to tracer
# too much refactor work, leave to the next PR
self._queue = Queue(capacity=options.queue_capacity)
# TODO: worker should not be created in the base exporter
self._worker = Worker(self._queue, self)
self._worker.start()
atexit.register(self._worker.stop, options.grace_period)
# Ideally we don't want to have `emit`
# Exporter will have one public method - `export`, which is a blocking
# method, running inside worker threads.
def emit(self, batch, event=None):
raise NotImplementedError # pragma: NO COVER
# TODO: we shouldn't have this at the beginning
# Tracer should own the queue, exporter shouldn't even know if the
# source is a queue or not.
# Tracer puts span_data into the queue.
# Worker gets span_data from the src (here is the queue) and feed into
# the dst (exporter).
# Exporter defines the MTU (max_batch_size) and export_interval.
# There can be one worker for each queue, or multiple workers for each
# queue, or shared workers among queues (e.g. queue for traces, queue
# for logs).
def export(self, items):
self._queue.puts(items, block=False) # pragma: NO COVER
def test_gets(self):
queue = Queue(capacity=10)
queue.puts((1, 2, 3))
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (1, 2, 3))
queue.puts((1, 2, 3, 4, 5))
result = queue.gets(count=3, timeout=TIMEOUT)
self.assertEqual(result, (1, 2, 3))
result = queue.gets(count=3, timeout=TIMEOUT)
self.assertEqual(result, (4, 5))
def test_gets_event(self):
queue = Queue(capacity=10)
event = QueueEvent('test')
queue.puts((event, 1, 2, 3, event))
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (event, ))
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (1, 2, 3, event))
task = PeriodicTask(TIMEOUT / 10, lambda: queue.put(1))
task.start()
try:
result = queue.gets(count=5, timeout=TIMEOUT)
self.assertEqual(result, (1, 1, 1, 1, 1))
finally:
task.cancel()
task.join()
def test_flush_timeout(self):
queue = Queue(capacity=10)
self.assertEqual(queue.flush(timeout=TIMEOUT), 0)
queue.put('test', timeout=TIMEOUT)
self.assertIsNone(queue.flush(timeout=TIMEOUT))
queue.puts(range(100), timeout=TIMEOUT)
self.assertIsNone(queue.flush(timeout=TIMEOUT))
def proc():
for item in queue.gets(count=1, timeout=TIMEOUT):
if isinstance(item, QueueEvent):
item.set()
task = PeriodicTask(TIMEOUT / 10, proc)
task.start()
try:
self.assertIsNotNone(queue.flush())
finally:
task.cancel()
task.join()
def test_puts_timeout(self):
queue = Queue(capacity=10)
queue.puts(range(100), timeout=TIMEOUT)