class BaseExporter(object): def __init__(self, **options): options = Options(**options) self.export_interval = options.export_interval self.max_batch_size = options.max_batch_size # TODO: queue should be moved to tracer # too much refactor work, leave to the next PR self._queue = Queue(capacity=options.queue_capacity) # TODO: worker should not be created in the base exporter self._worker = Worker(self._queue, self) self._worker.start() atexit.register(self._worker.stop, options.grace_period) # Ideally we don't want to have `emit` # Exporter will have one public method - `export`, which is a blocking # method, running inside worker threads. def emit(self, batch, event=None): raise NotImplementedError # pragma: NO COVER # TODO: we shouldn't have this at the beginning # Tracer should own the queue, exporter shouldn't even know if the # source is a queue or not. # Tracer puts span_data into the queue. # Worker gets span_data from the src (here is the queue) and feed into # the dst (exporter). # Exporter defines the MTU (max_batch_size) and export_interval. # There can be one worker for each queue, or multiple workers for each # queue, or shared workers among queues (e.g. queue for traces, queue # for logs). def export(self, items): self._queue.puts(items, block=False) # pragma: NO COVER
def test_gets(self): queue = Queue(capacity=10) queue.puts((1, 2, 3)) result = queue.gets(count=5, timeout=TIMEOUT) self.assertEqual(result, (1, 2, 3)) queue.puts((1, 2, 3, 4, 5)) result = queue.gets(count=3, timeout=TIMEOUT) self.assertEqual(result, (1, 2, 3)) result = queue.gets(count=3, timeout=TIMEOUT) self.assertEqual(result, (4, 5))
def test_gets_event(self): queue = Queue(capacity=10) event = QueueEvent('test') queue.puts((event, 1, 2, 3, event)) result = queue.gets(count=5, timeout=TIMEOUT) self.assertEqual(result, (event, )) result = queue.gets(count=5, timeout=TIMEOUT) self.assertEqual(result, (1, 2, 3, event)) task = PeriodicTask(TIMEOUT / 10, lambda: queue.put(1)) task.start() try: result = queue.gets(count=5, timeout=TIMEOUT) self.assertEqual(result, (1, 1, 1, 1, 1)) finally: task.cancel() task.join()
def test_flush_timeout(self): queue = Queue(capacity=10) self.assertEqual(queue.flush(timeout=TIMEOUT), 0) queue.put('test', timeout=TIMEOUT) self.assertIsNone(queue.flush(timeout=TIMEOUT)) queue.puts(range(100), timeout=TIMEOUT) self.assertIsNone(queue.flush(timeout=TIMEOUT)) def proc(): for item in queue.gets(count=1, timeout=TIMEOUT): if isinstance(item, QueueEvent): item.set() task = PeriodicTask(TIMEOUT / 10, proc) task.start() try: self.assertIsNotNone(queue.flush()) finally: task.cancel() task.join()
def test_puts_timeout(self): queue = Queue(capacity=10) queue.puts(range(100), timeout=TIMEOUT)