def test_process_stops_on_stop_token_in_queue(self):
token = 'STOP'
expected_flushed_metrics_count = 2
metrics = (Counter('user.login', 1),
Set('username', 'navdoon'),
Counter('user.login', 3),
token,
Counter('user.login', -1),
Counter('user.logout', 1),)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.flush_interval = 2
processor.stop_process_token = token
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
if metric is token:
sleep(processor.flush_interval) # make sure one flush happened before token
request = metric
else:
request = metric.to_request()
queue_.put(request)
# make sure the processor has process the queue
processor.wait_until_shutdown(5)
self.assertFalse(processor.is_processing())
destination.wait_until_expected_count_items(10)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.login', 4), destination.metrics[0][:2])
self.assertEqual(('username', 1), destination.metrics[1][:2])
def test_process(self):
expected_flushed_metrics_count = 2
metrics = (Counter('user.jump', 2),
Set('username', 'navdoon'),
Set('username', 'navdoon.test'),
Counter('user.jump', 4),
Set('username', 'navdoon'),
Counter('user.jump', -1),)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
processor.init_destinations()
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.jump', 5), destination.metrics[0][:2])
self.assertEqual(('username', 2), destination.metrics[1][:2])
def test_process_timers(self):
start_timestamp = time()
expected_flushed_metrics_count = 2 + 5 # each timer has 5 separate metrics
metrics = (Counter('user.jump', 2), Set('username', 'navdoon'),
Timer('db.query', 300), Set('username', 'navdoon2'),
Counter('user.jump',
-1), Timer('db.query', 309), Timer('db.query', 303))
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
metrics_dict = dict()
for (name, value, timestamp) in destination.metrics:
metrics_dict[name] = value
self.assertGreaterEqual(timestamp, start_timestamp)
self.assertEqual(metrics_dict['user.jump'], 1)
self.assertEqual(metrics_dict['username'], 2)
self.assertEqual(metrics_dict['db.query.count'], 3)
self.assertEqual(metrics_dict['db.query.max'], 309)
self.assertEqual(metrics_dict['db.query.min'], 300)
self.assertEqual(metrics_dict['db.query.mean'], 304)
self.assertEqual(metrics_dict['db.query.median'], 303)
def test_process(self):
expected_flushed_metrics_count = 2
metrics = (
Counter('user.jump', 2),
Set('username', 'navdoon'),
Set('username', 'navdoon.test'),
Counter('user.jump', 4),
Set('username', 'navdoon'),
Counter('user.jump', -1),
)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
processor.init_destinations()
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.jump', 5), destination.metrics[0][:2])
self.assertEqual(('username', 2), destination.metrics[1][:2])
def test_queue_can_only_change_when_not_processing(self):
processor = QueueProcessor(Queue())
orig_queue = processor.queue
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
def set_queue(new_queue):
processor.queue = new_queue
new_queue = Queue()
self.assertRaises(Exception, set_queue, new_queue)
self.assertEqual(orig_queue, processor.queue)
processor.shutdown()
processor.wait_until_shutdown(5)
processor.queue = new_queue
self.assertEqual(new_queue, processor.queue)
def test_queue_can_only_change_when_not_processing(self):
processor = QueueProcessor(Queue())
orig_queue = processor.queue
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
def set_queue(new_queue):
processor.queue = new_queue
new_queue = Queue()
self.assertRaises(Exception, set_queue, new_queue)
self.assertEqual(orig_queue, processor.queue)
processor.shutdown()
processor.wait_until_shutdown(5)
processor.queue = new_queue
self.assertEqual(new_queue, processor.queue)
def test_process_timers(self):
start_timestamp = time()
expected_flushed_metrics_count = 2 + 5 # each timer has 5 separate metrics
metrics = (Counter('user.jump', 2),
Set('username', 'navdoon'),
Timer('db.query', 300),
Set('username', 'navdoon2'),
Counter('user.jump', -1),
Timer('db.query', 309),
Timer('db.query', 303)
)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
metrics_dict = dict()
for (name, value, timestamp) in destination.metrics:
metrics_dict[name] = value
self.assertGreaterEqual(timestamp, start_timestamp)
self.assertEqual(metrics_dict['user.jump'], 1)
self.assertEqual(metrics_dict['username'], 2)
self.assertEqual(metrics_dict['db.query.count'], 3)
self.assertEqual(metrics_dict['db.query.max'], 309)
self.assertEqual(metrics_dict['db.query.min'], 300)
self.assertEqual(metrics_dict['db.query.mean'], 304)
self.assertEqual(metrics_dict['db.query.median'], 303)
def test_process_stops_on_stop_token_in_queue(self):
token = 'STOP'
expected_flushed_metrics_count = 2
metrics = (
Counter('user.login', 1),
Set('username', 'navdoon'),
Counter('user.login', 3),
token,
Counter('user.login', -1),
Counter('user.logout', 1),
)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.flush_interval = 2
processor.stop_process_token = token
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
if metric is token:
sleep(processor.flush_interval
) # make sure one flush happened before token
request = metric
else:
request = metric.to_request()
queue_.put(request)
# make sure the processor has process the queue
processor.wait_until_shutdown(5)
self.assertFalse(processor.is_processing())
destination.wait_until_expected_count_items(10)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.login', 4), destination.metrics[0][:2])
self.assertEqual(('username', 1), destination.metrics[1][:2])
