def test_process_timers(self):
start_timestamp = time()
expected_flushed_metrics_count = 2 + 5 # each timer has 5 separate metrics
metrics = (Counter('user.jump', 2), Set('username', 'navdoon'),
Timer('db.query', 300), Set('username', 'navdoon2'),
Counter('user.jump',
-1), Timer('db.query', 309), Timer('db.query', 303))
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
metrics_dict = dict()
for (name, value, timestamp) in destination.metrics:
metrics_dict[name] = value
self.assertGreaterEqual(timestamp, start_timestamp)
self.assertEqual(metrics_dict['user.jump'], 1)
self.assertEqual(metrics_dict['username'], 2)
self.assertEqual(metrics_dict['db.query.count'], 3)
self.assertEqual(metrics_dict['db.query.max'], 309)
self.assertEqual(metrics_dict['db.query.min'], 300)
self.assertEqual(metrics_dict['db.query.mean'], 304)
self.assertEqual(metrics_dict['db.query.median'], 303)
def test_process(self):
expected_flushed_metrics_count = 2
metrics = (
Counter('user.jump', 2),
Set('username', 'navdoon'),
Set('username', 'navdoon.test'),
Counter('user.jump', 4),
Set('username', 'navdoon'),
Counter('user.jump', -1),
)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.set_destinations([destination])
processor.init_destinations()
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
processor.wait_until_shutdown(5)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.jump', 5), destination.metrics[0][:2])
self.assertEqual(('username', 2), destination.metrics[1][:2])
def test_parse_set_metric_from_request(self):
self.assertEqual(
Set("host", '127.0.0.1'),
parse_metric_from_request("host:127.0.0.1|s")
)
self.assertEqual(
Set("user id?", '12345', 1),
parse_metric_from_request("user id?:12345|s|@1")
)
self.assertEqual(
Set("weird.set.with.rate", '25.7', 0.5),
parse_metric_from_request("weird.set.with.rate:25.7|s|@0.5")
)
def test_equality_check(self):
metric = Set('uid', 5)
with self.assertRaises(AssertionError):
metric != 'I am not a Set'
different_name = Set('userid', 5)
self.assertTrue(metric != different_name)
different_value = Set('uid', 25)
self.assertTrue(metric != different_value)
different_rate = Set('uid', 10, 0.5)
self.assertTrue(metric != different_rate)
same = Set('uid', 5)
self.assertTrue(same == metric)
def test_value_should_be_hashable(self):
self.assertRaises(AssertionError, Set, 'not_hashable', [])
set_ = Set('ok', 4)
set_.value = 2.0
self.assertEqual(set_.value, 2.0)
set_.value = 'something hashable'
self.assertEqual(set_.value, 'something hashable')
def test_clear_all_metrics(self):
shelf = StatsShelf()
shelf.add(Set("users", "me"))
shelf.add(Counter("mymetric", 3))
shelf.add(Timer("query", 4.12))
shelf.add(Gauge("cpu%", 38))
shelf.clear()
self.assertEqual(dict(), shelf.counters())
self.assertEqual(dict(), shelf.sets())
self.assertEqual(dict(), shelf.timers_data())
self.assertEqual(dict(), shelf.gauges())
def test_continues_processing_after_reload(self):
metrics = (Counter('user.login',
1), Set('username',
'navdoon'), Counter('user.login', 3))
queue_ = Queue()
destination = StubDestination()
destination.expected_count = 1
processor = QueueProcessor(queue_)
processor.flush_interval = 1
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
expected_flushed_metrics2 = 2
destination2 = StubDestination()
destination2.expected_count = expected_flushed_metrics2
processor.set_destinations([destination2])
for metric in metrics:
queue_.put(metric.to_request())
destination.wait_until_expected_count_items(5)
processor.shutdown()
process_thread.join(5)
self.assertFalse(processor.is_processing())
self.assertGreaterEqual(len(destination.metrics), 1)
self.assertLessEqual(queue_.qsize(), 2)
for metric in metrics:
queue_.put(metric.to_request())
self.assertGreaterEqual(queue_.qsize(), len(metrics))
resume_process_thread = Thread(target=processor.process)
resume_process_thread.start()
processor.wait_until_processing(5)
self.assertTrue(processor.is_processing())
self.assertEqual(processor.get_destinations(), [destination2])
destination2.wait_until_expected_count_items(5)
processor.shutdown()
resume_process_thread.join(5)
self.assertGreaterEqual(len(destination2.metrics),
expected_flushed_metrics2)
def test_sets(self):
shelf = StatsShelf()
self.assertEqual(dict(), shelf.sets())
shelf.add(Set("users", "me"))
shelf.add(Set("users", "me"))
shelf.add(Set("users", "you"))
shelf.add(Set("say.what?", "nothing"))
shelf.add(Set("users", "me"))
shelf.add(Set("say.what?", "nothing"))
shelf.add(Set("say.what?", "ok"))
expected = {"users": {"me", "you"}, "say.what?": {"nothing", "ok"}}
self.assertEqual(expected, shelf.sets())
sets = shelf.sets()
sets["sets should"] = set("not change")
self.assertEqual(expected, shelf.sets())
def test_process_stops_on_stop_token_in_queue(self):
token = 'STOP'
expected_flushed_metrics_count = 2
metrics = (
Counter('user.login', 1),
Set('username', 'navdoon'),
Counter('user.login', 3),
token,
Counter('user.login', -1),
Counter('user.logout', 1),
)
queue_ = Queue()
destination = StubDestination()
destination.expected_count = expected_flushed_metrics_count
processor = QueueProcessor(queue_)
processor.flush_interval = 2
processor.stop_process_token = token
processor.set_destinations([destination])
process_thread = Thread(target=processor.process)
process_thread.start()
processor.wait_until_processing(5)
for metric in metrics:
if metric is token:
sleep(processor.flush_interval
) # make sure one flush happened before token
request = metric
else:
request = metric.to_request()
queue_.put(request)
# make sure the processor has process the queue
processor.wait_until_shutdown(5)
self.assertFalse(processor.is_processing())
destination.wait_until_expected_count_items(10)
self.assertEqual(expected_flushed_metrics_count,
len(destination.metrics))
self.assertEqual(('user.login', 4), destination.metrics[0][:2])
self.assertEqual(('username', 1), destination.metrics[1][:2])
def test_constructor(self):
set_ = Set('unique', 5)
self.assertEqual(set_.name, 'unique')
self.assertEqual(set_.value, 5)