def test_publish_cleanup_noref(self):
r = Registry()
with r.start():
id = r.counter('test').meterId
r.counter('test').increment()
self.assertTrue(id in r._meters)
r._publish()
self.assertFalse(id in r._meters)
def test_stopwatch(self):
clock = ManualClock()
r = Registry(clock=clock)
t = PercentileTimer(r, "test", min=0, max=100)
with t.stopwatch():
clock.set_monotonic_time(1.0)
self._check_percentiles(t, 0)
def test_do_post_no_logging(self):
r = Registry()
client = HttpClient(r)
client.post_json(self._uri,
'{"status": 429}',
retry_delay=0,
disable_logging=False)
def test_add_metrics(self):
test_context = TestContext()
registry = Registry()
reporter = test_context.get_workload_manager()
reporter.set_registry(registry, {})
workload = get_test_workload(uuid.uuid4(), 2, STATIC)
reporter.add_workload(workload)
reporter.report_metrics({})
self.assertTrue(gauge_value_equals(registry, RUNNING, 1))
self.assertTrue(counter_value_equals(registry, ADDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, REMOVED_KEY, 0))
self.assertTrue(counter_value_equals(registry, SUCCEEDED_KEY, 1))
self.assertTrue(counter_value_equals(registry, FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 1))
self.assertTrue(gauge_value_equals(registry, PACKAGE_VIOLATIONS_KEY,
0))
self.assertTrue(gauge_value_equals(registry, CORE_VIOLATIONS_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
workload.get_thread_count()))
expected_unallocated_size = len(test_context.get_cpu().get_threads()
) - workload.get_thread_count()
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY,
expected_unallocated_size))
def test_unknown_workload_type_label(self):
registry = Registry()
test_context = TestContext()
unknown_event = get_event(
CONTAINER, CREATE, uuid.uuid4(), {
NAME: "container-name",
APP_NAME_LABEL_KEY: DEFAULT_TEST_APP_NAME,
CPU_LABEL_KEY: "1",
MEM_LABEL_KEY: str(DEFAULT_TEST_MEM),
DISK_LABEL_KEY: str(DEFAULT_TEST_DISK),
NETWORK_LABEL_KEY: str(DEFAULT_TEST_NETWORK),
JOB_TYPE_LABEL_KEY: DEFAULT_TEST_JOB_TYPE,
WORKLOAD_TYPE_LABEL_KEY: "unknown",
OWNER_EMAIL_LABEL_KEY: DEFAULT_TEST_OWNER_EMAIL,
IMAGE_LABEL_KEY: DEFAULT_TEST_IMAGE,
})
valid_event = get_container_create_event(1)
event_iterable = MockEventProvider([unknown_event, valid_event])
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: manager.get_error_count() == 1)
wait_until(lambda: manager.get_processed_count() == 2)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_SUCCEEDED_KEY, 5))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 1))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 2))
def test_absent_workload_type_label(self):
registry = Registry()
test_context = TestContext()
name = str(uuid.uuid4())
unknown_event = get_event(CONTAINER, CREATE, name, {
CPU_LABEL_KEY: "1",
NAME: name
})
event_handlers = test_context.get_event_handlers()
event_iterable = MockEventProvider([unknown_event])
manager = EventManager(event_iterable, event_handlers,
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: test_context.get_create_event_handler().
get_ignored_event_count() == 1)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_SUCCEEDED_KEY,
len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 1))
def test_empty_api_methods(self):
r = Registry(
config=SidecarConfig({"sidecar.output-location": "memory"}))
t = PercentileDistributionSummary(r, "test")
self.assertEqual(0, t.count())
self.assertEqual(0, t.total_amount())
self.assertEqual(0, t.percentile(1))
def test_thread_allocation_computation(self):
for allocator in [IntegerProgramCpuAllocator(), GreedyCpuAllocator()]:
static_thread_count = 2
burst_thread_count = 4
w_static = get_test_workload("s", static_thread_count, STATIC)
w_burst = get_test_workload("b", burst_thread_count, BURST)
cgroup_manager = MockCgroupManager()
registry = Registry()
workload_manager = WorkloadManager(get_cpu(), cgroup_manager,
allocator)
workload_manager.set_registry(registry, {})
workload_manager.add_workload(w_static)
workload_manager.add_workload(w_burst)
workload_manager.report_metrics({})
total_thread_count = len(workload_manager.get_cpu().get_threads())
expected_burst_allocation_size = total_thread_count - static_thread_count
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
expected_burst_allocation_size))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY, 0))
# Claim every thread for the burst workload which will oversubscribe the static threads
for t in workload_manager.get_cpu().get_threads():
t.claim(w_burst.get_id())
workload_manager.report_metrics({})
self.assertTrue(
gauge_value_equals(registry, ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, STATIC_ALLOCATED_SIZE_KEY,
static_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_ALLOCATED_SIZE_KEY,
total_thread_count))
self.assertTrue(
gauge_value_equals(registry, BURST_REQUESTED_SIZE_KEY,
burst_thread_count))
self.assertTrue(
gauge_value_equals(registry, OVERSUBSCRIBED_THREADS_KEY,
static_thread_count))
def test_record(self):
r = Registry(
config=SidecarConfig({"sidecar.output-location": "memory"}))
t = PercentileDistributionSummary(r, "test")
self.assertTrue(t._pct_distsummary._writer.is_empty())
t.record(42)
self.assertEqual("D:test:42", t._pct_distsummary._writer.last_line())
def test_stopwatch(self):
clock = ManualClock()
r = Registry(clock=clock,
config=SidecarConfig(
{"sidecar.output-location": "memory"}))
t = PercentileTimer(r, "test")
with t.stopwatch():
clock.set_monotonic_time(42)
self.assertEqual("T:test:42", t._pct_timer._writer.last_line())
def test_get_measurements_gauge_not_deleted_before_ttl(self):
clock = ManualClock()
r = Registry(clock=clock)
with r.start():
r.gauge('gauge').set(1)
ms = r._get_measurements()
self.assertEqual(1, len(ms))
# two cycles are required to delete a gauge after it expires
ms = r._get_measurements()
self.assertEqual(1, len(ms))
ms = r._get_measurements()
self.assertEqual(1, len(ms))
def test_counter_without_reference(self):
r = Registry(
config=SidecarConfig({"sidecar.output-location": "memory"}))
self.assertTrue(r.counter("test")._writer.is_empty())
r.counter("test").increment()
self.assertEqual("c:test:1", r.counter("test")._writer.last_line())
def test_iterate(self):
r = Registry()
r.counter("counter")
r.timer("timer")
meters = 0
for m in r:
meters += 1
self.assertEqual(2, meters)
def test_add_metrics(self):
test_context = TestContext()
workload_name = str(uuid.uuid4())
events = [
get_container_create_event(DEFAULT_CPU_COUNT, STATIC,
workload_name, workload_name)
]
event_count = len(events)
event_manager = EventManager(MockEventProvider(events),
test_context.get_event_handlers(),
get_mock_file_manager(), 5.0)
wait_until(lambda: event_count == event_manager.get_processed_count())
log.info("Event manager has processed {} events.".format(
event_manager.get_processed_count()))
workload_manager = test_context.get_workload_manager()
registry = Registry()
reporter = InternalMetricsReporter(workload_manager, event_manager)
reporter.set_registry(registry)
reporter.report_metrics({})
wait_until(lambda: self.__gauge_value_equals(registry, RUNNING, 1))
wait_until(lambda: self.__gauge_value_equals(registry, ADDED_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(registry, REMOVED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, SUCCEEDED_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(registry, FAILED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(
registry, PACKAGE_VIOLATIONS_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
CORE_VIOLATIONS_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
EVENT_SUCCEEDED_KEY, 3))
wait_until(
lambda: self.__gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
EVENT_PROCESSED_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(
registry, FALLBACK_ALLOCATOR_COUNT, 0))
wait_until(lambda: self.__gauge_value_equals(
registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
event_manager.stop_processing_events()
def test_free_cpu_on_container_die(self):
registry = Registry()
test_pod = get_simple_test_pod()
get_pod_manager().set_pod(test_pod)
workload_name = test_pod.metadata.name
events = [
get_container_create_event(DEFAULT_CPU_COUNT, STATIC,
workload_name, workload_name),
get_container_die_event(workload_name)
]
event_count = len(events)
event_iterable = MockEventProvider(
events, 1) # Force in order event processing for the test
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry, {})
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
1,
test_context.get_create_event_handler().get_handled_event_count())
self.assertEqual(
1,
test_context.get_free_event_handler().get_handled_event_count())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
counter_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(counter_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
counter_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_iterate(self):
"""Avoid breaking the API."""
r = Registry(
config=SidecarConfig({"sidecar.output-location": "memory"}))
for _ in r:
self.fail("registry should be empty")
r.counter("counter")
r.timer("timer")
for _ in r:
self.fail("registry no longer holds references to MeterIds")
def test_rebalance(self):
registry = Registry()
events = [REBALANCE_EVENT]
event_count = len(events)
event_iterable = MockEventProvider(events)
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
0,
test_context.get_create_event_handler().get_handled_event_count())
self.assertEqual(
0,
test_context.get_free_event_handler().get_handled_event_count())
self.assertEqual(
1,
test_context.get_rebalance_event_handler().get_handled_event_count(
))
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_measurements_to_json(self):
r = Registry()
with r.start({'common_tags': {'nf.app': 'app'}}):
c = r.counter('c')
c.increment()
r.gauge('g').set(42)
ms = r._get_measurements()
expected_counter = {
'op': 'sum',
'value': 1,
'tags': {'nf.app': 'app', 'name': 'c', 'statistic': 'count'}
}
expected_gauge = {
'op': 'max',
'value': 42,
'tags': {'nf.app': 'app', 'name': 'g', 'statistic': 'gauge'}
}
expected_entries = [expected_counter, expected_gauge]
payload = r._measurements_to_json(ms)
entries = self.payload_to_entries(payload)
self.assertEqual(expected_entries, entries)
def test_update_mock_container(self):
registry = Registry()
workload_name = str(uuid.uuid4())
events = [
get_container_create_event(DEFAULT_CPU_COUNT, STATIC,
workload_name, workload_name)
]
event_count = len(events)
event_iterable = MockEventProvider(events)
test_context = TestContext()
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: event_count == manager.get_processed_count())
self.assertEqual(0, manager.get_queue_depth())
self.assertEqual(
event_count,
test_context.get_workload_manager().get_success_count())
self.assertEqual(DEFAULT_TOTAL_THREAD_COUNT - DEFAULT_CPU_COUNT,
len(test_context.get_cpu().get_empty_threads()))
self.assertEqual(
1,
test_context.get_create_event_handler().get_handled_event_count())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(
registry, EVENT_SUCCEEDED_KEY,
event_count * len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_PROCESSED_KEY, event_count))
def test_empty_metrics(self):
test_context = TestContext()
registry = Registry()
reporter = test_context.get_workload_manager()
reporter.set_registry(registry, {})
reporter.report_metrics({})
self.assertTrue(gauge_value_equals(registry, RUNNING, 1))
self.assertTrue(counter_value_equals(registry, ADDED_KEY, 0))
self.assertTrue(counter_value_equals(registry, REMOVED_KEY, 0))
self.assertTrue(counter_value_equals(registry, SUCCEEDED_KEY, 0))
self.assertTrue(counter_value_equals(registry, FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 0))
self.assertTrue(gauge_value_equals(registry, PACKAGE_VIOLATIONS_KEY,
0))
self.assertTrue(gauge_value_equals(registry, CORE_VIOLATIONS_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
self.assertTrue(gauge_value_equals(registry, ALLOCATED_SIZE_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, UNALLOCATED_SIZE_KEY,
len(test_context.get_cpu().get_threads())))
def test_crash_ip_allocator_metrics(self):
cpu = get_cpu(2, 16, 2)
test_context = TestContext(cpu=cpu)
# now override the cpu seen by the allocator to crash it
test_context.get_workload_manager().get_allocator().set_cpu(
get_cpu(2, 2, 2))
events = [get_container_create_event(10, name="foo", id="bar")]
event_count = len(events)
event_manager = EventManager(MockEventProvider(events),
test_context.get_event_handlers(),
get_mock_file_manager(), 5.0)
wait_until(lambda: event_count == event_manager.get_processed_count())
log.info("Event manager has processed {} events.".format(
event_manager.get_processed_count()))
workload_manager = test_context.get_workload_manager()
registry = Registry()
reporter = InternalMetricsReporter(workload_manager, event_manager)
reporter.set_registry(registry)
reporter.report_metrics({})
wait_until(lambda: self.__gauge_value_equals(registry, RUNNING, 1))
wait_until(lambda: self.__gauge_value_equals(registry, ADDED_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(registry, REMOVED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, SUCCEEDED_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(registry, FAILED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 1))
wait_until(lambda: self.__gauge_value_equals(
registry, FALLBACK_ALLOCATOR_COUNT, 1))
event_manager.stop_processing_events()
def test_empty_metrics(self):
test_context = TestContext()
event_manager = EventManager(MockEventProvider([]), [],
get_mock_file_manager(), 0.01)
registry = Registry()
reporter = InternalMetricsReporter(test_context.get_workload_manager(),
event_manager)
reporter.set_registry(registry)
reporter.report_metrics({})
wait_until(lambda: self.__gauge_value_equals(registry, RUNNING, 1))
wait_until(lambda: self.__gauge_value_equals(registry, ADDED_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry, REMOVED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, SUCCEEDED_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry, FAILED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, WORKLOAD_COUNT_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(
registry, PACKAGE_VIOLATIONS_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
CORE_VIOLATIONS_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
EVENT_SUCCEEDED_KEY, 0))
wait_until(
lambda: self.__gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(registry,
EVENT_PROCESSED_KEY, 0))
wait_until(lambda: self.__gauge_value_equals(
registry, FALLBACK_ALLOCATOR_COUNT, 0))
wait_until(lambda: self.__gauge_value_equals(
registry, IP_ALLOCATOR_TIMEBOUND_COUNT, 0))
event_manager.stop_processing_events()
def test_unknown_action(self):
registry = Registry()
test_context = TestContext()
unknown_event = get_event(CONTAINER, "unknown", uuid.uuid4(), {})
event_iterable = MockEventProvider([unknown_event])
manager = EventManager(event_iterable,
test_context.get_event_handlers(),
DEFAULT_TEST_EVENT_TIMEOUT_SECS)
manager.set_registry(registry)
manager.start_processing_events()
wait_until(lambda: test_context.get_create_event_handler().
get_ignored_event_count() == 1)
self.assertEqual(0, manager.get_queue_depth())
manager.stop_processing_events()
manager.report_metrics({})
self.assertTrue(gauge_value_equals(registry, QUEUE_DEPTH_KEY, 0))
self.assertTrue(
gauge_value_equals(registry, EVENT_SUCCEEDED_KEY,
len(test_context.get_event_handlers())))
self.assertTrue(gauge_value_equals(registry, EVENT_FAILED_KEY, 0))
self.assertTrue(gauge_value_equals(registry, EVENT_PROCESSED_KEY, 1))
def test_duplicate_timer_different_type(self):
r = Registry()
c = r.counter("check_value")
t = r.timer("check_value")
self.assertIsNot(c, t)
self.assertIs(t, r.noopTimer)
def test_get_measurements_only_useful(self):
r = Registry()
with r.start():
# meters with reference held
c = r.counter('counter')
g = r.gauge('gauge')
c.increment()
g.set(1)
ms = r._get_measurements()
self.assertEqual(2, len(ms))
# meters without reference held; only the non-nan gauge will report
r.gauge('gauge-nan').set(float('nan'))
r.counter('counter-zero')
ms = r._get_measurements()
self.assertEqual(1, len(ms))
def test_counter(self):
r = Registry()
c = r.counter("test")
self.assertEqual(c.count(), 0)
c.increment()
self.assertEqual(c.count(), 1)
def test_duplicate_start(self):
r = Registry()
r.start()
t1 = r._timer
r.start()
self.assertEqual(r._timer, t1)
def test_iterate_empty(self):
r = Registry()
for m in r:
self.fail("registry should be empty")
def test_timer_and_counter_same_name_different_tags(self):
r = Registry()
c = r.counter("check_value", tags=dict(tag="a"))
t = r.timer("check_value", tags=dict(tag="b"))
self.assertIsNot(c, t)
self.assertIsNot(t, r.noopTimer)
def test_duplicate_distro_different_type(self):
r = Registry()
c = r.counter("check_value")
d = r.distribution_summary("check_value")
self.assertIsNot(c, d)
self.assertIs(d, r.noopDistributionSummary)