def test_merge(self):
observer1 = ObserverAggregator()
observer2 = ObserverAggregator()
mmsc_checkpoint1 = MinMaxSumCountAggregator._TYPE(3, 150, 101, 3)
mmsc_checkpoint2 = MinMaxSumCountAggregator._TYPE(1, 33, 44, 2)
checkpoint1 = ObserverAggregator._TYPE(*(mmsc_checkpoint1 + (23, )))
checkpoint2 = ObserverAggregator._TYPE(*(mmsc_checkpoint2 + (27, )))
observer1.mmsc.checkpoint = mmsc_checkpoint1
observer2.mmsc.checkpoint = mmsc_checkpoint2
observer1.checkpoint = checkpoint1
observer2.checkpoint = checkpoint2
observer1.merge(observer2)
self.assertEqual(
observer1.checkpoint,
(
min(checkpoint1.min, checkpoint2.min),
max(checkpoint1.max, checkpoint2.max),
checkpoint1.sum + checkpoint2.sum,
checkpoint1.count + checkpoint2.count,
checkpoint2.last,
),
)
def test_update(self):
mmsc = MinMaxSumCountAggregator()
# test current values without any update
self.assertEqual(mmsc.current, MinMaxSumCountAggregator._EMPTY)
# call update with some values
values = (3, 50, 3, 97)
for val in values:
mmsc.update(val)
self.assertEqual(mmsc.current,
(min(values), max(values), sum(values), len(values)))
def test_concurrent_update(self):
mmsc = MinMaxSumCountAggregator()
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
fut1 = ex.submit(self.call_update, mmsc)
fut2 = ex.submit(self.call_update, mmsc)
ret1 = fut1.result()
ret2 = fut2.result()
update_total = MinMaxSumCountAggregator._merge_checkpoint(
ret1, ret2)
mmsc.take_checkpoint()
self.assertEqual(update_total, mmsc.checkpoint)
def test_update(self, time_mock):
time_mock.return_value = 123
mmsc = MinMaxSumCountAggregator()
# test current values without any update
self.assertEqual(mmsc.current, MinMaxSumCountAggregator._EMPTY)
# call update with some values
values = (3, 50, 3, 97)
for val in values:
mmsc.update(val)
self.assertEqual(mmsc.current,
(min(values), max(values), sum(values), len(values)))
self.assertEqual(mmsc.last_update_timestamp, 123)
def test_concurrent_update(self):
mmsc0 = MinMaxSumCountAggregator()
mmsc1 = MinMaxSumCountAggregator()
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
mmsc0.checkpoint = ex.submit(self.call_update, mmsc0).result()
mmsc1.checkpoint = ex.submit(self.call_update, mmsc0).result()
mmsc0.merge(mmsc1)
mmsc0_checkpoint = mmsc0.checkpoint
mmsc0.take_checkpoint()
self.assertEqual(mmsc0_checkpoint, mmsc0.checkpoint)
self.assertIsNot(mmsc0_checkpoint, mmsc0.checkpoint)
def test_convert_from_min_max_sum_count(self):
min_max_sum_count_record = ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
MinMaxSumCountAggregator(),
Resource({}),
)
min_max_sum_count_record.aggregator.update(5)
min_max_sum_count_record.aggregator.update(1)
min_max_sum_count_record.aggregator.take_checkpoint()
expected_min_timeseries = self.exporter._create_timeseries(
min_max_sum_count_record, "testname_min", 1.0)
expected_max_timeseries = self.exporter._create_timeseries(
min_max_sum_count_record, "testname_max", 5.0)
expected_sum_timeseries = self.exporter._create_timeseries(
min_max_sum_count_record, "testname_sum", 6.0)
expected_count_timeseries = self.exporter._create_timeseries(
min_max_sum_count_record, "testname_count", 2.0)
timeseries = self.exporter._convert_from_min_max_sum_count(
min_max_sum_count_record)
self.assertEqual(timeseries[0], expected_min_timeseries)
self.assertEqual(timeseries[1], expected_max_timeseries)
self.assertEqual(timeseries[2], expected_sum_timeseries)
self.assertEqual(timeseries[3], expected_count_timeseries)
def test_concurrent_update_and_checkpoint(self):
mmsc = MinMaxSumCountAggregator()
checkpoint_total = MinMaxSumCountAggregator._TYPE(2**32, 0, 0, 0)
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
fut = ex.submit(self.call_update, mmsc)
while not fut.done():
mmsc.take_checkpoint()
checkpoint_total = MinMaxSumCountAggregator._merge_checkpoint(
checkpoint_total, mmsc.checkpoint)
mmsc.take_checkpoint()
checkpoint_total = MinMaxSumCountAggregator._merge_checkpoint(
checkpoint_total, mmsc.checkpoint)
self.assertEqual(checkpoint_total, fut.result())
def test_merge_with_empty(self):
mmsc1 = MinMaxSumCountAggregator()
mmsc2 = MinMaxSumCountAggregator()
checkpoint1 = MinMaxSumCountAggregator._TYPE(3, 150, 101, 3)
mmsc1.checkpoint = checkpoint1
mmsc1.merge(mmsc2)
self.assertEqual(mmsc1.checkpoint, checkpoint1)
def aggregator_for(self, metric_type: Type[MetricT]) -> Aggregator:
"""Returns an aggregator based on metric type.
Aggregators keep track of and updates values when metrics get updated.
"""
# pylint:disable=R0201
if issubclass(metric_type, Counter):
return CounterAggregator()
if issubclass(metric_type, Measure):
return MinMaxSumCountAggregator()
# TODO: Add other aggregators
return CounterAggregator()
def test_value_recorder_to_envelope(self):
aggregator = MinMaxSumCountAggregator()
aggregator.update(123)
aggregator.take_checkpoint()
record = MetricRecord(self._test_value_recorder, self._test_labels,
aggregator)
exporter = self._exporter
envelope = exporter._metric_to_envelope(record)
self.assertIsInstance(envelope, Envelope)
self.assertEqual(envelope.ver, 1)
self.assertEqual(envelope.name, "Microsoft.ApplicationInsights.Metric")
self.assertEqual(envelope.time,
ns_to_iso_str(aggregator.last_update_timestamp))
self.assertEqual(envelope.sample_rate, None)
self.assertEqual(envelope.seq, None)
self.assertEqual(envelope.ikey, "1234abcd-5678-4efa-8abc-1234567890ab")
self.assertEqual(envelope.flags, None)
self.assertIsInstance(envelope.data, Data)
self.assertIsInstance(envelope.data.base_data, MetricData)
self.assertEqual(envelope.data.base_data.ver, 2)
self.assertEqual(len(envelope.data.base_data.metrics), 1)
self.assertIsInstance(envelope.data.base_data.metrics[0], DataPoint)
self.assertEqual(envelope.data.base_data.metrics[0].ns, "testdesc")
self.assertEqual(envelope.data.base_data.metrics[0].name, "testname")
self.assertEqual(envelope.data.base_data.metrics[0].value, 1)
self.assertEqual(envelope.data.base_data.properties["environment"],
"staging")
self.assertIsNotNone(envelope.tags["ai.cloud.role"])
self.assertIsNotNone(envelope.tags["ai.cloud.roleInstance"])
self.assertIsNotNone(envelope.tags["ai.device.id"])
self.assertIsNotNone(envelope.tags["ai.device.locale"])
self.assertIsNotNone(envelope.tags["ai.device.osVersion"])
self.assertIsNotNone(envelope.tags["ai.device.type"])
self.assertIsNotNone(envelope.tags["ai.internal.sdkVersion"])
def test_merge_with_empty(self):
observer1 = ObserverAggregator()
observer2 = ObserverAggregator()
mmsc_checkpoint1 = MinMaxSumCountAggregator._TYPE(3, 150, 101, 3)
checkpoint1 = ObserverAggregator._TYPE(*(mmsc_checkpoint1 + (23, )))
observer1.mmsc.checkpoint = mmsc_checkpoint1
observer1.checkpoint = checkpoint1
observer1.merge(observer2)
self.assertEqual(observer1.checkpoint, checkpoint1)
def call_update(mmsc):
min_ = float("inf")
max_ = float("-inf")
sum_ = 0
count_ = 0
for _ in range(0, 100000):
val = random.getrandbits(32)
mmsc.update(val)
if val < min_:
min_ = val
if val > max_:
max_ = val
sum_ += val
count_ += 1
return MinMaxSumCountAggregator._TYPE(min_, max_, sum_, count_)
def aggregator_for(self, instrument_type: Type[InstrumentT]) -> Aggregator:
"""Returns an aggregator based on metric instrument type.
Aggregators keep track of and updates values when metrics get updated.
"""
# pylint:disable=R0201
if issubclass(instrument_type, (Counter, UpDownCounter)):
return SumAggregator()
if issubclass(instrument_type, (SumObserver, UpDownSumObserver)):
return LastValueAggregator()
if issubclass(instrument_type, ValueRecorder):
return MinMaxSumCountAggregator()
if issubclass(instrument_type, ValueObserver):
return ValueObserverAggregator()
# TODO: Add other aggregators
return SumAggregator()
def test_checkpoint(self):
mmsc = MinMaxSumCountAggregator()
# take checkpoint wihtout any update
mmsc.take_checkpoint()
self.assertEqual(mmsc.checkpoint, MinMaxSumCountAggregator._EMPTY)
# call update with some values
values = (3, 50, 3, 97)
for val in values:
mmsc.update(val)
mmsc.take_checkpoint()
self.assertEqual(
mmsc.checkpoint,
(min(values), max(values), sum(values), len(values)),
)
self.assertEqual(mmsc.current, MinMaxSumCountAggregator._EMPTY)
def test_valid_convert_to_timeseries(self):
test_records = [
ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
SumAggregator(),
Resource({}),
),
ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
MinMaxSumCountAggregator(),
Resource({}),
),
ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
HistogramAggregator(),
Resource({}),
),
ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
LastValueAggregator(),
Resource({}),
),
ExportRecord(
Counter("testname", "testdesc", "testunit", int, None),
None,
ValueObserverAggregator(),
Resource({}),
),
]
for record in test_records:
record.aggregator.update(5)
record.aggregator.take_checkpoint()
data = self.exporter._convert_to_timeseries(test_records)
self.assertIsInstance(data, list)
self.assertEqual(len(data), 13)
for timeseries in data:
self.assertIsInstance(timeseries, TimeSeries)
def test_min_max_sum_aggregator_to_prometheus(self):
meter = get_meter_provider().get_meter(__name__)
metric = meter.create_metric("test@name", "testdesc", "unit", int,
metrics.ValueRecorder, [])
labels = {}
key_labels = get_dict_as_key(labels)
aggregator = MinMaxSumCountAggregator()
aggregator.update(123)
aggregator.update(456)
aggregator.take_checkpoint()
record = MetricRecord(metric, key_labels, aggregator)
collector = CustomCollector("testprefix")
collector.add_metrics_data([record])
result_bytes = generate_latest(collector)
result = result_bytes.decode("utf-8")
self.assertIn("testprefix_test_name_count 2.0", result)
self.assertIn("testprefix_test_name_sum 579.0", result)
def test_live_metric_envelope_value_recorder(self):
aggregator = MinMaxSumCountAggregator()
aggregator.update(123)
aggregator.take_checkpoint()
record = MetricRecord(self._test_metric2, self._test_labels,
aggregator)
exporter = LiveMetricsExporter(
instrumentation_key=self._instrumentation_key,
span_processor=self._span_processor,
)
envelope = exporter._metric_to_live_metrics_envelope([record])
self.assertIsInstance(envelope, LiveMetricEnvelope)
self.assertEqual(envelope.documents, [])
self.assertEqual(envelope.metrics[0].name, "testname")
self.assertEqual(envelope.metrics[0].value, 1)
self.assertEqual(envelope.metrics[0].weight, 1)
def test_concurrent_update_and_checkpoint(self):
mmsc0 = MinMaxSumCountAggregator()
mmsc1 = MinMaxSumCountAggregator()
mmsc1.checkpoint = MinMaxSumCountAggregator._TYPE(2**32, 0, 0, 0)
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
fut = ex.submit(self.call_update, mmsc0)
while not fut.done():
mmsc0.take_checkpoint()
mmsc0.merge(mmsc1)
mmsc1.checkpoint = mmsc0.checkpoint
mmsc0.take_checkpoint()
mmsc0.merge(mmsc1)
self.assertEqual(mmsc0.checkpoint, fut.result())
def test_merge(self):
mmsc1 = MinMaxSumCountAggregator()
mmsc2 = MinMaxSumCountAggregator()
checkpoint1 = MinMaxSumCountAggregator._TYPE(3, 150, 101, 3)
checkpoint2 = MinMaxSumCountAggregator._TYPE(1, 33, 44, 2)
mmsc1.checkpoint = checkpoint1
mmsc2.checkpoint = checkpoint2
mmsc1.last_update_timestamp = 100
mmsc2.last_update_timestamp = 123
mmsc1.merge(mmsc2)
mmsc1_checkpoint = mmsc1.checkpoint
mmsc1.checkpoint = checkpoint1
mmsc2.checkpoint = checkpoint2
mmsc1.merge(mmsc2)
self.assertEqual(mmsc1_checkpoint, mmsc1.checkpoint)
self.assertEqual(mmsc1.last_update_timestamp, 123)
def test_merge_checkpoint(self):
type_ = MinMaxSumCountAggregator._TYPE
empty = MinMaxSumCountAggregator._EMPTY
mmsc0 = MinMaxSumCountAggregator()
mmsc1 = MinMaxSumCountAggregator()
mmsc0.checkpoint = empty
mmsc1.checkpoint = empty
mmsc0.merge(mmsc1)
self.assertEqual(mmsc0.checkpoint, mmsc1.checkpoint)
mmsc0.checkpoint = empty
mmsc1.checkpoint = type_(0, 0, 0, 0)
mmsc0.merge(mmsc1)
self.assertEqual(mmsc0.checkpoint, mmsc1.checkpoint)
mmsc0.checkpoint = type_(0, 0, 0, 0)
mmsc1.checkpoint = empty
mmsc1.merge(mmsc0)
self.assertEqual(mmsc1.checkpoint, mmsc0.checkpoint)
mmsc0.checkpoint = type_(0, 0, 0, 0)
mmsc1.checkpoint = type_(0, 0, 0, 0)
mmsc0.merge(mmsc1)
self.assertEqual(mmsc1.checkpoint, mmsc0.checkpoint)
mmsc0.checkpoint = type_(44, 23, 55, 86)
mmsc1.checkpoint = empty
mmsc0.merge(mmsc1)
self.assertEqual(mmsc0.checkpoint, type_(44, 23, 55, 86))
mmsc0.checkpoint = type_(3, 150, 101, 3)
mmsc1.checkpoint = type_(1, 33, 44, 2)
mmsc0.merge(mmsc1)
self.assertEqual(mmsc0.checkpoint, type_(1, 150, 101 + 44, 2 + 3))
def test_merge(self):
mmsc1 = MinMaxSumCountAggregator()
mmsc2 = MinMaxSumCountAggregator()
checkpoint1 = MinMaxSumCountAggregator._TYPE(3, 150, 101, 3)
checkpoint2 = MinMaxSumCountAggregator._TYPE(1, 33, 44, 2)
mmsc1.checkpoint = checkpoint1
mmsc2.checkpoint = checkpoint2
mmsc1.merge(mmsc2)
self.assertEqual(
mmsc1.checkpoint,
MinMaxSumCountAggregator._merge_checkpoint(checkpoint1,
checkpoint2),
)
