def test_translate_distribution_using_distribution_data(self):
metric_update = dataflow.CounterUpdate()
distribution_update = DistributionData(16, 2, 1, 15)
apiclient.translate_distribution(distribution_update, metric_update)
self.assertEqual(
metric_update.distribution.min.lowBits, distribution_update.min)
self.assertEqual(
metric_update.distribution.max.lowBits, distribution_update.max)
self.assertEqual(
metric_update.distribution.sum.lowBits, distribution_update.sum)
self.assertEqual(
metric_update.distribution.count.lowBits, distribution_update.count)
def test_commit_logical_no_filter(self):
metrics = DirectMetrics()
metrics.commit_logical(
self.bundle1,
MetricUpdates(
counters={
MetricKey('step1', self.name1): 5,
MetricKey('step1', self.name2): 8
},
distributions={
MetricKey('step1', self.name1): DistributionData(8, 2, 3, 5)
}))
metrics.commit_logical(
self.bundle1,
MetricUpdates(
counters={
MetricKey('step2', self.name1): 7,
MetricKey('step1', self.name2): 4
},
distributions={
MetricKey('step1', self.name1): DistributionData(4, 1, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(
*[
MetricResult(MetricKey('step1', self.name2), 12, 0),
MetricResult(MetricKey('step2', self.name1), 7, 0),
MetricResult(MetricKey('step1', self.name1), 5, 0)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(
MetricResult(
MetricKey('step1', self.name1),
DistributionResult(DistributionData(12, 3, 3, 5)),
DistributionResult(DistributionData(0, 0, None, None)))))
def extract_metric_result_map_value(monitoring_info_proto):
"""Returns the relevant GaugeResult, DistributionResult or int value.
These are the proper format for use in the MetricResult.query() result.
"""
# Returns a metric result (AKA the legacy format).
# from the MonitoringInfo
if is_counter(monitoring_info_proto):
return extract_counter_value(monitoring_info_proto)
if is_distribution(monitoring_info_proto):
(count, sum, min, max) = extract_distribution(monitoring_info_proto)
return DistributionResult(DistributionData(sum, count, min, max))
if is_gauge(monitoring_info_proto):
(timestamp, value) = extract_gauge_value(monitoring_info_proto)
return GaugeResult(GaugeData(value, timestamp))
def extract_metric_result_map_value(monitoring_info_proto):
"""Returns the relevant GaugeResult, DistributionResult or int value.
These are the proper format for use in the MetricResult.query() result.
"""
# Returns a metric result (AKA the legacy format).
# from the MonitoringInfo
if is_counter(monitoring_info_proto):
return extract_counter_value(monitoring_info_proto)
if is_distribution(monitoring_info_proto):
distribution_data = extract_distribution(monitoring_info_proto)
return DistributionResult(
DistributionData(distribution_data.sum, distribution_data.count,
distribution_data.min, distribution_data.max))
if is_gauge(monitoring_info_proto):
timestamp_secs = to_timestamp_secs(monitoring_info_proto.timestamp)
return GaugeResult(GaugeData(
extract_counter_value(monitoring_info_proto), timestamp_secs))
def monitoring_infos(self, transform_id):
# type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
infos = super(DoOperation, self).monitoring_infos(transform_id)
if self.tagged_receivers:
for tag, receiver in self.tagged_receivers.items():
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
ptransform=transform_id,
tag=str(tag))
infos[monitoring_infos.to_key(mi)] = mi
(unused_mean, sum, count, min, max) = (
receiver.opcounter.mean_byte_counter.value())
sampled_byte_count = monitoring_infos.int64_distribution(
monitoring_infos.SAMPLED_BYTE_SIZE_URN,
DistributionData(sum, count, min, max),
ptransform=transform_id,
tag=str(tag))
infos[monitoring_infos.to_key(sampled_byte_count)] = sampled_byte_count
return infos
def _get_metric_value(self, metric):
"""Get a metric result object from a MetricUpdate from Dataflow API."""
if metric is None:
return None
if metric.scalar is not None:
return metric.scalar.integer_value
elif metric.distribution is not None:
dist_count = _get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'count').value.integer_value
dist_min = _get_match(metric.distribution.object_value.properties,
lambda x: x.key == 'min').value.integer_value
dist_max = _get_match(metric.distribution.object_value.properties,
lambda x: x.key == 'max').value.integer_value
dist_sum = _get_match(metric.distribution.object_value.properties,
lambda x: x.key == 'sum').value.integer_value
return DistributionResult(
DistributionData(dist_sum, dist_count, dist_min, dist_max))
else:
return None
def test_parallel_access(self):
# We create NUM_THREADS threads that concurrently modify the distribution.
threads = []
d = DistributionCell()
for _ in range(TestDistributionCell.NUM_THREADS):
t = threading.Thread(
target=TestDistributionCell._modify_distribution, args=(d, ))
threads.append(t)
t.start()
for t in threads:
t.join()
total = (self.NUM_ITERATIONS * (self.NUM_ITERATIONS - 1) // 2 *
self.NUM_THREADS)
count = (self.NUM_ITERATIONS * self.NUM_THREADS)
self.assertEqual(
d.get_cumulative(),
DistributionData(total, count, 0, self.NUM_ITERATIONS - 1))
def _create_metric_result(data_dict):
step = data_dict['step'] if 'step' in data_dict else ''
labels = data_dict['labels'] if 'labels' in data_dict else dict()
values = {}
for key in ['attempted', 'committed']:
if key in data_dict:
if 'counter' in data_dict[key]:
values[key] = data_dict[key]['counter']
elif 'distribution' in data_dict[key]:
distribution = data_dict[key]['distribution']
values[key] = DistributionResult(
DistributionData(
distribution['sum'],
distribution['count'],
distribution['min'],
distribution['max'],
))
attempted = values['attempted'] if 'attempted' in values else None
committed = values['committed'] if 'committed' in values else None
metric_name = MetricName(data_dict['namespace'], data_dict['name'])
metric_key = MetricKey(step, metric_name, labels)
return MetricResult(metric_key, committed, attempted)
def test_create_counter_distribution(self):
sampler = statesampler.StateSampler('', counters.CounterFactory())
statesampler.set_current_tracker(sampler)
state1 = sampler.scoped_state(
'mystep', 'myState', metrics_container=MetricsContainer('mystep'))
try:
sampler.start()
with state1:
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(
isinstance(distro, Metrics.DelegatingDistribution))
del distro
del counter
container = MetricsEnvironment.current_container()
self.assertEqual(
container.get_counter(MetricName(counter_ns,
name)).get_cumulative(),
7)
self.assertEqual(
container.get_distribution(MetricName(
distro_ns, name)).get_cumulative(),
DistributionData(12, 2, 2, 10))
finally:
sampler.stop()
def _get_metric_value(self, metric):
"""Get a metric result object from a MetricUpdate from Dataflow API."""
if metric is None:
return None
if metric.scalar is not None:
return metric.scalar.integer_value
elif metric.distribution is not None:
dist_count = _get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'count').value.integer_value
dist_min = _get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'min').value.integer_value
dist_max = _get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'max').value.integer_value
dist_sum = _get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'sum').value.integer_value
if dist_sum is None:
# distribution metric is not meant to use on large values, but in case
# it is, the value can overflow and become double_value, the correctness
# of the value may not be guaranteed.
_LOGGER.info(
"Distribution metric sum value seems to have "
"overflowed integer_value range, the correctness of sum or mean "
"value may not be guaranteed: %s" % metric.distribution)
dist_sum = int(
_get_match(
metric.distribution.object_value.properties,
lambda x: x.key == 'sum').value.double_value)
return DistributionResult(
DistributionData(dist_sum, dist_count, dist_min, dist_max))
else:
return None
def test_create_counter_distribution(self):
MetricsEnvironment.set_current_container(MetricsContainer('mystep'))
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution))
del distro
del counter
container = MetricsEnvironment.current_container()
self.assertEqual(
container.counters[MetricName(counter_ns, name)].get_cumulative(),
7)
self.assertEqual(
container.distributions[MetricName(distro_ns, name)].get_cumulative(),
DistributionData(12, 2, 2, 10))
def test_apply_physical_logical(self):
metrics = DirectMetrics()
dist_zero = DistributionData(0, 0, None, None)
metrics.update_physical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): 7,
MetricKey('step1', self.name2): 5,
MetricKey('step2', self.name1): 1
},
distributions={
MetricKey('step1', self.name1):
DistributionData(3, 1, 3, 3),
MetricKey('step2', self.name3):
DistributionData(8, 2, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 7),
MetricResult(MetricKey('step1', self.name2), 0, 5),
MetricResult(MetricKey('step2', self.name1), 0, 1)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(dist_zero),
DistributionResult(DistributionData(3, 1, 3, 3))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 2, 4, 4)))
]))
metrics.commit_physical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): -3,
MetricKey('step2', self.name1): -5
},
distributions={
MetricKey('step1', self.name1):
DistributionData(8, 4, 1, 5),
MetricKey('step2', self.name2):
DistributionData(8, 8, 1, 1)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 4),
MetricResult(MetricKey('step1', self.name2), 0, 5),
MetricResult(MetricKey('step2', self.name1), 0, -4)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(dist_zero),
DistributionResult(DistributionData(11, 5, 1,
5))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 2, 4, 4))),
MetricResult(MetricKey('step2', self.name2),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 8, 1, 1)))
]))
metrics.commit_logical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): 3,
MetricKey('step1', self.name2): 5,
MetricKey('step2', self.name1): -3
},
distributions={
MetricKey('step1', self.name1):
DistributionData(11, 5, 1, 5),
MetricKey('step2', self.name2):
DistributionData(8, 8, 1, 1),
MetricKey('step2', self.name3):
DistributionData(4, 1, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 3, 4),
MetricResult(MetricKey('step1', self.name2), 5, 5),
MetricResult(MetricKey('step2', self.name1), -3, -4)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(DistributionData(11, 5, 1, 5)),
DistributionResult(DistributionData(11, 5, 1,
5))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(DistributionData(4, 1, 4, 4)),
DistributionResult(DistributionData(8, 2, 4, 4))),
MetricResult(MetricKey('step2', self.name2),
DistributionResult(DistributionData(8, 8, 1, 1)),
DistributionResult(DistributionData(8, 8, 1, 1)))
]))
def test_integer_only(self):
d = DistributionCell()
d.update(3.1)
d.update(3.2)
d.update(3.3)
self.assertEqual(d.get_cumulative(), DistributionData(9, 3, 3, 3))
