def test_translate_distribution_using_distribution_data(self): metric_update = dataflow.CounterUpdate() distribution_update = DistributionData(16, 2, 1, 15) apiclient.translate_distribution(distribution_update, metric_update) self.assertEqual( metric_update.distribution.min.lowBits, distribution_update.min) self.assertEqual( metric_update.distribution.max.lowBits, distribution_update.max) self.assertEqual( metric_update.distribution.sum.lowBits, distribution_update.sum) self.assertEqual( metric_update.distribution.count.lowBits, distribution_update.count)
def test_commit_logical_no_filter(self): metrics = DirectMetrics() metrics.commit_logical( self.bundle1, MetricUpdates( counters={ MetricKey('step1', self.name1): 5, MetricKey('step1', self.name2): 8 }, distributions={ MetricKey('step1', self.name1): DistributionData(8, 2, 3, 5) })) metrics.commit_logical( self.bundle1, MetricUpdates( counters={ MetricKey('step2', self.name1): 7, MetricKey('step1', self.name2): 4 }, distributions={ MetricKey('step1', self.name1): DistributionData(4, 1, 4, 4) })) results = metrics.query() hc.assert_that( results['counters'], hc.contains_inanyorder( *[ MetricResult(MetricKey('step1', self.name2), 12, 0), MetricResult(MetricKey('step2', self.name1), 7, 0), MetricResult(MetricKey('step1', self.name1), 5, 0) ])) hc.assert_that( results['distributions'], hc.contains_inanyorder( MetricResult( MetricKey('step1', self.name1), DistributionResult(DistributionData(12, 3, 3, 5)), DistributionResult(DistributionData(0, 0, None, None)))))
def extract_metric_result_map_value(monitoring_info_proto): """Returns the relevant GaugeResult, DistributionResult or int value. These are the proper format for use in the MetricResult.query() result. """ # Returns a metric result (AKA the legacy format). # from the MonitoringInfo if is_counter(monitoring_info_proto): return extract_counter_value(monitoring_info_proto) if is_distribution(monitoring_info_proto): (count, sum, min, max) = extract_distribution(monitoring_info_proto) return DistributionResult(DistributionData(sum, count, min, max)) if is_gauge(monitoring_info_proto): (timestamp, value) = extract_gauge_value(monitoring_info_proto) return GaugeResult(GaugeData(value, timestamp))
def extract_metric_result_map_value(monitoring_info_proto): """Returns the relevant GaugeResult, DistributionResult or int value. These are the proper format for use in the MetricResult.query() result. """ # Returns a metric result (AKA the legacy format). # from the MonitoringInfo if is_counter(monitoring_info_proto): return extract_counter_value(monitoring_info_proto) if is_distribution(monitoring_info_proto): distribution_data = extract_distribution(monitoring_info_proto) return DistributionResult( DistributionData(distribution_data.sum, distribution_data.count, distribution_data.min, distribution_data.max)) if is_gauge(monitoring_info_proto): timestamp_secs = to_timestamp_secs(monitoring_info_proto.timestamp) return GaugeResult(GaugeData( extract_counter_value(monitoring_info_proto), timestamp_secs))
def monitoring_infos(self, transform_id): # type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo] infos = super(DoOperation, self).monitoring_infos(transform_id) if self.tagged_receivers: for tag, receiver in self.tagged_receivers.items(): mi = monitoring_infos.int64_counter( monitoring_infos.ELEMENT_COUNT_URN, receiver.opcounter.element_counter.value(), ptransform=transform_id, tag=str(tag)) infos[monitoring_infos.to_key(mi)] = mi (unused_mean, sum, count, min, max) = ( receiver.opcounter.mean_byte_counter.value()) sampled_byte_count = monitoring_infos.int64_distribution( monitoring_infos.SAMPLED_BYTE_SIZE_URN, DistributionData(sum, count, min, max), ptransform=transform_id, tag=str(tag)) infos[monitoring_infos.to_key(sampled_byte_count)] = sampled_byte_count return infos
def _get_metric_value(self, metric): """Get a metric result object from a MetricUpdate from Dataflow API.""" if metric is None: return None if metric.scalar is not None: return metric.scalar.integer_value elif metric.distribution is not None: dist_count = _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'count').value.integer_value dist_min = _get_match(metric.distribution.object_value.properties, lambda x: x.key == 'min').value.integer_value dist_max = _get_match(metric.distribution.object_value.properties, lambda x: x.key == 'max').value.integer_value dist_sum = _get_match(metric.distribution.object_value.properties, lambda x: x.key == 'sum').value.integer_value return DistributionResult( DistributionData(dist_sum, dist_count, dist_min, dist_max)) else: return None
def test_parallel_access(self): # We create NUM_THREADS threads that concurrently modify the distribution. threads = [] d = DistributionCell() for _ in range(TestDistributionCell.NUM_THREADS): t = threading.Thread( target=TestDistributionCell._modify_distribution, args=(d, )) threads.append(t) t.start() for t in threads: t.join() total = (self.NUM_ITERATIONS * (self.NUM_ITERATIONS - 1) // 2 * self.NUM_THREADS) count = (self.NUM_ITERATIONS * self.NUM_THREADS) self.assertEqual( d.get_cumulative(), DistributionData(total, count, 0, self.NUM_ITERATIONS - 1))
def _create_metric_result(data_dict): step = data_dict['step'] if 'step' in data_dict else '' labels = data_dict['labels'] if 'labels' in data_dict else dict() values = {} for key in ['attempted', 'committed']: if key in data_dict: if 'counter' in data_dict[key]: values[key] = data_dict[key]['counter'] elif 'distribution' in data_dict[key]: distribution = data_dict[key]['distribution'] values[key] = DistributionResult( DistributionData( distribution['sum'], distribution['count'], distribution['min'], distribution['max'], )) attempted = values['attempted'] if 'attempted' in values else None committed = values['committed'] if 'committed' in values else None metric_name = MetricName(data_dict['namespace'], data_dict['name']) metric_key = MetricKey(step, metric_name, labels) return MetricResult(metric_key, committed, attempted)
def test_create_counter_distribution(self): sampler = statesampler.StateSampler('', counters.CounterFactory()) statesampler.set_current_tracker(sampler) state1 = sampler.scoped_state( 'mystep', 'myState', metrics_container=MetricsContainer('mystep')) try: sampler.start() with state1: counter_ns = 'aCounterNamespace' distro_ns = 'aDistributionNamespace' name = 'a_name' counter = Metrics.counter(counter_ns, name) distro = Metrics.distribution(distro_ns, name) counter.inc(10) counter.dec(3) distro.update(10) distro.update(2) self.assertTrue(isinstance(counter, Metrics.DelegatingCounter)) self.assertTrue( isinstance(distro, Metrics.DelegatingDistribution)) del distro del counter container = MetricsEnvironment.current_container() self.assertEqual( container.get_counter(MetricName(counter_ns, name)).get_cumulative(), 7) self.assertEqual( container.get_distribution(MetricName( distro_ns, name)).get_cumulative(), DistributionData(12, 2, 2, 10)) finally: sampler.stop()
def _get_metric_value(self, metric): """Get a metric result object from a MetricUpdate from Dataflow API.""" if metric is None: return None if metric.scalar is not None: return metric.scalar.integer_value elif metric.distribution is not None: dist_count = _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'count').value.integer_value dist_min = _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'min').value.integer_value dist_max = _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'max').value.integer_value dist_sum = _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'sum').value.integer_value if dist_sum is None: # distribution metric is not meant to use on large values, but in case # it is, the value can overflow and become double_value, the correctness # of the value may not be guaranteed. _LOGGER.info( "Distribution metric sum value seems to have " "overflowed integer_value range, the correctness of sum or mean " "value may not be guaranteed: %s" % metric.distribution) dist_sum = int( _get_match( metric.distribution.object_value.properties, lambda x: x.key == 'sum').value.double_value) return DistributionResult( DistributionData(dist_sum, dist_count, dist_min, dist_max)) else: return None
def test_create_counter_distribution(self): MetricsEnvironment.set_current_container(MetricsContainer('mystep')) counter_ns = 'aCounterNamespace' distro_ns = 'aDistributionNamespace' name = 'a_name' counter = Metrics.counter(counter_ns, name) distro = Metrics.distribution(distro_ns, name) counter.inc(10) counter.dec(3) distro.update(10) distro.update(2) self.assertTrue(isinstance(counter, Metrics.DelegatingCounter)) self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution)) del distro del counter container = MetricsEnvironment.current_container() self.assertEqual( container.counters[MetricName(counter_ns, name)].get_cumulative(), 7) self.assertEqual( container.distributions[MetricName(distro_ns, name)].get_cumulative(), DistributionData(12, 2, 2, 10))
def test_apply_physical_logical(self): metrics = DirectMetrics() dist_zero = DistributionData(0, 0, None, None) metrics.update_physical( object(), MetricUpdates(counters={ MetricKey('step1', self.name1): 7, MetricKey('step1', self.name2): 5, MetricKey('step2', self.name1): 1 }, distributions={ MetricKey('step1', self.name1): DistributionData(3, 1, 3, 3), MetricKey('step2', self.name3): DistributionData(8, 2, 4, 4) })) results = metrics.query() hc.assert_that( results['counters'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), 0, 7), MetricResult(MetricKey('step1', self.name2), 0, 5), MetricResult(MetricKey('step2', self.name1), 0, 1) ])) hc.assert_that( results['distributions'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), DistributionResult(dist_zero), DistributionResult(DistributionData(3, 1, 3, 3))), MetricResult(MetricKey('step2', self.name3), DistributionResult(dist_zero), DistributionResult(DistributionData(8, 2, 4, 4))) ])) metrics.commit_physical( object(), MetricUpdates(counters={ MetricKey('step1', self.name1): -3, MetricKey('step2', self.name1): -5 }, distributions={ MetricKey('step1', self.name1): DistributionData(8, 4, 1, 5), MetricKey('step2', self.name2): DistributionData(8, 8, 1, 1) })) results = metrics.query() hc.assert_that( results['counters'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), 0, 4), MetricResult(MetricKey('step1', self.name2), 0, 5), MetricResult(MetricKey('step2', self.name1), 0, -4) ])) hc.assert_that( results['distributions'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), DistributionResult(dist_zero), DistributionResult(DistributionData(11, 5, 1, 5))), MetricResult(MetricKey('step2', self.name3), DistributionResult(dist_zero), DistributionResult(DistributionData(8, 2, 4, 4))), MetricResult(MetricKey('step2', self.name2), DistributionResult(dist_zero), DistributionResult(DistributionData(8, 8, 1, 1))) ])) metrics.commit_logical( object(), MetricUpdates(counters={ MetricKey('step1', self.name1): 3, MetricKey('step1', self.name2): 5, MetricKey('step2', self.name1): -3 }, distributions={ MetricKey('step1', self.name1): DistributionData(11, 5, 1, 5), MetricKey('step2', self.name2): DistributionData(8, 8, 1, 1), MetricKey('step2', self.name3): DistributionData(4, 1, 4, 4) })) results = metrics.query() hc.assert_that( results['counters'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), 3, 4), MetricResult(MetricKey('step1', self.name2), 5, 5), MetricResult(MetricKey('step2', self.name1), -3, -4) ])) hc.assert_that( results['distributions'], hc.contains_inanyorder(*[ MetricResult(MetricKey('step1', self.name1), DistributionResult(DistributionData(11, 5, 1, 5)), DistributionResult(DistributionData(11, 5, 1, 5))), MetricResult(MetricKey('step2', self.name3), DistributionResult(DistributionData(4, 1, 4, 4)), DistributionResult(DistributionData(8, 2, 4, 4))), MetricResult(MetricKey('step2', self.name2), DistributionResult(DistributionData(8, 8, 1, 1)), DistributionResult(DistributionData(8, 8, 1, 1))) ]))
def test_integer_only(self): d = DistributionCell() d.update(3.1) d.update(3.2) d.update(3.3) self.assertEqual(d.get_cumulative(), DistributionData(9, 3, 3, 3))