def __init__(self, project, fixed_batch_size=None): """ Args: project: str, the cloud project id. fixed_batch_size: int, for testing only, this forces all batches of writes to be a fixed size, for easier unittesting. """ self._project = project self._datastore = None self._fixed_batch_size = fixed_batch_size self._rpc_successes = Metrics.counter( _Mutate.DatastoreWriteFn, "datastoreRpcSuccesses") self._rpc_errors = Metrics.counter( _Mutate.DatastoreWriteFn, "datastoreRpcErrors")
def test_create_counter_distribution(self): MetricsEnvironment.set_current_container(MetricsContainer('mystep')) counter_ns = 'aCounterNamespace' distro_ns = 'aDistributionNamespace' gauge_ns = 'aGaugeNamespace' name = 'a_name' counter = Metrics.counter(counter_ns, name) distro = Metrics.distribution(distro_ns, name) gauge = Metrics.gauge(gauge_ns, name) counter.inc(10) counter.dec(3) distro.update(10) distro.update(2) gauge.set(10) self.assertTrue(isinstance(counter, Metrics.DelegatingCounter)) self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution)) self.assertTrue(isinstance(gauge, Metrics.DelegatingGauge)) del distro del counter del gauge container = MetricsEnvironment.current_container() self.assertEqual( container.counters[MetricName(counter_ns, name)].get_cumulative(), 7) self.assertEqual( container.distributions[MetricName(distro_ns, name)].get_cumulative(), DistributionData(12, 2, 2, 10)) self.assertEqual( container.gauges[MetricName(gauge_ns, name)].get_cumulative().value, 10)
def test_create_counter_distribution(self): sampler = statesampler.StateSampler('', counters.CounterFactory()) statesampler.set_current_tracker(sampler) state1 = sampler.scoped_state('mystep', 'myState', metrics_container=MetricsContainer('mystep')) sampler.start() with state1: counter_ns = 'aCounterNamespace' distro_ns = 'aDistributionNamespace' name = 'a_name' counter = Metrics.counter(counter_ns, name) distro = Metrics.distribution(distro_ns, name) counter.inc(10) counter.dec(3) distro.update(10) distro.update(2) self.assertTrue(isinstance(counter, Metrics.DelegatingCounter)) self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution)) del distro del counter container = MetricsEnvironment.current_container() self.assertEqual( container.counters[MetricName(counter_ns, name)].get_cumulative(), 7) self.assertEqual( container.distributions[MetricName(distro_ns, name)].get_cumulative(), DistributionData(12, 2, 2, 10)) sampler.stop()
def process(self, element): gauge = Metrics.gauge(self.__class__, 'latest_element') gauge.set(element) count = Metrics.counter(self.__class__, 'elements') count.inc() distro = Metrics.distribution(self.__class__, 'element_dist') distro.update(element) return [element]
def __init__(self, project, fixed_batch_size=None): """ Args: project: str, the cloud project id. fixed_batch_size: int, for testing only, this forces all batches of writes to be a fixed size, for easier unittesting. """ self._project = project self._datastore = None self._fixed_batch_size = fixed_batch_size self._rpc_successes = Metrics.counter( _Mutate.DatastoreWriteFn, "datastoreRpcSuccesses") self._rpc_errors = Metrics.counter( _Mutate.DatastoreWriteFn, "datastoreRpcErrors") self._throttled_secs = Metrics.counter( _Mutate.DatastoreWriteFn, "cumulativeThrottlingSeconds") self._throttler = AdaptiveThrottler(window_ms=120000, bucket_ms=1000, overload_ratio=1.25)
def test_run_api(self): my_metric = Metrics.counter('namespace', 'my_metric') runner = DirectRunner() result = runner.run( beam.Create([1, 10, 100]) | beam.Map(lambda x: my_metric.inc(x))) result.wait_until_finish() # Use counters to assert the pipeline actually ran. my_metric_value = result.metrics().query()['counters'][0].committed self.assertEqual(my_metric_value, 111)
def test_scoped_container(self): c1 = MetricsContainer('mystep') c2 = MetricsContainer('myinternalstep') with ScopedMetricsContainer(c1): self.assertEqual(c1, MetricsEnvironment.current_container()) counter = Metrics.counter('ns', 'name') counter.inc(2) with ScopedMetricsContainer(c2): self.assertEqual(c2, MetricsEnvironment.current_container()) counter = Metrics.counter('ns', 'name') counter.inc(3) self.assertEqual( c2.get_cumulative().counters.items(), [(MetricKey('myinternalstep', MetricName('ns', 'name')), 3)]) self.assertEqual(c1, MetricsEnvironment.current_container()) counter = Metrics.counter('ns', 'name') counter.inc(4) self.assertEqual( c1.get_cumulative().counters.items(), [(MetricKey('mystep', MetricName('ns', 'name')), 6)])
def test_uses_right_container(self): c1 = MetricsContainer('step1') c2 = MetricsContainer('step2') counter = Metrics.counter('ns', 'name') MetricsEnvironment.set_current_container(c1) counter.inc() MetricsEnvironment.set_current_container(c2) counter.inc(3) MetricsEnvironment.unset_current_container() self.assertEqual( c1.get_cumulative().counters.items(), [(MetricKey('step1', MetricName('ns', 'name')), 1)]) self.assertEqual( c2.get_cumulative().counters.items(), [(MetricKey('step2', MetricName('ns', 'name')), 3)])
def __init__(self): super(ParseGameEventFn, self).__init__() self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
def test_counter_empty_namespace(self): with self.assertRaises(ValueError): Metrics.counter("", "names")
def __init__(self): super(ParseHastagData, self).__init__() self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
def __init__(self): self.processed_records = Metrics.counter(self.__class__, 'processed_records') self.parsing_errors = Metrics.counter(self.__class__, 'parsing_errors') self.schema = 'ancestor_id,authors,cleaned_content,content,conversation_id,id,indentation,page_id,page_title,parent_id,replyTo_id,rev_id,timestamp,type,user_id,user_text' self.fields = self.schema.split(',')
def meter(self, name: str, time_span_in_seconds: int = 60) -> 'Meter': from apache_beam.metrics.metric import Metrics # There is no meter type in Beam, use counter to implement meter return Meter( Metrics.counter(self._get_namespace(time_span_in_seconds), name))
def finish_bundle(self): count = Metrics.counter(self.__class__, 'finished_bundles') count.inc()
def process(self, element): count = Metrics.counter(self.__class__, 'elements') count.inc() distro = Metrics.distribution(self.__class__, 'element_dist') distro.update(element) return [element]
def start_bundle(self): count = Metrics.counter(self.__class__, 'bundles') count.inc()
def __init__(self): # TODO(BEAM-6158): Revert the workaround once we can pickle super() on py3. # super(ParseGameEventFn, self).__init__() beam.DoFn.__init__(self) self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
def __init__(self): beam.DoFn.__init__(self) self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
def counter(self, name: str) -> 'Counter': from apache_beam.metrics.metric import Metrics return Counter(Metrics.counter(self._get_namespace(), name))
def __init__(self): self.processed_revisions = Metrics.counter(self.__class__, 'processed_revisions') self.pov_pages = Metrics.counter(self.__class__, 'pov_pages')