def __init__(self, project, fixed_batch_size=None):
"""
Args:
project: str, the cloud project id.
fixed_batch_size: int, for testing only, this forces all batches of
writes to be a fixed size, for easier unittesting.
"""
self._project = project
self._datastore = None
self._fixed_batch_size = fixed_batch_size
self._rpc_successes = Metrics.counter(
_Mutate.DatastoreWriteFn, "datastoreRpcSuccesses")
self._rpc_errors = Metrics.counter(
_Mutate.DatastoreWriteFn, "datastoreRpcErrors")
def test_create_counter_distribution(self):
MetricsEnvironment.set_current_container(MetricsContainer('mystep'))
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
gauge_ns = 'aGaugeNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
gauge = Metrics.gauge(gauge_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
gauge.set(10)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution))
self.assertTrue(isinstance(gauge, Metrics.DelegatingGauge))
del distro
del counter
del gauge
container = MetricsEnvironment.current_container()
self.assertEqual(
container.counters[MetricName(counter_ns, name)].get_cumulative(),
7)
self.assertEqual(
container.distributions[MetricName(distro_ns, name)].get_cumulative(),
DistributionData(12, 2, 2, 10))
self.assertEqual(
container.gauges[MetricName(gauge_ns, name)].get_cumulative().value,
10)
def test_create_counter_distribution(self):
sampler = statesampler.StateSampler('', counters.CounterFactory())
statesampler.set_current_tracker(sampler)
state1 = sampler.scoped_state('mystep', 'myState',
metrics_container=MetricsContainer('mystep'))
sampler.start()
with state1:
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution))
del distro
del counter
container = MetricsEnvironment.current_container()
self.assertEqual(
container.counters[MetricName(counter_ns, name)].get_cumulative(),
7)
self.assertEqual(
container.distributions[MetricName(distro_ns, name)].get_cumulative(),
DistributionData(12, 2, 2, 10))
sampler.stop()
def process(self, element): gauge = Metrics.gauge(self.__class__, 'latest_element') gauge.set(element) count = Metrics.counter(self.__class__, 'elements') count.inc() distro = Metrics.distribution(self.__class__, 'element_dist') distro.update(element) return [element]
def __init__(self, project, fixed_batch_size=None):
"""
Args:
project: str, the cloud project id.
fixed_batch_size: int, for testing only, this forces all batches of
writes to be a fixed size, for easier unittesting.
"""
self._project = project
self._datastore = None
self._fixed_batch_size = fixed_batch_size
self._rpc_successes = Metrics.counter(
_Mutate.DatastoreWriteFn, "datastoreRpcSuccesses")
self._rpc_errors = Metrics.counter(
_Mutate.DatastoreWriteFn, "datastoreRpcErrors")
self._throttled_secs = Metrics.counter(
_Mutate.DatastoreWriteFn, "cumulativeThrottlingSeconds")
self._throttler = AdaptiveThrottler(window_ms=120000, bucket_ms=1000,
overload_ratio=1.25)
def test_run_api(self):
my_metric = Metrics.counter('namespace', 'my_metric')
runner = DirectRunner()
result = runner.run(
beam.Create([1, 10, 100]) | beam.Map(lambda x: my_metric.inc(x)))
result.wait_until_finish()
# Use counters to assert the pipeline actually ran.
my_metric_value = result.metrics().query()['counters'][0].committed
self.assertEqual(my_metric_value, 111)
def test_scoped_container(self):
c1 = MetricsContainer('mystep')
c2 = MetricsContainer('myinternalstep')
with ScopedMetricsContainer(c1):
self.assertEqual(c1, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(2)
with ScopedMetricsContainer(c2):
self.assertEqual(c2, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(3)
self.assertEqual(
c2.get_cumulative().counters.items(),
[(MetricKey('myinternalstep', MetricName('ns', 'name')), 3)])
self.assertEqual(c1, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(4)
self.assertEqual(
c1.get_cumulative().counters.items(),
[(MetricKey('mystep', MetricName('ns', 'name')), 6)])
def test_uses_right_container(self):
c1 = MetricsContainer('step1')
c2 = MetricsContainer('step2')
counter = Metrics.counter('ns', 'name')
MetricsEnvironment.set_current_container(c1)
counter.inc()
MetricsEnvironment.set_current_container(c2)
counter.inc(3)
MetricsEnvironment.unset_current_container()
self.assertEqual(
c1.get_cumulative().counters.items(),
[(MetricKey('step1', MetricName('ns', 'name')), 1)])
self.assertEqual(
c2.get_cumulative().counters.items(),
[(MetricKey('step2', MetricName('ns', 'name')), 3)])
def __init__(self): super(ParseGameEventFn, self).__init__() self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')
def test_counter_empty_namespace(self):
with self.assertRaises(ValueError):
Metrics.counter("", "names")
def __init__(self):
super(ParseHastagData, self).__init__()
self.num_parse_errors = Metrics.counter(self.__class__,
'num_parse_errors')
def __init__(self):
self.processed_records = Metrics.counter(self.__class__,
'processed_records')
self.parsing_errors = Metrics.counter(self.__class__, 'parsing_errors')
self.schema = 'ancestor_id,authors,cleaned_content,content,conversation_id,id,indentation,page_id,page_title,parent_id,replyTo_id,rev_id,timestamp,type,user_id,user_text'
self.fields = self.schema.split(',')
def meter(self, name: str, time_span_in_seconds: int = 60) -> 'Meter':
from apache_beam.metrics.metric import Metrics
# There is no meter type in Beam, use counter to implement meter
return Meter(
Metrics.counter(self._get_namespace(time_span_in_seconds), name))
def finish_bundle(self): count = Metrics.counter(self.__class__, 'finished_bundles') count.inc()
def process(self, element):
count = Metrics.counter(self.__class__, 'elements')
count.inc()
distro = Metrics.distribution(self.__class__, 'element_dist')
distro.update(element)
return [element]
def finish_bundle(self):
count = Metrics.counter(self.__class__, 'finished_bundles')
count.inc()
def start_bundle(self):
count = Metrics.counter(self.__class__, 'bundles')
count.inc()
def __init__(self):
# TODO(BEAM-6158): Revert the workaround once we can pickle super() on py3.
# super(ParseGameEventFn, self).__init__()
beam.DoFn.__init__(self)
self.num_parse_errors = Metrics.counter(self.__class__,
'num_parse_errors')
def __init__(self):
beam.DoFn.__init__(self)
self.num_parse_errors = Metrics.counter(self.__class__,
'num_parse_errors')
def __init__(self):
super(ParseGameEventFn, self).__init__()
self.num_parse_errors = Metrics.counter(self.__class__,
'num_parse_errors')
def counter(self, name: str) -> 'Counter':
from apache_beam.metrics.metric import Metrics
return Counter(Metrics.counter(self._get_namespace(), name))
def start_bundle(self): count = Metrics.counter(self.__class__, 'bundles') count.inc()
def test_counter_empty_namespace(self):
with self.assertRaises(ValueError):
Metrics.counter("", "names")
def process(self, element): count = Metrics.counter(self.__class__, 'elements') count.inc() distro = Metrics.distribution(self.__class__, 'element_dist') distro.update(element) return [element]
def __init__(self):
self.processed_revisions = Metrics.counter(self.__class__,
'processed_revisions')
self.pov_pages = Metrics.counter(self.__class__, 'pov_pages')
