def test_create_counter_distribution(self):
sampler = statesampler.StateSampler('', counters.CounterFactory())
statesampler.set_current_tracker(sampler)
state1 = sampler.scoped_state('mystep', 'myState',
metrics_container=MetricsContainer('mystep'))
sampler.start()
with state1:
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution))
del distro
del counter
container = MetricsEnvironment.current_container()
self.assertEqual(
container.counters[MetricName(counter_ns, name)].get_cumulative(),
7)
self.assertEqual(
container.distributions[MetricName(distro_ns, name)].get_cumulative(),
DistributionData(12, 2, 2, 10))
sampler.stop()
def test_metrics(self):
sampler = statesampler.StateSampler('', counters.CounterFactory())
statesampler.set_current_tracker(sampler)
state1 = sampler.scoped_state(
'mystep', 'myState', metrics_container=MetricsContainer('mystep'))
try:
sampler.start()
with state1:
counter = MetricTests.base_metric_group.counter("my_counter")
meter = MetricTests.base_metric_group.meter("my_meter")
distribution = MetricTests.base_metric_group.distribution("my_distribution")
container = MetricsEnvironment.current_container()
self.assertEqual(0, counter.get_count())
self.assertEqual(0, meter.get_count())
self.assertEqual(
DistributionData(
0, 0, 0, 0), container.get_distribution(
MetricName(
'[]', 'my_distribution')).get_cumulative())
counter.inc(-2)
meter.mark_event(3)
distribution.update(10)
distribution.update(2)
self.assertEqual(-2, counter.get_count())
self.assertEqual(3, meter.get_count())
self.assertEqual(
DistributionData(
12, 2, 2, 10), container.get_distribution(
MetricName(
'[]', 'my_distribution')).get_cumulative())
finally:
sampler.stop()
def test_query_structured_metrics(self):
mock_client, mock_job_result = self.setup_mock_client_result(
self.STRUCTURED_COUNTER_LIST)
dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
dm._translate_step_name = types.MethodType(lambda self, x: 'split', dm)
query_result = dm.query()
expected_counters = [
MetricResult(
MetricKey(
'split',
MetricName('__main__.WordExtractingDoFn', 'word_lengths'),
), 109475, 109475),
]
self.assertEqual(query_result['counters'], expected_counters)
expected_distributions = [
MetricResult(
MetricKey(
'split',
MetricName('__main__.WordExtractingDoFn',
'word_length_dist'),
), DistributionResult(DistributionData(18, 2, 2, 16)),
DistributionResult(DistributionData(18, 2, 2, 16))),
]
self.assertEqual(query_result['distributions'], expected_distributions)
def test_get_cumulative_or_updates(self):
mc = MetricsContainer('astep')
all_values = []
for i in range(1, 11):
counter = mc.get_counter(MetricName('namespace', 'name{}'.format(i)))
distribution = mc.get_distribution(
MetricName('namespace', 'name{}'.format(i)))
gauge = mc.get_gauge(MetricName('namespace', 'name{}'.format(i)))
counter.inc(i)
distribution.update(i)
gauge.set(i)
all_values.append(i)
# Retrieve ALL updates.
cumulative = mc.get_cumulative()
self.assertEqual(len(cumulative.counters), 10)
self.assertEqual(len(cumulative.distributions), 10)
self.assertEqual(len(cumulative.gauges), 10)
self.assertEqual(
set(all_values), set([v for _, v in cumulative.counters.items()]))
self.assertEqual(
set(all_values), set([v.value for _, v in cumulative.gauges.items()]))
def test_create_counter_distribution(self):
MetricsEnvironment.set_current_container(MetricsContainer('mystep'))
counter_ns = 'aCounterNamespace'
distro_ns = 'aDistributionNamespace'
gauge_ns = 'aGaugeNamespace'
name = 'a_name'
counter = Metrics.counter(counter_ns, name)
distro = Metrics.distribution(distro_ns, name)
gauge = Metrics.gauge(gauge_ns, name)
counter.inc(10)
counter.dec(3)
distro.update(10)
distro.update(2)
gauge.set(10)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
self.assertTrue(isinstance(distro, Metrics.DelegatingDistribution))
self.assertTrue(isinstance(gauge, Metrics.DelegatingGauge))
del distro
del counter
del gauge
container = MetricsEnvironment.current_container()
self.assertEqual(
container.counters[MetricName(counter_ns, name)].get_cumulative(),
7)
self.assertEqual(
container.distributions[MetricName(distro_ns,
name)].get_cumulative(),
DistributionData(12, 2, 2, 10))
self.assertEqual(
container.gauges[MetricName(gauge_ns,
name)].get_cumulative().value, 10)
def test_direct_runner_metrics(self):
class MyDoFn(beam.DoFn):
def start_bundle(self):
count = Metrics.counter(self.__class__, 'bundles')
count.inc()
def finish_bundle(self):
count = Metrics.counter(self.__class__, 'finished_bundles')
count.inc()
def process(self, element):
gauge = Metrics.gauge(self.__class__, 'latest_element')
gauge.set(element)
count = Metrics.counter(self.__class__, 'elements')
count.inc()
distro = Metrics.distribution(self.__class__, 'element_dist')
distro.update(element)
return [element]
p = Pipeline(DirectRunner())
pcoll = (p | beam.Create([1, 2, 3, 4, 5])
| 'Do' >> beam.ParDo(MyDoFn()))
assert_that(pcoll, equal_to([1, 2, 3, 4, 5]))
result = p.run()
result.wait_until_finish()
metrics = result.metrics().query()
namespace = '{}.{}'.format(MyDoFn.__module__,
MyDoFn.__name__)
hc.assert_that(
metrics['counters'],
hc.contains_inanyorder(
MetricResult(
MetricKey('Do', MetricName(namespace, 'elements')),
5, 5),
MetricResult(
MetricKey('Do', MetricName(namespace, 'bundles')),
1, 1),
MetricResult(
MetricKey('Do', MetricName(namespace, 'finished_bundles')),
1, 1)))
hc.assert_that(
metrics['distributions'],
hc.contains_inanyorder(
MetricResult(
MetricKey('Do', MetricName(namespace, 'element_dist')),
DistributionResult(DistributionData(15, 5, 1, 5)),
DistributionResult(DistributionData(15, 5, 1, 5)))))
gauge_result = metrics['gauges'][0]
hc.assert_that(
gauge_result.key,
hc.equal_to(MetricKey('Do', MetricName(namespace, 'latest_element'))))
hc.assert_that(gauge_result.committed.value, hc.equal_to(5))
hc.assert_that(gauge_result.attempted.value, hc.equal_to(5))
def test_metric_filter_name_matching(self):
filter = MetricsFilter().with_name('name1').with_namespace('ns1')
name = MetricName('ns1', 'name1')
key = MetricKey('step1', name)
self.assertTrue(MetricResults.matches(filter, key))
filter = MetricsFilter().with_name('name1')
name = MetricName('ns1', 'name1')
key = MetricKey('step1', name)
self.assertTrue(MetricResults.matches(filter, key))
def test_get_cumulative_or_updates(self):
mc = MetricsContainer('astep')
clean_values = []
dirty_values = []
for i in range(1, 11):
counter = mc.get_counter(
MetricName('namespace', 'name{}'.format(i)))
distribution = mc.get_distribution(
MetricName('namespace', 'name{}'.format(i)))
gauge = mc.get_gauge(MetricName('namespace', 'name{}'.format(i)))
counter.inc(i)
distribution.update(i)
gauge.set(i)
if i % 2 == 0:
# Some are left to be DIRTY (i.e. not yet committed).
# Some are left to be CLEAN (i.e. already committed).
dirty_values.append(i)
continue
# Assert: Counter/Distribution is DIRTY or COMMITTING (not CLEAN)
self.assertEqual(distribution.commit.before_commit(), True)
self.assertEqual(counter.commit.before_commit(), True)
self.assertEqual(gauge.commit.before_commit(), True)
distribution.commit.after_commit()
counter.commit.after_commit()
gauge.commit.after_commit()
# Assert: Counter/Distribution has been committed, therefore it's CLEAN
self.assertEqual(counter.commit.state, CellCommitState.CLEAN)
self.assertEqual(distribution.commit.state, CellCommitState.CLEAN)
self.assertEqual(gauge.commit.state, CellCommitState.CLEAN)
clean_values.append(i)
# Retrieve NON-COMMITTED updates.
logical = mc.get_updates()
self.assertEqual(len(logical.counters), 5)
self.assertEqual(len(logical.distributions), 5)
self.assertEqual(len(logical.gauges), 5)
self.assertEqual(set(dirty_values),
set([v.value for _, v in logical.gauges.items()]))
self.assertEqual(set(dirty_values),
set([v for _, v in logical.counters.items()]))
# Retrieve ALL updates.
cumulative = mc.get_cumulative()
self.assertEqual(len(cumulative.counters), 10)
self.assertEqual(len(cumulative.distributions), 10)
self.assertEqual(len(cumulative.gauges), 10)
self.assertEqual(set(dirty_values + clean_values),
set([v for _, v in cumulative.counters.items()]))
self.assertEqual(set(dirty_values + clean_values),
set([v.value for _, v in cumulative.gauges.items()]))
def test_basic_metric_name(self):
name = MetricName('namespace1', 'name1')
self.assertEqual(name.namespace, 'namespace1')
self.assertEqual(name.name, 'name1')
self.assertEqual(name, MetricName('namespace1', 'name1'))
key = MetricKey('step1', name)
self.assertEqual(key.step, 'step1')
self.assertEqual(key.metric.namespace, 'namespace1')
self.assertEqual(key.metric.name, 'name1')
self.assertEqual(key, MetricKey('step1', MetricName('namespace1', 'name1')))
def test_equality_for_key_with_labels(self):
test_labels = {'label1', 'value1'}
test_object = MetricKey(
'step', MetricName('namespace', 'name'), labels=test_labels)
same_labels = MetricKey(
'step', MetricName('namespace', 'name'), labels={'label1', 'value1'})
same_label_reference = MetricKey(
'step', MetricName('namespace', 'name'), labels=test_labels)
self.assertEqual(test_object, same_labels)
self.assertEqual(test_object, same_label_reference)
self.assertEqual(hash(test_object), hash(same_labels))
self.assertEqual(hash(test_object), hash(same_label_reference))
def test_direct_runner_metrics(self):
from apache_beam.metrics.metric import Metrics
class MyDoFn(beam.DoFn):
def start_bundle(self):
count = Metrics.counter(self.__class__, 'bundles')
count.inc()
def finish_bundle(self):
count = Metrics.counter(self.__class__, 'finished_bundles')
count.inc()
def process(self, element):
count = Metrics.counter(self.__class__, 'elements')
count.inc()
distro = Metrics.distribution(self.__class__, 'element_dist')
distro.update(element)
return [element]
runner = DirectRunner()
p = Pipeline(runner,
options=PipelineOptions(self.default_properties))
# pylint: disable=expression-not-assigned
(p | ptransform.Create([1, 2, 3, 4, 5])
| 'Do' >> beam.ParDo(MyDoFn()))
result = p.run()
result.wait_until_finish()
metrics = result.metrics().query()
namespace = '{}.{}'.format(MyDoFn.__module__,
MyDoFn.__name__)
hc.assert_that(
metrics['counters'],
hc.contains_inanyorder(
MetricResult(
MetricKey('Do', MetricName(namespace, 'elements')),
5, 5),
MetricResult(
MetricKey('Do', MetricName(namespace, 'bundles')),
1, 1),
MetricResult(
MetricKey('Do', MetricName(namespace, 'finished_bundles')),
1, 1)))
hc.assert_that(
metrics['distributions'],
hc.contains_inanyorder(
MetricResult(
MetricKey('Do', MetricName(namespace, 'element_dist')),
DistributionResult(DistributionData(15, 5, 1, 5)),
DistributionResult(DistributionData(15, 5, 1, 5)))))
def test_inequality_for_key_with_labels(self):
test_labels = {'label1', 'value1'}
test_object = MetricKey(
'step', MetricName('namespace', 'name'), labels=test_labels)
no_labels = MetricKey('step', MetricName('namespace', 'name'))
diff_label_key = MetricKey(
'step', MetricName('namespace', 'name'), labels={'l1_diff', 'value1'})
diff_label_value = MetricKey(
'step', MetricName('namespace', 'name'), labels={'label1', 'v1_diff'})
self.assertNotEqual(test_object, no_labels)
self.assertNotEqual(test_object, diff_label_key)
self.assertNotEqual(test_object, diff_label_value)
self.assertNotEqual(hash(test_object), hash(no_labels))
self.assertNotEqual(hash(test_object), hash(diff_label_key))
self.assertNotEqual(hash(test_object), hash(diff_label_value))
def test_uses_right_container(self):
c1 = MetricsContainer('step1')
c2 = MetricsContainer('step2')
counter = Metrics.counter('ns', 'name')
MetricsEnvironment.set_current_container(c1)
counter.inc()
MetricsEnvironment.set_current_container(c2)
counter.inc(3)
MetricsEnvironment.unset_current_container()
self.assertEqual(list(c1.get_cumulative().counters.items()),
[(MetricKey('step1', MetricName('ns', 'name')), 1)])
self.assertEqual(list(c2.get_cumulative().counters.items()),
[(MetricKey('step2', MetricName('ns', 'name')), 3)])
def test_equality_for_key_with_no_labels(self):
test_object = MetricKey('step', MetricName('namespace', 'name'))
same = MetricKey('step', MetricName('namespace', 'name'))
self.assertEqual(test_object, same)
self.assertEqual(hash(test_object), hash(same))
diff_step = MetricKey('step_diff', MetricName('namespace', 'name'))
diff_namespace = MetricKey('step', MetricName('namespace_diff', 'name'))
diff_name = MetricKey('step', MetricName('namespace', 'name_diff'))
self.assertNotEqual(test_object, diff_step)
self.assertNotEqual(test_object, diff_namespace)
self.assertNotEqual(test_object, diff_name)
self.assertNotEqual(hash(test_object), hash(diff_step))
self.assertNotEqual(hash(test_object), hash(diff_namespace))
self.assertNotEqual(hash(test_object), hash(diff_name))
def test_downloader_fail_to_get_project_number(self, mock_get):
# Raising an error when listing GCS Bucket so that project number fails to
# be retrieved.
mock_get.side_effect = HttpError({'status': 403}, None, None)
# Clear the process wide metric container.
MetricsEnvironment.process_wide_container().reset()
file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
file_size = 5 * 1024 * 1024 + 100
random_file = self._insert_random_file(self.client, file_name,
file_size)
self.gcs.open(file_name, 'r')
resource = resource_identifiers.GoogleCloudStorageBucket(
random_file.bucket)
labels = {
monitoring_infos.SERVICE_LABEL: 'Storage',
monitoring_infos.METHOD_LABEL: 'Objects.get',
monitoring_infos.RESOURCE_LABEL: resource,
monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
monitoring_infos.GCS_PROJECT_ID_LABEL: str(DEFAULT_PROJECT_NUMBER),
monitoring_infos.STATUS_LABEL: 'ok'
}
metric_name = MetricName(None,
None,
urn=monitoring_infos.API_REQUEST_COUNT_URN,
labels=labels)
metric_value = MetricsEnvironment.process_wide_container().get_counter(
metric_name).get_cumulative()
self.assertEqual(metric_value, 0)
labels_without_project_id = {
monitoring_infos.SERVICE_LABEL: 'Storage',
monitoring_infos.METHOD_LABEL: 'Objects.get',
monitoring_infos.RESOURCE_LABEL: resource,
monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
monitoring_infos.STATUS_LABEL: 'ok'
}
metric_name = MetricName(None,
None,
urn=monitoring_infos.API_REQUEST_COUNT_URN,
labels=labels_without_project_id)
metric_value = MetricsEnvironment.process_wide_container().get_counter(
metric_name).get_cumulative()
self.assertEqual(metric_value, 2)
def test_uploader_monitoring_info(self):
# Clear the process wide metric container.
MetricsEnvironment.process_wide_container().reset()
file_name = 'gs://gcsio-metrics-test/dummy_mode_file'
file_size = 5 * 1024 * 1024 + 100
random_file = self._insert_random_file(self.client, file_name,
file_size)
f = self.gcs.open(file_name, 'w')
resource = resource_identifiers.GoogleCloudStorageBucket(
random_file.bucket)
labels = {
monitoring_infos.SERVICE_LABEL: 'Storage',
monitoring_infos.METHOD_LABEL: 'Objects.insert',
monitoring_infos.RESOURCE_LABEL: resource,
monitoring_infos.GCS_BUCKET_LABEL: random_file.bucket,
monitoring_infos.GCS_PROJECT_ID_LABEL: str(DEFAULT_PROJECT_NUMBER),
monitoring_infos.STATUS_LABEL: 'ok'
}
f.close()
metric_name = MetricName(None,
None,
urn=monitoring_infos.API_REQUEST_COUNT_URN,
labels=labels)
metric_value = MetricsEnvironment.process_wide_container().get_counter(
metric_name).get_cumulative()
self.assertEqual(metric_value, 1)
def test_create_process_wide(self):
sampler = statesampler.StateSampler('', counters.CounterFactory())
statesampler.set_current_tracker(sampler)
state1 = sampler.scoped_state(
'mystep', 'myState', metrics_container=MetricsContainer('mystep'))
try:
sampler.start()
with state1:
urn = "my:custom:urn"
labels = {'key': 'value'}
counter = InternalMetrics.counter(
urn=urn, labels=labels, process_wide=True)
# Test that if process_wide is set, that it will be set
# on the process_wide container.
counter.inc(10)
self.assertTrue(isinstance(counter, Metrics.DelegatingCounter))
del counter
metric_name = MetricName(None, None, urn=urn, labels=labels)
# Expect a value set on the current container.
self.assertEqual(
MetricsEnvironment.process_wide_container().get_counter(
metric_name).get_cumulative(),
10)
# Expect no value set on the current container.
self.assertEqual(
MetricsEnvironment.current_container().get_counter(
metric_name).get_cumulative(),
0)
finally:
sampler.stop()
def _get_metric_key(self, metric):
"""Populate the MetricKey object for a queried metric result."""
try:
# If ValueError is thrown within this try-block, it is because of
# one of the following:
# 1. Unable to translate the step name. Only happening with improperly
# formatted job graph (unlikely), or step name not being the internal
# step name (only happens for unstructured-named metrics).
# 2. Unable to unpack [step] or [namespace]; which should only happen
# for unstructured names.
[step] = [
prop.value for prop in metric.name.context.additionalProperties
if prop.key == 'step'
]
step = self._translate_step_name(step)
[namespace] = [
prop.value for prop in metric.name.context.additionalProperties
if prop.key == 'namespace'
]
name = metric.name.name
except ValueError:
# An unstructured metric name is "step/namespace/name", but step names
# can (and often do) contain slashes. Must only split on the right-most
# two slashes, to preserve the full step name.
[step, namespace, name] = metric.name.name.rsplit('/', 2)
return MetricKey(step, MetricName(namespace, name))
def test_metric_filter_step_matching(self):
name = MetricName('ns1', 'name1')
filter = MetricsFilter().with_step('Step1')
key = MetricKey('Step1', name)
self.assertTrue(MetricResults.matches(filter, key))
key = MetricKey('Step10', name)
self.assertFalse(MetricResults.matches(filter, key))
key = MetricKey('Step10/Step1', name)
self.assertTrue(MetricResults.matches(filter, key))
key = MetricKey('Top1/Outer1/Inner1', name)
filter = MetricsFilter().with_step('Top1/Outer1/Inner1')
self.assertTrue(MetricResults.matches(filter, key))
filter = MetricsFilter().with_step('Top1/Outer1')
self.assertTrue(MetricResults.matches(filter, key))
filter = MetricsFilter().with_step('Outer1/Inner1')
self.assertTrue(MetricResults.matches(filter, key))
filter = MetricsFilter().with_step('Top1/Inner1')
self.assertFalse(MetricResults.matches(filter, key))
def test_start_time_set(self):
c = CounterCell()
c.inc(2)
name = MetricName('namespace', 'name1')
mi = c.to_runner_api_monitoring_info(name, 'transform_id')
self.assertGreater(mi.start_time.seconds, 0)
def _get_metric_key(self, metric):
"""Populate the MetricKey object for a queried metric result."""
step = ""
name = metric.name.name # Always extract a name
labels = dict()
try: # Try to extract the user step name.
# If ValueError is thrown within this try-block, it is because of
# one of the following:
# 1. Unable to translate the step name. Only happening with improperly
# formatted job graph (unlikely), or step name not being the internal
# step name (only happens for unstructured-named metrics).
# 2. Unable to unpack [step] or [namespace]; which should only happen
# for unstructured names.
step = _get_match(metric.name.context.additionalProperties,
lambda x: x.key == STEP_LABEL).value
step = self._translate_step_name(step)
except ValueError:
pass
namespace = "dataflow/v1b3" # Try to extract namespace or add a default.
try:
namespace = _get_match(metric.name.context.additionalProperties,
lambda x: x.key == 'namespace').value
except ValueError:
pass
for kv in metric.name.context.additionalProperties:
if kv.key in STRUCTURED_NAME_LABELS:
labels[kv.key] = kv.value
# Package everything besides namespace and name the labels as well,
# including unmodified step names to assist in integration the exact
# unmodified values which come from dataflow.
return MetricKey(step, MetricName(namespace, name), labels=labels)
def test_start_time_set(self):
d = DistributionCell()
d.update(3.1)
name = MetricName('namespace', 'name1')
mi = d.to_runner_api_monitoring_info(name, 'transform_id')
self.assertGreater(mi.start_time.seconds, 0)
def test_start_time_set(self):
g1 = GaugeCell()
g1.set(3)
name = MetricName('namespace', 'name1')
mi = g1.to_runner_api_monitoring_info(name, 'transform_id')
self.assertGreater(mi.start_time.seconds, 0)
def assert_counter_exists(metrics, namespace, name, step):
found = 0
metric_key = MetricKey(step, MetricName(namespace, name))
for m in metrics['counters']:
if m.key == metric_key:
found = found + 1
self.assertEqual(
1, found, "Did not find exactly 1 metric for %s." % metric_key)
def test_general_urn_metric_name_str(self):
mn = MetricName("my_namespace",
"my_name",
urn='my_urn',
labels={'key': 'value'})
expected_str = ("MetricName(namespace=my_namespace, name=my_name, "
"urn=my_urn, labels={'key': 'value'})")
self.assertEqual(str(mn), expected_str)
def test_query_counters(self):
mock_client, mock_job_result = self.setup_mock_client_result()
dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
query_result = dm.query()
expected_counters = [
MetricResult(
MetricKey(
'split',
MetricName('__main__.WordExtractingDoFn', 'empty_lines')),
1080, 1080),
MetricResult(
MetricKey('longstepname/split',
MetricName('__main__.WordExtractingDoFn', 'words')),
26181, 26185),
]
self.assertEqual(
sorted(query_result['counters'], key=lambda x: x.key.metric.name),
sorted(expected_counters, key=lambda x: x.key.metric.name))
def test_pardo_metrics(self):
class MyDoFn(beam.DoFn):
def start_bundle(self):
self.count = Metrics.counter(self.__class__, 'elements')
def process(self, element):
self.count.inc(element)
return [element]
class MyOtherDoFn(beam.DoFn):
def start_bundle(self):
self.count = Metrics.counter(self.__class__, 'elementsplusone')
def process(self, element):
self.count.inc(element + 1)
return [element]
with self.create_pipeline() as p:
res = (p | beam.Create([1, 2, 3])
| 'mydofn' >> beam.ParDo(MyDoFn())
| 'myotherdofn' >> beam.ParDo(MyOtherDoFn()))
p.run()
if not MetricsEnvironment.METRICS_SUPPORTED:
self.skipTest('Metrics are not supported.')
counter_updates = [
{
'key': key,
'value': val
} for container in p.runner.metrics_containers()
for key, val in container.get_updates().counters.items()
]
counter_values = [update['value'] for update in counter_updates]
counter_keys = [update['key'] for update in counter_updates]
assert_that(res, equal_to([1, 2, 3]))
self.assertEqual(counter_values, [6, 9])
self.assertEqual(counter_keys, [
MetricKey('mydofn', MetricName(__name__ + '.MyDoFn',
'elements')),
MetricKey(
'myotherdofn',
MetricName(__name__ + '.MyOtherDoFn', 'elementsplusone'))
])
def test_query_counters(self):
mock_client, mock_job_result = self.setup_mock_client_result(
self.ONLY_COUNTERS_LIST)
dm = dataflow_metrics.DataflowMetrics(mock_client, mock_job_result)
dm._translate_step_name = types.MethodType(lambda self, x: 'split', dm)
query_result = dm.query()
expected_counters = [
MetricResult(
MetricKey(
'split',
MetricName('__main__.WordExtractingDoFn', 'empty_lines')),
1080, 1080),
MetricResult(
MetricKey('split',
MetricName('__main__.WordExtractingDoFn', 'words')),
26181, 26185),
]
self.assertEqual(
sorted(query_result['counters'], key=lambda x: x.key.metric.name),
sorted(expected_counters, key=lambda x: x.key.metric.name))
def counter(namespace, name):
"""Obtains or creates a Counter metric.
Args:
namespace: A class or string that gives the namespace to a metric
name: A string that gives a unique name to a metric
Returns:
A Counter object.
"""
namespace = Metrics.get_namespace(namespace)
return Metrics.DelegatingCounter(MetricName(namespace, name))
def test_scoped_container(self):
c1 = MetricsContainer('mystep')
c2 = MetricsContainer('myinternalstep')
with ScopedMetricsContainer(c1):
self.assertEqual(c1, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(2)
with ScopedMetricsContainer(c2):
self.assertEqual(c2, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(3)
self.assertEqual(list(c2.get_cumulative().counters.items()),
[(MetricKey('myinternalstep',
MetricName('ns', 'name')), 3)])
self.assertEqual(c1, MetricsEnvironment.current_container())
counter = Metrics.counter('ns', 'name')
counter.inc(4)
self.assertEqual(
list(c1.get_cumulative().counters.items()),
[(MetricKey('mystep', MetricName('ns', 'name')), 6)])
