def test_apply_physical_no_filter(self):
metrics = DirectMetrics()
metrics.update_physical(
object(),
MetricUpdates(
counters={
MetricKey('step1', self.name1): 5,
MetricKey('step1', self.name3): 8
}))
metrics.update_physical(
object(),
MetricUpdates(
counters={
MetricKey('step2', self.name1): 7,
MetricKey('step1', self.name3): 4
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 5),
MetricResult(MetricKey('step1', self.name3), 0, 12),
MetricResult(MetricKey('step2', self.name1), 0, 7)
]))
metrics.commit_physical(object(), MetricUpdates())
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 5),
MetricResult(MetricKey('step1', self.name3), 0, 12),
MetricResult(MetricKey('step2', self.name1), 0, 7)
]))
def __init__(
self,
pipeline_options,
bundle_factory, # type: BundleFactory
root_transforms,
value_to_consumers,
step_names,
views, # type: Iterable[pvalue.AsSideInput]
clock):
self.pipeline_options = pipeline_options
self._bundle_factory = bundle_factory
self._root_transforms = root_transforms
self._value_to_consumers = value_to_consumers
self._step_names = step_names
self.views = views
self._pcollection_to_views = collections.defaultdict(
list) # type: DefaultDict[pvalue.PValue, List[pvalue.AsSideInput]]
for view in views:
self._pcollection_to_views[view.pvalue].append(view)
self._transform_keyed_states = self._initialize_keyed_states(
root_transforms, value_to_consumers)
self._side_inputs_container = _SideInputsContainer(views)
self._watermark_manager = WatermarkManager(
clock, root_transforms, value_to_consumers,
self._transform_keyed_states)
self._pending_unblocked_tasks = [
] # type: List[Tuple[TransformExecutor, Timestamp]]
self._counter_factory = counters.CounterFactory()
self._metrics = DirectMetrics()
self._lock = threading.Lock()
self.shutdown_requested = False
def test_combiner_functions(self):
metrics = DirectMetrics()
counter = metrics._counters['anykey']
counter.commit_logical(self.bundle1, 5)
self.assertEqual(counter.extract_committed(), 5)
with self.assertRaises(TypeError):
counter.commit_logical(self.bundle1, None)
distribution = metrics._distributions['anykey']
distribution.commit_logical(self.bundle1, DistributionData(4, 1, 4, 4))
self.assertEqual(distribution.extract_committed(),
DistributionResult(DistributionData(4, 1, 4, 4)))
with self.assertRaises(AttributeError):
distribution.commit_logical(self.bundle1, None)
def __init__(self, pipeline_options, bundle_factory, root_transforms,
value_to_consumers, step_names, views):
self.pipeline_options = pipeline_options
self._bundle_factory = bundle_factory
self._root_transforms = root_transforms
self._value_to_consumers = value_to_consumers
self._step_names = step_names
self.views = views
# AppliedPTransform -> Evaluator specific state objects
self._application_state_interals = {}
self._watermark_manager = WatermarkManager(Clock(), root_transforms,
value_to_consumers)
self._side_inputs_container = _SideInputsContainer(views)
self._pending_unblocked_tasks = []
self._counter_factory = counters.CounterFactory()
self._cache = None
self._metrics = DirectMetrics()
self._lock = threading.Lock()
def test_commit_logical_no_filter(self):
metrics = DirectMetrics()
metrics.commit_logical(
self.bundle1,
MetricUpdates(
counters={
MetricKey('step1', self.name1): 5,
MetricKey('step1', self.name2): 8
},
distributions={
MetricKey('step1', self.name1): DistributionData(8, 2, 3, 5)
}))
metrics.commit_logical(
self.bundle1,
MetricUpdates(
counters={
MetricKey('step2', self.name1): 7,
MetricKey('step1', self.name2): 4
},
distributions={
MetricKey('step1', self.name1): DistributionData(4, 1, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(
*[
MetricResult(MetricKey('step1', self.name2), 12, 0),
MetricResult(MetricKey('step2', self.name1), 7, 0),
MetricResult(MetricKey('step1', self.name1), 5, 0)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(
MetricResult(
MetricKey('step1', self.name1),
DistributionResult(DistributionData(12, 3, 3, 5)),
DistributionResult(DistributionData(0, 0, None, None)))))
def __init__(self, pipeline_options, bundle_factory, root_transforms,
value_to_consumers, step_names, views, clock):
self.pipeline_options = pipeline_options
self._bundle_factory = bundle_factory
self._root_transforms = root_transforms
self._value_to_consumers = value_to_consumers
self._step_names = step_names
self.views = views
self._pcollection_to_views = collections.defaultdict(list)
for view in views:
self._pcollection_to_views[view.pvalue].append(view)
self._transform_keyed_states = self._initialize_keyed_states(
root_transforms, value_to_consumers)
self._side_inputs_container = _SideInputsContainer(views)
self._watermark_manager = WatermarkManager(
clock, root_transforms, value_to_consumers,
self._transform_keyed_states)
self._pending_unblocked_tasks = []
self._counter_factory = counters.CounterFactory()
self._metrics = DirectMetrics()
self._lock = threading.Lock()
def test_apply_physical_logical(self):
metrics = DirectMetrics()
dist_zero = DistributionData(0, 0, None, None)
metrics.update_physical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): 7,
MetricKey('step1', self.name2): 5,
MetricKey('step2', self.name1): 1
},
distributions={
MetricKey('step1', self.name1):
DistributionData(3, 1, 3, 3),
MetricKey('step2', self.name3):
DistributionData(8, 2, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 7),
MetricResult(MetricKey('step1', self.name2), 0, 5),
MetricResult(MetricKey('step2', self.name1), 0, 1)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(dist_zero),
DistributionResult(DistributionData(3, 1, 3, 3))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 2, 4, 4)))
]))
metrics.commit_physical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): -3,
MetricKey('step2', self.name1): -5
},
distributions={
MetricKey('step1', self.name1):
DistributionData(8, 4, 1, 5),
MetricKey('step2', self.name2):
DistributionData(8, 8, 1, 1)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 0, 4),
MetricResult(MetricKey('step1', self.name2), 0, 5),
MetricResult(MetricKey('step2', self.name1), 0, -4)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(dist_zero),
DistributionResult(DistributionData(11, 5, 1,
5))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 2, 4, 4))),
MetricResult(MetricKey('step2', self.name2),
DistributionResult(dist_zero),
DistributionResult(DistributionData(8, 8, 1, 1)))
]))
metrics.commit_logical(
object(),
MetricUpdates(counters={
MetricKey('step1', self.name1): 3,
MetricKey('step1', self.name2): 5,
MetricKey('step2', self.name1): -3
},
distributions={
MetricKey('step1', self.name1):
DistributionData(11, 5, 1, 5),
MetricKey('step2', self.name2):
DistributionData(8, 8, 1, 1),
MetricKey('step2', self.name3):
DistributionData(4, 1, 4, 4)
}))
results = metrics.query()
hc.assert_that(
results['counters'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1), 3, 4),
MetricResult(MetricKey('step1', self.name2), 5, 5),
MetricResult(MetricKey('step2', self.name1), -3, -4)
]))
hc.assert_that(
results['distributions'],
hc.contains_inanyorder(*[
MetricResult(MetricKey('step1', self.name1),
DistributionResult(DistributionData(11, 5, 1, 5)),
DistributionResult(DistributionData(11, 5, 1,
5))),
MetricResult(MetricKey('step2', self.name3),
DistributionResult(DistributionData(4, 1, 4, 4)),
DistributionResult(DistributionData(8, 2, 4, 4))),
MetricResult(MetricKey('step2', self.name2),
DistributionResult(DistributionData(8, 8, 1, 1)),
DistributionResult(DistributionData(8, 8, 1, 1)))
]))