def testCombineOnBatchSimple(self):
batch_1 = [np.ones((2, 6))]
batch_2 = [np.ones((1, 6))]
out = [3 for _ in range(6)]
analyzer = impl._CombineFnWrapper(
analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False))
self.assertCombine(analyzer, [batch_1, batch_2], out)
def testCombineOnBatchLotsOfData(self):
shards = [[np.ones(3)] for _ in range(
beam_impl._DEFAULT_DESIRED_BATCH_SIZE * 2)]
out = [1 for _ in range(3)]
analyzer = impl._CombineFnWrapper(
analyzers._NumPyCombinerSpec(np.min, reduce_instance_dims=False))
self.assertCombine(analyzer, shards, out)
def testCombineOnBatchLotsOfData(self):
shards = [[np.ones((1, 3))] for _ in range(2000)]
out = [1 for _ in range(3)]
analyzer = impl._CombineFnWrapper(
analyzers._NumPyCombinerSpec(np.min,
reduce_instance_dims=False,
output_dtypes=[np.int64]))
self.assertCombine(analyzer, shards, out)
def testCombineOnBatchWithBeamPipeline(self):
# Test with a real Beam pipeline instead of calling the Combiner methods
# directly. This surfaces bugs that only occur within a Beam pipeline, e.g.
# due to Beam passing iterators to merge_accumulators instead of lists.
with beam.Pipeline() as p:
batch_1 = [np.ones((2, 6), dtype=np.int)]
batch_2 = [np.ones((1, 6), dtype=np.int)]
expected_output = np.ones(6) * 3
def assert_equals_expected(outputs):
output, = outputs # Expect exactly one analyzer output
return np.array_equal(output, expected_output)
analyzer = impl._CombineFnWrapper(
analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False))
assert_that(p
| beam.Create([batch_1, batch_2])
| beam.CombineGlobally(analyzer)
| beam.Map(assert_equals_expected),
equal_to([True]))
from __future__ import division
from __future__ import print_function
import numpy as np
from tensorflow_transform import analyzers
from tensorflow_transform import test_case
import unittest
_NP_TYPES = (np.float32, np.float64, np.int32, np.int64)
_SUM_TEST = dict(
testcase_name='Sum',
combiner_spec=analyzers._NumPyCombinerSpec(np.sum,
reduce_instance_dims=False,
output_dtypes=[np.int64]),
batches=[
[np.ones((2, 6))],
[np.ones((1, 6))],
],
expected_outputs=[np.ones((6, ), np.int64) * 3],
)
_SUM_OF_SIZE_ZERO_TENSORS_TEST = dict(
testcase_name='SumOfSizeZeroTensors',
combiner_spec=analyzers._NumPyCombinerSpec(np.sum,
reduce_instance_dims=False,
output_dtypes=[np.int64]),
batches=[
[np.ones((2, 0))],
def testCombineOnBatchAllEmptyRow(self):
analyzer = impl._CombineFnWrapper(
analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False))
self.assertCombine(analyzer, [[[[]]], [[[]]], [[[]]]], [])
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import tensorflow as tf
from tensorflow_transform import analyzers
from tensorflow_transform import test_case
_NP_TYPES = (np.float32, np.float64, np.int32, np.int64)
_SUM_TEST = dict(
testcase_name='Sum',
combiner_spec=analyzers._NumPyCombinerSpec(np.sum,
output_dtypes=[np.int64]),
batches=[
(np.array([1, 2, 3, 4, 5, 6]), ),
(np.array([1, 2, 3, 4, 5, 6]), ),
],
expected_outputs=[np.array([2, 4, 6, 8, 10, 12])],
)
_SUM_SCALAR_TEST = dict(
testcase_name='SumScalar',
combiner_spec=analyzers._NumPyCombinerSpec(np.sum,
output_dtypes=[np.int64]),
batches=[
(np.array(1), ),
(np.array(2), ),
],
def testSumOfSizeZeroTensors(self):
self._test_combiner_spec_helper(
combiner_spec=analyzers._NumPyCombinerSpec(
np.sum, reduce_instance_dims=False, output_dtypes=[np.int64]),
batches=[[np.ones((2, 0))], [np.ones((1, 0))]],
expected_outputs=[np.ones((0, ), np.int64) * 3])