def testCombineOnBatchSimple(self): batch_1 = [np.ones((2, 6))] batch_2 = [np.ones((1, 6))] out = [3 for _ in range(6)] analyzer = impl._CombineFnWrapper( analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False)) self.assertCombine(analyzer, [batch_1, batch_2], out)
def testCombineOnBatchLotsOfData(self): shards = [[np.ones(3)] for _ in range( beam_impl._DEFAULT_DESIRED_BATCH_SIZE * 2)] out = [1 for _ in range(3)] analyzer = impl._CombineFnWrapper( analyzers._NumPyCombinerSpec(np.min, reduce_instance_dims=False)) self.assertCombine(analyzer, shards, out)
def testCombineOnBatchLotsOfData(self): shards = [[np.ones((1, 3))] for _ in range(2000)] out = [1 for _ in range(3)] analyzer = impl._CombineFnWrapper( analyzers._NumPyCombinerSpec(np.min, reduce_instance_dims=False, output_dtypes=[np.int64])) self.assertCombine(analyzer, shards, out)
def testCombineOnBatchWithBeamPipeline(self): # Test with a real Beam pipeline instead of calling the Combiner methods # directly. This surfaces bugs that only occur within a Beam pipeline, e.g. # due to Beam passing iterators to merge_accumulators instead of lists. with beam.Pipeline() as p: batch_1 = [np.ones((2, 6), dtype=np.int)] batch_2 = [np.ones((1, 6), dtype=np.int)] expected_output = np.ones(6) * 3 def assert_equals_expected(outputs): output, = outputs # Expect exactly one analyzer output return np.array_equal(output, expected_output) analyzer = impl._CombineFnWrapper( analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False)) assert_that(p | beam.Create([batch_1, batch_2]) | beam.CombineGlobally(analyzer) | beam.Map(assert_equals_expected), equal_to([True]))
from __future__ import division from __future__ import print_function import numpy as np from tensorflow_transform import analyzers from tensorflow_transform import test_case import unittest _NP_TYPES = (np.float32, np.float64, np.int32, np.int64) _SUM_TEST = dict( testcase_name='Sum', combiner_spec=analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False, output_dtypes=[np.int64]), batches=[ [np.ones((2, 6))], [np.ones((1, 6))], ], expected_outputs=[np.ones((6, ), np.int64) * 3], ) _SUM_OF_SIZE_ZERO_TENSORS_TEST = dict( testcase_name='SumOfSizeZeroTensors', combiner_spec=analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False, output_dtypes=[np.int64]), batches=[ [np.ones((2, 0))],
def testCombineOnBatchAllEmptyRow(self): analyzer = impl._CombineFnWrapper( analyzers._NumPyCombinerSpec(np.sum, reduce_instance_dims=False)) self.assertCombine(analyzer, [[[[]]], [[[]]], [[[]]]], [])
from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np import tensorflow as tf from tensorflow_transform import analyzers from tensorflow_transform import test_case _NP_TYPES = (np.float32, np.float64, np.int32, np.int64) _SUM_TEST = dict( testcase_name='Sum', combiner_spec=analyzers._NumPyCombinerSpec(np.sum, output_dtypes=[np.int64]), batches=[ (np.array([1, 2, 3, 4, 5, 6]), ), (np.array([1, 2, 3, 4, 5, 6]), ), ], expected_outputs=[np.array([2, 4, 6, 8, 10, 12])], ) _SUM_SCALAR_TEST = dict( testcase_name='SumScalar', combiner_spec=analyzers._NumPyCombinerSpec(np.sum, output_dtypes=[np.int64]), batches=[ (np.array(1), ), (np.array(2), ), ],
def testSumOfSizeZeroTensors(self): self._test_combiner_spec_helper( combiner_spec=analyzers._NumPyCombinerSpec( np.sum, reduce_instance_dims=False, output_dtypes=[np.int64]), batches=[[np.ones((2, 0))], [np.ones((1, 0))]], expected_outputs=[np.ones((0, ), np.int64) * 3])