def reduce_fn(x, y):
name, input_fn, output_fn = y
return x + combinations.combine(input_fn=combinations.NamedObject(
"input_fn.{}".format(name), input_fn),
output_fn=combinations.NamedObject(
"output_fn.{}".format(name),
output_fn))
def reduce_fn(x, y):
name, value_fn, element_0_fn = y
return x + combinations.combine(value_fn=combinations.NamedObject(
"value_fn.{}".format(name), value_fn),
element_0_fn=combinations.NamedObject(
"element_0_fn.{}".format(name),
element_0_fn))
def _test_objects():
Item = namedtuple("Item", "id name")
return [
combinations.NamedObject("int", 1),
combinations.NamedObject("string", "dog"),
combinations.NamedObject("tuple", (1, 1)),
combinations.NamedObject("nested_tuple", ((1, 1), (2, 2))),
combinations.NamedObject("named_tuple", Item(id=1, name="item1")),
combinations.NamedObject("unicode", "アヒル"),
combinations.NamedObject(
"nested_named_tuple",
(Item(id=1, name="item1"), Item(id=2, name="item2"))),
combinations.NamedObject("int_string_tuple", (1, "dog")),
combinations.NamedObject(
"sparse",
sparse_tensor.SparseTensorValue(
indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])),
combinations.NamedObject(
"sparse_structured", {
"a":
sparse_tensor.SparseTensorValue(
indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4]),
"b": (1, 2, "dog")
})
]
def reduce_fn(x, y):
name, classes_fn, expected_fn = y
return x + combinations.combine(classes_fn=combinations.NamedObject(
"classes_fn.{}".format(name), classes_fn),
expected_fn=combinations.NamedObject(
"expected_fn.{}".format(name),
expected_fn))
def reduce_fn(x, y):
name, value1_fn, value2_fn, value3_fn = y
return x + combinations.combine(
value1_fn=combinations.NamedObject("value1_fn.{}".format(name),
value1_fn),
value2_fn=combinations.NamedObject("value2_fn.{}".format(name),
value2_fn),
value3_fn=combinations.NamedObject("value3_fn.{}".format(name),
value3_fn))
def reduce_fn(x, y):
name, original_value_fn, compatible_values_fn, incompatible_values_fn = y
return x + combinations.combine(
original_value_fn=combinations.NamedObject(
"original_value_fn.{}".format(name), original_value_fn),
compatible_values_fn=combinations.NamedObject(
"compatible_values_fn.{}".format(name), compatible_values_fn),
incompatible_values_fn=combinations.NamedObject(
"incompatible_values_fn.{}".format(name), incompatible_values_fn))
def reduce_fn(x, y):
name, value1_fn, value2_fn, *not_equal_value_fns = y
return x + combinations.combine(
value1_fn=combinations.NamedObject("value1_fn.{}".format(name),
value1_fn),
value2_fn=combinations.NamedObject("value2_fn.{}".format(name),
value2_fn),
not_equal_value_fns=combinations.NamedObject(
"not_equal_value_fns.{}".format(name), not_equal_value_fns))
def reduce_fn(x, y):
# workaround for long line
name, value_fn = y[:2]
expected_structure_fn, expected_types_fn, expected_shapes_fn = y[2:]
return x + combinations.combine(
value_fn=combinations.NamedObject("value_fn.{}".format(name), value_fn),
expected_structure_fn=combinations.NamedObject(
"expected_structure_fn.{}".format(name), expected_structure_fn),
expected_types_fn=combinations.NamedObject(
"expected_types_fn.{}".format(name), expected_types_fn),
expected_shapes_fn=combinations.NamedObject(
"expected_shapes_fn.{}".format(name), expected_shapes_fn))
def new_and_legacy_filter_fn_combinations():
def new_filter_fn(dataset, predicate):
return dataset.filter(predicate)
def legacy_filter_fn(dataset, predicate):
return dataset.filter_with_legacy_function(predicate)
return (combinations.combine(tf_api_version=[1, 2],
mode=["eager", "graph"],
apply_filter=combinations.NamedObject(
"new_filter_fn", new_filter_fn)) +
combinations.combine(tf_api_version=1,
mode=["eager", "graph"],
apply_filter=combinations.NamedObject(
"legacy_filter_fn", legacy_filter_fn)))
def _test_v2_eager_only_objects():
return [
combinations.NamedObject(
"ragged",
ragged_factory_ops.constant([[0, 1, 2, 3], [4, 5], [6, 7, 8], [9]])),
combinations.NamedObject(
"sparse_ragged_structured", {
"sparse":
sparse_tensor.SparseTensorValue(
indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4]),
"ragged":
ragged_factory_ops.constant([[0, 1, 2, 3], [9]])
})
]
def _test_combinations():
def filter_fn(dataset, predicate):
return dataset.filter(predicate)
def legacy_filter_fn(dataset, predicate):
return dataset.filter_with_legacy_function(predicate)
filter_combinations = combinations.combine(
tf_api_version=[1, 2],
mode=["eager", "graph"],
apply_filter=combinations.NamedObject("filter_fn", filter_fn))
legacy_filter_combinations = combinations.combine(
tf_api_version=1,
mode=["eager", "graph"],
apply_filter=combinations.NamedObject("legacy_filter_fn",
legacy_filter_fn))
return filter_combinations + legacy_filter_combinations
def _test_objects():
return [
combinations.NamedObject("int", 1),
combinations.NamedObject("string", "dog"),
combinations.NamedObject("tuple", (1, 1)),
combinations.NamedObject("int_string_tuple", (1, "dog")),
combinations.NamedObject(
"sparse",
sparse_tensor.SparseTensorValue(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4])),
combinations.NamedObject(
"sparse_structured", {
"a":
sparse_tensor.SparseTensorValue(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4]),
"b": (1, 2, "dog")
})
]
def reduce_fn(x, y):
function, name, predicates = y
return x + combinations.combine(function=function,
predicates=combinations.NamedObject(
name, predicates))
class FromListTest(test_base.DatasetTestBase, parameterized.TestCase):
@combinations.generate(
combinations.times(
test_base.default_test_combinations(),
combinations.combine(elements=[
combinations.NamedObject("empty_input", []),
combinations.NamedObject("non-list_input", (1, 2, 3)),
])))
def testInvalidInputs(self, elements):
with self.assertRaises(ValueError):
from_list.from_list(elements._obj)
def testLargeNInputs(self):
elements = list(range(1000))
dataset = dataset_ops.Dataset.from_tensor_slices(elements)
self.assertDatasetProduces(dataset, expected_output=elements)
@combinations.generate(test_base.default_test_combinations())
def testTupleInputs(self):
elements = [(1, 2), (3, 4)]
dataset = from_list.from_list(elements)
self.assertEqual(
[np.shape(c) for c in elements[0]],
[shape for shape in dataset_ops.get_legacy_output_shapes(dataset)])
self.assertDatasetProduces(dataset, expected_output=elements)
@combinations.generate(
combinations.times(
test_base.default_test_combinations(),
combinations.combine(map_fn=[
combinations.NamedObject("simple", lambda x: x),
combinations.NamedObject(
"ordered_dicts",
lambda x: collections.OrderedDict([("x", x)]))
])))
def testDatasetInputs(self, map_fn):
dss = [dataset_ops.Dataset.range(10).map(map_fn) for _ in range(10)]
ds = dataset_ops.Dataset.from_tensor_slices(dss)
ds = ds.flat_map(lambda x: x)
self.assertDatasetProduces(
ds, expected_output=[map_fn(x) for x in list(range(10)) * 10])
@combinations.generate(test_base.default_test_combinations())
def testNonRectangularInputs(self):
elements = [[[1]], [[2, 3]], [[4, 5, 6]]]
dataset = from_list.from_list(elements)
self.assertEqual(tensor_shape.Dimension(1),
dataset_ops.get_legacy_output_shapes(dataset)[0])
self.assertDatasetProduces(dataset, expected_output=elements)
@combinations.generate(test_base.default_test_combinations())
def testDictInputs(self):
elements = [{
"foo": [1, 2, 3],
"bar": [[4.0], [5.0], [6.0]]
}, {
"foo": [4, 5, 6],
"bar": [[7.0], [8.0], [9.0]]
}]
dataset = from_list.from_list(elements)
self.assertEqual(dtypes.int32,
dataset_ops.get_legacy_output_types(dataset)["foo"])
self.assertEqual(dtypes.float32,
dataset_ops.get_legacy_output_types(dataset)["bar"])
self.assertEqual((3, ),
dataset_ops.get_legacy_output_shapes(dataset)["foo"])
self.assertEqual((3, 1),
dataset_ops.get_legacy_output_shapes(dataset)["bar"])
self.assertDatasetProduces(dataset, expected_output=elements)
@combinations.generate(test_base.default_test_combinations())
def testUintInputs(self):
elements = [(np.tile(np.array([[0], [1]], dtype=np.uint8), 2),
np.tile(np.array([[2], [256]], dtype=np.uint16), 2),
np.tile(np.array([[4], [65536]], dtype=np.uint32), 2),
np.tile(np.array([[8], [4294967296]], dtype=np.uint64),
2))]
dataset = from_list.from_list(elements)
self.assertEqual(
(dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64),
dataset_ops.get_legacy_output_types(dataset))
self.assertDatasetProduces(dataset, elements)
def reduce_fn(x, y):
name, map_fn, should_optimize = y
return x + combinations.combine(map_fn=combinations.NamedObject(
name, map_fn),
should_optimize=should_optimize)
def reduce_fn(x, y):
name, functions = y
return x + combinations.combine(
functions=combinations.NamedObject(name, functions))
def reduce_fn(x, y):
name, dataset_fn, expected_output = y
return x + combinations.combine(dataset_fn=combinations.NamedObject(
name, dataset_fn),
expected_output=[expected_output])
def reduce_fn(x, y):
name, fn = y
return x + combinations.combine(
map_fn=combinations.NamedObject(name, fn))
def reduce_fn(result, case):
name, transformation, expected = case
return result + combinations.combine(
init_dataset_fn=make_range,
transformation=combinations.NamedObject(name, transformation),
expected_name=expected)
from tensorflow.python.platform import test
class CollectiveOpsV1(object):
all_reduce = _collective_ops.all_reduce
all_gather = _collective_ops.all_gather
class CollectiveOpsV2(object):
all_reduce = _collective_ops.all_reduce_v2
all_gather = _collective_ops.all_gather_v2
@combinations.generate(
combinations.combine(collective_ops=[
combinations.NamedObject('v1', CollectiveOpsV1),
combinations.NamedObject('v2', CollectiveOpsV2)
],
mode='eager'))
class CollectiveOpsTest(test.TestCase, parameterized.TestCase):
def setUp(self):
_setup_context()
super().setUp()
def testReduce(self, collective_ops):
@def_function.function
def run_all_reduce_1cpu():
with ops.device('/device:CPU:0'):
in_value = constant_op.constant([1.])
group_size = 1
group_key = 1
def default_test_combinations():
"""Returns the default test combinations for testing checkpointing."""
def disable_optimizations(ds_fn):
options = options_lib.Options()
options.experimental_optimization.apply_default_optimizations = False
def ds_fn_no_opt():
return ds_fn().with_options(options)
return ds_fn_no_opt
def verify_unused_iterator(obj, ds_fn, num_outputs, sparse_tensors=False):
obj.verify_unused_iterator(ds_fn=disable_optimizations(ds_fn=ds_fn),
num_outputs=num_outputs,
sparse_tensors=sparse_tensors)
verify_unused_iterator_combination = combinations.combine(
verify_fn=combinations.NamedObject("verify_unused_iterator",
verify_unused_iterator))
def verify_fully_used_iterator(obj,
ds_fn,
num_outputs,
sparse_tensors=False):
obj.verify_fully_used_iterator(
ds_fn=disable_optimizations(ds_fn=ds_fn),
num_outputs=num_outputs,
sparse_tensors=sparse_tensors)
verify_fully_used_iterator_combination = combinations.combine(
verify_fn=combinations.NamedObject("verify_fully_used_iterator",
verify_fully_used_iterator))
def verify_exhausted_iterator(obj,
ds_fn,
num_outputs,
sparse_tensors=False):
obj.verify_exhausted_iterator(ds_fn=disable_optimizations(ds_fn=ds_fn),
num_outputs=num_outputs,
sparse_tensors=sparse_tensors)
verify_exhausted_iterator_combination = combinations.combine(
verify_fn=combinations.NamedObject("verify_exhausted_iterator",
verify_exhausted_iterator))
def verify_multiple_breaks(obj, ds_fn, num_outputs, sparse_tensors=False):
obj.verify_multiple_breaks(ds_fn=disable_optimizations(ds_fn=ds_fn),
num_outputs=num_outputs,
sparse_tensors=sparse_tensors)
verify_multiple_breaks_combination = combinations.combine(
verify_fn=combinations.NamedObject("verify_multiple_breaks",
verify_multiple_breaks))
def verify_reset_restored_iterator(obj,
ds_fn,
num_outputs,
sparse_tensors=False):
obj.verify_reset_restored_iterator(
ds_fn=disable_optimizations(ds_fn=ds_fn),
num_outputs=num_outputs,
sparse_tensors=sparse_tensors)
verify_reset_restored_iterator_combination = combinations.combine(
verify_fn=combinations.NamedObject("verify_reset_restored_iterator",
verify_reset_restored_iterator))
return (verify_unused_iterator_combination +
verify_fully_used_iterator_combination +
verify_exhausted_iterator_combination +
verify_multiple_breaks_combination +
verify_reset_restored_iterator_combination)
def reduce_fn(x, y):
name, value_fn, expected_structure = y
return x + combinations.combine(
tf_value_fn=combinations.NamedObject(name, value_fn),
expected_value_structure=expected_structure)
def reduce_fn(x, y):
name, value, value_fn, gpu_compatible = y
return x + combinations.combine(np_value=value,
tf_value_fn=combinations.NamedObject(
name, value_fn),
gpu_compatible=gpu_compatible)
from tensorflow.python.data.ops import multi_device_iterator_ops
from tensorflow.python.data.ops import options as options_lib
from tensorflow.python.eager import cancellation
from tensorflow.python.eager import context
from tensorflow.python.eager import def_function
from tensorflow.python.eager import executor
from tensorflow.python.framework import combinations
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_spec
from tensorflow.python.ops import data_flow_ops
from tensorflow.python.platform import test
cls_combination = combinations.combine(cls=[
combinations.NamedObject("MultiDeviceIterator",
multi_device_iterator_ops.MultiDeviceIterator),
combinations.NamedObject(
"OwnedMultiDeviceIterator",
multi_device_iterator_ops.OwnedMultiDeviceIterator)
])
class MultiDeviceIteratorCommonTest(test_base.DatasetTestBase,
parameterized.TestCase):
"""Tests that are common to MultiDeviceIterator and OwnedMultiDeviceIterator."""
def setUp(self):
super().setUp()
self._devices = self.configureDevicesForMultiDeviceTest(3)
@combinations.generate(
combinations.times(test_base.eager_only_combinations(),
def reduce_fn(x, y):
name, dataset_fn, expected_result = y
return x + combinations.combine(dataset_fn=combinations.NamedObject(
name, dataset_fn),
expected_result=expected_result)
def reduce_fn(x, y):
name, function, should_optimize = y
return x + combinations.combine(function=combinations.NamedObject(
name, function),
should_optimize=should_optimize)
def reduce_fn(x, y):
name, dataset_fn = y
return x + combinations.combine(
dataset_fn=combinations.NamedObject(name, dataset_fn))
def _reduce_cases_to_combinations(result, case):
name, dataset_fn, sharding_policy, expected_result = case
return result + combinations.combine(
dataset_fn=combinations.NamedObject(name, dataset_fn),
sharding_policy=sharding_policy,
expected_result=expected_result)