def test(self):
mask = core.LabeledTensor(math_ops.range(7) > 3, [self.a0])
masked_lt = ops.boolean_mask(self.original_lt, mask)
golden_lt = core.LabeledTensor(
array_ops.boolean_mask(self.original_lt.tensor, mask.tensor),
['x', self.a1, self.a2, self.a3])
self.assertLabeledTensorsEqual(masked_lt, golden_lt)
def test(self):
expected_a = core.LabeledTensor(constant_op.constant([1, 5]),
['batch'])
expected_b = core.LabeledTensor(
constant_op.constant([[2, 3, 4], [6, 7, 8]]), ['batch', 'x'])
parsed = io_ops.parse_example(self.serialized, self.features)
self.assertLabeledTensorsEqual(expected_a, parsed['a'])
self.assertLabeledTensorsEqual(expected_b, parsed['b'])
def setUp(self):
super(VerifyTensorAllFiniteTest, self).setUp()
self.finite_lt = core.LabeledTensor(constant_op.constant(42.0), [])
self.nan_lt = core.LabeledTensor(constant_op.constant(np.nan), [])
self.checked_finite_lt = ops.verify_tensor_all_finite(
self.finite_lt, '')
self.checked_nan_lt = ops.verify_tensor_all_finite(self.nan_lt, '')
def test(self):
condition = core.LabeledTensor(math_ops.range(5) < 3, ['x'])
x = core.LabeledTensor(array_ops.ones(5), ['x'])
y = core.LabeledTensor(array_ops.zeros(5), ['x'])
where_lt = ops.where(condition, x, y)
golden_lt = core.LabeledTensor(
array_ops.concat(
[array_ops.ones(3), array_ops.zeros(2)], 0), ['x'])
self.assertLabeledTensorsEqual(where_lt, golden_lt)
def test_identity(self):
axis_order = ['w', 'x', 'y', 'z']
lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order)
actual = core.impose_axis_order(lt, axis_order)
self.assertLabeledTensorsEqual(lt, actual)
lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3])
actual = core.impose_axis_order(lt, axis_order)
self.assertLabeledTensorsEqual(lt, actual)
def test_passes(self):
axis_order = ['w', 'x', 'y', 'z']
lt = core.LabeledTensor(array_ops.ones((1, 1, 1, 1)), axis_order)
core.check_axis_order(lt, axis_order)
lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[1:])
core.check_axis_order(lt, axis_order)
lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[:-1])
core.check_axis_order(lt, axis_order)
def test_matrix_vector(self):
xy_lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y'])
y_lt = core.LabeledTensor(math_ops.range(3), ['y'])
matmul_lt = ops.matmul(xy_lt, y_lt)
golden_lt = core.LabeledTensor(
math_ops.matmul(xy_lt.tensor,
array_ops.reshape(y_lt.tensor, (-1, 1)))[:, 0],
['x'])
self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
matmul_lt = ops.matmul(y_lt, xy_lt)
self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
def test_reverse(self):
axis_order = ['w', 'x', 'y', 'z']
lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order)
actual = core.impose_axis_order(lt, axis_order[::-1])
expected = core.transpose(lt, axis_order[::-1])
self.assertLabeledTensorsEqual(expected, actual)
lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3])
actual = core.impose_axis_order(lt, axis_order[::-1])
expected = core.transpose(lt, ['y', 'x', 'w'])
self.assertLabeledTensorsEqual(expected, actual)
def test_indexing_scalars(self):
actual = self.lt[:, :, :, 0]
expected = core.LabeledTensor(self.lt.tensor[:, :, :, 0],
list(self.lt.axes.values())[:-1])
self.assertLabeledTensorsEqual(actual, expected)
actual = self.lt[1, :, :, 0]
expected = core.LabeledTensor(self.lt.tensor[1, :, :, 0],
list(self.lt.axes.values())[1:-1])
self.assertLabeledTensorsEqual(actual, expected)
actual = self.lt[1, 2, :, 0]
expected = core.LabeledTensor(self.lt.tensor[1, 2, :, 0],
list(self.lt.axes.values())[2:-1])
self.assertLabeledTensorsEqual(actual, expected)
def setUp(self):
super(Base, self).setUp()
self.x_size = 7
self.channel_size = 3
self.z_size = 4
self.probs_size = 11
tensor = math_ops.range(
0, self.x_size * self.channel_size * self.z_size * self.probs_size)
tensor = array_ops.reshape(
tensor,
[self.x_size, self.channel_size, self.z_size, self.probs_size])
a0 = ('x', range(self.x_size))
a1 = ('channel', ['red', 'green', 'blue'])
a2 = 'z'
a3 = ('probs', np.linspace(0.0, 1.0, self.probs_size))
self.tensor = tensor
self.a0 = a0
self.a1 = a1
self.a2 = a2
self.a2_resolved = ('z', self.z_size)
self.a3 = a3
self.original_lt = core.LabeledTensor(tensor, [a0, a1, a2, a3])
self.x_probs_lt = core.slice_function(self.original_lt, {'z': 0})
self.x_probs_lt = ops.select(self.x_probs_lt, {'channel': 'red'})
self.channel_probs_lt = core.slice_function(self.original_lt, {
'x': 3,
'z': 0
})
def test_new_name(self):
rename_axis_lt = ops.rename_axis(self.original_lt, 'channel', 'foo')
expected_axes = [(name if name != 'channel' else 'foo', axis.value)
for name, axis in self.original_lt.axes.items()]
expected_lt = core.LabeledTensor(self.original_lt.tensor,
expected_axes)
self.assertLabeledTensorsEqual(rename_axis_lt, expected_lt)
def test_unknown_size(self):
reshape_lt = ops.reshape(self.original_lt, ['channel', 'z', 'probs'],
['new_dim'])
golden_lt = core.LabeledTensor(
array_ops.reshape(self.original_lt.tensor, [self.x_size, -1]),
[self.original_lt.axes['x'], 'new_dim'])
self.assertLabeledTensorsEqual(reshape_lt, golden_lt)
def test_axis(self):
unpack_lt = ops.unpack(self.original_lt, axis_name='z')[0]
golden_lt = core.LabeledTensor(
array_ops.unstack(self.original_lt.tensor, axis=2)[0],
[self.a0, self.a1, self.a3])
self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def test(self):
unpack_lt = ops.unpack(self.original_lt)[0]
golden_lt = core.LabeledTensor(
array_ops.unstack(self.original_lt.tensor)[0],
[self.a1, self.a2, self.a3])
self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def setUp(self):
super(ReshapeCoderTest, self).setUp()
self.batch_size = 8
self.num_rows = 50
self.num_columns = 100
self.channels = ['red', 'green', 'blue']
self.masks = [False, True]
tensor = math_ops.range(
0,
self.batch_size * self.num_rows * self.num_columns *
len(self.channels) * len(self.masks))
tensor = array_ops.reshape(tensor, [
self.batch_size, self.num_rows, self.num_columns,
len(self.channels),
len(self.masks)
])
self.batch_axis = ('batch', range(self.batch_size))
self.row_axis = ('row', range(self.num_rows))
self.column_axis = ('column', range(self.num_columns))
self.channel_axis = ('channel', self.channels)
self.mask_axis = ('mask', self.masks)
axes = [
self.batch_axis, self.row_axis, self.column_axis,
self.channel_axis, self.mask_axis
]
self.masked_image_lt = core.LabeledTensor(tensor, axes)
def parse_example(serialized, features, name=None, example_names=None):
"""Parse `Example` protos into a `dict` of labeled tensors.
See tf.parse_example.
Args:
serialized: A 1-D LabeledTensor of strings, a batch of binary serialized
`Example` protos.
features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature`
values.
name: A name for this operation (optional).
example_names: A vector (1-D Tensor) of strings (optional), the names of
the serialized protos in the batch.
Returns:
A `dict` mapping feature keys to `LabeledTensor` values. The single axis
from `serialized` will be prepended to the axes provided by each feature.
Raises:
ValueError: if any feature is invalid.
"""
serialized = core.convert_to_labeled_tensor(serialized)
unlabeled_features = _labeled_to_unlabeled_features(features)
unlabeled_parsed = parsing_ops.parse_example(
serialized.tensor, unlabeled_features, name, example_names)
parsed = {}
for name, parsed_feature in unlabeled_parsed.items():
axes = list(serialized.axes.values()) + features[name].axes
parsed[name] = core.LabeledTensor(parsed_feature, axes)
return parsed
def concat(labeled_tensors, axis_name, name=None):
"""Concatenate tensors along a dimension.
See tf.concat.
Args:
labeled_tensors: A list of input LabeledTensors.
axis_name: The name of the axis along which to concatenate.
name: Optional op name.
Returns:
The concatenated tensor.
The coordinate labels for the concatenation dimension are also concatenated,
if they are available for every tensor.
Raises:
ValueError: If fewer than one tensor inputs is provided, if the tensors
have incompatible axes, or if `axis_name` isn't the name of an axis.
"""
with ops.name_scope(name, 'lt_concat', labeled_tensors) as scope:
labeled_tensors = [
core.convert_to_labeled_tensor(lt) for lt in labeled_tensors
]
if len(labeled_tensors) < 1:
raise ValueError('concat expects at least 1 tensor, but received %s' %
labeled_tensors)
# All tensors must have these axes.
axes_0 = labeled_tensors[0].axes
axis_names = list(axes_0.keys())
if axis_name not in axis_names:
raise ValueError('%s not in %s' % (axis_name, axis_names))
shared_axes = axes_0.remove(axis_name)
tensors = [labeled_tensors[0].tensor]
concat_axis_list = [axes_0[axis_name]]
for labeled_tensor in labeled_tensors[1:]:
current_shared_axes = labeled_tensor.axes.remove(axis_name)
if current_shared_axes != shared_axes:
# TODO(shoyer): add more specific checks about what went wrong,
# including raising AxisOrderError when appropriate
raise ValueError('Mismatched shared axes: the first tensor '
'had axes %r but this tensor has axes %r.' %
(shared_axes, current_shared_axes))
# Accumulate the axis labels, if they're available.
concat_axis_list.append(labeled_tensor.axes[axis_name])
tensors.append(labeled_tensor.tensor)
concat_axis = core.concat_axes(concat_axis_list)
concat_dimension = axis_names.index(axis_name)
concat_tensor = array_ops.concat(tensors, concat_dimension, name=scope)
values = list(axes_0.values())
concat_axes = (values[:concat_dimension] + [concat_axis] +
values[concat_dimension + 1:])
return core.LabeledTensor(concat_tensor, concat_axes)
def test_matrix_matrix_axis_order(self):
xy_lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y'])
yz_lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(12), (3, 4)), ['y', 'z'])
golden_lt = core.LabeledTensor(
math_ops.matmul(xy_lt.tensor, yz_lt.tensor), ['x', 'z'])
with core.axis_order_scope(['x', 'y', 'z']):
matmul_lt = ops.matmul(xy_lt, yz_lt)
self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
matmul_lt = ops.matmul(yz_lt, xy_lt)
self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
def test_core_op(self):
for op_name, _, tf_op, lt_op in self.ops:
golden_tensor = tf_op(self.test_lt_1_broadcast, self.test_lt_2_broadcast)
golden_lt = core.LabeledTensor(golden_tensor, self.broadcast_axes)
actual_lt = lt_op(self.test_lt_1, self.test_lt_2)
self.assertIn(op_name, actual_lt.name)
self.assertLabeledTensorsEqual(golden_lt, actual_lt)
def setUp(self):
super(CoreUnaryOpsTest, self).setUp()
self.ops = [
('abs', operator.abs, math_ops.abs, core.abs_function),
('neg', operator.neg, math_ops.negative, core.neg),
# TODO(shoyer): add unary + to core TensorFlow
('pos', None, None, None),
('sign', None, math_ops.sign, core.sign),
('reciprocal', None, math_ops.reciprocal, core.reciprocal),
('square', None, math_ops.square, core.square),
('round', None, math_ops.round, core.round_function),
('sqrt', None, math_ops.sqrt, core.sqrt),
('rsqrt', None, math_ops.rsqrt, core.rsqrt),
('log', None, math_ops.log, core.log),
('exp', None, math_ops.exp, core.exp),
('log', None, math_ops.log, core.log),
('ceil', None, math_ops.ceil, core.ceil),
('floor', None, math_ops.floor, core.floor),
('cos', None, math_ops.cos, core.cos),
('sin', None, math_ops.sin, core.sin),
('tan', None, math_ops.tan, core.tan),
('acos', None, math_ops.acos, core.acos),
('asin', None, math_ops.asin, core.asin),
('atan', None, math_ops.atan, core.atan),
('lgamma', None, math_ops.lgamma, core.lgamma),
('digamma', None, math_ops.digamma, core.digamma),
('erf', None, math_ops.erf, core.erf),
('erfc', None, math_ops.erfc, core.erfc),
('lgamma', None, math_ops.lgamma, core.lgamma),
]
total_size = np.prod([v.size for v in self.original_lt.axes.values()])
self.test_lt = core.LabeledTensor(
math_ops.cast(self.original_lt, dtypes.float32) / total_size,
self.original_lt.axes)
def test_invalid(self):
lt = core.LabeledTensor(
array_ops.reshape(math_ops.range(2), (1, 2)), ['x', 'y'])
with self.assertRaises(ValueError):
core.impose_axis_order(lt)
with self.assertRaises(ValueError):
core.impose_axis_order(lt, ['x'])
def parse_single_example(serialized, features, name=None, example_names=None):
"""Parses a single `Example` proto.
See tf.parse_single_example.
Args:
serialized: A scalar string Tensor or LabeledTensor, a single serialized
Example.
features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature`
values.
name: A name for this operation (optional).
example_names: (Optional) A scalar string Tensor, the associated name.
Returns:
A `dict` mapping feature keys to `LabeledTensor` values.
Raises:
ValueError: if any feature is invalid.
"""
serialized = core.convert_to_labeled_tensor(serialized)
unlabeled_features = _labeled_to_unlabeled_features(features)
unlabeled_parsed = parsing_ops.parse_single_example(
serialized.tensor, unlabeled_features, name, example_names)
parsed = {}
for name, parsed_feature in unlabeled_parsed.items():
parsed[name] = core.LabeledTensor(parsed_feature, features[name].axes)
return parsed
def foldl(fn, labeled_tensor, initial_value, name=None):
"""Left fold on the list of tensors unpacked from labeled_tensor.
See tf.foldl.
Args:
fn: The function to apply to each unpacked LabeledTensor.
It should have type (LabeledTensor, LabeledTensor) -> LabeledTensor.
Its arguments are (accumulated_value, next_value).
labeled_tensor: The input tensor.
initial_value: The initial value of the accumulator.
name: Optional op name.
Returns:
The accumulated value.
"""
with ops.name_scope(name, 'lt_foldl',
[labeled_tensor, initial_value]) as scope:
labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor)
initial_value = core.convert_to_labeled_tensor(initial_value)
@tc.returns(ops.Tensor)
@tc.accepts(ops.Tensor, ops.Tensor)
def tf_fn(accumulator, next_element):
accumulator_lt = core.LabeledTensor(accumulator, initial_value.axes)
next_element_lt = core.LabeledTensor(
next_element, list(labeled_tensor.axes.values())[1:])
return fn(accumulator_lt, next_element_lt).tensor
foldl_op = functional_ops.foldl(
tf_fn, labeled_tensor.tensor, initializer=initial_value.tensor)
foldl_lt = core.LabeledTensor(foldl_op, initial_value.axes)
return core.identity(foldl_lt, name=scope)
def boolean_mask(labeled_tensor, mask, name=None):
"""Apply a boolean mask to a labeled tensor.
Unlike `tf.boolean_mask`, this currently only works on 1-dimensional masks.
The mask is applied to the first axis of `labeled_tensor`. Labels on the first
axis are removed, because True indices in `mask` may not be known dynamically.
Args:
labeled_tensor: The input tensor.
mask: The type of the returned tensor.
name: Optional op name.
Returns:
The masked labeled tensor.
Raises:
ValueError: if the first axis of the mask
"""
with ops.name_scope(name, 'lt_boolean_mask', [labeled_tensor, mask]) as scope:
labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor)
mask = core.convert_to_labeled_tensor(mask)
if len(mask.axes) > 1:
raise NotImplementedError(
"LabeledTensor's boolean_mask currently only supports 1D masks")
mask_axis = list(mask.axes.values())[0]
lt_axis = list(labeled_tensor.axes.values())[0]
if mask_axis != lt_axis:
raise ValueError('the first axis of the labeled tensor and the mask '
'are not equal:\n%r\n%r' % (lt_axis, mask_axis))
op = array_ops.boolean_mask(labeled_tensor.tensor, mask.tensor, name=scope)
# TODO(shoyer): attempt to infer labels for the masked values, by calling
# tf.get_static_value on the mask?
axes = [lt_axis.name] + list(labeled_tensor.axes.values())[1:]
return core.LabeledTensor(op, axes)
def test_default_axis_order(self):
transpose_lt = core.transpose(self.original_lt)
golden_lt = core.LabeledTensor(
array_ops.transpose(self.tensor, [3, 2, 1, 0]),
list(reversed(list(self.original_lt.axes.values()))))
self.assertLabeledTensorsEqual(transpose_lt, golden_lt)
def test_slice(self):
select_lt = ops.select(self.original_lt,
{'channel': slice('red', 'green')})
a1_sliced = ('channel', ['red', 'green'])
golden_lt = core.LabeledTensor(self.tensor[:, :2, :, :],
[self.a0, a1_sliced, self.a2, self.a3])
self.assertLabeledTensorsEqual(select_lt, golden_lt)
def where(condition, x, y, name=None):
"""Return elements from x or y depending on condition.
See `tf.where` for more details. This function currently only implements the
three argument version of where.
Args:
condition: LabeledTensor of type `bool`.
x: LabeledTensor for values where condition is true.
y: LabeledTensor for values where condition is false.
name: Optional op name.
Returns:
The labeled tensor with values according to condition.
Raises:
ValueError: if `x` and `y` have different axes, or if the axes of `x` do not
start with the axes of `condition`.
"""
with ops.name_scope(name, 'lt_where', [condition, x, y]) as scope:
condition = core.convert_to_labeled_tensor(condition)
x = core.convert_to_labeled_tensor(x)
y = core.convert_to_labeled_tensor(y)
if not condition.axes == x.axes == y.axes:
raise ValueError('all inputs to `where` must have equal axes')
op = array_ops.where(condition.tensor, x.tensor, y.tensor, name=scope)
return core.LabeledTensor(op, x.axes)
def unpack(labeled_tensor, axis_name=None, name=None):
"""Unpack the tensor.
See tf.unpack.
Args:
labeled_tensor: The input tensor.
axis_name: Optional name of axis to unpack. By default, the first axis is
used.
name: Optional op name.
Returns:
The list of unpacked LabeledTensors.
Raises:
ValueError: If `axis_name` is not an axis on the input.
"""
with ops.name_scope(name, 'lt_unpack', [labeled_tensor]) as scope:
labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor)
axis_names = list(labeled_tensor.axes.keys())
if axis_name is None:
axis_name = axis_names[0]
if axis_name not in axis_names:
raise ValueError('%s not in %s' % (axis_name, axis_names))
axis = axis_names.index(axis_name)
unpack_ops = array_ops.unstack(labeled_tensor.tensor, axis=axis, name=scope)
axes = [a for i, a in enumerate(labeled_tensor.axes.values()) if i != axis]
return [core.LabeledTensor(t, axes) for t in unpack_ops]
def setUp(self):
super(ParseBase, self).setUp()
examples = [
example_pb2.Example(features=feature_pb2.Features(
feature={
'a':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[1])),
'b':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[2, 3, 4])),
})),
example_pb2.Example(features=feature_pb2.Features(
feature={
'a':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[5])),
'b':
feature_pb2.Feature(int64_list=feature_pb2.Int64List(
value=[6, 7, 8])),
})),
]
self.serialized = core.LabeledTensor(
constant_op.constant([ex.SerializeToString() for ex in examples]),
['batch'])
self.features = {
'a': io_ops.FixedLenFeature([], dtypes.int64),
'b': io_ops.FixedLenFeature([('x', 3)], dtypes.int64)
}
def setUp(self):
super(ShuffleBatchTest, self).setUp()
tensors = []
for i in range(10):
offset_lt = core.LabeledTensor(constant_op.constant(i), [])
tensors.append(core.add(self.original_lt, offset_lt))
self.pack_lt = ops.pack(tensors, 'batch')
