def test(self): mask = core.LabeledTensor(math_ops.range(7) > 3, [self.a0]) masked_lt = ops.boolean_mask(self.original_lt, mask) golden_lt = core.LabeledTensor( array_ops.boolean_mask(self.original_lt.tensor, mask.tensor), ['x', self.a1, self.a2, self.a3]) self.assertLabeledTensorsEqual(masked_lt, golden_lt)
def test(self): expected_a = core.LabeledTensor(constant_op.constant([1, 5]), ['batch']) expected_b = core.LabeledTensor( constant_op.constant([[2, 3, 4], [6, 7, 8]]), ['batch', 'x']) parsed = io_ops.parse_example(self.serialized, self.features) self.assertLabeledTensorsEqual(expected_a, parsed['a']) self.assertLabeledTensorsEqual(expected_b, parsed['b'])
def setUp(self): super(VerifyTensorAllFiniteTest, self).setUp() self.finite_lt = core.LabeledTensor(constant_op.constant(42.0), []) self.nan_lt = core.LabeledTensor(constant_op.constant(np.nan), []) self.checked_finite_lt = ops.verify_tensor_all_finite( self.finite_lt, '') self.checked_nan_lt = ops.verify_tensor_all_finite(self.nan_lt, '')
def test(self): condition = core.LabeledTensor(math_ops.range(5) < 3, ['x']) x = core.LabeledTensor(array_ops.ones(5), ['x']) y = core.LabeledTensor(array_ops.zeros(5), ['x']) where_lt = ops.where(condition, x, y) golden_lt = core.LabeledTensor( array_ops.concat( [array_ops.ones(3), array_ops.zeros(2)], 0), ['x']) self.assertLabeledTensorsEqual(where_lt, golden_lt)
def test_identity(self): axis_order = ['w', 'x', 'y', 'z'] lt = core.LabeledTensor( array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order) actual = core.impose_axis_order(lt, axis_order) self.assertLabeledTensorsEqual(lt, actual) lt = core.LabeledTensor( array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3]) actual = core.impose_axis_order(lt, axis_order) self.assertLabeledTensorsEqual(lt, actual)
def test_passes(self): axis_order = ['w', 'x', 'y', 'z'] lt = core.LabeledTensor(array_ops.ones((1, 1, 1, 1)), axis_order) core.check_axis_order(lt, axis_order) lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[1:]) core.check_axis_order(lt, axis_order) lt = core.LabeledTensor(array_ops.ones((1, 1, 1)), axis_order[:-1]) core.check_axis_order(lt, axis_order)
def test_matrix_vector(self): xy_lt = core.LabeledTensor( array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y']) y_lt = core.LabeledTensor(math_ops.range(3), ['y']) matmul_lt = ops.matmul(xy_lt, y_lt) golden_lt = core.LabeledTensor( math_ops.matmul(xy_lt.tensor, array_ops.reshape(y_lt.tensor, (-1, 1)))[:, 0], ['x']) self.assertLabeledTensorsEqual(matmul_lt, golden_lt) matmul_lt = ops.matmul(y_lt, xy_lt) self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
def test_reverse(self): axis_order = ['w', 'x', 'y', 'z'] lt = core.LabeledTensor( array_ops.reshape(math_ops.range(24), (1, 2, 3, 4)), axis_order) actual = core.impose_axis_order(lt, axis_order[::-1]) expected = core.transpose(lt, axis_order[::-1]) self.assertLabeledTensorsEqual(expected, actual) lt = core.LabeledTensor( array_ops.reshape(math_ops.range(6), (1, 2, 3)), axis_order[:3]) actual = core.impose_axis_order(lt, axis_order[::-1]) expected = core.transpose(lt, ['y', 'x', 'w']) self.assertLabeledTensorsEqual(expected, actual)
def test_indexing_scalars(self): actual = self.lt[:, :, :, 0] expected = core.LabeledTensor(self.lt.tensor[:, :, :, 0], list(self.lt.axes.values())[:-1]) self.assertLabeledTensorsEqual(actual, expected) actual = self.lt[1, :, :, 0] expected = core.LabeledTensor(self.lt.tensor[1, :, :, 0], list(self.lt.axes.values())[1:-1]) self.assertLabeledTensorsEqual(actual, expected) actual = self.lt[1, 2, :, 0] expected = core.LabeledTensor(self.lt.tensor[1, 2, :, 0], list(self.lt.axes.values())[2:-1]) self.assertLabeledTensorsEqual(actual, expected)
def setUp(self): super(Base, self).setUp() self.x_size = 7 self.channel_size = 3 self.z_size = 4 self.probs_size = 11 tensor = math_ops.range( 0, self.x_size * self.channel_size * self.z_size * self.probs_size) tensor = array_ops.reshape( tensor, [self.x_size, self.channel_size, self.z_size, self.probs_size]) a0 = ('x', range(self.x_size)) a1 = ('channel', ['red', 'green', 'blue']) a2 = 'z' a3 = ('probs', np.linspace(0.0, 1.0, self.probs_size)) self.tensor = tensor self.a0 = a0 self.a1 = a1 self.a2 = a2 self.a2_resolved = ('z', self.z_size) self.a3 = a3 self.original_lt = core.LabeledTensor(tensor, [a0, a1, a2, a3]) self.x_probs_lt = core.slice_function(self.original_lt, {'z': 0}) self.x_probs_lt = ops.select(self.x_probs_lt, {'channel': 'red'}) self.channel_probs_lt = core.slice_function(self.original_lt, { 'x': 3, 'z': 0 })
def test_new_name(self): rename_axis_lt = ops.rename_axis(self.original_lt, 'channel', 'foo') expected_axes = [(name if name != 'channel' else 'foo', axis.value) for name, axis in self.original_lt.axes.items()] expected_lt = core.LabeledTensor(self.original_lt.tensor, expected_axes) self.assertLabeledTensorsEqual(rename_axis_lt, expected_lt)
def test_unknown_size(self): reshape_lt = ops.reshape(self.original_lt, ['channel', 'z', 'probs'], ['new_dim']) golden_lt = core.LabeledTensor( array_ops.reshape(self.original_lt.tensor, [self.x_size, -1]), [self.original_lt.axes['x'], 'new_dim']) self.assertLabeledTensorsEqual(reshape_lt, golden_lt)
def test_axis(self): unpack_lt = ops.unpack(self.original_lt, axis_name='z')[0] golden_lt = core.LabeledTensor( array_ops.unstack(self.original_lt.tensor, axis=2)[0], [self.a0, self.a1, self.a3]) self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def test(self): unpack_lt = ops.unpack(self.original_lt)[0] golden_lt = core.LabeledTensor( array_ops.unstack(self.original_lt.tensor)[0], [self.a1, self.a2, self.a3]) self.assertLabeledTensorsEqual(unpack_lt, golden_lt)
def setUp(self): super(ReshapeCoderTest, self).setUp() self.batch_size = 8 self.num_rows = 50 self.num_columns = 100 self.channels = ['red', 'green', 'blue'] self.masks = [False, True] tensor = math_ops.range( 0, self.batch_size * self.num_rows * self.num_columns * len(self.channels) * len(self.masks)) tensor = array_ops.reshape(tensor, [ self.batch_size, self.num_rows, self.num_columns, len(self.channels), len(self.masks) ]) self.batch_axis = ('batch', range(self.batch_size)) self.row_axis = ('row', range(self.num_rows)) self.column_axis = ('column', range(self.num_columns)) self.channel_axis = ('channel', self.channels) self.mask_axis = ('mask', self.masks) axes = [ self.batch_axis, self.row_axis, self.column_axis, self.channel_axis, self.mask_axis ] self.masked_image_lt = core.LabeledTensor(tensor, axes)
def parse_example(serialized, features, name=None, example_names=None): """Parse `Example` protos into a `dict` of labeled tensors. See tf.parse_example. Args: serialized: A 1-D LabeledTensor of strings, a batch of binary serialized `Example` protos. features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature` values. name: A name for this operation (optional). example_names: A vector (1-D Tensor) of strings (optional), the names of the serialized protos in the batch. Returns: A `dict` mapping feature keys to `LabeledTensor` values. The single axis from `serialized` will be prepended to the axes provided by each feature. Raises: ValueError: if any feature is invalid. """ serialized = core.convert_to_labeled_tensor(serialized) unlabeled_features = _labeled_to_unlabeled_features(features) unlabeled_parsed = parsing_ops.parse_example( serialized.tensor, unlabeled_features, name, example_names) parsed = {} for name, parsed_feature in unlabeled_parsed.items(): axes = list(serialized.axes.values()) + features[name].axes parsed[name] = core.LabeledTensor(parsed_feature, axes) return parsed
def concat(labeled_tensors, axis_name, name=None): """Concatenate tensors along a dimension. See tf.concat. Args: labeled_tensors: A list of input LabeledTensors. axis_name: The name of the axis along which to concatenate. name: Optional op name. Returns: The concatenated tensor. The coordinate labels for the concatenation dimension are also concatenated, if they are available for every tensor. Raises: ValueError: If fewer than one tensor inputs is provided, if the tensors have incompatible axes, or if `axis_name` isn't the name of an axis. """ with ops.name_scope(name, 'lt_concat', labeled_tensors) as scope: labeled_tensors = [ core.convert_to_labeled_tensor(lt) for lt in labeled_tensors ] if len(labeled_tensors) < 1: raise ValueError('concat expects at least 1 tensor, but received %s' % labeled_tensors) # All tensors must have these axes. axes_0 = labeled_tensors[0].axes axis_names = list(axes_0.keys()) if axis_name not in axis_names: raise ValueError('%s not in %s' % (axis_name, axis_names)) shared_axes = axes_0.remove(axis_name) tensors = [labeled_tensors[0].tensor] concat_axis_list = [axes_0[axis_name]] for labeled_tensor in labeled_tensors[1:]: current_shared_axes = labeled_tensor.axes.remove(axis_name) if current_shared_axes != shared_axes: # TODO(shoyer): add more specific checks about what went wrong, # including raising AxisOrderError when appropriate raise ValueError('Mismatched shared axes: the first tensor ' 'had axes %r but this tensor has axes %r.' % (shared_axes, current_shared_axes)) # Accumulate the axis labels, if they're available. concat_axis_list.append(labeled_tensor.axes[axis_name]) tensors.append(labeled_tensor.tensor) concat_axis = core.concat_axes(concat_axis_list) concat_dimension = axis_names.index(axis_name) concat_tensor = array_ops.concat(tensors, concat_dimension, name=scope) values = list(axes_0.values()) concat_axes = (values[:concat_dimension] + [concat_axis] + values[concat_dimension + 1:]) return core.LabeledTensor(concat_tensor, concat_axes)
def test_matrix_matrix_axis_order(self): xy_lt = core.LabeledTensor( array_ops.reshape(math_ops.range(6), (2, 3)), ['x', 'y']) yz_lt = core.LabeledTensor( array_ops.reshape(math_ops.range(12), (3, 4)), ['y', 'z']) golden_lt = core.LabeledTensor( math_ops.matmul(xy_lt.tensor, yz_lt.tensor), ['x', 'z']) with core.axis_order_scope(['x', 'y', 'z']): matmul_lt = ops.matmul(xy_lt, yz_lt) self.assertLabeledTensorsEqual(matmul_lt, golden_lt) matmul_lt = ops.matmul(yz_lt, xy_lt) self.assertLabeledTensorsEqual(matmul_lt, golden_lt)
def test_core_op(self): for op_name, _, tf_op, lt_op in self.ops: golden_tensor = tf_op(self.test_lt_1_broadcast, self.test_lt_2_broadcast) golden_lt = core.LabeledTensor(golden_tensor, self.broadcast_axes) actual_lt = lt_op(self.test_lt_1, self.test_lt_2) self.assertIn(op_name, actual_lt.name) self.assertLabeledTensorsEqual(golden_lt, actual_lt)
def setUp(self): super(CoreUnaryOpsTest, self).setUp() self.ops = [ ('abs', operator.abs, math_ops.abs, core.abs_function), ('neg', operator.neg, math_ops.negative, core.neg), # TODO(shoyer): add unary + to core TensorFlow ('pos', None, None, None), ('sign', None, math_ops.sign, core.sign), ('reciprocal', None, math_ops.reciprocal, core.reciprocal), ('square', None, math_ops.square, core.square), ('round', None, math_ops.round, core.round_function), ('sqrt', None, math_ops.sqrt, core.sqrt), ('rsqrt', None, math_ops.rsqrt, core.rsqrt), ('log', None, math_ops.log, core.log), ('exp', None, math_ops.exp, core.exp), ('log', None, math_ops.log, core.log), ('ceil', None, math_ops.ceil, core.ceil), ('floor', None, math_ops.floor, core.floor), ('cos', None, math_ops.cos, core.cos), ('sin', None, math_ops.sin, core.sin), ('tan', None, math_ops.tan, core.tan), ('acos', None, math_ops.acos, core.acos), ('asin', None, math_ops.asin, core.asin), ('atan', None, math_ops.atan, core.atan), ('lgamma', None, math_ops.lgamma, core.lgamma), ('digamma', None, math_ops.digamma, core.digamma), ('erf', None, math_ops.erf, core.erf), ('erfc', None, math_ops.erfc, core.erfc), ('lgamma', None, math_ops.lgamma, core.lgamma), ] total_size = np.prod([v.size for v in self.original_lt.axes.values()]) self.test_lt = core.LabeledTensor( math_ops.cast(self.original_lt, dtypes.float32) / total_size, self.original_lt.axes)
def test_invalid(self): lt = core.LabeledTensor( array_ops.reshape(math_ops.range(2), (1, 2)), ['x', 'y']) with self.assertRaises(ValueError): core.impose_axis_order(lt) with self.assertRaises(ValueError): core.impose_axis_order(lt, ['x'])
def parse_single_example(serialized, features, name=None, example_names=None): """Parses a single `Example` proto. See tf.parse_single_example. Args: serialized: A scalar string Tensor or LabeledTensor, a single serialized Example. features: A `dict` mapping feature keys to `labeled_tensor.FixedLenFeature` values. name: A name for this operation (optional). example_names: (Optional) A scalar string Tensor, the associated name. Returns: A `dict` mapping feature keys to `LabeledTensor` values. Raises: ValueError: if any feature is invalid. """ serialized = core.convert_to_labeled_tensor(serialized) unlabeled_features = _labeled_to_unlabeled_features(features) unlabeled_parsed = parsing_ops.parse_single_example( serialized.tensor, unlabeled_features, name, example_names) parsed = {} for name, parsed_feature in unlabeled_parsed.items(): parsed[name] = core.LabeledTensor(parsed_feature, features[name].axes) return parsed
def foldl(fn, labeled_tensor, initial_value, name=None): """Left fold on the list of tensors unpacked from labeled_tensor. See tf.foldl. Args: fn: The function to apply to each unpacked LabeledTensor. It should have type (LabeledTensor, LabeledTensor) -> LabeledTensor. Its arguments are (accumulated_value, next_value). labeled_tensor: The input tensor. initial_value: The initial value of the accumulator. name: Optional op name. Returns: The accumulated value. """ with ops.name_scope(name, 'lt_foldl', [labeled_tensor, initial_value]) as scope: labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) initial_value = core.convert_to_labeled_tensor(initial_value) @tc.returns(ops.Tensor) @tc.accepts(ops.Tensor, ops.Tensor) def tf_fn(accumulator, next_element): accumulator_lt = core.LabeledTensor(accumulator, initial_value.axes) next_element_lt = core.LabeledTensor( next_element, list(labeled_tensor.axes.values())[1:]) return fn(accumulator_lt, next_element_lt).tensor foldl_op = functional_ops.foldl( tf_fn, labeled_tensor.tensor, initializer=initial_value.tensor) foldl_lt = core.LabeledTensor(foldl_op, initial_value.axes) return core.identity(foldl_lt, name=scope)
def boolean_mask(labeled_tensor, mask, name=None): """Apply a boolean mask to a labeled tensor. Unlike `tf.boolean_mask`, this currently only works on 1-dimensional masks. The mask is applied to the first axis of `labeled_tensor`. Labels on the first axis are removed, because True indices in `mask` may not be known dynamically. Args: labeled_tensor: The input tensor. mask: The type of the returned tensor. name: Optional op name. Returns: The masked labeled tensor. Raises: ValueError: if the first axis of the mask """ with ops.name_scope(name, 'lt_boolean_mask', [labeled_tensor, mask]) as scope: labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) mask = core.convert_to_labeled_tensor(mask) if len(mask.axes) > 1: raise NotImplementedError( "LabeledTensor's boolean_mask currently only supports 1D masks") mask_axis = list(mask.axes.values())[0] lt_axis = list(labeled_tensor.axes.values())[0] if mask_axis != lt_axis: raise ValueError('the first axis of the labeled tensor and the mask ' 'are not equal:\n%r\n%r' % (lt_axis, mask_axis)) op = array_ops.boolean_mask(labeled_tensor.tensor, mask.tensor, name=scope) # TODO(shoyer): attempt to infer labels for the masked values, by calling # tf.get_static_value on the mask? axes = [lt_axis.name] + list(labeled_tensor.axes.values())[1:] return core.LabeledTensor(op, axes)
def test_default_axis_order(self): transpose_lt = core.transpose(self.original_lt) golden_lt = core.LabeledTensor( array_ops.transpose(self.tensor, [3, 2, 1, 0]), list(reversed(list(self.original_lt.axes.values())))) self.assertLabeledTensorsEqual(transpose_lt, golden_lt)
def test_slice(self): select_lt = ops.select(self.original_lt, {'channel': slice('red', 'green')}) a1_sliced = ('channel', ['red', 'green']) golden_lt = core.LabeledTensor(self.tensor[:, :2, :, :], [self.a0, a1_sliced, self.a2, self.a3]) self.assertLabeledTensorsEqual(select_lt, golden_lt)
def where(condition, x, y, name=None): """Return elements from x or y depending on condition. See `tf.where` for more details. This function currently only implements the three argument version of where. Args: condition: LabeledTensor of type `bool`. x: LabeledTensor for values where condition is true. y: LabeledTensor for values where condition is false. name: Optional op name. Returns: The labeled tensor with values according to condition. Raises: ValueError: if `x` and `y` have different axes, or if the axes of `x` do not start with the axes of `condition`. """ with ops.name_scope(name, 'lt_where', [condition, x, y]) as scope: condition = core.convert_to_labeled_tensor(condition) x = core.convert_to_labeled_tensor(x) y = core.convert_to_labeled_tensor(y) if not condition.axes == x.axes == y.axes: raise ValueError('all inputs to `where` must have equal axes') op = array_ops.where(condition.tensor, x.tensor, y.tensor, name=scope) return core.LabeledTensor(op, x.axes)
def unpack(labeled_tensor, axis_name=None, name=None): """Unpack the tensor. See tf.unpack. Args: labeled_tensor: The input tensor. axis_name: Optional name of axis to unpack. By default, the first axis is used. name: Optional op name. Returns: The list of unpacked LabeledTensors. Raises: ValueError: If `axis_name` is not an axis on the input. """ with ops.name_scope(name, 'lt_unpack', [labeled_tensor]) as scope: labeled_tensor = core.convert_to_labeled_tensor(labeled_tensor) axis_names = list(labeled_tensor.axes.keys()) if axis_name is None: axis_name = axis_names[0] if axis_name not in axis_names: raise ValueError('%s not in %s' % (axis_name, axis_names)) axis = axis_names.index(axis_name) unpack_ops = array_ops.unstack(labeled_tensor.tensor, axis=axis, name=scope) axes = [a for i, a in enumerate(labeled_tensor.axes.values()) if i != axis] return [core.LabeledTensor(t, axes) for t in unpack_ops]
def setUp(self): super(ParseBase, self).setUp() examples = [ example_pb2.Example(features=feature_pb2.Features( feature={ 'a': feature_pb2.Feature(int64_list=feature_pb2.Int64List( value=[1])), 'b': feature_pb2.Feature(int64_list=feature_pb2.Int64List( value=[2, 3, 4])), })), example_pb2.Example(features=feature_pb2.Features( feature={ 'a': feature_pb2.Feature(int64_list=feature_pb2.Int64List( value=[5])), 'b': feature_pb2.Feature(int64_list=feature_pb2.Int64List( value=[6, 7, 8])), })), ] self.serialized = core.LabeledTensor( constant_op.constant([ex.SerializeToString() for ex in examples]), ['batch']) self.features = { 'a': io_ops.FixedLenFeature([], dtypes.int64), 'b': io_ops.FixedLenFeature([('x', 3)], dtypes.int64) }
def setUp(self): super(ShuffleBatchTest, self).setUp() tensors = [] for i in range(10): offset_lt = core.LabeledTensor(constant_op.constant(i), []) tensors.append(core.add(self.original_lt, offset_lt)) self.pack_lt = ops.pack(tensors, 'batch')