def _SparseReorderGrad(op, unused_output_indices_grad, output_values_grad):
"""Gradients for the SparseReorder op.
Args:
op: the SparseReorder op
unused_output_indices_grad: the incoming gradients of the output indices
output_values_grad: the incoming gradients of the output values
Returns:
Gradient for each of the 3 input tensors:
(input_indices, input_values, input_shape)
The gradients for input_indices and input_shape is None.
"""
input_indices = op.inputs[0]
input_shape = op.inputs[2]
num_entries = array_ops.shape(input_indices)[0]
entry_indices = math_ops.range(num_entries)
sp_unordered = sparse_tensor.SparseTensor(
input_indices, entry_indices, input_shape)
sp_ordered = sparse_ops.sparse_reorder(sp_unordered)
inverted_permutation = array_ops.invert_permutation(sp_ordered.values)
return (None,
array_ops.gather(output_values_grad, inverted_permutation),
None)
def testAlreadyInOrder(self):
with self.test_session(use_gpu=False) as sess:
input_val = self._SparseTensorValue_5x6(np.arange(6))
sp_output = sparse_ops.sparse_reorder(input_val)
output_val = sess.run(sp_output)
self.assertAllEqual(output_val.indices, input_val.indices)
self.assertAllEqual(output_val.values, input_val.values)
self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
def testFeedAlreadyInOrder(self):
with self.session(use_gpu=False) as sess:
sp_input = self._SparseTensorPlaceholder()
input_val = self._SparseTensorValue_5x6(np.arange(6))
sp_output = sparse_ops.sparse_reorder(sp_input)
output_val = sess.run(sp_output, {sp_input: input_val})
self.assertAllEqual(output_val.indices, input_val.indices)
self.assertAllEqual(output_val.values, input_val.values)
self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
def testOutOfOrder(self):
expected_output_val = self._SparseTensorValue_5x6(np.arange(6))
with self.test_session(use_gpu=False) as sess:
for _ in range(5): # To test various random permutations
input_val = self._SparseTensorValue_5x6(np.random.permutation(6))
sp_output = sparse_ops.sparse_reorder(input_val)
output_val = sess.run(sp_output)
self.assertAllEqual(output_val.indices, expected_output_val.indices)
self.assertAllEqual(output_val.values, expected_output_val.values)
self.assertAllEqual(output_val.dense_shape,
expected_output_val.dense_shape)
def testGradients(self):
with self.test_session(use_gpu=False):
for _ in range(5): # To test various random permutations
input_val = self._SparseTensorValue_5x6(np.random.permutation(6))
sp_input = sparse_tensor.SparseTensor(input_val.indices,
input_val.values,
input_val.dense_shape)
sp_output = sparse_ops.sparse_reorder(sp_input)
err = gradient_checker.compute_gradient_error(
sp_input.values,
input_val.values.shape,
sp_output.values,
input_val.values.shape,
x_init_value=input_val.values)
self.assertLess(err, 1e-11)
def _make_sparse_mask(self, mask_shape, nnz, sort=False):
"""Creates a sparse tensor to be used as a mask in masked_matmul.
Args:
mask_shape: int list, the shape of the mask.
nnz: int, the number of non-zero elements in the mask.
sort: boolean, whether to sort the indices of the mask (in lexicographic
order).
Returns:
A sparse tensor, with nnz indices, drawn uniformly at random.
"""
num_rows = mask_shape[0]
num_cols = mask_shape[1]
row_idx = random_ops.random_uniform(
[nnz], minval=0, maxval=num_rows, dtype=dtypes.int64)
col_idx = random_ops.random_uniform(
[nnz], minval=0, maxval=num_cols, dtype=dtypes.int64)
indices = array_ops.stack([row_idx, col_idx], axis=1)
values = array_ops.ones([nnz])
unordered_mask = sparse_tensor.SparseTensor(indices, values, mask_shape)
return sparse_ops.sparse_reorder(unordered_mask) if sort else unordered_mask
def _SparseReorderGrad(op, unused_output_indices_grad, output_values_grad):
"""Gradients for the SparseReorder op.
Args:
op: the SparseReorder op
unused_output_indices_grad: the incoming gradients of the output indices
output_values_grad: the incoming gradients of the output values
Returns:
Gradient for each of the 3 input tensors:
(input_indices, input_values, input_shape)
The gradients for input_indices and input_shape is None.
"""
input_indices = op.inputs[0]
input_shape = op.inputs[2]
num_entries = array_ops.shape(input_indices)[0]
entry_indices = math_ops.range(num_entries)
sp_unordered = ops.SparseTensor(input_indices, entry_indices, input_shape)
sp_ordered = sparse_ops.sparse_reorder(sp_unordered)
inverted_permutation = array_ops.invert_permutation(sp_ordered.values)
return (None, array_ops.gather(output_values_grad,
inverted_permutation), None)
def testStaticShapeInfoPreserved(self):
sp_input = sparse_tensor.SparseTensor.from_value(
self._SparseTensorValue_5x6(np.arange(6)))
self.assertAllEqual((5, 6), sp_input.get_shape())
sp_output = sparse_ops.sparse_reorder(sp_input)
self.assertAllEqual((5, 6), sp_output.get_shape())
def testStaticShapeInfoPreserved(self):
sp_input = sparse_tensor.SparseTensor.from_value(
self._SparseTensorValue_5x6(np.arange(6)))
self.assertAllEqual((5, 6), sp_input.get_shape())
sp_output = sparse_ops.sparse_reorder(sp_input)
self.assertAllEqual((5, 6), sp_output.get_shape())
def _flatten_tensor(tensor, sequence_mask, expected_length):
"""Flattens the two first dimensions and reshapes a tensor or sparse tensor.
If `tensor` is a dense tensor, the sequence_mask is used to infer valid
inputs.
Note: If `tensor` is a `SparseTensor` and the indices are not sorted, they
will be reordered.
Args:
tensor: A `Tensor` or `SparseTensor` of dimension at least 2, of shape
[batch_size, seq_length, D0, D1, ..., DN].
sequence_mask: A boolean `Tensor` of shape [batch_size, seq_length].
expected_length: A integer scalar `Tensor` with the expected length of the
resulting flattenned Tensor.
Returns:
A `Tensor` object of shape [expected_length, D0, D1, ..., DN].
Raises:
ValueError: If `tensor` has not at least 2 dimensions.
ValueError: If `tensor` is not a `Tensor` or `SparseTensor` object.
InvalidArgumentError: If the resulting `Tensor` doesn't have the expected
length.
"""
shape = tensor.get_shape()
if shape.ndims < 2:
raise ValueError(
'Input tensor expected to have at least 2 dimensions, '
'got {} instead.'.format(shape.ndims))
if isinstance(tensor, sparse_tensor.SparseTensor):
# What follows depends on the indices ordering. Hence we reorder the indices
# to ensure correctness.
flat_tensor = sparse_ops.sparse_reorder(tensor).values
if shape.ndims > 2:
new_shape = array_ops.concat([[-1], shape[2:]], axis=0)
flat_tensor = array_ops.reshape(tensor.values, new_shape)
elif isinstance(tensor, ops.Tensor):
flat_tensor = array_ops.boolean_mask_v2(tensor, sequence_mask)
else:
raise ValueError(
'`tensor` expected to be a `Tensor` or `SparseTensor` '
'got `{}` instead.'.format(tensor))
if shape.ndims == 2:
flat_tensor = array_ops.expand_dims(flat_tensor, -1)
expected_shape = array_ops.concat([[expected_length], [1]], axis=0)
else:
expected_shape = array_ops.concat([[expected_length], shape[2:]],
axis=0)
# TODO(b/119617064): Unify eager and graph implementations.
err_message = 'Tensor shape is incompatible with provided mask.'
if context.executing_eagerly():
if flat_tensor._shape_tuple() != tuple(expected_shape.numpy()): # pylint: disable=protected-access
raise ValueError(err_message)
return flat_tensor
with ops.control_dependencies([
check_ops.assert_equal(array_ops.shape(flat_tensor),
expected_shape,
message=err_message)
]):
return array_ops.identity(flat_tensor)
def sparse_random_mask(dense_shape,
density=0.5,
mask_values=[1],
symmetrical=True,
dtype=dtypes.float32,
seed=None):
"""Uses values to create a sparse random mask according to a given density
a density of 0 returns an empty sparse tensor
Note:
if symmetrical the mask has always the same number of mask_values per row
which means that if ``density * dense_shape[1] < len(mask_values)``, the mask will be an empty ``SparseTensor``.
It also means that if ``dense_shape[1] % len(mask_values) != 0`` and ``density = 1.0``, not all values will be
corrupted because we can't fill every entry with a symmetrical mask.
There are other ways to fill a dense tensor with random values though so a density of 1 defeats the purpose of
this operation.
if not symmetrical the number of mask_values will not be the same per row. If we need to fill 2 extra entries
with values 2 masked values are picked at random to fill the excess.
Example:
if **not** symmetrical and
``shape = [1,10]]``
``density = 0.5``
``mask_values = [1,2,3]``
the result could be something like::
[[1. 1. 2. 3. 0. 0. 0. 2. 0. 0.]]
Args:
seed: int32 to te used as seed
dtype: tensor tensor value type
dense_shape: a 1-D tensor tensor with shape [2]
density: desired density
mask_values: the values to be used to generate the random mask
Returns:
A sparse random mask with a density of the original shape corrupted using the mask values
"""
# total number of corrupted indices
num_values = len(mask_values)
num_corrupted = int(density * dense_shape[1])
num_mask_values = num_corrupted // num_values * num_values
if num_mask_values == 0:
return empty_sparse_tensor(dense_shape)
else:
# num corrupted indices per value
if not symmetrical:
mask_values = random_ops.random_shuffle(mask_values, seed)
extra_corrupted = num_corrupted - num_mask_values
if not symmetrical:
num_mask_values = num_corrupted
samples = sample(dense_shape[1],
num_mask_values,
dense_shape[0],
unique=True,
seed=seed)
indices = batch_to_matrix_indices(samples, dtype=dtypes.int64)
value_tensors = []
for i in range(num_values):
num_vi = num_mask_values // num_values
# spread the extra to be corrupted by n mask_values
if not symmetrical and i < extra_corrupted:
num_vi = num_vi + 1
vi_shape = math_ops.cast([dense_shape[0], num_vi], dtypes.int32)
vi_tensor = array_ops.fill(vi_shape, mask_values[i])
value_tensors.append(vi_tensor)
values = array_ops.concat(value_tensors, axis=-1)
values = array_ops.reshape(values, [-1])
if values.dtype != dtype:
values = math_ops.cast(values, dtype)
dense_shape = math_ops.cast([dense_shape[0], dense_shape[1]],
dtypes.int64)
sp_tensor = SparseTensor(indices, values, dense_shape)
# the indices were generated at random so
sp_tensor = sparse_ops.sparse_reorder(sp_tensor)
return sp_tensor
def __init__(self,
loss,
penalty,
At,
D,
b,
tau=None,
sigma=None,
sess=None,
dtype=dtypes.float32,
devices='',
aggregate=False,
init_var=None):
if not tau:
raise ValueError("Must set tau")
if not sigma:
raise ValueError("Must set sigma")
if sess is None:
sess = session.Session()
self.tau = tau
self.sigma = sigma
self.aggregate = aggregate
#print(type(tau), type(sigma))
#assert type(tau)==type(sigma)
if isinstance(self.tau, float):
self.parammode = 'static'
else:
self.parammode = 'variable'
self.sess = sess
self.loss = loss
self.penalty = penalty
self.dtype = dtype
self.devices = devices
self.init_var = True if init_var else False
if isinstance(self.devices, list):
self.master = self.devices[0]
else:
self.master = self.devices
if not isinstance(devices, list):
self.matmul = tf.matmul
self.spmatmul = tf.sparse_tensor_dense_matmul
else:
self.matmul = distops.matmul
self.spmatmul = distops.spmatmul
# check shape.
self.m, self.n = At.T.shape
self.l, _ = D.shape
print(At.T.shape)
print(D.shape)
print(self.n, D.shape[1])
assert (self.n == D.shape[1])
# setup variables.
# for A, we need to consider the case where A is larger than 2GB. (to be distributed)
if not isinstance(At, distmat.DistMat):
if not isinstance(devices, list):
with tf.device(self.devices):
Ap = tf.placeholder(dtype, shape=At.shape)
self.At = variables.Variable(Ap)
sess.run(self.At.initializer, feed_dict={Ap: At})
self.A = None
else:
self.At = distmat.DistMat.from_dataset(At,
devices=self.devices,
sess=sess,
dtype=self.dtype)
self.A = None
else:
assert all([d1 == d2 for d1, d2 in zip(At.devices, self.devices)])
self.At = At
if not isinstance(D, distmat.DistSpMat):
# D should be COO format.
# dtype induced from original matrix.
# avoid recomputation of transpose.
if isinstance(D, coo_matrix):
pass
elif isinstance(D, spmatrix):
D = D.tocoo()
else:
raise ValueError("must be a scipy sparse matrix")
if not isinstance(devices, list):
D_tensor = coo_to_sparsetensor(D)
D_sorted_op = sparse_ops.sparse_reorder(D_tensor)
Dt_sorted_op = sparse_ops.sparse_reorder(
sparse_ops.sparse_transpose(D_tensor))
D_sorted, Dt_sorted = sess.run([D_sorted_op, Dt_sorted_op])
with tf.device(self.devices):
self.D = sparse_tensor.SparseTensor.from_value(D_sorted)
self.Dt = sparse_tensor.SparseTensor.from_value(Dt_sorted)
# b is a constant. (to duplicate)
else:
self.D = distmat.DistSpMat.from_spmatrix(
D, devices_r=self.devices)
self.Dt = None
else:
assert all([d1 == d2 for d1, d2 in zip(D.devices_r, self.devices)])
self.D = D
self.Dt = None
with tf.device(self.master):
self.b = constant_op.constant(b, dtype=dtype)
self._setup_variables(init_var)
self._setup_evals()