def SparseLogSoftMax(logits, indices):
with tf.name_scope("SparseLogSoftMax"):
if len(indices.shape) == 1:
indices = tf.expand_dims(indices, 1)
batch_size = tf.math.reduce_max(indices) + 1
num_values = tf.cast(tf.shape(logits)[0], tf.int64)
dense_shape_2d = [batch_size, num_values]
indices_2d = tf.concat(
[indices, tf.expand_dims(tf.range(num_values), 1)], axis=1)
sparse_logits = tf.SparseTensor(indices=indices_2d,
values=logits,
dense_shape=dense_shape_2d)
logits_max = tf.sparse_reduce_max(sp_input=sparse_logits,
axis=-1,
keepdims=True)
logits_max = tf.reshape(tf.manip.gather_nd(logits_max, indices), [-1])
# Propagating the gradient through logits_max should be a no-op, so to accelrate this
# we just prune it,
# Also tf.sparse_reduce_max doesn't have a gradient implemented in TensorFlow (as of Nov/2018).
logits_max = tf.stop_gradient(logits_max)
normalized_logits = logits - logits_max
normalized_exp_values = tf.exp(normalized_logits)
normalized_exp_sum = tf.manip.scatter_nd(indices,
updates=normalized_exp_values,
shape=[batch_size])
normalized_log_exp_sum = tf.manip.gather_nd(
params=tf.log(normalized_exp_sum), indices=indices)
return normalized_logits - normalized_log_exp_sum
def test_salt_pepper_noise(self):
batch_size = 8
dim = 12
noise_amount = 0.5
noise_tensor = random.salt_pepper_noise([batch_size, dim],
density=noise_amount,
salt_value=1,
pepper_value=0,
dtype=tf.float32)
sum_tensor = tf.sparse_reduce_sum(noise_tensor)
max_tensor = tf.sparse_reduce_max(noise_tensor)
self.assertEqual(sum_tensor.eval(),
int(dim * noise_amount) // 2 * batch_size)
self.assertEqual(max_tensor.eval(), 1)
# self.assertEqual(min_tensor,0)
# use negative pepper
noise_tensor = random.salt_pepper_noise([batch_size, dim],
density=noise_amount,
salt_value=1,
pepper_value=-1,
dtype=tf.float32)
sum_tensor = tf.sparse_reduce_sum(noise_tensor)
self.assertEqual(sum_tensor.eval(), 0)
dim = 10
noise_tensor = random.salt_pepper_noise([batch_size, dim],
density=noise_amount,
salt_value=1,
pepper_value=-1,
dtype=tf.float32)
sum_tensor = tf.sparse_reduce_sum(noise_tensor)
self.assertEqual(sum_tensor.eval(), 0)
def _sparse_minus_reduce_min_and_reduce_max(x):
"""Computes the -min and max of a SparseTensor x.
It differs from sparse_reduce_max in that sparse_reduce_max returns 0 when all
elements are missing along axis 0.
We replace the 0 with NaN when x's dtype is float and dtype.min+1 when it's
int.
Args:
x: A `SparseTensor`.
Returns:
Two `Tensors' which are the -min and max.
Raises:
TypeError: If the type of `x` is not supported.
"""
if not isinstance(x, tf.SparseTensor):
raise TypeError('Expected a SparseTensor, but got %r' % x)
minus_x = tf.SparseTensor(indices=x.indices,
values=0 - x.values,
dense_shape=x.dense_shape)
x_count = reduce_batch_count(x, reduce_instance_dims=False)
batch_has_no_values = tf.equal(x_count, tf.constant(0, dtype=tf.int64))
x_batch_max = tf.sparse_reduce_max(x, axis=0)
x_batch_minus_min = tf.sparse_reduce_max(minus_x, axis=0)
if x.dtype.is_floating:
missing_value = tf.constant(_FLOATING_NAN, x.dtype)
else:
missing_value = tf.constant(x.dtype.min + 1, x.dtype)
x_batch_max = tf.where(batch_has_no_values,
tf.fill(tf.shape(x_batch_max), missing_value),
x_batch_max)
x_batch_minus_min = tf.where(
batch_has_no_values, tf.fill(tf.shape(x_batch_minus_min),
missing_value), x_batch_minus_min)
return x_batch_minus_min, x_batch_max
def get_sp_topk(adj_pred, sp_adj_train, nb_nodes, k):
"""Returns binary matrix with topK."""
_, indices = tf.nn.top_k(tf.reshape(adj_pred, (-1,)), k)
indices = tf.reshape(tf.cast(indices, tf.int64), (-1, 1))
sp_adj_pred = tf.SparseTensor(
indices=indices,
values=tf.ones(k),
dense_shape=(nb_nodes * nb_nodes,))
sp_adj_pred = tf.sparse_reshape(sp_adj_pred,
shape=(nb_nodes, nb_nodes, 1))
sp_adj_train = tf.SparseTensor(
indices=sp_adj_train.indices,
values=tf.ones_like(sp_adj_train.values),
dense_shape=sp_adj_train.dense_shape)
sp_adj_train = tf.sparse_reshape(sp_adj_train,
shape=(nb_nodes, nb_nodes, 1))
sp_adj_pred = tf.sparse_concat(
sp_inputs=[sp_adj_pred, sp_adj_train], axis=-1)
return tf.sparse_reduce_max(sp_adj_pred, axis=-1)
def call(self, batch):
feat, ct = batch # gic, gin -> gnc
bool_ct = tf.cast(ct, tf.bool)
tiling = tf.stack([1, 1, 1, tf.shape(feat)[-1]], axis=0)
ext_ct = tf.tile(tf.expand_dims(ct, -1), tiling) # ginc
w = tf.where(ext_ct)
values = tf.gather_nd(feat, tf.gather(w, [0, 1, 3], axis=1))
masked = tf.SparseTensor(indices=w,
values=values,
dense_shape=tf.cast(
tf.shape(ext_ct), tf.int64))
nb = tf.SparseTensor(indices=w,
values=tf.ones_like(values, dtype=tf.float32),
dense_shape=tf.cast(tf.shape(ext_ct),
tf.int64))
return tf.reshape(tf.sparse_reduce_max(masked, axis=1),
tf.shape(feat))
def sparse_log_soft_max(logits, indices):
"""Sparse version of a `log(softmax)` function.
There are `BATCH_SIZE` entries, each with an arbitrary number of logits. For
each value in `logits` there is a corresponding value in `indices` from `0` to `BATCH_SIZE-1`
which indicate which entry the logit is participating.
Args:
logits: dense collection of logits, a concatenation of logits for all entries, of shape `[N]`.
indices: indices of values `0` to `BATCH_SIZE-1`, that indicates which entry the
corresponding logit participate. Shape `[N]`.
Returns `log(softmax)` of the logits, such that the sum of all `exp(values)`
for the same entry sums to 1. So the `log(probabilities)` depending on how their values are trained.
A Tensor of shape `[N]`,
"""
with tf.name_scope("SparseLogSoftMax"):
if len(indices.shape) == 1:
indices = tf.expand_dims(indices, 1)
batch_size = tf.math.reduce_max(indices) + 1
num_values = tf.cast(tf.shape(logits)[0], tf.int64)
dense_shape_2d = [batch_size, num_values]
indices_2d = tf.concat([indices,
tf.expand_dims(tf.range(num_values), 1)], axis=1)
sparse_logits = tf.SparseTensor(
indices=indices_2d, values=logits, dense_shape=dense_shape_2d)
logits_max = tf.sparse_reduce_max(
sp_input=sparse_logits, axis=-1, keepdims=True)
logits_max = tf.reshape(tf.manip.gather_nd(logits_max, indices), [-1])
# Propagating the gradient through logits_max should be a no-op, so to accelrate this
# we just prune it,
# Also tf.sparse_reduce_max doesn't have a gradient implemented in TensorFlow (as of Nov/2018).
logits_max = tf.stop_gradient(logits_max)
normalized_logits = logits - logits_max
normalized_exp_values = tf.exp(normalized_logits)
normalized_exp_sum = tf.manip.scatter_nd(
indices, updates=normalized_exp_values, shape=[batch_size])
normalized_log_exp_sum = tf.manip.gather_nd(
params=tf.log(normalized_exp_sum), indices=indices)
return normalized_logits - normalized_log_exp_sum
def max_counts(dataset):
maxdataset = dataset.map(lambda x: tf.sparse_reduce_max_sparse(x, axis=0))
maxdataset = maxdataset.batch(100)
maxdataset = maxdataset.map(lambda x: tf.sparse_reduce_max(x, axis=0))
maxdata_iter = maxdataset.make_one_shot_iterator()
maxdata = maxdata_iter.get_next()
counter = np.zeros(202498, )
i = 0
while True:
try:
print(i, end=', ')
# Sum of 1000 pages
maxbatch = maxdata.eval()
counter = np.vstack([counter, maxbatch]).max(axis=0)
i += 1
except:
break
return counter
def next_batch(self):
batch = tf.train.batch([self.serialized_example], self.batch_size)
features = tf.parse_example(batch,
features={
'fea_indices_row':
tf.VarLenFeature(tf.int64),
'fea_indices_col':
tf.VarLenFeature(tf.int64),
'fea_values':
tf.VarLenFeature(tf.float32),
'seq_len':
tf.FixedLenFeature([1], tf.int64),
'label':
tf.FixedLenFeature([1], tf.int64),
'forloeb':
tf.FixedLenFeature([1], tf.int64),
'ctx_indices':
tf.VarLenFeature(tf.int64),
'ctx_values':
tf.VarLenFeature(tf.float32),
'text_indices_row':
tf.VarLenFeature(tf.int64),
'text_indices_col':
tf.VarLenFeature(tf.int64),
'text_indices_vals':
tf.VarLenFeature(tf.int64),
'text_len_idx':
tf.VarLenFeature(tf.int64),
'text_len':
tf.VarLenFeature(tf.int64),
'doc_ids':
tf.VarLenFeature(tf.int64)
})
seq_len = tf.sparse_reduce_max(features['fea_indices_row'],
axis=1) + 10
max_seq_len = tf.reduce_max(
tf.stack([
tf.reduce_max(seq_len) - 1,
tf.reduce_max(features['text_indices_row'].values)
])) + 1
fea_dim = [max_seq_len, self.feature_size]
# Read values and construct sparse tensor
sparse_indices_row = features['fea_indices_row']
sparse_indices_col = features['fea_indices_col']
sparse_vals = features['fea_values']
feature_tensor = sparse_merge.sparse_merge(
[sparse_indices_row, sparse_indices_col], sparse_vals, fea_dim)
text_indices_row = features['text_indices_row']
text_indices_col = features['text_indices_col']
text_indices_vals = features['text_indices_vals']
max_text_len = tf.cond(
tf.equal(text_indices_col.dense_shape[1],
tf.constant(0,
tf.int64)), lambda: tf.constant(0, tf.int64),
lambda: tf.add(tf.reduce_max(text_indices_col.values), 1))
text_tensor = sparse_merge.sparse_merge(
[text_indices_row, text_indices_col], text_indices_vals,
[max_seq_len, max_text_len])
text_len_tensor = sparse_merge.sparse_merge([features['text_len_idx']],
features['text_len'],
[max_seq_len])
doc_ids_tensor = sparse_merge.sparse_merge([features['text_len_idx']],
features['doc_ids'],
[max_seq_len])
ctx_tensor = tf.sparse_merge(features['ctx_indices'],
features['ctx_values'], self.context_size)
return {
'features': feature_tensor,
'context': ctx_tensor,
'seq_len': seq_len,
'label': features['label'],
'forloeb': features['forloeb'],
'text': text_tensor,
'text_len': text_len_tensor,
'doc_ids': doc_ids_tensor
}
import sparse_reduce_max_gpu_py as sparse_reduce
from tensorflow.python.ops import math_ops
from tensorflow.python.framework import dtypes
import numpy as np
sys.settrace
indices = math_ops.cast([[0, 0, 0], [0, 0, 2], [0, 1, 1], [0, 1, 2], [0, 2, 2],
[1, 0, 0], [1, 0, 1], [1, 1, 2], [1, 2, 0], [1, 2, 1],
[2, 0, 1], [2, 0, 2], [2, 1, 0], [2, 2, 2]],
dtype=dtypes.int64)
values = math_ops.cast(
[3, 1, 2, 6, 1, 1.5, 2.5, 3.5, 2.5, 5.5, 0.5, 0.75, 0.35, 0.1],
dtype=dtypes.float32)
sp_input = tf.SparseTensor(indices=indices,
values=values,
dense_shape=[3, 3, 3])
sp_reduced = sparse_reduce.sparse_reduce_max_gpu(sp_input, axis=2)
sp_reduced_cpu = tf.sparse_reduce_max(sp_input, axis=2)
config = tf.ConfigProto(log_device_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
output = sess.run(sp_reduced)
outputCPU = sess.run(sp_reduced_cpu)
print "output GPU", output
print "output CPU", outputCPU
def sparse_reduce_logsumexp(sparse_tensor,
dense_shape,
axis=None,
keep_dims=False,
mode='FAST',
name=None):
"""
Sparse version of tf.reduce_logsumexp that take a SparseTensor as input and returns a dense tensor. The design of
this function takes advantage some properties that may be specific to this situation so care should be taken using
this in other projects.
:param sparse_tensor:
:param dense_shape:
:param axis:
:param keep_dims:
:param mode:
:param name:
:return: full_tensor:
"""
if name is None:
scope_name = 'sparse_reduce_logsumexp'
else:
scope_name = name
with tf.variable_scope(scope_name):
if mode is 'FAST':
# First convert sparse to dense for reduce max (assumes tensor is no negative?)
dense_tensor = tf.sparse_tensor_to_dense(sparse_tensor,
validate_indices=False)
# Rehape sparse_full to regain shape info
dense_tensor = tf.reshape(dense_tensor, dense_shape)
# Now compute raw max using standard rather than sparse reduce max
raw_max = tf.reduce_max(dense_tensor, axis=axis, keep_dims=True)
# Compute my_max
conditional = tf.where(tf.is_finite(raw_max), raw_max,
tf.zeros_like(raw_max))
my_max = tf.stop_gradient(conditional)
# Subtract my_max
exp_values = tf.exp(dense_tensor - my_max)
else:
raw_max = tf.sparse_reduce_max(sparse_tensor,
axis=axis,
keep_dims=keep_dims)
# Reshape to regain shape info for raw_max
reduced_shape = dense_shape.copy()
for ax in axis:
reduced_shape[ax] = 1
raw_max = tf.reshape(raw_max, reduced_shape)
conditional = tf.where(tf.is_finite(raw_max), raw_max,
tf.zeros_like(raw_max))
my_max = tf.stop_gradient(conditional)
# Exp
# Convert sparse_tensor to full so that we can subtract my_max
dense_tensor = tf.sparse_tensor_to_dense(sparse_tensor,
validate_indices=False)
# Rehape sparse_full to regain shape info
dense_tensor = tf.reshape(dense_tensor, dense_shape)
# Subtract my_max
exp_values = tf.exp(dense_tensor - my_max)
# Extract only required values from exp_values (according to sparse_tensor.indices)
new_dense_shape = tf.reduce_prod(sparse_tensor.dense_shape,
keep_dims=True)
reshape_sparse_tensor = tf.sparse_reshape(sparse_tensor,
new_dense_shape)
exp_values = tf.reshape(exp_values, [-1])
exp_values = tf.gather_nd(exp_values, reshape_sparse_tensor.indices)
# Convert back to sparse
exp_sparse = tf.SparseTensor(sparse_tensor.indices, exp_values,
sparse_tensor.dense_shape)
# Sum
sum_sparse = sparse_reduce_sum_nd(exp_sparse,
dense_shape,
axis=axis,
keep_dims=True)
# Log
full_tensor = tf.log(sum_sparse) + my_max
return full_tensor
tf.truncated_normal() tf.truediv() tf.sparse_transpose() tf.sparse_tensor_dense_matmul() tf.sparse_accumulator_apply_gradient() tf.sparse_accumulator_take_gradient() tf.sparse_add() tf.sparse_concat() tf.sparse_conditional_accumulator() tf.sparse_mask() tf.sparse_matmul() tf.sparse_maximum() tf.sparse_merge() tf.sparse_minimum() tf.sparse_reduce_max() tf.sparse_reduce_max_sparse() tf.reduce_all() tf.reduce_any() tf.reduce_join() tf.reduce_logsumexp() tf.reduce_max() tf.reduce_mean() tf.reduce_min() tf.reduce_prod() tf.reduce_sum() tf.reduced_shape() tf.random_crop() tf.random_gamma()
