def SparseLogSoftMax(logits, indices): with tf.name_scope("SparseLogSoftMax"): if len(indices.shape) == 1: indices = tf.expand_dims(indices, 1) batch_size = tf.math.reduce_max(indices) + 1 num_values = tf.cast(tf.shape(logits)[0], tf.int64) dense_shape_2d = [batch_size, num_values] indices_2d = tf.concat( [indices, tf.expand_dims(tf.range(num_values), 1)], axis=1) sparse_logits = tf.SparseTensor(indices=indices_2d, values=logits, dense_shape=dense_shape_2d) logits_max = tf.sparse_reduce_max(sp_input=sparse_logits, axis=-1, keepdims=True) logits_max = tf.reshape(tf.manip.gather_nd(logits_max, indices), [-1]) # Propagating the gradient through logits_max should be a no-op, so to accelrate this # we just prune it, # Also tf.sparse_reduce_max doesn't have a gradient implemented in TensorFlow (as of Nov/2018). logits_max = tf.stop_gradient(logits_max) normalized_logits = logits - logits_max normalized_exp_values = tf.exp(normalized_logits) normalized_exp_sum = tf.manip.scatter_nd(indices, updates=normalized_exp_values, shape=[batch_size]) normalized_log_exp_sum = tf.manip.gather_nd( params=tf.log(normalized_exp_sum), indices=indices) return normalized_logits - normalized_log_exp_sum
def test_salt_pepper_noise(self): batch_size = 8 dim = 12 noise_amount = 0.5 noise_tensor = random.salt_pepper_noise([batch_size, dim], density=noise_amount, salt_value=1, pepper_value=0, dtype=tf.float32) sum_tensor = tf.sparse_reduce_sum(noise_tensor) max_tensor = tf.sparse_reduce_max(noise_tensor) self.assertEqual(sum_tensor.eval(), int(dim * noise_amount) // 2 * batch_size) self.assertEqual(max_tensor.eval(), 1) # self.assertEqual(min_tensor,0) # use negative pepper noise_tensor = random.salt_pepper_noise([batch_size, dim], density=noise_amount, salt_value=1, pepper_value=-1, dtype=tf.float32) sum_tensor = tf.sparse_reduce_sum(noise_tensor) self.assertEqual(sum_tensor.eval(), 0) dim = 10 noise_tensor = random.salt_pepper_noise([batch_size, dim], density=noise_amount, salt_value=1, pepper_value=-1, dtype=tf.float32) sum_tensor = tf.sparse_reduce_sum(noise_tensor) self.assertEqual(sum_tensor.eval(), 0)
def _sparse_minus_reduce_min_and_reduce_max(x): """Computes the -min and max of a SparseTensor x. It differs from sparse_reduce_max in that sparse_reduce_max returns 0 when all elements are missing along axis 0. We replace the 0 with NaN when x's dtype is float and dtype.min+1 when it's int. Args: x: A `SparseTensor`. Returns: Two `Tensors' which are the -min and max. Raises: TypeError: If the type of `x` is not supported. """ if not isinstance(x, tf.SparseTensor): raise TypeError('Expected a SparseTensor, but got %r' % x) minus_x = tf.SparseTensor(indices=x.indices, values=0 - x.values, dense_shape=x.dense_shape) x_count = reduce_batch_count(x, reduce_instance_dims=False) batch_has_no_values = tf.equal(x_count, tf.constant(0, dtype=tf.int64)) x_batch_max = tf.sparse_reduce_max(x, axis=0) x_batch_minus_min = tf.sparse_reduce_max(minus_x, axis=0) if x.dtype.is_floating: missing_value = tf.constant(_FLOATING_NAN, x.dtype) else: missing_value = tf.constant(x.dtype.min + 1, x.dtype) x_batch_max = tf.where(batch_has_no_values, tf.fill(tf.shape(x_batch_max), missing_value), x_batch_max) x_batch_minus_min = tf.where( batch_has_no_values, tf.fill(tf.shape(x_batch_minus_min), missing_value), x_batch_minus_min) return x_batch_minus_min, x_batch_max
def get_sp_topk(adj_pred, sp_adj_train, nb_nodes, k): """Returns binary matrix with topK.""" _, indices = tf.nn.top_k(tf.reshape(adj_pred, (-1,)), k) indices = tf.reshape(tf.cast(indices, tf.int64), (-1, 1)) sp_adj_pred = tf.SparseTensor( indices=indices, values=tf.ones(k), dense_shape=(nb_nodes * nb_nodes,)) sp_adj_pred = tf.sparse_reshape(sp_adj_pred, shape=(nb_nodes, nb_nodes, 1)) sp_adj_train = tf.SparseTensor( indices=sp_adj_train.indices, values=tf.ones_like(sp_adj_train.values), dense_shape=sp_adj_train.dense_shape) sp_adj_train = tf.sparse_reshape(sp_adj_train, shape=(nb_nodes, nb_nodes, 1)) sp_adj_pred = tf.sparse_concat( sp_inputs=[sp_adj_pred, sp_adj_train], axis=-1) return tf.sparse_reduce_max(sp_adj_pred, axis=-1)
def call(self, batch): feat, ct = batch # gic, gin -> gnc bool_ct = tf.cast(ct, tf.bool) tiling = tf.stack([1, 1, 1, tf.shape(feat)[-1]], axis=0) ext_ct = tf.tile(tf.expand_dims(ct, -1), tiling) # ginc w = tf.where(ext_ct) values = tf.gather_nd(feat, tf.gather(w, [0, 1, 3], axis=1)) masked = tf.SparseTensor(indices=w, values=values, dense_shape=tf.cast( tf.shape(ext_ct), tf.int64)) nb = tf.SparseTensor(indices=w, values=tf.ones_like(values, dtype=tf.float32), dense_shape=tf.cast(tf.shape(ext_ct), tf.int64)) return tf.reshape(tf.sparse_reduce_max(masked, axis=1), tf.shape(feat))
def sparse_log_soft_max(logits, indices): """Sparse version of a `log(softmax)` function. There are `BATCH_SIZE` entries, each with an arbitrary number of logits. For each value in `logits` there is a corresponding value in `indices` from `0` to `BATCH_SIZE-1` which indicate which entry the logit is participating. Args: logits: dense collection of logits, a concatenation of logits for all entries, of shape `[N]`. indices: indices of values `0` to `BATCH_SIZE-1`, that indicates which entry the corresponding logit participate. Shape `[N]`. Returns `log(softmax)` of the logits, such that the sum of all `exp(values)` for the same entry sums to 1. So the `log(probabilities)` depending on how their values are trained. A Tensor of shape `[N]`, """ with tf.name_scope("SparseLogSoftMax"): if len(indices.shape) == 1: indices = tf.expand_dims(indices, 1) batch_size = tf.math.reduce_max(indices) + 1 num_values = tf.cast(tf.shape(logits)[0], tf.int64) dense_shape_2d = [batch_size, num_values] indices_2d = tf.concat([indices, tf.expand_dims(tf.range(num_values), 1)], axis=1) sparse_logits = tf.SparseTensor( indices=indices_2d, values=logits, dense_shape=dense_shape_2d) logits_max = tf.sparse_reduce_max( sp_input=sparse_logits, axis=-1, keepdims=True) logits_max = tf.reshape(tf.manip.gather_nd(logits_max, indices), [-1]) # Propagating the gradient through logits_max should be a no-op, so to accelrate this # we just prune it, # Also tf.sparse_reduce_max doesn't have a gradient implemented in TensorFlow (as of Nov/2018). logits_max = tf.stop_gradient(logits_max) normalized_logits = logits - logits_max normalized_exp_values = tf.exp(normalized_logits) normalized_exp_sum = tf.manip.scatter_nd( indices, updates=normalized_exp_values, shape=[batch_size]) normalized_log_exp_sum = tf.manip.gather_nd( params=tf.log(normalized_exp_sum), indices=indices) return normalized_logits - normalized_log_exp_sum
def max_counts(dataset): maxdataset = dataset.map(lambda x: tf.sparse_reduce_max_sparse(x, axis=0)) maxdataset = maxdataset.batch(100) maxdataset = maxdataset.map(lambda x: tf.sparse_reduce_max(x, axis=0)) maxdata_iter = maxdataset.make_one_shot_iterator() maxdata = maxdata_iter.get_next() counter = np.zeros(202498, ) i = 0 while True: try: print(i, end=', ') # Sum of 1000 pages maxbatch = maxdata.eval() counter = np.vstack([counter, maxbatch]).max(axis=0) i += 1 except: break return counter
def next_batch(self): batch = tf.train.batch([self.serialized_example], self.batch_size) features = tf.parse_example(batch, features={ 'fea_indices_row': tf.VarLenFeature(tf.int64), 'fea_indices_col': tf.VarLenFeature(tf.int64), 'fea_values': tf.VarLenFeature(tf.float32), 'seq_len': tf.FixedLenFeature([1], tf.int64), 'label': tf.FixedLenFeature([1], tf.int64), 'forloeb': tf.FixedLenFeature([1], tf.int64), 'ctx_indices': tf.VarLenFeature(tf.int64), 'ctx_values': tf.VarLenFeature(tf.float32), 'text_indices_row': tf.VarLenFeature(tf.int64), 'text_indices_col': tf.VarLenFeature(tf.int64), 'text_indices_vals': tf.VarLenFeature(tf.int64), 'text_len_idx': tf.VarLenFeature(tf.int64), 'text_len': tf.VarLenFeature(tf.int64), 'doc_ids': tf.VarLenFeature(tf.int64) }) seq_len = tf.sparse_reduce_max(features['fea_indices_row'], axis=1) + 10 max_seq_len = tf.reduce_max( tf.stack([ tf.reduce_max(seq_len) - 1, tf.reduce_max(features['text_indices_row'].values) ])) + 1 fea_dim = [max_seq_len, self.feature_size] # Read values and construct sparse tensor sparse_indices_row = features['fea_indices_row'] sparse_indices_col = features['fea_indices_col'] sparse_vals = features['fea_values'] feature_tensor = sparse_merge.sparse_merge( [sparse_indices_row, sparse_indices_col], sparse_vals, fea_dim) text_indices_row = features['text_indices_row'] text_indices_col = features['text_indices_col'] text_indices_vals = features['text_indices_vals'] max_text_len = tf.cond( tf.equal(text_indices_col.dense_shape[1], tf.constant(0, tf.int64)), lambda: tf.constant(0, tf.int64), lambda: tf.add(tf.reduce_max(text_indices_col.values), 1)) text_tensor = sparse_merge.sparse_merge( [text_indices_row, text_indices_col], text_indices_vals, [max_seq_len, max_text_len]) text_len_tensor = sparse_merge.sparse_merge([features['text_len_idx']], features['text_len'], [max_seq_len]) doc_ids_tensor = sparse_merge.sparse_merge([features['text_len_idx']], features['doc_ids'], [max_seq_len]) ctx_tensor = tf.sparse_merge(features['ctx_indices'], features['ctx_values'], self.context_size) return { 'features': feature_tensor, 'context': ctx_tensor, 'seq_len': seq_len, 'label': features['label'], 'forloeb': features['forloeb'], 'text': text_tensor, 'text_len': text_len_tensor, 'doc_ids': doc_ids_tensor }
import sparse_reduce_max_gpu_py as sparse_reduce from tensorflow.python.ops import math_ops from tensorflow.python.framework import dtypes import numpy as np sys.settrace indices = math_ops.cast([[0, 0, 0], [0, 0, 2], [0, 1, 1], [0, 1, 2], [0, 2, 2], [1, 0, 0], [1, 0, 1], [1, 1, 2], [1, 2, 0], [1, 2, 1], [2, 0, 1], [2, 0, 2], [2, 1, 0], [2, 2, 2]], dtype=dtypes.int64) values = math_ops.cast( [3, 1, 2, 6, 1, 1.5, 2.5, 3.5, 2.5, 5.5, 0.5, 0.75, 0.35, 0.1], dtype=dtypes.float32) sp_input = tf.SparseTensor(indices=indices, values=values, dense_shape=[3, 3, 3]) sp_reduced = sparse_reduce.sparse_reduce_max_gpu(sp_input, axis=2) sp_reduced_cpu = tf.sparse_reduce_max(sp_input, axis=2) config = tf.ConfigProto(log_device_placement=True) config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: output = sess.run(sp_reduced) outputCPU = sess.run(sp_reduced_cpu) print "output GPU", output print "output CPU", outputCPU
def sparse_reduce_logsumexp(sparse_tensor, dense_shape, axis=None, keep_dims=False, mode='FAST', name=None): """ Sparse version of tf.reduce_logsumexp that take a SparseTensor as input and returns a dense tensor. The design of this function takes advantage some properties that may be specific to this situation so care should be taken using this in other projects. :param sparse_tensor: :param dense_shape: :param axis: :param keep_dims: :param mode: :param name: :return: full_tensor: """ if name is None: scope_name = 'sparse_reduce_logsumexp' else: scope_name = name with tf.variable_scope(scope_name): if mode is 'FAST': # First convert sparse to dense for reduce max (assumes tensor is no negative?) dense_tensor = tf.sparse_tensor_to_dense(sparse_tensor, validate_indices=False) # Rehape sparse_full to regain shape info dense_tensor = tf.reshape(dense_tensor, dense_shape) # Now compute raw max using standard rather than sparse reduce max raw_max = tf.reduce_max(dense_tensor, axis=axis, keep_dims=True) # Compute my_max conditional = tf.where(tf.is_finite(raw_max), raw_max, tf.zeros_like(raw_max)) my_max = tf.stop_gradient(conditional) # Subtract my_max exp_values = tf.exp(dense_tensor - my_max) else: raw_max = tf.sparse_reduce_max(sparse_tensor, axis=axis, keep_dims=keep_dims) # Reshape to regain shape info for raw_max reduced_shape = dense_shape.copy() for ax in axis: reduced_shape[ax] = 1 raw_max = tf.reshape(raw_max, reduced_shape) conditional = tf.where(tf.is_finite(raw_max), raw_max, tf.zeros_like(raw_max)) my_max = tf.stop_gradient(conditional) # Exp # Convert sparse_tensor to full so that we can subtract my_max dense_tensor = tf.sparse_tensor_to_dense(sparse_tensor, validate_indices=False) # Rehape sparse_full to regain shape info dense_tensor = tf.reshape(dense_tensor, dense_shape) # Subtract my_max exp_values = tf.exp(dense_tensor - my_max) # Extract only required values from exp_values (according to sparse_tensor.indices) new_dense_shape = tf.reduce_prod(sparse_tensor.dense_shape, keep_dims=True) reshape_sparse_tensor = tf.sparse_reshape(sparse_tensor, new_dense_shape) exp_values = tf.reshape(exp_values, [-1]) exp_values = tf.gather_nd(exp_values, reshape_sparse_tensor.indices) # Convert back to sparse exp_sparse = tf.SparseTensor(sparse_tensor.indices, exp_values, sparse_tensor.dense_shape) # Sum sum_sparse = sparse_reduce_sum_nd(exp_sparse, dense_shape, axis=axis, keep_dims=True) # Log full_tensor = tf.log(sum_sparse) + my_max return full_tensor
tf.truncated_normal() tf.truediv() tf.sparse_transpose() tf.sparse_tensor_dense_matmul() tf.sparse_accumulator_apply_gradient() tf.sparse_accumulator_take_gradient() tf.sparse_add() tf.sparse_concat() tf.sparse_conditional_accumulator() tf.sparse_mask() tf.sparse_matmul() tf.sparse_maximum() tf.sparse_merge() tf.sparse_minimum() tf.sparse_reduce_max() tf.sparse_reduce_max_sparse() tf.reduce_all() tf.reduce_any() tf.reduce_join() tf.reduce_logsumexp() tf.reduce_max() tf.reduce_mean() tf.reduce_min() tf.reduce_prod() tf.reduce_sum() tf.reduced_shape() tf.random_crop() tf.random_gamma()