def relu(x, alpha=0., max_value=None): """ Rectified linear unit [1]. References: [1] (Vinod & Hinton, 2010) "Rectified linear units improve restricted boltzmann machines." Args: x (Tensor): Tensor or variable to compute the activation function for. alpha: Slope for negative input, usually between 0 and 1. The default value of 0 will lead to the standard rectifier, 1 will lead to a linear activation function, and any value in between will give a leaky rectifier max_value: Saturation threshold. Returns: ´´Tensor´´: A tensor that results in element-wise rectifier applied to x. """ x = ops.convert_to_tensor(x) if alpha != 0.: negative_part = nn.relu(-x) x = nn.relu(x) if max_value is not None: max_value = to_tensor_cast(max_value, x.dtype) zero = to_tensor_cast(0., x.dtype) x = clip_ops.clip_by_value(x, zero, max_value) if alpha != 0.: alpha = to_tensor_cast(alpha, x.dtype) x -= alpha * negative_part return x
def hard_sigmoid(x): """Segment-wise linear approximation of sigmoid. Faster than sigmoid. Note: Approximates in 3 parts: 0, scaled linear, 1. returns `0.` if `x < -2.5`, `1.` if `x > 2.5`. In `-2.5 <= x <= 2.5`, returns `0.2 * x + 0.5`. Args: x: A tensor or variable. Returns: a float32 tensor resulting in an approximated element-wise sigmoid applied to x """ x = ops.convert_to_tensor(x) slope = to_tensor_cast(0.2, x.dtype) shift = to_tensor_cast(0.5, x.dtype) x = (slope * x) + shift zero = to_tensor_cast(0., x.dtype) one = to_tensor_cast(1., x.dtype) x = clip_ops.clip_by_value(x, zero, one) return x
def dense_one_hot(indices, dense_shape, dtype=dtypes.float32): """Transforms a batch of indices to a dense ``Tensor`` by adding the `one-hot` encoding for each index. Example:: indices = [[0],[1]] dense_shape = [2,2] dense_one_hot = [[1,0],[0,1]] Args: indices: a dense ``Tensor`` with the active indices for each sample (row). dense_shape: a list, array or `Tensor` with the shape for the output dense one_hot encoding tensor. dtype: the type for the output tensor. Returns: ``Tensor``: A dense ``Tensor`` with a `one-hot encoding` for the given indices. """ indices = to_tensor_cast(indices, dtypes.int64) dense_shape = ops.convert_to_tensor(dense_shape) encoding = array_ops.one_hot(indices, depth=dense_shape[1], dtype=dtype) one_hot_dense = math_ops.reduce_sum(encoding, axis=1) return one_hot_dense
def sparse_one_hot(indices, dense_shape, dtype=dtypes.float32): """Transforms a batch of indices to a one-hot encoding ``SparseTensor``. Example:: indices = [[0,1,4], [1,2,6]] dense_shape = [2,10] sp_one_hot = sparse_one_hot(indices,dense_shape) expected = SparseTensor(indices=[[0,0],[0,1],[0,4],[1,1],[1,2],[1,6]], values=[1,1,1,1,1,1], dense_shape=[2,10]) Args: indices: a dense ``Tensor`` with the indices to be active for each sample (row) dense_shape: a list, array or `Tensor` with the shape for output dense one_hot encoding tensor. dtype: the type for the output values. Returns: `SparseTensor`: a ``Sparse Tensor`` with the one hot encoding for the given indices """ flat_indices = to_tensor_cast(indices, dtypes.int64) indices = batch_to_matrix_indices(flat_indices, dtype=dtypes.int64) return sparse_ones(indices, dense_shape, dtype)
def sparse_ones(indices, dense_shape, dtype=dtypes.float32): """ Creates a new ``SparseTensor`` where the values are 1 Args: indices: a 2-D ``Tensor`` with the indices for the resulting sparse tensor dense_shape: the ``SparseTensor`` dense shape dtype: the tensor type for the values Returns: ``SparseTensor``: a new SparseTensor with the values set to 1. """ indices = to_tensor_cast(indices, dtypes.int64) dense_shape = to_tensor_cast(dense_shape, dtypes.int64) indices_shape = complete_shape(indices) values = array_ops.ones([indices_shape[0]], dtype) return SparseTensor(indices, values, dense_shape)
def gather(self, ids): with tf.name_scope("gather"): ids = to_tensor_cast(ids, tf.int64) ids = tf.reshape(ids, [-1]) indices = tf.gather(self.indices, ids) values = tf.gather(self.values, ids) return RandomIndexTensor(indices, values, self.k, self.s)
def batch_to_matrix_indices(tensor, name="batch_to_matrix", dtype=dtypes.int32): """ Converts a batch of indices to a 2-D tensor of matrix indices. For a given batch of indices of shape [n,m] this op outputs a 2-D ``Tensor`` with the `row-index pairs`:: [[r1,i1],[r1,i2],...,[r1,im], [r2,i1],[r2,i2],...,[r2,im], ... [rn,i1],[rn,i2],...,[rn,im]] Example:: tensor = [[1,2], [2,5]] batch_to_matrix(tensor) [[0,1], [0,2], [1,2], [1,5]] Use Case: Convert a batch of indices (used to slice another tensor with embedding lookup or gather) to be used in a SparseTensor, so that we can change the weights of each slice. Args: dtype: int32 or int64, the output tensor type name: name for batch_to_matrix_indices op tensor: the tensor to be converted Returns: ``Tensor``: a 2-D tensor with (row index,value) for each index in the input tensor. """ with ops.name_scope(name): tensor = to_tensor_cast(tensor, dtype) if tensor.dtype != dtypes.int32 and tensor.dtype != dtypes.int64: raise TypeError( "Invalid tensor type: expected {t1} or {t2}, found {t3}". format(t1=dtypes.int32, t2=dtypes.int64, t3=tensor.dtype)) shape = tensor.get_shape().with_rank(2) if shape.is_fully_defined(): shape = shape.as_list() else: shape = array_ops.shape(tensor) rows = math_ops.range(math_ops.cast(shape[0], tensor.dtype)) rows = array_ops.expand_dims(rows, 1) multiples = array_ops.stack([1, shape[1]]) rows = array_ops.tile(rows, multiples) enum = array_ops.stack([rows, tensor], axis=-1) enum = array_ops.reshape(enum, shape=[-1, 2]) return enum
def __init__(self, indices, values, k, s, dtype=tf.float32): self.indices = to_tensor_cast(indices, tf.int64) self.values = to_tensor_cast(values) self.k = k self.s = s