def relu(x, alpha=0., max_value=None):
""" Rectified linear unit [1].
References:
[1] (Vinod & Hinton, 2010) "Rectified linear units improve restricted boltzmann machines."
Args:
x (Tensor): Tensor or variable to compute the activation function for.
alpha: Slope for negative input, usually between 0 and 1. The default value of 0 will lead to the standard
rectifier, 1 will lead to a linear activation function, and any value in between will give a leaky rectifier
max_value: Saturation threshold.
Returns:
´´Tensor´´: A tensor that results in element-wise rectifier applied to x.
"""
x = ops.convert_to_tensor(x)
if alpha != 0.:
negative_part = nn.relu(-x)
x = nn.relu(x)
if max_value is not None:
max_value = to_tensor_cast(max_value, x.dtype)
zero = to_tensor_cast(0., x.dtype)
x = clip_ops.clip_by_value(x, zero, max_value)
if alpha != 0.:
alpha = to_tensor_cast(alpha, x.dtype)
x -= alpha * negative_part
return x
def hard_sigmoid(x):
"""Segment-wise linear approximation of sigmoid.
Faster than sigmoid.
Note: Approximates in 3 parts: 0, scaled linear, 1.
returns `0.` if `x < -2.5`, `1.` if `x > 2.5`.
In `-2.5 <= x <= 2.5`, returns `0.2 * x + 0.5`.
Args:
x: A tensor or variable.
Returns:
a float32 tensor resulting in an approximated element-wise sigmoid applied to x
"""
x = ops.convert_to_tensor(x)
slope = to_tensor_cast(0.2, x.dtype)
shift = to_tensor_cast(0.5, x.dtype)
x = (slope * x) + shift
zero = to_tensor_cast(0., x.dtype)
one = to_tensor_cast(1., x.dtype)
x = clip_ops.clip_by_value(x, zero, one)
return x
def dense_one_hot(indices, dense_shape, dtype=dtypes.float32):
"""Transforms a batch of indices to a dense ``Tensor`` by adding the `one-hot` encoding for each index.
Example::
indices = [[0],[1]]
dense_shape = [2,2]
dense_one_hot = [[1,0],[0,1]]
Args:
indices: a dense ``Tensor`` with the active indices for each sample (row).
dense_shape: a list, array or `Tensor` with the shape for the output dense one_hot encoding tensor.
dtype: the type for the output tensor.
Returns:
``Tensor``: A dense ``Tensor`` with a `one-hot encoding` for the given indices.
"""
indices = to_tensor_cast(indices, dtypes.int64)
dense_shape = ops.convert_to_tensor(dense_shape)
encoding = array_ops.one_hot(indices, depth=dense_shape[1], dtype=dtype)
one_hot_dense = math_ops.reduce_sum(encoding, axis=1)
return one_hot_dense
def sparse_one_hot(indices, dense_shape, dtype=dtypes.float32):
"""Transforms a batch of indices to a one-hot encoding ``SparseTensor``.
Example::
indices = [[0,1,4],
[1,2,6]]
dense_shape = [2,10]
sp_one_hot = sparse_one_hot(indices,dense_shape)
expected = SparseTensor(indices=[[0,0],[0,1],[0,4],[1,1],[1,2],[1,6]],
values=[1,1,1,1,1,1],
dense_shape=[2,10])
Args:
indices: a dense ``Tensor`` with the indices to be active for each sample (row)
dense_shape: a list, array or `Tensor` with the shape for output dense one_hot encoding tensor.
dtype: the type for the output values.
Returns:
`SparseTensor`: a ``Sparse Tensor`` with the one hot encoding for the given indices
"""
flat_indices = to_tensor_cast(indices, dtypes.int64)
indices = batch_to_matrix_indices(flat_indices, dtype=dtypes.int64)
return sparse_ones(indices, dense_shape, dtype)
def sparse_ones(indices, dense_shape, dtype=dtypes.float32):
""" Creates a new ``SparseTensor`` where the values are 1
Args:
indices: a 2-D ``Tensor`` with the indices for the resulting sparse tensor
dense_shape: the ``SparseTensor`` dense shape
dtype: the tensor type for the values
Returns:
``SparseTensor``: a new SparseTensor with the values set to 1.
"""
indices = to_tensor_cast(indices, dtypes.int64)
dense_shape = to_tensor_cast(dense_shape, dtypes.int64)
indices_shape = complete_shape(indices)
values = array_ops.ones([indices_shape[0]], dtype)
return SparseTensor(indices, values, dense_shape)
def gather(self, ids):
with tf.name_scope("gather"):
ids = to_tensor_cast(ids, tf.int64)
ids = tf.reshape(ids, [-1])
indices = tf.gather(self.indices, ids)
values = tf.gather(self.values, ids)
return RandomIndexTensor(indices, values, self.k, self.s)
def batch_to_matrix_indices(tensor,
name="batch_to_matrix",
dtype=dtypes.int32):
""" Converts a batch of indices to a 2-D tensor of matrix indices.
For a given batch of indices of shape [n,m] this op outputs a 2-D ``Tensor``
with the `row-index pairs`::
[[r1,i1],[r1,i2],...,[r1,im],
[r2,i1],[r2,i2],...,[r2,im],
...
[rn,i1],[rn,i2],...,[rn,im]]
Example::
tensor = [[1,2],
[2,5]]
batch_to_matrix(tensor)
[[0,1],
[0,2],
[1,2],
[1,5]]
Use Case:
Convert a batch of indices (used to slice another tensor with embedding lookup or gather)
to be used in a SparseTensor, so that we can change the weights of each slice.
Args:
dtype: int32 or int64, the output tensor type
name: name for batch_to_matrix_indices op
tensor: the tensor to be converted
Returns:
``Tensor``: a 2-D tensor with (row index,value) for each index in the input tensor.
"""
with ops.name_scope(name):
tensor = to_tensor_cast(tensor, dtype)
if tensor.dtype != dtypes.int32 and tensor.dtype != dtypes.int64:
raise TypeError(
"Invalid tensor type: expected {t1} or {t2}, found {t3}".
format(t1=dtypes.int32, t2=dtypes.int64, t3=tensor.dtype))
shape = tensor.get_shape().with_rank(2)
if shape.is_fully_defined():
shape = shape.as_list()
else:
shape = array_ops.shape(tensor)
rows = math_ops.range(math_ops.cast(shape[0], tensor.dtype))
rows = array_ops.expand_dims(rows, 1)
multiples = array_ops.stack([1, shape[1]])
rows = array_ops.tile(rows, multiples)
enum = array_ops.stack([rows, tensor], axis=-1)
enum = array_ops.reshape(enum, shape=[-1, 2])
return enum
def __init__(self, indices, values, k, s, dtype=tf.float32):
self.indices = to_tensor_cast(indices, tf.int64)
self.values = to_tensor_cast(values)
self.k = k
self.s = s