def tanh(inputs, **kwargs):
r"""Compute the tanh of input.
The **Tanh** function is defined as:
.. math:: \text{Tanh}(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}
Examples:
```python
x = dragon.constant([0.2, 0.4, 0.6, 0.8, 1.0], 'float32')
print(dragon.math.tanh(x))
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inplace = args.pop('inplace') if 'inplace' in args else False
op_lib = activation_ops_lib.Activation
if context.executing_eagerly():
return op_lib \
.instantiate(op_type='Tanh') \
.apply([inputs], inplace=inplace)
else:
return op_lib.blend('Tanh', **args)
def sigmoid(inputs, **kwargs):
r"""Compute the sigmoid result of input.
The **Sigmoid** function is defined as:
.. math:: \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}
Examples:
```python
x = dragon.constant([0.2, 0.4, 0.6, 0.8, 1.0])
print(dragon.math.sigmoid(x, inplace=False))
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor
"""
args = ArgHelper.parse(locals())
inplace = args.pop('inplace') if 'inplace' in args else False
op_lib = activation_ops_lib.Activation
if context.executing_eagerly():
return op_lib \
.instantiate(op_type='Sigmoid') \
.apply([inputs], inplace=inplace)
else:
return op_lib.blend('Sigmoid', **args)
def swish(inputs, **kwargs):
r"""Apply the swish function.
`[Ramachandran et.al, 2017] <https://arxiv.org/abs/1710.05941>`_.
The **Swish** function is defined as:
.. math:: \text{Swish}(x) = x \cdot \frac{1}{1 + \exp(-x)}
Examples:
```python
x = dragon.constant([-2.5, -1.0, 0.0, 1.0, 2.5])
print(dragon.nn.swish(x))
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = activation_ops_lib.Activation
if context.executing_eagerly():
return op_lib \
.instantiate(op_type='Swish') \
.apply([inputs])
else:
return op_lib.blend('Swish', **args)
def drop_path(inputs, ratio=0.2, **kwargs):
r"""Set the examples over the input to zero randomly.
`[Larsson et.al, 2016] <https://arxiv.org/abs/1605.07648>`_.
The **DropPath** function is defined as:
.. math:: \text{DropPath}(x_{ij}) = x_{ij} * (r_{i} \sim \mathcal{B}(1, 1 - \text{ratio}))
Parameters
----------
inputs : dragon.Tensor
The input tensor.
ratio : Union[float, dragon.Tensor], optional, default=0.2
The dropping ratio.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inplace = args.pop('inplace') if 'inplace' in args else False
op_lib = activation_ops_lib.DropPath
if context.executing_eagerly():
return op_lib \
.instantiate() \
.apply([inputs], args['ratio'], inplace=inplace)
else:
return op_lib.blend(**args)
def square(inputs, **kwargs):
r"""Compute the square of input.
.. math:: \text{out} = \text{input}^{2}
Examples:
```python
x = dragon.constant([2, 3, 4])
print(dragon.math.square(x))
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Square').apply([inputs])
else:
return op_lib.blend('Square', **args)
def bias_add(inputs, data_format='NCHW', **kwargs):
"""Add the bias across channels to input.
Parameters
----------
inputs : Sequence[dragon.Tensor]
The ``x`` and ``bias``.
data_format : str, optional, default='NCHW'
``'NCHW'`` or ``'NHWC'``.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
if data_format not in ('NCHW', 'NHWC'):
raise ValueError('Unsupported data format: %s' % data_format)
op_lib = vision_ops_lib.BiasAdd
if context.executing_eagerly():
return op_lib \
.instantiate(data_format=data_format) \
.apply(inputs, args.get('inplace', False))
else:
return op_lib.blend(**args)
def rsqrt(inputs, **kwargs):
r"""Compute the reciprocal square root of input.
.. math:: \text{out} = \frac{1}{\sqrt{\text{input}}}
Examples:
```python
x = dragon.constant([0., 4., 16.])
print(dragon.math.rsqrt(x)) # [inf, 0.5, 0.25]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Rsqrt').apply([inputs])
else:
return op_lib.blend('Rsqrt', **args)
def get_variable(
name,
shape=None,
dtype=None,
initializer=None,
regularizer=None,
trainable=True,
use_resource=True,
):
if shape is None:
raise ValueError('Must specific a shape to create a Variable.')
if initializer is None:
initializer = get_default_initializer(name, shape=shape, dtype=dtype)
if use_resource or eager_context.executing_eagerly():
with eager_context.eager_mode():
if callable(initializer):
initial_value = initializer(shape, dtype=dtype)
else:
initial_value = initializer
variable = Variable(
initial_value=initial_value,
trainable=trainable,
name=name,
dtype=dtype,
)
else:
raise RuntimeError('VariableV1 has been removed.')
if regularizer is not None:
variable = regularizer(variable)
return variable
def pow(inputs, **kwargs):
r"""Compute the power of input.
.. math:: \text{out} = \text{input}^{\text{exponent}}
The two inputs should be broadcast to each other:
```python
x = dragon.fill(shape=(1, 2), value=2)
print(dragon.math.pow([x, x])) # [[4, 4]]
print(dragon.math.pow([x, 3])) # [[8, 8]]
print(dragon.math.pow([3, x])) # [[9, 9]]
```
Parameters
----------
inputs : Sequence[dragon.Tensor]
The input and exponent tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inputs = ops.remove_binary_scalar(inputs)
op_lib = math_ops_lib.BinaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Pow').apply(inputs)
else:
return op_lib.blend('Pow', **args)
def broadcast(inputs, root=0, group=None, **kwargs):
"""Broadcast the input from root node in a group.
Parameters
----------
inputs : dragon.Tensor
The tensor to broadcast.
root : int, optional, default=0
The node index in the group.
group : ProcessGroup, optional
The group for communication.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
if group is None:
group = distributed.get_group()
if group is None:
raise ValueError('<group> is required.')
args.update(group.arguments)
args.pop('group')
op_lib = distributed_ops_lib.Collective
if context.executing_eagerly():
return op_lib \
.instantiate(
root=root,
communication='BROADCAST',
group=group,
).apply(inputs)
else:
return op_lib.blend(communication='BROADCAST', **args)
def accuracy(inputs, axis=-1, top_k=1, ignore_index=None, **kwargs):
"""Compute the top-k accuracy.
Parameters
----------
inputs : Sequence[dragon.Tensor]
The tensor ``logit`` and ``label``.
axis : int, optional, default=-1
The axis of classes.
top_k : int, optional, default=1
The top-k accuracy to compute.
ignore_index : int, optional
The label index to ignore.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = metric_ops_lib.Accuracy
if context.executing_eagerly():
return op_lib \
.instantiate(
axis=axis,
top_k=top_k,
ignore_index=ignore_index,
).apply(inputs)
else:
return op_lib.blend(**args)
def forward(self, inputs, **kwargs):
"""Compute the output of RNN."""
inputs = nest.flatten(inputs)
op_lib = rnn_ops_lib.Recurrent
if context.executing_eagerly():
inputs.insert(1, self._weights)
return op_lib \
.instantiate(
mode=self.mode,
num_layers=self.num_layers,
hidden_size=self.hidden_size,
bidirectional=self.bidirectional,
dropout_ratio=self.dropout,
is_training=kwargs.get('is_training', False),
).apply(inputs)
else:
inputs.insert(1, self._weights_ref)
return op_lib.blend(
inputs=inputs,
rnn_mode=self.mode,
num_layers=self.num_layers,
hidden_size=self.hidden_size,
bidirectional=self.bidirectional,
dropout_ratio=self.dropout,
)
def sign(inputs, **kwargs):
r"""Compute the sign indication of input.
.. math::
\text{out}_[i] =
\begin{cases}
-1, & \text{ if } \text{input}_{i} < 0 \\
0, & \text{ if } \text{input}_{i} = 0 \\
1, & \text{ if } \text{input}_{i} > 0
\end{cases}
Examples:
```python
x = dragon.constant([-2, 0, 2])
print(dragon.math.sign(x)) # [-1, 0, 1]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Sign').apply([inputs])
else:
return op_lib.blend('Sign', **args)
def clip(inputs, low=None, high=None, **kwargs):
r"""Compute the clipped input according to the given bounds.
.. math:: \text{out} = \min(\max(\text{input}, \text{low}), \text{high})
Parameters
----------
inputs : dragon.Tensor
The input tensor.
low : number, optional
The value to :math:`\text{low}`.
high : number, optional
The value to :math:`\text{high}`.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
if low is not None:
args['low'] = float(args['low'])
if high is not None:
args['high'] = float(args['high'])
op_lib = math_ops_lib.Clip
if context.executing_eagerly():
return op_lib \
.instantiate(
low=args['low'],
high=args['high'],
).apply([inputs])
else:
return op_lib.blend(**args)
def zeros_like(other, dtype='float32', **kwargs):
r"""Return a tensor of zeros with shape as the other.
.. math:: \text{out} \leftarrow 0
Examples:
```python
x = dragon.zeros(shape=(2, 3))
y = dragon.zeros_like(x)
```
Parameters
----------
other : dragon.Tensor
The tensor to hint the shape.
dtype : str, optional, default='float32'
The optional data type.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
trainable = args.pop('trainable') if 'trainable' in args else False
op_lib = init_ops_lib.Fill
if context.executing_eagerly():
return op_lib \
.instantiate(value=0, dtype=dtype) \
.apply([], other, trainable=trainable)
else:
args.pop('other')
return op_lib.blend(inputs=[other], value=0., **args)
def not_equal(inputs, **kwargs):
r"""Compute the element-wise not-equal comparison.
.. math:: \text{out} = (\text{input1} \neq \text{input2})
Examples:
```python
a = dragon.constant(2)
b = dragon.constant(3)
c = dragon.constant(3)
print(dragon.math.not_equal([a, b]))
print(dragon.math.not_equal([b, c]))
```
Parameters
----------
inputs : Sequence[dragon.Tensor]
The input1 and input2 tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inputs = ops.remove_binary_scalar(inputs)
op_lib = math_ops_lib.BinaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='NotEqual').apply(inputs)
else:
return op_lib.blend('NotEqual', **args)
def mul(inputs, **kwargs):
r"""Compute the element-wise multiplication.
.. math:: \text{out} = \text{input1} \times \text{input2}
Examples:
```python
a = dragon.constant(4)
b = dragon.constant(2)
print(dragon.math.mul([a, b]))
print(a * b) # Equivalent operation
```
Parameters
----------
inputs : Sequence[dragon.Tensor]
The input1 and input2 tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inputs = ops.remove_binary_scalar(inputs)
op_lib = math_ops_lib.BinaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Mul').apply(inputs)
else:
return op_lib.blend('Mul', **args)
def negative(inputs, **kwargs):
r"""Compute the element-wise negative.
.. math:: \text{out} = -\text{input}
```python
x = dragon.constant([-1, 0, 1])
print(dragon.math.negative(x))
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Neg').apply([inputs])
else:
return op_lib.blend('Neg', **args)
def log(inputs, **kwargs):
r"""Compute the logarithm of input.
.. math:: \text{out} = \log(\text{input})
Examples:
```python
x = dragon.constant([1., 2., 3.])
print(dragon.math.log(x)) # [0., 0.69314718, 1.09861229]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Log').apply([inputs])
else:
return op_lib.blend('Log', **args)
def axpby(inputs, outputs=None, alpha=1., beta=1., **kwargs):
r"""Compute the element-wise addition from input to output.
.. math:: \text{out} = \alpha * \text{input} + \beta * \text{out}
If ``outputs`` is not provided, **zeros** will be used instead.
Parameters
----------
inputs : dragon.Tensor
The input tensor.
outputs : dragon.Tensor, optional
The output tensor.
alpha : number, optional, default=1.
The value to :math:`\alpha`.
beta : number, optional, default=1.
The value to :math:`\beta`.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
args['alpha'], args['beta'] = float(alpha), float(beta)
op_lib = math_ops_lib.Axpby
if context.executing_eagerly():
return op_lib \
.instantiate(alpha=args['alpha'], beta=args['beta']) \
.apply([inputs], [outputs])
else:
return op_lib.blend(**args)
def invert(inputs, **kwargs):
r"""Invert each bit of input.
.. math:: \text{out} = \,\,\sim \text{input}
Examples:
```python
# Typically, ``x`` is a bool tensor
print(dragon.bitwise.invert(dragon.constant([0, 1], 'bool'))) # [True, False]
# Otherwise, integral types are required (unsigned or signed)
# 00001101 (13) -> 11110010 (?)
print(dragon.bitwise.invert(dragon.constant(13, 'uint8'))) # 242
print(dragon.bitwise.invert(dragon.constant(13, 'int8'))) # -14
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Invert').apply([inputs])
else:
return op_lib.blend('Invert', **args)
def is_nan(inputs, **kwargs):
r"""Check if the elements of input are NaN.
.. math:: \text{out} = \text{isnan}(\text{input})
Examples:
```python
x = dragon.constant([0., 1., float('nan')])
print(dragon.math.is_nan(x)) # [False, False, True]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='IsNaN').apply([inputs])
else:
return op_lib.blend('IsNaN', **args)
def exp(inputs, **kwargs):
r"""Compute the exponential of input.
.. math:: \text{out} = \exp(\text{input})
Examples:
```python
x = dragon.constant([1., 2., 3.])
print(dragon.math.exp(x)) # [2.71828183, 7.3890561, 20.08553692]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Exp').apply([inputs])
else:
return op_lib.blend('Exp', **args)
def floor(inputs, **kwargs):
r"""Compute the largest integer not greater than input.
.. math:: \text{out} = \lfloor \text{input} \rfloor
Examples:
```python
x = dragon.constant([0.9, 1.4, 1.9])
print(dragon.math.floor(x)) # [0., 1., 1.]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Floor').apply([inputs])
else:
return op_lib.blend('Floor', **args)
def cos(inputs, **kwargs):
r"""Compute the cos of input.
.. math:: \text{out} = \cos(\text{input})
Examples:
```python
x = dragon.constant([0., math.pi])
print(dragon.math.cos(x)) # [1., -1.]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Cos').apply([inputs])
else:
return op_lib.blend('Cos', **args)
def ctc_loss(inputs, padding_mask=-1, **kwargs):
r"""Compute the ctc loss with batched labels.
`[Graves & Gomez, 2006] <http://www.cs.utoronto.ca/~graves/icml_2006.pdf>`_.
The shape of ``logit`` and ``label`` should be :math:`(T, N, C)`,
:math:`(N, C)` respectively, where :math:`T` is the sequence length,
:math:`N` is the batch size, and :math:`C` is max label length. The range
of ``labels`` should be :math:`[1, C)`, as :math:`0` is reserved for blank.
Use ``padding_mask`` to fill it when length is not sufficient.
Parameters
----------
inputs : Sequence[dragon.Tensor]
The tensor ``logit`` and ``label``.
padding_mask : int, optional, default=-1
The mask for padding the redundant labels.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
inputs[0] = activation_ops.softmax(inputs[0], axis=2)
op_lib = loss_ops_lib.Operator
if context.executing_eagerly():
raise NotImplementedError
else:
return op_lib.blend('CTCLoss', **args)
def abs(inputs, **kwargs):
r"""Compute the absolute value of input.
.. math:: \text{out} = \left| \text{input} \right|
Examples:
```python
x = dragon.constant([-1, 0, 1])
print(dragon.math.abs(x)) # [1, 0, 1]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Abs').apply([inputs])
else:
return op_lib.blend('Abs', **args)
def ceil(inputs, **kwargs):
r"""Compute the smallest integer not less than input.
.. math:: \text{out} = \lceil \text{input} \rceil
Examples:
```python
x = dragon.constant([1.4, 1.7, 2.0])
print(dragon.math.ceil(x)) # [2., 2., 2.]
```
Parameters
----------
inputs : dragon.Tensor
The input tensor.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
op_lib = math_ops_lib.UnaryOp
if context.executing_eagerly():
return op_lib.instantiate(op_type='Ceil').apply([inputs])
else:
return op_lib.blend('Ceil', **args)
def eye(n, m=None, k=0, dtype='float32', **kwargs):
r"""Return a tensor constructed as the identity matrix.
.. math:: \text{out} \leftarrow \text{diag}(1, 1, ..., 1)
The rows and cols of matrix are determined by ``n`` and ``m``:
```python
print(dragon.eye(2)) # [[1., 0.], [0., 1.]]
print(dragon.eye(2, 3)) # [[1., 0., 0.], [0., 1., 0.]]
```
The diagonal could be controlled by ``k``:
* k > 0: Populate upper diagonal
* k = 0: Populate main diagonal
* k < 0: Populate lower diagonal
Parameters
----------
n : Union[int, dragon.Tensor]
The number output rows.
m : Union[int, dragon.Tensor], optional
The number output cols.
k : int, optional, default=0
The index of diagonal.
dtype : str, optional, default='float32'
The optional data type.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
m = n if m is None else m
trainable = args.pop('trainable') if 'trainable' in args else False
op_lib = init_ops_lib.Eye
if context.executing_eagerly():
if types.is_tensor(n):
n = int(n.get_value())
if types.is_tensor(m):
m = int(m.get_value())
return op_lib \
.instantiate(k=k, ndim=2, dtype=dtype) \
.apply([n, m], trainable=trainable)
else:
args['n'] = args['m'] = None
if types.is_tensor(n) or types.is_tensor(m):
n = ops.scalar_to_tensor(n, 'int64')
m = ops.scalar_to_tensor(m, 'int64')
args['dims_descs'] = [n.id, m.id]
args['extra_inputs'] = [n, m]
else:
args['dims'] = [n, m]
return op_lib.blend(**args)
def sync_batch_norm(inputs,
axis=-1,
momentum=0.9,
epsilon=1e-5,
use_stats=-1,
process_group=None,
**kwargs):
r"""Apply the batch normalization with synced statistics.
`[Ioffe & Szegedy, 2015] <https://arxiv.org/abs/1502.03167>`_.
The normalization is defined as:
.. math:: \text{out} = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta
The running average of statistics are calculated as:
.. math:: x_{\text{running}} = \text{momentum} * x_{\text{running}} +
(1 - \text{momentum}) * x_{\text{batch}}
Parameters
----------
inputs : Sequence[dragon.Tensor]
The tensor ``x``, ``gamma``, ``beta``, ``mean`` and ``var``.
axis : int, optional, default=-1
The channel axis.
momentum : Union[float, dragon.Tensor], optional
The value to :math:`\text{momentum}`.
epsilon : float, optional, default=1e-5
The value to :math:`\epsilon`.
use_stats : int, optional, default=-1
Whether to use estimated statistics or not.
process_group : ProcessGroup, optional
The group for communication.
Returns
-------
dragon.Tensor
The output tensor.
"""
args = ArgHelper.parse(locals())
args['epsilon'] = float(epsilon)
if process_group is None:
process_group = distributed.get_group()
if process_group is None:
raise ValueError('<process_group> is required.')
op_lib = normalization_ops_lib.SyncBatchNorm
if context.executing_eagerly():
return op_lib \
.instantiate(
axis=axis,
epsilon=args['epsilon'],
use_stats=use_stats,
process_group=process_group,
).apply(inputs, args['momentum'])
else:
args.update(process_group.arguments)
return op_lib.blend(**args)
