def maximum(t1, t2): r"""Get the maximum between two tensors. Args: t1 (Tensor): input tensor to compare. t2 (Tensor): input tensor to compare. Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) return where(t1 > t2, t1, t2)
def set(t, key, value): r"""Set new value(s) to a tensor. .. warning:: Setting manually values of a tensor will invalidate its gradients. Args: t (Tensor): tensor to compare. key (scalar or tensor): indices to set. value (scalar or tensor): new values. Returns: Tensor """ t = nets.to_tensor(t) value = nets.to_tensor(value) # To device if t.device == 'cpu' and value.device != 'cpu': t.cuda() elif value.device == 'cpu' and t.device != 'cpu': value.cuda() cpu = True if isinstance(key, tuple): for k in key: if isinstance(k, nets.Tensor): if k.device != 'cpu': cpu = False if not cpu: t.cuda() value.cuda() for k in key: if isinstance(k, nets.Tensor): k.cuda() if isinstance(key, nets.Tensor): key = key.data elif isinstance(key, tuple): keys = [] for k in key: if isinstance(k, nets.Tensor): keys.append(k.data) else: keys.append(k) key = tuple(keys) t.data[key] = value.data # Setting a tensor invalidate its gradient t.detach() return t
def max(t, axis=None): r"""Get the maximum from a ``Tensor``. Args: t (Tensor): tensor to transform axis (int, optional): index of the axis to search. Default is ``None``. Returns: Tensor """ t = nets.to_tensor(t) data = np.max(t.data, axis=axis) requires_grad = t.requires_grad hooks = [] if requires_grad: def grad_fn(grad): bigger_grad = np.zeros_like(t.data) if axis is None: # If there is no axis, the argmax is the location of he maximum single element max_indices = np.unravel_index(np.argmax(t.data), t.shape) bigger_grad[max_indices] = grad else: # If there is an axis, we reconstruct the bigger matrix by 'rolling' on this axis max_indices = np.argmax(t.data, axis=axis) for i, roll in enumerate(np.rollaxis(bigger_grad, axis)): roll += (max_indices == i).astype(int) * grad return bigger_grad hooks.append(Hook(t, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def concatenate(iterable): r"""Concatenate multiples ``Tensor`` from an iterable. .. note:: The ``Tensor`` in ``iterable`` should and must have the same shape. Args: iterable (tuple, list): list containing ``Tensor`` to concatenate. Returns: Tensor: the concatenation of all ``Tensor``. """ assert isinstance(iterable, ITERABLE), f'iterable type {type(iterable)} unsupported for `concatenate` function.' \ f'Types currently supported are list, tuple.' requires_grad = False hooks = [] data = np.array([]) for idx, t in enumerate(iterable): t = nets.to_tensor(t) requires_grad = t.requires_grad or requires_grad if data.size == 0: data = t.data else: data = np.concatenate((data, t.data)) if t.requires_grad: def grad_fn(grad): return grad[idx:idx + t.shape[0]] hooks.append(Hook(t, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def transpose(t, indices=None): r"""Permutation a tensor object. Args: t (Tensor): indices (tuple, optional): index to transpose. Returns: Tensor """ t = nets.to_tensor(t) if indices is None: indices = tuple(range(t.ndim - 1, -1, -1)) data = t.data.transpose(indices) requires_grad = t.requires_grad hooks = [] if requires_grad: def grad_fn(grad): indices_back = tuple(inv_permutation(indices)) grad = grad.transpose(indices_back) return grad hooks.append(Hook(t, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def pow(t, power): r"""Power a tensor-like object. .. math:: T_{out} = T^2 Args: t (Tensor like): reference tensor power (int): power to elevate a tensor Returns: Tensor """ assert type( power ) == int, "unsupported type {} for power. Currently supported type: int".format( type(power)) t = nets.to_tensor(t) data = t.data**power requires_grad = t.requires_grad hooks = [] # Update the gradient if requires_grad: hooks.append(Hook(t, lambda grad: grad * power * t.data**(power - 1))) return nets.Tensor(data, requires_grad, hooks)
def sub(t1, t2): r"""Subtract two tensor-like object .. math:: \text{sub}(t_1, t_2) = t_1 - t_2 Args: t1 (Tensor like): tensor to subtract. t2 (Tensor like): second tensor to subtract with. Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) return add(t1, neg(t2))
def inverse(t): r"""Inverse a tensor-like object. .. math:: T_{out} = \frac{1}{T} Args: t (Tensor like): tensor to inverse. Returns: Tensor """ t = nets.to_tensor(t) requires_grad = t.requires_grad hooks = [] if requires_grad: def grad_fn(grad): r"""Update the gradient for the inverse operation, :math:`grad = grad \times \frac{-1}{T^2}`. Shape: - inputs (np.ndarray): upstream gradient. - outputs (np.ndarray): downstream gradient. """ return -1 / (t.data**2) * grad hooks.append(Hook(t, grad_fn)) return nets.Tensor(1 / t.data, requires_grad, hooks)
def leaky_relu_prime(t, alpha=0.01): r"""First order derivative of ``leaky_relu`` function. .. math:: \text{leaky_relu'(x)} = \begin{cases} 1, &\quad x \ge 0 \\ \alpha, &\quad x < 0. \end{cases} Args: t (Tensor): input tensor. alpha (float, optional): slope towards :math:`-\infty`. .. image:: /images/functional_leaky_relu_prime.png Example: >>> import nets >>> tensor = nets.tensor([-5, 2, 6, -2, 4]) >>> alpha = 0.1 >>> leaky_relu_prime(tensor, alpha) See :class:`~nets.nn.activation.LeakyReLU` for the activation implementation. """ t = nets.to_tensor(t) return where(t > 0, nets.ones_like(t), alpha * nets.ones_like(t))
def slice(t, indices): r"""Slice a tensor from given indices. Args: t (Tensor): tensor to slice idxs (tuple, int, :): indices to extract data Returns: Tensor """ t = nets.to_tensor(t) if isinstance(indices, nets.Tensor): indices = indices.data data = t.data[indices] requires_grad = t.requires_grad hooks = [] if requires_grad: def grad_fn(grad): bigger_grad = np.zeros_like(t.data) if grad.shape != bigger_grad.shape: bigger_grad[indices] = grad else: bigger_grad = grad return bigger_grad hooks.append(Hook(t, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def multiply(t1, t2): r"""Elementwise multiplication of two tensors. .. math:: \text{multiply}(t_1, t_2) = t_1 \times t_2 Args: t1 (Tensor like): tensor to multiply. t2 (Tensor like): second tensor to multiply with. Returns: Tensor: the elementwise multiplication. """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) operation = op.Multiply() return operation(t1, t2)
def add(t1, t2): r"""Add two tensor-like together. .. math:: \text{add}(t_1, t_2) = t_1 + t_2 Args: t1 (Tensor like): tensor to add. t2 (Tensor like): second tensor to add with. Returns: Tensor: the sum of two Tensor-like object. """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) operation = op.Add() return operation(t1, t2)
def ge(t, other): r"""Return a boolean tensor for *greater or equal* condition. .. math:: \text{gt}_{\text{other}}(t) = t \ge other Args: t (Tensor): tensor to compare other (Tensor like): object to compare the tensor Returns: Tensor """ t = nets.to_tensor(t) other = nets.to_tensor(other) data = t.data >= other.data return nets.Tensor(data, device=t.device)
def lt(t, other): r"""Return a boolean tensor for *lower than* condition. .. math:: \text{gt}_{\text{other}}(t) = t < other Args: t (Tensor): tensor to compare other (Tensor like): object to compare the tensor Returns: Tensor """ t = nets.to_tensor(t) other = nets.to_tensor(other) data = t.data < other.data return nets.Tensor(data, device=t.device)
def eq(t, other): r"""Return a boolean tensor for *equal* condition. .. math:: \text{gt}_{\text{other}}(t) = t == other Args: t (Tensor): tensor to compare other (Tensor like): object to compare the tensor Returns: Tensor """ t = nets.to_tensor(t) other = nets.to_tensor(other) cond = t.data == other.data return nets.Tensor(cond, device=t.device)
def div(t1, t2): r"""Divide two tensor-like object. .. math:: T_{out} = T_1 \times \frac{1}{T_2} Args: t1 (Tensor like): tensor to multiply t2 (Tensor like): tensor to invert Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) return multiply(t1, inverse(t2))
def div(t1, t2): r"""Divide all elements of two tensors. .. math:: \text{div} = t_1 \times \frac{1}{t_2} .. note:: The two input tensors must have the same shape. Args: t1 (Tensor like): tensor to multiply. t2 (Tensor like): tensor to invert. Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) return multiply(t1, inverse(t2))
def where(cond, t1, t2): r"""Transformation regarding a condition. .. math:: T_{out} = \begin{cases} T_1, &\quad if \quad condition \\ T_2, &\quad else. \end{cases} Args: cond (bool): condition to merge two tensors t1 (Tensor): input tensor to merge t2 (Tensor): input tensor to merge Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) # TODO: handle broadcasting with where(): sum across the broadcast dimension assert t1.shape == t2.shape, f"tensors should have the same shape. Got t1.shape={t1.shape}, t2.shape={t2.shape}" cond = nets.to_array(cond) data = np.where(cond, t1.data, t2.data) requires_grad = t1.requires_grad or t2.requires_grad hooks = [] if t1.requires_grad: def grad_fn(grad): return grad * np.where(cond, 1, 0) hooks.append(Hook(t1, grad_fn)) if t2.requires_grad: def grad_fn(grad): return grad * np.where(cond, 0, 1) hooks.append(Hook(t2, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def append(t, value): r"""Append multiples ``Tensor`` from an iterable. .. note:: The ``Tensor`` in ``iterable`` should and must have the same shape. Args: t (Tensor): list containing ``Tensor`` to concatenate. Returns: Tensor: the concatenation of all ``Tensor``. """ t = nets.to_tensor(t) value = nets.to_tensor(value) requires_grad = False hooks = [] requires_grad = t.requires_grad or value.requires_grad if t.size == 0: data = [value.data] elif value.size == 0: data = [t.data] else: data = t.data.tolist() data.append(value.data) if t.requires_grad: def grad_fn(grad): return grad[:-1] hooks.append(Hook(t, grad_fn)) if value.requires_grad: def grad_fn(grad): return grad[-1] hooks.append(Hook(value, grad_fn)) return nets.Tensor(data, requires_grad, hooks)
def log(t): r"""Logarithm of a tensor. Args: t (Tensor): input tensor. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Log() return func(t)
def sqrt(t): r"""Square root of a tensor-like object. Args: t (Tensor): input tensor. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Sqrt() return func(t)
def transpose(t, indices=None): r"""Transpose a tensor regarding indices. Args: t (Tensor like): tensor to transpose. indices (iterable): indices to perform the permutation. Default to ``None``. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Transpose(indices) return func(t)
def dot(t1, t2): r"""Dot product of two matrices. .. math:: \begin{*align} \text{dot}(t1, t2) &= t_{out} &= t_1 \dot t_2 \\ \quad where \quad t_{i, j}^{[out]} = \sum_{k=1}^{n} t_{i, k}^{[1]} \times t_{k, j}^{[2]} \end{*align} Args: t1 (Tensor like): tensor to multiply. t2 (Tensor like): second tensor to multiply with. Returns: Tensor: the elementwise multiplication. """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) operation = op.Dot() return operation(t1, t2)
def slice(t, indices): r"""Slice a tensor from given indices. Args: t (Tensor): tensor to slice. indices (tuple, int, :): indices to extract data. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Slice(indices) return func(t)
def where(cond, t1, t2): r"""Transformation regarding a condition. .. math:: \text{where}(t) = \begin{cases} t_1, &\quad if \quad condition \\ t_2, &\quad else. \end{cases} Args: cond (bool): condition to merge two tensors. t1 (Tensor): input tensor to compare. t2 (Tensor): input tensor to compare. Returns: Tensor """ t1 = nets.to_tensor(t1) t2 = nets.to_tensor(t2) operation = op.Where(cond) return operation(t1, t2)
def max(t, axis=None): r"""pad a tensor. Args: t (Tensor like): tensor to transform. axis (tuple): index of the axis to search. Default is ``None``. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Max(axis) return func(t)
def pad(t, padding): r"""pad a tensor. Args: t (Tensor like): tensor to transform. padding (tuple): padding size of the tensor. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Pad(padding) return func(t)
def reshape(t, shape): r"""Reshape a tensor. Args: t (Tensor like): tensor to transform. shape (tuple): new shape of the input tensor. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Reshape(shape) return func(t)
def exp(t): r"""Exponentiation of a tensor. .. math:: \text{exp}(t) = e^{t} Args: t (Tensor): input tensor. Returns: Tensor """ t = nets.to_tensor(t) func = fc.Exp() return func(t)
def argmax(t, axis=None): r"""Get the indices of maximum elements from a ``Tensor``. Args: t (Tensor): tensor get maximum indices from axis (int, optional): index of the axis. Default is ``None``. Returns: Tensor """ t = nets.to_tensor(t) if axis is None: return nets.Tensor(np.unravel_index(np.argmax(t.data), t.shape)) else: return nets.Tensor(np.argmax(t.data, axis=axis))