def mean_squared_loss(outputs, targets):
""" Returns the mean squared error Σ(xᵢ - yᵢ)² over the data points.
Parameters
----------
outputs : mygrad.Tensor, shape=(N, any)
The model outputs, where `N` is the number of items.
targets : mygrad.Tensor, shape=(N, any)
The target values, where `N` is the number of items.
Returns
-------
mygrad.Tensor, shape=()
The mean squared error.
Extended Description
--------------------
The mean squared error is given by
.. math::
\frac{1}{N}\sum\limits_{1}^{N}(x_i - y_i)^2
where :math:`N` is the number of elements in `x` and `y`.
"""
return mean((outputs - targets)**2)
def focal_loss(scores, targets, *, alpha=1, gamma=0):
""" Return the focal loss.
Parameters
----------
scores : mygrad.Tensor, shape=(N, C)
The C class scores for each of the N pieces of data.
targets : Sequence[int], shape=(N,)
The correct class indices, in [0, C), for each datum.
alpha : Real, optional (default=1)
The ɑ weighting factor in the loss formulation.
gamma : Real, optional (default=0)
The ɣ focusing parameter. Note that for Ɣ=0 and ɑ=1, this is cross-entropy loss.
Returns
-------
mygrad.Tensor
The average focal loss.
Notes
-----
This function does not perform a softmax before computing the loss. If you ned to take the
softmax before computing the loss, see :class:`SoftmaxFocalLoss` instead.
"""
if isinstance(targets, Tensor):
targets = targets.data
label_locs = (range(len(targets)), targets)
pc = scores[label_locs]
return -mean(alpha * (1 - pc + 1e-14)**gamma * log(pc))
def l1_loss(outputs, targets):
''' Returns the L¹ loss Σ|xᵢ - yᵢ| averaged over the number of data points.
Parameters
----------
outputs : mygrad.Tensor, shape=(N,)
The predictions for each of the N pieces of data.
targets : numpy.ndarray, shape=(N,)
The correct value for each of the N pieces of data.
Returns
-------
mygrad.Tensor, shape=()
The average L¹ loss.
Extended Description
--------------------
The L1 loss is given by
.. math::
\frac{1}{N}\sum\limits_{1}^{N}|x_i - y_i|
where :math:`N` is the number of elements in `x` and `y`.
'''
return mean(abs(outputs - targets))
def _var(x, keepdims=False, axis=None, ddof=0):
"""Defines variance without using abs. Permits use of
complex-step numerical derivative."""
def mean(y, keepdims=False, axis=None, ddof=0):
N = y.size if axis is None else np.prod([y.shape[i] for i in axis])
return y.sum(keepdims=keepdims, axis=axis) / (N - ddof)
return mean(
(x - x.mean(axis=axis, keepdims=True))**2,
keepdims=keepdims,
axis=axis,
ddof=ddof,
)
def _std(x, keepdims=False, axis=None, ddof=0):
"""Defines standard dev without using abs. Permits use of
complex-step numerical derivative."""
def mean(y, keepdims=False, axis=None, ddof=0):
if isinstance(axis, int):
axis = (axis, )
N = y.size if axis is None else np.prod([y.shape[i] for i in axis])
return y.sum(keepdims=keepdims, axis=axis) / (N - ddof)
return np.sqrt(
mean(
(x - x.mean(axis=axis, keepdims=True))**2,
keepdims=keepdims,
axis=axis,
ddof=ddof,
))
def l2loss(pred, actual): # L2 loss function (mean square distance)
"""
Parameters
----------
pred: Union[mygrad.Tensor, numpy.ndarray]
A tensor or numpy array containing the model's predicted values
actual: Union[mygrad.Tensor, numpy.ndarray]
A tensor or numpy array containing the actual values
Returns
-------
mg.Tensor
A tensor containing the mean square distance between the prediction and actual values.
"""
return mg.mean(mg.square(pred - actual))
def binary_cross_entropy(y_pred, y_truth):
""" Calculates the binary cross entropy loss for a given set of predictions.
Parameters
----------
y_pred: mg.Tensor, shape=
The Tensor of class scores output from the model
y_truth: mg.Tensor, shape=
A constant Tensor or a NumPy array that contains the truth values for each prediction
Returns
-------
mg.Tensor, shape=()
A zero-dimensional tensor that is the loss
"""
return -mg.mean(y_truth * mg.log(y_pred + 1e-08) + (1 - y_truth) * mg.log(1 - y_pred + 1e-08)) # <COGLINE>
def simple_batchnorm(x, gamma, beta, eps):
axes = [i for i in range(x.ndim)]
axes.pop(1) # every axis except 1
axes = tuple(axes)
keepdims_shape = tuple(1 if n != 1 else d for n, d in enumerate(x.shape))
mean = mg.mean(x, axis=axes, keepdims=True)
var = mg.var(x, axis=axes, keepdims=True)
norm = (x - mean) / mg.sqrt(var + eps)
if gamma is not None:
gamma = gamma.reshape(keepdims_shape)
norm *= gamma
if beta is not None:
beta = beta.reshape(keepdims_shape)
norm += beta
return norm
def simple_loss(x1, x2, y, margin):
"""
x1 : mygrad.Tensor, shape=(N, D)
x2 : mygrad.Tensor, shape=(N, D)
y : Union[int, numpy.ndarray], scalar or shape=(N,)
margin : float
Returns
-------
mygrad.Tensor, shape=()
"""
y = np.asarray(y)
if y.ndim:
assert y.size == 1 or len(y) == len(x1)
if x1.ndim == 2:
y = y.reshape(-1, 1)
return mg.mean(mg.maximum(0, margin - y * (x1 - x2)))
def kl_divergence(outputs, targets):
''' Returns the Kullback-Leibler divergence loss from the outputs to the targets.
The KL-Divergence loss for a single sample is given by yᵢ⊙(log(yᵢ) - xᵢ)
Parameters
----------
outputs : mygrad.Tensor, shape=(N, any)
The model outputs for each of the N pieces of data.
targets : numpy.ndarray, shape=(N, any)
The correct vaue for each datum.
Returns
-------
mygrad.Tensor, shape=()
The mean Kullback-Leibler divergence.
'''
return mean(targets * (log(targets) - outputs))
def negative_log_likelihood(x, y_true, *, weights=None, constant=False):
""" Returns the (weighted) negative log-likelihood loss between log-probabilities and y_true.
Note that this does not compute a softmax, so you should input log-probabilities to this.
See ``softmax_crossentropy`` if you need your loss to compute a softmax.
Parameters
----------
x : array_like, shape=(N, C)
The C log-probabilities for each of the N pieces of data.
y_true : array_like, shape=(N,)
The correct class indices, in [0, C), for each datum.
weights : array_like, shape=(C,) optional (default=None)
The weighting factor to use on each class, or None.
constant : bool, optional(default=False)
If ``True``, the returned tensor is a constant (it
does not back-propagate a gradient)
Returns
-------
mygrad.Tensor, shape=()
The average (weighted) negative log-likelihood loss.
Examples
--------
>>> import mygrad as mg
>>> from mygrad.nnet import negative_log_likelihood
Let's take a simple case where N=1, and C=3. We'll thus make up classification
scores for a single datum. Suppose the scores are identical for the three classes
and that the true class is class-0, so that the log-probs are each 1/3:
>>> logprob = mg.log(1 / 3).item()
>>> x = mg.Tensor([[logprob, logprob, logprob]]) # a shape-(1, 3) tensor of log-probabilities
>>> y_true = mg.Tensor([0]) # the correct class for this datum is class-0
>>> negative_log_likelihood(x, y_true)
Tensor(1.09861229)
# log-probabilities where the prediction is highly-confident and correct
>>> x = mg.Tensor([[0, -20, -20]])
>>> negative_log_likelihood(x, y_true)
Tensor(0.)
# adding a class-weighting
>>> x = mg.Tensor([[-4.6, -4.6, -0.02]])
>>> weights = mg.Tensor([2, 1, 1])
>>> negative_log_likelihood(x, y_true, weights=weights)
Tensor(9.2)
"""
if isinstance(y_true, Tensor):
y_true = y_true.data
check_loss_inputs(x, y_true)
if weights is None:
weights = np.ones(x.shape[1])
weights = asarray(weights)
if weights.ndim != 1 or weights.shape[0] != x.shape[1]:
raise ValueError("`weights` must be a shape-(C,) array: \n"
f"\tExpected shape-{x.shape[1]}\n"
f"\tGot shape-{y_true.shape}")
label_locs = (range(len(y_true)), y_true)
factors = weights[y_true]
return -mean(x[label_locs] * factors, constant=constant)
