class LossStorer:
def __init__(self, X, y):
self.loss = np.inf # initialize the loss to very high
# Initialize a fake NCA and variables needed to compute the loss:
self.fake_nca = NeighborhoodComponentsAnalysis()
self.fake_nca.n_iter_ = np.inf
self.X, y, _ = self.fake_nca._validate_params(X, y)
self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
def callback(self, transformation, n_iter):
"""Stores the last value of the loss function"""
self.loss, _ = self.fake_nca._loss_grad_lbfgs(
transformation, self.X, self.same_class_mask, -1.0)
class TransformationStorer:
def __init__(self, X, y):
# Initialize a fake NCA and variables needed to call the loss
# function:
self.fake_nca = NeighborhoodComponentsAnalysis()
self.fake_nca.n_iter_ = np.inf
self.X, y, _ = self.fake_nca._validate_params(X, y)
self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
def callback(self, transformation, n_iter):
"""Stores the last value of the transformation taken as input by
the optimizer"""
self.transformation = transformation
class TransformationStorer:
def __init__(self, X, y):
# Initialize a fake NCA and variables needed to call the loss
# function:
self.fake_nca = NeighborhoodComponentsAnalysis()
self.fake_nca.n_iter_ = np.inf
self.X, y, _ = self.fake_nca._validate_params(X, y)
self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
def callback(self, transformation, n_iter):
"""Stores the last value of the transformation taken as input by
the optimizer"""
self.transformation = transformation
class LossStorer:
def __init__(self, X, y):
self.loss = np.inf # initialize the loss to very high
# Initialize a fake NCA and variables needed to compute the loss:
self.fake_nca = NeighborhoodComponentsAnalysis()
self.fake_nca.n_iter_ = np.inf
self.X, y, _ = self.fake_nca._validate_params(X, y)
self.same_class_mask = y[:, np.newaxis] == y[np.newaxis, :]
def callback(self, transformation, n_iter):
"""Stores the last value of the loss function"""
self.loss, _ = self.fake_nca._loss_grad_lbfgs(transformation,
self.X,
self.same_class_mask,
-1.0)
def test_finite_differences():
r"""Test gradient of loss function
Test if the gradient is correct by computing the relative difference
between the projected gradient PG:
.. math::
PG = \mathbf d^{\top} \cdot \nabla
\mathcal L(\mathbf x)
and the finite differences FD:
.. math::
FD = \frac{\mathcal L(\mathbf x + \epsilon \mathbf d) -
\mathcal L(\mathbf x - \epsilon \mathbf d)}{2 \epsilon}
where :math:`d` is a random direction (random vector of shape `n_features`,
and norm 1), :math:`\epsilon` is a very small number, :math:`\mathcal L` is
the loss function and :math:`\nabla \mathcal L` is its gradient. This
relative difference should be zero:
.. math ::
\frac{|PG -FD|}{|PG|} = 0
"""
# Initialize `transformation`, `X` and `y` and `NCA`
X = iris_data
y = iris_target
point = rng.randn(rng.randint(1, X.shape[1] + 1), X.shape[1])
nca = NeighborhoodComponentsAnalysis(init=point)
X, y, init = nca._validate_params(X, y)
mask = y[:, np.newaxis] == y[np.newaxis, :] # (n_samples, n_samples)
nca.n_iter_ = 0
point = nca._initialize(X, init)
# compute the gradient at `point`
_, gradient = nca._loss_grad_lbfgs(point, X, mask)
# create a random direction of norm 1
random_direction = rng.randn(*point.shape)
random_direction /= np.linalg.norm(random_direction)
# computes projected gradient
projected_gradient = random_direction.ravel().dot(
gradient.ravel())
# compute finite differences
eps = 1e-5
right_loss, _ = nca._loss_grad_lbfgs(point + eps * random_direction, X,
mask)
left_loss, _ = nca._loss_grad_lbfgs(point - eps * random_direction, X,
mask)
finite_differences = 1 / (2 * eps) * (right_loss - left_loss)
# compute relative error
relative_error = np.abs(finite_differences - projected_gradient) / \
np.abs(projected_gradient)
np.testing.assert_almost_equal(relative_error, 0.)
