"""Test class for module Batch_l2_grad (squared ℓ₂ norm of batch gradients)""" from test.automated_test import check_sizes_and_values from test.extensions.firstorder.batch_l2_grad.batchl2grad_settings import ( BATCHl2GRAD_SETTINGS, ) from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import make_test_problems import pytest PROBLEMS = make_test_problems(BATCHl2GRAD_SETTINGS) IDS = [problem.make_id() for problem in PROBLEMS] @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_batch_l2_grad(problem): """Test squared ℓ₂ norm of individual gradients. Args: problem (ExtensionsTestProblem): Problem for extension test. """ problem.set_up() backpack_res = BackpackExtensions(problem).batch_l2_grad() autograd_res = AutogradExtensions(problem).batch_l2_grad() check_sizes_and_values(autograd_res, backpack_res) problem.tear_down()
Test individual gradients for the following layers: - sum of the square of batch gradients of linear layers - sum of the square of batch gradients of convolutional layers """ from test.automated_test import check_sizes_and_values from test.extensions.firstorder.sum_grad_squared.sumgradsquared_settings import ( SUMGRADSQUARED_SETTINGS, ) from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import make_test_problems import pytest PROBLEMS = make_test_problems(SUMGRADSQUARED_SETTINGS) IDS = [problem.make_id() for problem in PROBLEMS] @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_sum_grad_squared(problem): """Test sum of square of individual gradients Args: problem (ExtensionsTestProblem): Problem for extension test. """ problem.set_up() backpack_res = BackpackExtensions(problem).sgs() autograd_res = AutogradExtensions(problem).sgs()
"""Test BackPACK's KFAC extension.""" from test.automated_test import check_sizes_and_values from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import ExtensionsTestProblem, make_test_problems from test.extensions.secondorder.hbp.kfac_settings import ( BATCH_SIZE_1_SETTINGS, NOT_SUPPORTED_SETTINGS, ) import pytest from backpack.utils.kroneckers import kfacs_to_mat NOT_SUPPORTED_PROBLEMS = make_test_problems(NOT_SUPPORTED_SETTINGS) NOT_SUPPORTED_IDS = [problem.make_id() for problem in NOT_SUPPORTED_PROBLEMS] BATCH_SIZE_1_PROBLEMS = make_test_problems(BATCH_SIZE_1_SETTINGS) BATCH_SIZE_1_IDS = [problem.make_id() for problem in BATCH_SIZE_1_PROBLEMS] @pytest.mark.parametrize("problem", NOT_SUPPORTED_PROBLEMS, ids=NOT_SUPPORTED_IDS) def test_kfac_not_supported(problem): """Check that the KFAC extension does not allow specific hyperparameters/modules. Args: problem (ExtensionsTestProblem): Test case. """ problem.set_up()
"""Tests BackPACK's ``SqrtGGNExact`` and ``SqrtGGNMC`` extension.""" from math import isclose from test.automated_test import check_sizes_and_values from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import ExtensionsTestProblem, make_test_problems from test.extensions.secondorder.sqrt_ggn.sqrt_ggn_settings import SQRT_GGN_SETTINGS from test.utils.skip_test import skip_large_parameters, skip_subsampling_conflict from typing import List, Union from pytest import fixture, mark PROBLEMS = make_test_problems(SQRT_GGN_SETTINGS) SUBSAMPLINGS = [None, [0, 0], [2, 0]] SUBSAMPLING_IDS = [f"subsampling={s}".replace(" ", "") for s in SUBSAMPLINGS] @fixture(params=PROBLEMS, ids=lambda p: p.make_id()) def problem(request) -> ExtensionsTestProblem: """Set seed, create tested model, loss, data. Finally clean up. Args: request (SubRequest): Request for the fixture from a test/fixture function. Yields: Test case with deterministically constructed attributes. """ case = request.param case.set_up()
"""Test DiagGGN extension.""" from test.automated_test import check_sizes_and_values from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import make_test_problems from test.extensions.secondorder.diag_ggn.diag_ggn_settings import DiagGGN_SETTINGS from test.utils.skip_test import skip_adaptive_avg_pool3d_cuda import pytest PROBLEMS = make_test_problems(DiagGGN_SETTINGS) IDS = [problem.make_id() for problem in PROBLEMS] @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_diag_ggn(problem, request): """Test the diagonal of generalized Gauss-Newton. Args: problem (ExtensionsTestProblem): Problem for extension test. request: problem request """ skip_adaptive_avg_pool3d_cuda(request) problem.set_up() backpack_res = BackpackExtensions(problem).diag_ggn() autograd_res = AutogradExtensions(problem).diag_ggn() check_sizes_and_values(autograd_res, backpack_res) problem.tear_down()
"""Test BackPACK's KFAC extension.""" from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import make_test_problems from test.extensions.secondorder.hbp.kfac_settings import NOT_SUPPORTED_SETTINGS import pytest NOT_SUPPORTED_PROBLEMS = make_test_problems(NOT_SUPPORTED_SETTINGS) NOT_SUPPORTED_IDS = [problem.make_id() for problem in NOT_SUPPORTED_PROBLEMS] @pytest.mark.parametrize("problem", NOT_SUPPORTED_PROBLEMS, ids=NOT_SUPPORTED_IDS) def test_kfac_not_supported(problem): """Check that the KFAC extension does not allow specific hyperparameters/modules. Args: problem (ExtensionsTestProblem): Test case. """ problem.set_up() with pytest.raises(NotImplementedError): BackpackExtensions(problem).kfac() problem.tear_down()
"""Test BackPACK's ``Variance`` extension.""" from test.automated_test import check_sizes_and_values from test.extensions.firstorder.variance.variance_settings import VARIANCE_SETTINGS from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import ExtensionsTestProblem, make_test_problems import pytest PROBLEMS = make_test_problems(VARIANCE_SETTINGS) IDS = [problem.make_id() for problem in PROBLEMS] @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_variance(problem: ExtensionsTestProblem) -> None: """Test variance of individual gradients. Args: problem: Test case. """ problem.set_up() backpack_res = BackpackExtensions(problem).variance() autograd_res = AutogradExtensions(problem).variance() rtol = 5e-5 check_sizes_and_values(autograd_res, backpack_res, rtol=rtol) problem.tear_down()
from test.automated_test import check_sizes_and_values from test.extensions.implementation.autograd import AutogradExtensions from test.extensions.implementation.backpack import BackpackExtensions from test.extensions.problem import make_test_problems from test.extensions.secondorder.diag_hessian.diagh_settings import DiagHESSIAN_SETTINGS import pytest PROBLEMS = make_test_problems(DiagHESSIAN_SETTINGS) IDS = [problem.make_id() for problem in PROBLEMS] @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_diag_h(problem): """Test Diagonal of Hessian Args: problem (ExtensionsTestProblem): Problem for extension test. """ problem.set_up() backpack_res = BackpackExtensions(problem).diag_h() autograd_res = AutogradExtensions(problem).diag_h() check_sizes_and_values(autograd_res, backpack_res) problem.tear_down() @pytest.mark.parametrize("problem", PROBLEMS, ids=IDS) def test_diag_h_batch(problem): """Test Diagonal of Hessian