def backpropagate_batch_average(self, ext, module, g_inp, g_out, H):
ggn = self.derivatives.ea_jac_t_mat_jac_prod(module, g_inp, g_out, H)
residual = self.second_order_module_effects(module, g_inp, g_out)
residual_mod = Curvature.modify_residual(residual, ext.get_curv_type())
if residual_mod is not None:
ggn = self.add_diag_to_mat(residual_mod, ggn)
return ggn
def __make_diagonal_R_mat_prod(self, ext, module, g_inp, g_out, backproped):
# TODO Refactor core: hessian_diagonal -> residual_diagonal
R = self.derivatives.hessian_diagonal(module, g_inp, g_out)
R_mod = Curvature.modify_residual(R, ext.get_curv_type())
@R_mat_prod_accept_vectors
@R_mat_prod_check_shapes
def make_residual_mat_prod(self, module, g_inp, g_out):
def R_mat_prod(mat):
"""Multiply with the residual: mat Рєњ [РѕЉ_{k} Hz_k(x) ЮЏ┐z_k] mat.
Second term of the module input Hessian backpropagation equation.
"""
return einsum("n...,vn...->vn...", (R_mod, mat))
return R_mat_prod
return make_residual_mat_prod(self, module, g_inp, g_out)
def _modify_residual(self, ext, residual):
return Curvature.modify_residual(residual, ext.get_curv_type())