def run_GN_iter(self, num_cg_iter): """Runs a single GN iteration.""" self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Create copy with graph detached self.g = self.f0.vdetach() self.g.requires_grad(True) # Get df/dx^t @ f0 self.dfdxt_g = TensorList( torch.autograd.grad(self.f0, self.x, self.g, create_graph=True)) # Get the right hand side self.b = -self.dfdxt_g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x)
def __init__(self, training_samples: TensorList, y: TensorList, filter_reg: torch.Tensor, sample_weights: TensorList, response_activation): self.training_samples = training_samples.variable() self.y = y.variable() self.filter_reg = filter_reg self.sample_weights = sample_weights self.response_activation = response_activation
def get_attribute(self, name: str, ignore_missing: bool = False): if ignore_missing: return TensorList([ getattr(f, name) for f in self.features if self._return_feature(f) and hasattr(f, name) ]) else: return TensorList([ getattr(f, name, None) for f in self.features if self._return_feature(f) ])
def run(self, num_iter, dummy=None): if num_iter == 0: return for i in range(num_iter): self.x.requires_grad(True) # Evaluate function at current estimate loss = self.problem(self.x) # Compute grad grad = TensorList(torch.autograd.grad(loss, self.x)) # Update direction if self.dir is None: self.dir = grad else: self.dir = grad + self.momentum * self.dir self.x.detach() self.x -= self.step_legnth * self.dir self.x.detach() self.clear_temp()
def A(self, x): dfdx_x = torch.autograd.grad(self.dfdxt_g, self.g, x, retain_graph=True) return TensorList( torch.autograd.grad(self.f0, self.x, dfdx_x, retain_graph=True))
def evaluate_CG_iteration(self, delta_x): if self.analyze_convergence: x = (self.x + delta_x).detach() x.requires_grad_(True) # compute loss and gradient loss = self.problem(x) grad = TensorList(torch.autograd.grad(loss, x)) # store in the vectors self.losses = torch.cat( (self.losses, loss.detach().cpu().view(-1))) self.gradient_mags = torch.cat( (self.gradient_mags, sum(grad.view(-1) @ grad.view(-1)).cpu().sqrt().detach().view(-1)))
def __init__(self, problem: L2Problem, variable: TensorList, cg_eps=0.0, fletcher_reeves=True, standard_alpha=True, direction_forget_factor=0, debug=False, plotting=False, fig_num=(10, 11)): super().__init__(fletcher_reeves, standard_alpha, direction_forget_factor, debug or plotting) self.problem = problem self.x = variable.variable() self.plotting = plotting self.fig_num = fig_num self.cg_eps = cg_eps self.f0 = None self.g = None self.dfdxt_g = None self.residuals = torch.zeros(0) self.losses = torch.zeros(0)
def get_fparams(self, name: str = None): if name is None: return [ f.fparams for f in self.features if self._return_feature(f) ] return TensorList([ getattr(f.fparams, name) for f in self.features if self._return_feature(f) ]).unroll()
def run_newton_iter(self, num_cg_iter): self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Gradient of loss self.g = TensorList( torch.autograd.grad(self.f0, self.x, create_graph=True)) # Get the right hand side self.b = -self.g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x)
def extract_patches(self, im_patches): """Extract features. args: im: Image. """ # Compute features # print (im_patches.shape) feature_map = TensorList( [f.get_feature(im_patches) for f in self.features]).unroll() return feature_map
def __init__(self, training_samples: TensorList, y: TensorList, filter_reg: torch.Tensor, projection_reg, params, sample_weights: TensorList, projection_activation, response_activation): self.training_samples = training_samples self.y = y.variable() self.filter_reg = filter_reg self.sample_weights = sample_weights self.params = params self.projection_reg = projection_reg self.projection_activation = projection_activation self.response_activation = response_activation self.diag_M = self.filter_reg.concat(projection_reg)
def run(self, num_cg_iter): """Run the oprimizer with the provided number of iterations.""" if num_cg_iter == 0: return lossvec = None if self.debug: lossvec = torch.zeros(2) self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Create copy with graph detached self.g = self.f0.vdetach() if self.debug: lossvec[0] = self.problem.ip_output(self.g, self.g) self.g.requires_grad(True) # Get df/dx^t @ f0 self.dfdxt_g = TensorList( torch.autograd.grad(self.f0, self.x, self.g, create_graph=True)) # Get the right hand side self.b = -self.dfdxt_g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x) self.x.vdetach_() self.clear_temp()
def extract(self, im, pos, scales, image_sz): if isinstance(scales, (int, float)): scales = [scales] # Get image patches im_patches = torch.cat( [sample_patch(im, pos, s * image_sz, image_sz) for s in scales]) # Compute features feature_map = torch.cat(TensorList( [f.get_feature(im_patches) for f in self.features]).unroll(), dim=1) return feature_map
def extract(self, im, pos, scales, image_sz): """Extract features. args: im: Image. pos: Center position for extraction. scales: Image scales to extract features from. image_sz: Size to resize the image samples to before extraction. """ if isinstance(scales, (int, float)): scales = [scales] # Get image patches im_patches = torch.cat( [sample_patch(im, pos, s * image_sz, image_sz) for s in scales]) # Compute features feature_map = TensorList( [f.get_feature(im_patches) for f in self.features]).unroll() return feature_map
def extract_transformed(self, im, pos, scale, image_sz, transforms): """Extract features from a set of transformed image samples. args: im: Image. pos: Center position for extraction. scale: Image scale to extract features from. image_sz: Size to resize the image samples to before extraction. transforms: A set of image transforms to apply. """ # Get image patche im_patch = sample_patch(im, pos, scale * image_sz, image_sz).data # Apply transforms im_patches = torch.cat([T(im_patch).data for T in transforms]) # Compute features feature_map = TensorList( [f.get_feature(im_patches) for f in self.features]).unroll() return feature_map
def A(self, x): return TensorList( torch.autograd.grad(self.g, self.x, x, retain_graph=True)) + self.hessian_reg * x
def stride(self): return torch.Tensor( TensorList([ f.stride() for f in self.features if self._return_feature(f) ]).unroll())
class NewtonCG(ConjugateGradientBase): """Newton with Conjugate Gradient. Handels general minimization problems.""" def __init__(self, problem: MinimizationProblem, variable: TensorList, init_hessian_reg=0.0, hessian_reg_factor=1.0, cg_eps=0.0, fletcher_reeves=True, standard_alpha=True, direction_forget_factor=0, debug=False, analyze=False, plotting=False, fig_num=(10, 11, 12)): super().__init__(fletcher_reeves, standard_alpha, direction_forget_factor, debug or analyze or plotting) self.problem = problem self.x = variable self.analyze_convergence = analyze self.plotting = plotting self.fig_num = fig_num self.hessian_reg = init_hessian_reg self.hessian_reg_factor = hessian_reg_factor self.cg_eps = cg_eps self.f0 = None self.g = None self.residuals = torch.zeros(0) self.losses = torch.zeros(0) self.gradient_mags = torch.zeros(0) def clear_temp(self): self.f0 = None self.g = None def run(self, num_cg_iter, num_newton_iter=None): if isinstance(num_cg_iter, int): if num_cg_iter == 0: return if num_newton_iter is None: num_newton_iter = 1 num_cg_iter = [num_cg_iter] * num_newton_iter num_newton_iter = len(num_cg_iter) if num_newton_iter == 0: return if self.analyze_convergence: self.evaluate_CG_iteration(0) for cg_iter in num_cg_iter: self.run_newton_iter(cg_iter) self.hessian_reg *= self.hessian_reg_factor self.x.vdetach_() self.clear_temp() return self.losses, self.residuals def run_newton_iter(self, num_cg_iter): self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Gradient of loss self.g = TensorList( torch.autograd.grad(self.f0, self.x, create_graph=True)) # Get the right hand side self.b = -self.g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x) def A(self, x): return TensorList( torch.autograd.grad(self.g, self.x, x, retain_graph=True)) + self.hessian_reg * x def ip(self, a, b): # Implements the inner product return self.problem.ip_input(a, b) def M1(self, x): return self.problem.M1(x) def M2(self, x): return self.problem.M2(x) def evaluate_CG_iteration(self, delta_x): if self.analyze_convergence: x = (self.x + delta_x).detach() x.requires_grad_(True) # compute loss and gradient loss = self.problem(x) grad = TensorList(torch.autograd.grad(loss, x)) # store in the vectors self.losses = torch.cat( (self.losses, loss.detach().cpu().view(-1))) self.gradient_mags = torch.cat( (self.gradient_mags, sum(grad.view(-1) @ grad.view(-1)).cpu().sqrt().detach().view(-1)))
def size(self, input_sz): return TensorList([ f.size(input_sz) for f in self.features if self._return_feature(f) ]).unroll()
def dim(self): return TensorList([ f.dim() for f in self.features if self._return_feature(f) ]).unroll()
class GaussNewtonCG(ConjugateGradientBase): """Gauss-Newton with Conjugate Gradient optimizer.""" def __init__(self, problem: L2Problem, variable: TensorList, cg_eps=0.0, fletcher_reeves=True, standard_alpha=True, direction_forget_factor=0, debug=False, analyze=False, plotting=False, fig_num=(10, 11, 12)): super().__init__(fletcher_reeves, standard_alpha, direction_forget_factor, debug or analyze or plotting) self.problem = problem self.x = variable self.analyze_convergence = analyze self.plotting = plotting self.fig_num = fig_num self.cg_eps = cg_eps self.f0 = None self.g = None self.dfdxt_g = None self.residuals = torch.zeros(0) self.losses = torch.zeros(0) self.gradient_mags = torch.zeros(0) def clear_temp(self): self.f0 = None self.g = None self.dfdxt_g = None def run_GN(self, *args, **kwargs): return self.run(*args, **kwargs) def run(self, num_cg_iter, num_gn_iter=None): """Run the optimizer. args: num_cg_iter: Number of CG iterations per GN iter. If list, then each entry specifies number of CG iterations and number of GN iterations is given by the length of the list. num_gn_iter: Number of GN iterations. Shall only be given if num_cg_iter is an integer. """ if isinstance(num_cg_iter, int): if num_gn_iter is None: raise ValueError( 'Must specify number of GN iter if CG iter is constant') num_cg_iter = [num_cg_iter] * num_gn_iter num_gn_iter = len(num_cg_iter) if num_gn_iter == 0: return if self.analyze_convergence: self.evaluate_CG_iteration(0) # Outer loop for running the GN iterations. for cg_iter in num_cg_iter: self.run_GN_iter(cg_iter) self.x.vdetach_() self.clear_temp() return self.losses, self.residuals def run_GN_iter(self, num_cg_iter): """Runs a single GN iteration.""" self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Create copy with graph detached self.g = self.f0.vdetach() self.g.requires_grad(True) # Get df/dx^t @ f0 self.dfdxt_g = TensorList( torch.autograd.grad(self.f0, self.x, self.g, create_graph=True)) # Get the right hand side self.b = -self.dfdxt_g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x) def A(self, x): dfdx_x = torch.autograd.grad(self.dfdxt_g, self.g, x, retain_graph=True) return TensorList( torch.autograd.grad(self.f0, self.x, dfdx_x, retain_graph=True)) def ip(self, a, b): return self.problem.ip_input(a, b) def M1(self, x): return self.problem.M1(x) def M2(self, x): return self.problem.M2(x) def evaluate_CG_iteration(self, delta_x): if self.analyze_convergence: x = (self.x + delta_x).detach() x.requires_grad_(True) # compute loss and gradient f = self.problem(x) loss = self.problem.ip_output(f, f) grad = TensorList(torch.autograd.grad(loss, x)) # store in the vectors self.losses = torch.cat( (self.losses, loss.detach().cpu().view(-1))) self.gradient_mags = torch.cat( (self.gradient_mags, sum(grad.view(-1) @ grad.view(-1)).cpu().sqrt().detach().view(-1)))
class ConjugateGradient(ConjugateGradientBase): """Conjugate Gradient optimizer, performing single linearization of the residuals in the start.""" def __init__(self, problem: L2Problem, variable: TensorList, cg_eps=0.0, fletcher_reeves=True, standard_alpha=True, direction_forget_factor=0, debug=False, plotting=False, fig_num=(10, 11)): super().__init__(fletcher_reeves, standard_alpha, direction_forget_factor, debug or plotting) self.problem = problem self.x = variable.variable() self.plotting = plotting self.fig_num = fig_num self.cg_eps = cg_eps self.f0 = None self.g = None self.dfdxt_g = None self.residuals = torch.zeros(0) self.losses = torch.zeros(0) def clear_temp(self): self.f0 = None self.g = None self.dfdxt_g = None def run(self, num_cg_iter): """Run the oprimizer with the provided number of iterations.""" if num_cg_iter == 0: return lossvec = None if self.debug: lossvec = torch.zeros(2) self.x.requires_grad(True) # Evaluate function at current estimate self.f0 = self.problem(self.x) # Create copy with graph detached self.g = self.f0.vdetach() if self.debug: lossvec[0] = self.problem.ip_output(self.g, self.g) self.g.requires_grad(True) # Get df/dx^t @ f0 self.dfdxt_g = TensorList( torch.autograd.grad(self.f0, self.x, self.g, create_graph=True)) # Get the right hand side self.b = -self.dfdxt_g.vdetach() # Run CG delta_x, res = self.run_CG(num_cg_iter, eps=self.cg_eps) self.x.vdetach_() self.x.plus_(delta_x) self.x.vdetach_() self.clear_temp() def A(self, x): dfdx_x = torch.autograd.grad(self.dfdxt_g, self.g, x, retain_graph=True) return TensorList( torch.autograd.grad(self.f0, self.x, dfdx_x, retain_graph=True)) def ip(self, a, b): return self.problem.ip_input(a, b) def M1(self, x): return self.problem.M1(x) def M2(self, x): return self.problem.M2(x)
def size(self, im_sz): if self.output_size is None: return TensorList([im_sz / s for s in self.stride()]) if isinstance(im_sz, torch.Tensor): return TensorList([im_sz / s if sz is None else torch.Tensor([sz[0], sz[1]]) for sz, s in zip(self.output_size, self.stride())])