def __init__(self, oracle, x_0, H_0=None, tolerance=1e-4, line_search_options=None, stopping_criteria='grad_rel', trace=True): super(BFGS, self).__init__(oracle, x_0, stopping_criteria, trace) self.tolerance = tolerance self.line_search_tool = get_line_search_tool(line_search_options) self.x_k = x_0.copy() self.grad_norm_0 = np.linalg.norm(self.oracle.grad(x_0)) if H_0 is None: self.H_k = np.eye(x_0.shape[0], dtype=x_0.dtype) else: self.H_k = H_0.copy() if self.line_search_tool._method == 'Constant': self.alpha_0 = self.line_search_tool.c else: self.alpha_0 = 1. self.grad_k = self.oracle.grad(self.x_k)
def __init__(self, oracle, x_0, s_distr, tolerance=1e-10, B_0=None, line_search_options=None, stopping_criteria='grad_rel', store_search_directions=False, update_every_iteration=False, trace=True): super(RBFGS, self).__init__(oracle, x_0, stopping_criteria, trace) if store_search_directions and \ not isinstance(s_distr, (ConstantDistribution, CustomDiscrete)): raise ValueError('Storing search directions only available for ConstantDistribution and CustomDiscrete') self.s_distr = s_distr self.d = x_0.shape[0] if B_0 is not None: self.B_0 = B_0.copy() else: self.B_0 = np.eye(self.d) self.B_k = self.B_0.copy() self.x_0 = x_0.copy() self.x_k = x_0.copy() self.id_mat = np.eye(self.d) self.line_search_tool = get_line_search_tool(line_search_options) self.grad_norm_0 = np.linalg.norm(self.oracle.grad(x_0)) self.tolerance = tolerance self.store_search_directions = store_search_directions self.update_every_iteration = update_every_iteration self.iter_count = 0 if store_search_directions: self.prev_directions = np.random.normal( 0., 1., (self.d, self.s_distr.mat.shape[1]))
def __init__(self, oracle, x_0, tolerance=1e-5, line_search_options=None, stopping_criteria='grad_rel', trace=True): super(GradientDescent, self).__init__(oracle, x_0, stopping_criteria, trace) self.x_k = self.x_0.copy() self.grad_norm_0 = np.linalg.norm(self.oracle.grad(x_0)) self.tolerance = tolerance self.line_search_tool = get_line_search_tool(line_search_options) try: self.alpha_k = self.line_search_tool.alpha_0 except AttributeError: self.alpha_k = self.line_search_tool.c
def __init__(self, oracle, x_0, tolerance=1e-5, line_search_options=None, stopping_criteria='grad_rel', trace=True): super(Nesterov, self).__init__(oracle, x_0, stopping_criteria, trace) self.x_k = self.x_0.copy() self.y_k = self.x_0.copy() self.grad_norm_0 = np.linalg.norm(self.oracle.grad(x_0)) self.line_search_tool = get_line_search_tool(line_search_options) self.tolerance = tolerance self.lam = 0.5
def __init__(self, oracle, x_0, tolerance=1e-4, memory_size=10, line_search_options=None, stopping_criteria='grad_rel', trace=True): super(LBFGS, self).__init__(oracle, x_0, stopping_criteria, trace) self.x_k = self.x_0.copy() self.grad_norm_0 = np.linalg.norm(self.oracle.grad(x_0)) self.tolerance = tolerance self.line_search_tool = get_line_search_tool(line_search_options) self.memory_size = memory_size self.lbfgs_queue = deque(maxlen=memory_size) if self.line_search_tool._method == 'Constant': self.alpha_0 = self.line_search_tool.c else: self.alpha_0 = 1.