class Nelder_Mead_Algo: def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_problem_wrapper() def run(self, initial_lambdas_set, num_iters=10, log_file=None): self.log_file = log_file start = time.time() total_calls = 0 self.fmodel = Fitted_Model(initial_lambdas_set[0].size) for initial_lambdas in initial_lambdas_set: res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter":num_iters}) model_params = self.problem_wrapper.solve(res.x, quick_run=True) total_calls += res.nfev self.fmodel.update(res.x, model_params, res.fun) self.log("fmodel %s" % self.fmodel) runtime = time.time() - start self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls)) self.fmodel.set_runtime(runtime) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str)
def run(self, initial_lambdas_set, num_iters=10, log_file=None): self.log_file = log_file start = time.time() total_calls = 0 self.fmodel = Fitted_Model(initial_lambdas_set[0].size) for initial_lambdas in initial_lambdas_set: self.log("initial_lambdas %s" % initial_lambdas) res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter": num_iters}) model_params = self.problem_wrapper.solve(res.x, quick_run=True) total_calls += res.nfev self.fmodel.update(res.x, model_params, res.fun) self.log("fmodel %s" % self.fmodel) runtime = time.time() - start self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls)) self.log("%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_runtime(runtime) self.fmodel.set_num_solves(total_calls)
def run(self, lambdas1, lambdas2=None, log_file=None): self.log_file = log_file start = time.time() best_cost = self.MAX_COST # if only one lambda to tune if lambdas2 is None: self.fmodel = Fitted_Model(num_lambdas=1) self.log("%s lambda values: %s" % (self.method_label, lambdas1)) for l1 in lambdas1: curr_lambdas = np.array([l1]) model_params = self.problem_wrapper.solve(curr_lambdas, quick_run=True) cost = self.get_validation_cost(model_params) self.log("%s: cost %f, lambda: %f, %f" % (self.method_label, cost, l1)) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f, lambda: %f" % (self.method_label, best_cost, l1)) self.fmodel.update(curr_lambdas, model_params, cost) self.log( "%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_num_solves(len(lambdas1)) else: self.fmodel = Fitted_Model(num_lambdas=2) self.log("%s lambda1 values: %s" % (self.method_label, lambdas1)) self.log("%s lambda2 values: %s" % (self.method_label, lambdas2)) # if two lambdas to tune for l1 in lambdas1: for l2 in lambdas2: curr_lambdas = np.array([l1, l2]) model_params = self.problem_wrapper.solve(curr_lambdas, quick_run=True) cost = self.get_validation_cost(model_params) self.log("%s: cost %f, lambdas: %f, %f" % (self.method_label, cost, l1, l2)) if best_cost > cost: best_cost = cost self.log( "%s: best_validation_error %f, lambdas: %f, %f" % (self.method_label, best_cost, l1, l2)) self.fmodel.update(curr_lambdas, model_params, cost) self.log( "%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2)) runtime = time.time() - start self.fmodel.set_runtime(runtime) self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas)) self.log("%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params))
class Grid_Search: MAX_COST = 1e5 def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_problem_wrapper() def run(self, lambdas1, lambdas2=None, log_file=None): self.log_file = log_file start = time.time() best_cost = self.MAX_COST # if only one lambda to tune if lambdas2 is None: self.fmodel = Fitted_Model(num_lambdas=1) self.log("%s lambda values: %s" % (self.method_label, lambdas1)) for l1 in lambdas1: curr_lambdas = np.array([l1]) model_params = self.problem_wrapper.solve(curr_lambdas) cost = self.get_validation_cost(model_params) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f" % (self.method_label, best_cost)) self.fmodel.update(curr_lambdas, model_params, cost) self.fmodel.set_num_solves(len(lambdas1)) else: self.fmodel = Fitted_Model(num_lambdas=2) self.log("%s lambda1 values: %s" % (self.method_label, lambdas1)) self.log("%s lambda2 values: %s" % (self.method_label, lambdas2)) # if two lambdas to tune for l1 in lambdas1: for l2 in lambdas2: curr_lambdas = np.array([l1,l2]) model_params = self.problem_wrapper.solve(curr_lambdas) cost = self.get_validation_cost(model_params) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f" % (self.method_label, best_cost)) self.fmodel.update(curr_lambdas, model_params, cost) self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2)) runtime = time.time() - start self.fmodel.set_runtime(runtime) self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas)) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str)
def run(self, num_runs, log_file=None): self.log_file = log_file # Run spearmint to get next experiment parameters self.fmodel = Fitted_Model(self.num_lambdas) # Find new experiment best_cost = None runtime = 0 for i in range(num_runs): start_time = time.time() self.log("%s: iter %d" % (self.method_label, i)) self.run_spearmint_command(self.result_folder) self.log("Spearmint command completed") with open(self.result_file,'r') as resfile: newlines = [] for line in resfile.readlines(): values = line.split() if len(values) < 3: continue val = values.pop(0) dur = values.pop(0) lambdas = np.array([10**float(v) for v in values[:self.num_lambdas]]) if (val == 'P'): # P means pending experiment to run # Run experiment self.log("lambdas %s" % lambdas) model_params = self.problem_wrapper.solve(lambdas, quick_run=True) if model_params is None: current_cost = self.MAX_COST else: current_cost = self.get_validation_cost(model_params) if best_cost is None or best_cost > current_cost: best_cost = current_cost self.fmodel.update(lambdas, model_params, current_cost) self.log("fmodel: %s" % self.fmodel) newlines.append(str(current_cost) + " 0 " + " ".join(values) + "\n") else: # Otherwise these are previous experiment results newlines.append(line) runtime += time.time() - start_time # Don't record time spent on writing files? with open(self.result_file, 'w') as resfile: resfile.writelines(newlines) sys.stdout.flush() self.fmodel.set_runtime(runtime) self.fmodel.set_num_solves(num_runs) self.log("%s: runtime %s" % (self.method_label, runtime)) self.log("fmodel: %s" % self.fmodel) # VERY IMPORTANT to clean spearmint results self.run_spearmint_clean(self.result_folder)
def run(self, initial_lambda_set, debug=True, log_file=None): self.log_file = log_file start_time = time.time() self.fmodel = Fitted_Model(initial_lambda_set[0].size) best_cost = None best_initial_lambdas = None for initial_lambdas in initial_lambda_set: self.log("%s: initial_lambdas %s" % (self.method_label, initial_lambdas)) self._run_lambdas(initial_lambdas, debug=debug) if best_cost is None or best_cost > self.fmodel.best_cost: best_cost = self.fmodel.best_cost best_initial_lambdas = initial_lambdas self.log("%s: best start lambda %s" % (self.method_label, best_initial_lambdas)) runtime = time.time() - start_time self.log("%s: runtime %s" % (self.method_label, runtime)) self.fmodel.set_runtime(runtime)
def run(self, lambdas1, lambdas2=None, log_file=None): self.log_file = log_file start = time.time() best_cost = self.MAX_COST # if only one lambda to tune if lambdas2 is None: self.fmodel = Fitted_Model(num_lambdas=1) self.log("%s lambda values: %s" % (self.method_label, lambdas1)) for l1 in lambdas1: curr_lambdas = np.array([l1]) model_params = self.problem_wrapper.solve(curr_lambdas) cost = self.get_validation_cost(model_params) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f" % (self.method_label, best_cost)) self.fmodel.update(curr_lambdas, model_params, cost) self.fmodel.set_num_solves(len(lambdas1)) else: self.fmodel = Fitted_Model(num_lambdas=2) self.log("%s lambda1 values: %s" % (self.method_label, lambdas1)) self.log("%s lambda2 values: %s" % (self.method_label, lambdas2)) # if two lambdas to tune for l1 in lambdas1: for l2 in lambdas2: curr_lambdas = np.array([l1,l2]) model_params = self.problem_wrapper.solve(curr_lambdas) cost = self.get_validation_cost(model_params) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f" % (self.method_label, best_cost)) self.fmodel.update(curr_lambdas, model_params, cost) self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2)) runtime = time.time() - start self.fmodel.set_runtime(runtime) self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas))
def run(self, initial_lambda_set, debug=True, log_file=None): self.log_file = log_file start_time = time.time() self.fmodel = Fitted_Model(initial_lambda_set[0].size) best_cost = None for initial_lambdas in initial_lambda_set: self._run_lambdas(initial_lambdas, debug=debug) #, max_cost_at_iter=best_cost, check_iter=self.check_iter) if best_cost < self.fmodel.best_cost: self.log("%s: best start lambda %s" % (self.method_label, initial_lambdas)) runtime = time.time() - start_time self.log("%s: runtime %s" % (self.method_label, runtime)) self.fmodel.set_runtime(runtime)
def run(self, initial_lambdas_set, num_iters=10, log_file=None): self.log_file = log_file start = time.time() total_calls = 0 self.fmodel = Fitted_Model(initial_lambdas_set[0].size) for initial_lambdas in initial_lambdas_set: res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter":num_iters}) model_params = self.problem_wrapper.solve(res.x, quick_run=True) total_calls += res.nfev self.fmodel.update(res.x, model_params, res.fun) self.log("fmodel %s" % self.fmodel) runtime = time.time() - start self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls)) self.fmodel.set_runtime(runtime)
class Nelder_Mead_Algo: def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_problem_wrapper() def run(self, initial_lambdas_set, num_iters=10, log_file=None): self.log_file = log_file start = time.time() total_calls = 0 self.fmodel = Fitted_Model(initial_lambdas_set[0].size) for initial_lambdas in initial_lambdas_set: self.log("initial_lambdas %s" % initial_lambdas) res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter": num_iters}) model_params = self.problem_wrapper.solve(res.x, quick_run=True) total_calls += res.nfev self.fmodel.update(res.x, model_params, res.fun) self.log("fmodel %s" % self.fmodel) runtime = time.time() - start self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls)) self.log("%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_runtime(runtime) self.fmodel.set_num_solves(total_calls) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str)
class Spearmint_Algo: MAX_COST = 1e6 def __init__(self, data, folder_suffix, settings=None): self.data = data self.settings = settings # Call problem wrapper first to determine number of lambdas self._create_problem_wrapper() self._check_make_configs(folder_suffix) self.result_file = "%s/results.dat" % self.result_folder def run(self, num_runs, log_file=None): self.log_file = log_file # Run spearmint to get next experiment parameters self.fmodel = Fitted_Model(self.num_lambdas) # Find new experiment best_cost = None runtime = 0 for i in range(num_runs): start_time = time.time() self.log("%s: iter %d" % (self.method_label, i)) self.run_spearmint_command(self.result_folder) self.log("Spearmint command completed") with open(self.result_file,'r') as resfile: newlines = [] for line in resfile.readlines(): values = line.split() if len(values) < 3: continue val = values.pop(0) dur = values.pop(0) lambdas = np.array([10**float(v) for v in values[:self.num_lambdas]]) if (val == 'P'): # P means pending experiment to run # Run experiment self.log("lambdas %s" % lambdas) model_params = self.problem_wrapper.solve(lambdas, quick_run=True) if model_params is None: current_cost = self.MAX_COST else: current_cost = self.get_validation_cost(model_params) if best_cost is None or best_cost > current_cost: best_cost = current_cost self.fmodel.update(lambdas, model_params, current_cost) self.log("fmodel: %s" % self.fmodel) newlines.append(str(current_cost) + " 0 " + " ".join(values) + "\n") else: # Otherwise these are previous experiment results newlines.append(line) runtime += time.time() - start_time # Don't record time spent on writing files? with open(self.result_file, 'w') as resfile: resfile.writelines(newlines) sys.stdout.flush() self.fmodel.set_runtime(runtime) self.log("%s: runtime %s" % (self.method_label, runtime)) self.log("fmodel: %s" % self.fmodel) # VERY IMPORTANT to clean spearmint results self.run_spearmint_clean(self.result_folder) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) def _check_make_configs(self, folder_suffix): self.result_folder = "%s%s" % (self.result_folder_prefix, folder_suffix) if not os.path.exists(self.result_folder): os.makedirs(self.result_folder) config_file_name = "%s/config.json" % self.result_folder if not os.path.exists(config_file_name): with open(config_file_name, 'w') as config_file: config_file.write(self._create_config_string(self.num_lambdas)) @staticmethod def run_spearmint_command(experiment_folder, use_multiprocessing=True, gridsize=20000): ## Note: not all machines have the capability of running spearmint without crashing. ## Gosset servers will crash! multiprocessing_option = "" if not use_multiprocessing: multiprocessing_option = ",use_multiprocessing=0" options = Namespace( chooser_module="GPEIOptChooser", chooser_args="noiseless=1%s" % multiprocessing_option, grid_size=gridsize, config_file="config.json", results_file="results.dat", grid_seed=1, num_jobs=1, max_finished_jobs=1000 ) spearmint_lite.main_controller(options, [experiment_folder]) @staticmethod def run_spearmint_clean(experiment_folder): cmd = "rm %s/chooser* %s/results.dat" % (experiment_folder, experiment_folder) os.system(cmd) @staticmethod def _create_config_string(num_lambdas): return '{"lambda": {"name": "lambda","type": "float","size":%d,"min": -6,"max": 2}}' % num_lambdas
class Grid_Search: MAX_COST = 1e5 def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_problem_wrapper() def run(self, lambdas1, lambdas2=None, log_file=None): self.log_file = log_file start = time.time() best_cost = self.MAX_COST # if only one lambda to tune if lambdas2 is None: self.fmodel = Fitted_Model(num_lambdas=1) self.log("%s lambda values: %s" % (self.method_label, lambdas1)) for l1 in lambdas1: curr_lambdas = np.array([l1]) model_params = self.problem_wrapper.solve(curr_lambdas, quick_run=True) cost = self.get_validation_cost(model_params) self.log("%s: cost %f, lambda: %f, %f" % (self.method_label, cost, l1)) if best_cost > cost: best_cost = cost self.log("%s: best_validation_error %f, lambda: %f" % (self.method_label, best_cost, l1)) self.fmodel.update(curr_lambdas, model_params, cost) self.log( "%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_num_solves(len(lambdas1)) else: self.fmodel = Fitted_Model(num_lambdas=2) self.log("%s lambda1 values: %s" % (self.method_label, lambdas1)) self.log("%s lambda2 values: %s" % (self.method_label, lambdas2)) # if two lambdas to tune for l1 in lambdas1: for l2 in lambdas2: curr_lambdas = np.array([l1, l2]) model_params = self.problem_wrapper.solve(curr_lambdas, quick_run=True) cost = self.get_validation_cost(model_params) self.log("%s: cost %f, lambdas: %f, %f" % (self.method_label, cost, l1, l2)) if best_cost > cost: best_cost = cost self.log( "%s: best_validation_error %f, lambdas: %f, %f" % (self.method_label, best_cost, l1, l2)) self.fmodel.update(curr_lambdas, model_params, cost) self.log( "%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2)) runtime = time.time() - start self.fmodel.set_runtime(runtime) self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas)) self.log("%s: best_model_params %s" % (self.method_label, self.fmodel.best_model_params)) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) self.log_file.flush()
class Gradient_Descent_Algo: def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_descent_settings() self._create_problem_wrapper() self._create_lambda_configs() def run(self, initial_lambda_set, debug=True, log_file=None): self.log_file = log_file start_time = time.time() self.fmodel = Fitted_Model(initial_lambda_set[0].size) best_cost = None best_initial_lambdas = None for initial_lambdas in initial_lambda_set: self.log("%s: initial_lambdas %s" % (self.method_label, initial_lambdas)) self._run_lambdas(initial_lambdas, debug=debug) if best_cost is None or best_cost > self.fmodel.best_cost: best_cost = self.fmodel.best_cost best_initial_lambdas = initial_lambdas self.log("%s: best start lambda %s" % (self.method_label, best_initial_lambdas)) runtime = time.time() - start_time self.log("%s: runtime %s" % (self.method_label, runtime)) self.fmodel.set_runtime(runtime) def _run_lambdas(self, initial_lambdas, debug=True): start_history_idx = len(self.fmodel.cost_history) # warm up the problem self._solve_wrapper(initial_lambdas, quick_run=True) # do a real run now model_params = self._solve_wrapper(initial_lambdas, quick_run=False) # Check that no model params are None if self._any_model_params_none(model_params): self.log("ERROR: No model params fit for initial lambda values") self.fmodel.update(initial_lambdas, None, None) return current_cost = self.get_validate_cost(model_params) self.fmodel.update(initial_lambdas, model_params, current_cost) self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost) self._print_model_details() step_size = self.step_size_init for i in range(0, self.num_iters): lambda_derivatives = self._get_lambda_derivatives_wrapper() potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True) self.log( "potential_lambdas %s, potential_cost %s, curr cost %s" % (potential_lambdas, potential_cost, self.fmodel.current_cost)) while self._check_should_backtrack( potential_cost, step_size, lambda_derivatives) and step_size > self.step_size_min: if potential_cost is None: # If can't find a solution, shrink faster step_size *= self.shrink_factor**3 else: step_size *= self.shrink_factor potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True) if potential_cost is not None: self.log( "(shrinking) potential_lambdas %s, cost %f, step, %f" % (potential_lambdas, potential_cost, step_size)) else: self.log("(shrinking) potential_lambdas None!") if self.fmodel.current_cost < potential_cost: self.log("COST IS INCREASING! %f" % potential_cost) break else: potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=False) self.fmodel.update(potential_lambdas, potential_model_params, potential_cost) self.log("%s iter: %d step_size %f" % (self.method_label, i, step_size)) self.log("current model %s" % self.fmodel) self.log("cost_history %s" % self.fmodel.cost_history[start_history_idx:]) self.log("current test cost %s" % self.get_test_cost(self.fmodel.best_model_params)) self._print_model_details() if self.fmodel.get_cost_diff() < self.decr_enough_threshold: self.log("decrease amount too small %f" % self.fmodel.get_cost_diff()) break if step_size < self.step_size_min: self.log("STEP SIZE TOO SMALL %f" % step_size) break sys.stdout.flush() self.log("TOTAL ITERS %d" % i) self.log("full_cost_hist: %s" % self.fmodel.cost_history[start_history_idx:]) self.log("current_test_cost: %s" % self.get_test_cost(self.fmodel.best_model_params)) def get_test_cost(self, model): return None def _print_model_details(self): # fill in if you want to print more things return def _check_should_backtrack(self, potential_cost, step_size, lambda_derivatives): if potential_cost is None: return True backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm( lambda_derivatives)**2 backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw return potential_cost > backtrack_thres def _run_potential_lambdas(self, step_size, lambda_derivatives, quick_run=False): potential_lambdas = self._get_updated_lambdas(step_size, lambda_derivatives) try: potential_model_params = self._solve_wrapper(potential_lambdas, quick_run=quick_run) except cvxpy.error.SolverError: potential_model_params = None if self._any_model_params_none(potential_model_params): potential_cost = None else: potential_cost = self.get_validate_cost(potential_model_params) return potential_lambdas, potential_model_params, potential_cost def _solve_wrapper(self, lambdas, quick_run): start_solve_time = time.time() model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run) if quick_run is False: self.fmodel.incr_num_solves() self.log("solve runtime %f" % (time.time() - start_solve_time)) return model_params def _get_lambda_derivatives_wrapper(self): start_solve_time = time.time() lambda_derivatives = self._get_lambda_derivatives() self.log("lambda_derivatives runtime %f" % (time.time() - start_solve_time)) self.log("lambda_derivatives %s" % lambda_derivatives) return lambda_derivatives def _get_updated_lambdas(self, method_step_size, lambda_derivatives): current_lambdas = self.fmodel.current_lambdas new_step_size = method_step_size if self.use_boundary: potential_lambdas = current_lambdas - method_step_size * lambda_derivatives for idx in range(0, current_lambdas.size): if current_lambdas[idx] > self.lambda_mins[ idx] and potential_lambdas[idx] < self.lambda_mins[idx]: smaller_step_size = self.boundary_factor * ( current_lambdas[idx] - self.lambda_mins[idx]) / lambda_derivatives[idx] new_step_size = min(new_step_size, smaller_step_size) self.log("USING THE BOUNDARY %f" % new_step_size) return np.maximum(current_lambdas - new_step_size * lambda_derivatives, self.lambda_mins) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) self.log_file.flush() @staticmethod def _any_model_params_none(model_params): if model_params is None: return True else: return any([m is None for m in model_params])
class Gradient_Descent_Algo: def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_descent_settings() self._create_problem_wrapper() self._create_lambda_configs() def run(self, initial_lambda_set, debug=True, log_file=None): self.log_file = log_file start_time = time.time() self.fmodel = Fitted_Model(initial_lambda_set[0].size) best_cost = None for initial_lambdas in initial_lambda_set: self._run_lambdas(initial_lambdas, debug=debug) #, max_cost_at_iter=best_cost, check_iter=self.check_iter) if best_cost < self.fmodel.best_cost: self.log("%s: best start lambda %s" % (self.method_label, initial_lambdas)) runtime = time.time() - start_time self.log("%s: runtime %s" % (self.method_label, runtime)) self.fmodel.set_runtime(runtime) def _run_lambdas(self, initial_lambdas, debug=True): #, max_cost_at_iter=None, check_iter=None): self.log("%s: initial_lambdas %s" % (self.method_label, initial_lambdas)) start_history_idx = len(self.fmodel.cost_history) # warm up the problem self._solve_wrapper(initial_lambdas, quick_run=True) # do a real run now model_params = self._solve_wrapper(initial_lambdas, quick_run=False) # Check that no model params are None if self._any_model_params_none(model_params): self.log("ERROR: No model params fit for initial lambda values") self.fmodel.update(initial_lambdas, None, None) return current_cost = self.get_validate_cost(model_params) self.fmodel.update(initial_lambdas, model_params, current_cost) self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost) step_size = self.step_size_init for i in range(0, self.num_iters): lambda_derivatives = self._get_lambda_derivatives_wrapper() if debug: self._double_check_derivative(lambda_derivatives) potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True ) # TODO: Do backtracking while self._check_should_backtrack(potential_cost, step_size, lambda_derivatives) and step_size > self.step_size_min: if potential_cost is None: # Then cvxpy couldn't find a solution. Shrink faster step_size *= self.shrink_factor**3 else: step_size *= self.shrink_factor potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True ) if potential_cost is not None: self.log("(shrinking) potential_lambdas %s, cost %f, step, %f" % (potential_lambdas, potential_cost, step_size)) else: self.log("(shrinking) potential_lambdas None!") if self.fmodel.current_cost < potential_cost: self.log("COST IS INCREASING! %f" % potential_cost) break else: potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=False ) self.fmodel.update(potential_lambdas, potential_model_params, potential_cost) self.log("%s iter: %d step_size %f" % (self.method_label, i, step_size)) self.log("current model %s" % self.fmodel) self.log("cost_history %s" % self.fmodel.cost_history[start_history_idx:]) if self.fmodel.get_cost_diff() < self.decr_enough_threshold: self.log("decrease amount too small %f" % self.fmodel.get_cost_diff()) break if step_size < self.step_size_min: self.log("STEP SIZE TOO SMALL %f" % step_size) break # if check_iter is not None and max_cost_at_iter is not None and check_iter == i and max_cost_at_iter < potential_cost: # self.log("Cost %f higher than threshold %f" % (potential_cost, max_cost_at_iter)) # break sys.stdout.flush() self.log("TOTAL ITERS %d" % i) self.log("%s" % self.fmodel.cost_history[start_history_idx:]) def _check_should_backtrack(self, potential_cost, step_size, lambda_derivatives): if potential_cost is None: return True backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm(lambda_derivatives)**2 backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw return potential_cost > backtrack_thres def _run_potential_lambdas(self, step_size, lambda_derivatives, quick_run=False): potential_lambdas = self._get_updated_lambdas( step_size, lambda_derivatives ) try: potential_model_params = self._solve_wrapper(potential_lambdas, quick_run=quick_run) except cvxpy.error.SolverError: potential_model_params = None if self._any_model_params_none(potential_model_params): potential_cost = None else: potential_cost = self.get_validate_cost(potential_model_params) return potential_lambdas, potential_model_params, potential_cost def _solve_wrapper(self, lambdas, quick_run): start_solve_time = time.time() model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run) self.log("CVX runtime %f" % (time.time() - start_solve_time)) return model_params def _get_lambda_derivatives_wrapper(self): start_solve_time = time.time() lambda_derivatives = self._get_lambda_derivatives() self.log("lambda_derivatives runtime %f" % (time.time() - start_solve_time)) self.log("lambda_derivatives %s" % lambda_derivatives) return lambda_derivatives def _get_updated_lambdas(self, method_step_size, lambda_derivatives): current_lambdas = self.fmodel.current_lambdas new_step_size = method_step_size if self.use_boundary: potential_lambdas = current_lambdas - method_step_size * lambda_derivatives for idx in range(0, current_lambdas.size): if current_lambdas[idx] > self.lambda_mins[idx] and potential_lambdas[idx] < (1 - self.boundary_factor) * current_lambdas[idx]: smaller_step_size = self.boundary_factor * current_lambdas[idx] / lambda_derivatives[idx] new_step_size = min(new_step_size, smaller_step_size) self.log("USING THE BOUNDARY %f" % new_step_size) return np.maximum(current_lambdas - new_step_size * lambda_derivatives, self.lambda_mins) def _double_check_derivative(self, calculated_derivative, accept_diff=1e-1, epsilon=1e-5): # Returns the numeral derivative if you want it deriv = [] num_lambdas = len(self.fmodel.current_lambdas) for i in range(num_lambdas): print "===========CHECK I= %d ===============" % i # don't allow the discrete derivative perturb too much if the lambda value is low already eps = min(epsilon, self.fmodel.current_lambdas[i]/100) reg1 = np.copy(self.fmodel.current_lambdas) reg1[i] += eps model1 = self.problem_wrapper.solve(np.array(reg1), quick_run=False) error1 = self.get_validate_cost(model1) reg2 = np.copy(self.fmodel.current_lambdas) reg2[i] -= eps model2 = self.problem_wrapper.solve(np.array(reg2), quick_run=False) error2 = self.get_validate_cost(model2) i_deriv = (error1 - error2)/(epsilon * 2) # print "numerical sum_dthetas_dlambda", np.sum((model1 - thetas2)/(epsilon * 2), axis=1) print "calculated_derivative[i]", calculated_derivative[i] print "numerical deriv", i_deriv deriv.append(i_deriv) print "np.abs(calculated_derivative[i] - i_deriv)", np.abs(calculated_derivative[i] - i_deriv) relative_ok = np.abs((calculated_derivative[i] - i_deriv)/i_deriv) < accept_diff absolute_ok = np.abs(calculated_derivative[i] - i_deriv) < accept_diff assert(relative_ok or absolute_ok) return np.hstack(deriv) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) @staticmethod def _any_model_params_none(model_params): if model_params is None: return True else: return any([m is None for m in model_params])
class Gradient_Descent_Algo: def __init__(self, data, settings=None): self.data = data self.settings = settings self._create_descent_settings() self._create_problem_wrapper() self._create_lambda_configs() def run(self, initial_lambda_set, debug=True, log_file=None): self.log_file = log_file start_time = time.time() self.fmodel = Fitted_Model(initial_lambda_set[0].size) best_cost = None best_initial_lambdas = None for initial_lambdas in initial_lambda_set: self.log("%s: initial_lambdas %s" % (self.method_label, initial_lambdas)) self._run_lambdas( initial_lambdas, debug=debug ) #, max_cost_at_iter=best_cost, check_iter=self.check_iter) if best_cost is None or best_cost > self.fmodel.best_cost: best_cost = self.fmodel.best_cost best_initial_lambdas = initial_lambdas self.log("%s: best start lambda %s" % (self.method_label, best_initial_lambdas)) runtime = time.time() - start_time self.log("%s: runtime %s" % (self.method_label, runtime)) self.fmodel.set_runtime(runtime) # self.fmodel.set_num_solves(len(self.fmodel.cost_history)) def _check_optimality_conditions(self, model_params, lambdas, thres=1): return def _run_lambdas(self, initial_lambdas, debug=True): #, max_cost_at_iter=None, check_iter=None): start_history_idx = len(self.fmodel.cost_history) # warm up the problem self._solve_wrapper(initial_lambdas, quick_run=True) # do a real run now model_params = self._solve_wrapper(initial_lambdas, quick_run=False) if debug: self._check_optimality_conditions(model_params, initial_lambdas) # Check that no model params are None if self._any_model_params_none(model_params): self.log("ERROR: No model params fit for initial lambda values") self.fmodel.update(initial_lambdas, None, None) return current_cost = self.get_validate_cost(model_params) self.fmodel.update(initial_lambdas, model_params, current_cost) self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost) self._print_model_details() step_size = self.step_size_init for i in range(0, self.num_iters): lambda_derivatives = self._get_lambda_derivatives_wrapper() if debug: self._double_check_derivative(lambda_derivatives) 1 / 0 potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True) self.log( "potential_lambdas %s, potential_cost %s, curr cost %s" % (potential_lambdas, potential_cost, self.fmodel.current_cost)) while self._check_should_backtrack( potential_cost, step_size, lambda_derivatives) and step_size > self.step_size_min: if potential_cost is None: # Then cvxpy couldn't find a solution. Shrink faster step_size *= self.shrink_factor**3 else: step_size *= self.shrink_factor potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=True) if potential_cost is not None: self.log( "(shrinking) potential_lambdas %s, cost %f, step, %f" % (potential_lambdas, potential_cost, step_size)) else: self.log("(shrinking) potential_lambdas None!") if self.fmodel.current_cost < potential_cost: self.log("COST IS INCREASING! %f" % potential_cost) break else: # Note to self: it is possible that solving to a lower accuracy results in lower validation loss # but higher accuracy gives higher validation loss. potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas( step_size, lambda_derivatives, quick_run=False) if debug: self._check_optimality_conditions(potential_model_params, potential_lambdas) self.fmodel.update(potential_lambdas, potential_model_params, potential_cost) self.log("%s iter: %d step_size %f" % (self.method_label, i, step_size)) self.log("current model %s" % self.fmodel) self.log("cost_history %s" % self.fmodel.cost_history[start_history_idx:]) self.log("current test cost %s" % self.get_test_cost(self.fmodel.best_model_params)) self._print_model_details() if self.fmodel.get_cost_diff() < self.decr_enough_threshold: self.log("decrease amount too small %f" % self.fmodel.get_cost_diff()) break if step_size < self.step_size_min: self.log("STEP SIZE TOO SMALL %f" % step_size) break # if check_iter is not None and max_cost_at_iter is not None and check_iter == i and max_cost_at_iter < potential_cost: # self.log("Cost %f higher than threshold %f" % (potential_cost, max_cost_at_iter)) # break sys.stdout.flush() self.log("TOTAL ITERS %d" % i) self.log("full_cost_hist: %s" % self.fmodel.cost_history[start_history_idx:]) self.log("current_test_cost: %s" % self.get_test_cost(self.fmodel.best_model_params)) def get_test_cost(self, model): return None def _print_model_details(self): # fill in if you want to print more things return def _check_should_backtrack(self, potential_cost, step_size, lambda_derivatives): if potential_cost is None: return True backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm( lambda_derivatives)**2 backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw return potential_cost > backtrack_thres def _run_potential_lambdas(self, step_size, lambda_derivatives, quick_run=False): potential_lambdas = self._get_updated_lambdas(step_size, lambda_derivatives) try: potential_model_params = self._solve_wrapper(potential_lambdas, quick_run=quick_run) except cvxpy.error.SolverError: potential_model_params = None if self._any_model_params_none(potential_model_params): potential_cost = None else: potential_cost = self.get_validate_cost(potential_model_params) return potential_lambdas, potential_model_params, potential_cost def _solve_wrapper(self, lambdas, quick_run): start_solve_time = time.time() model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run) if quick_run is False: self.fmodel.incr_num_solves() self.log("CVX runtime %f" % (time.time() - start_solve_time)) return model_params def _get_lambda_derivatives_wrapper(self): start_solve_time = time.time() lambda_derivatives = self._get_lambda_derivatives() self.log("lambda_derivatives runtime %f" % (time.time() - start_solve_time)) self.log("lambda_derivatives %s" % lambda_derivatives) return lambda_derivatives def _get_updated_lambdas(self, method_step_size, lambda_derivatives): current_lambdas = self.fmodel.current_lambdas new_step_size = method_step_size if self.use_boundary: potential_lambdas = current_lambdas - method_step_size * lambda_derivatives for idx in range(0, current_lambdas.size): if current_lambdas[idx] > self.lambda_mins[ idx] and potential_lambdas[idx] < self.lambda_mins[idx]: smaller_step_size = self.boundary_factor * ( current_lambdas[idx] - self.lambda_mins[idx]) / lambda_derivatives[idx] new_step_size = min(new_step_size, smaller_step_size) self.log("USING THE BOUNDARY %f" % new_step_size) return np.maximum(current_lambdas - new_step_size * lambda_derivatives, self.lambda_mins) def _double_check_derivative_indepth(self, i, model1, model2, model0, eps): # override this function if you want to do more detailed checking of the derivatives # useful when you want to also double check the derivatives of the model parameters # wrt to lambda penalty parameters return def _double_check_derivative(self, calculated_derivative, accept_diff=1e-1, epsilon=1e-6): num_lambdas = len(self.fmodel.current_lambdas) print "self.fmodel.current_lambdas", self.fmodel.current_lambdas numerical_derivs = [] for i in range(num_lambdas): print "===========CHECK I= %d ===============" % i # don't allow the discrete derivative perturb too much if the lambda value is low already eps = min(epsilon, self.fmodel.current_lambdas[i] / 100) reg1 = np.copy(self.fmodel.current_lambdas) reg1[i] += eps model1 = self.problem_wrapper.solve(np.array(reg1), quick_run=False, warm_start=False) reg2 = np.copy(self.fmodel.current_lambdas) reg2[i] -= eps model2 = self.problem_wrapper.solve(np.array(reg2), quick_run=False, warm_start=False) # Calculate derivative of validation error error1 = self.get_validate_cost(model1) error2 = self.get_validate_cost(model2) i_deriv = (error1 - error2) / (eps * 2) numerical_derivs.append(i_deriv) print "********** jean calc derivative[i]", calculated_derivative[ i] print "********** i_deriv", i_deriv print "np.abs(calculated_derivative[i] - i_deriv)", np.abs( calculated_derivative[i] - i_deriv) relative_ok = np.abs( (calculated_derivative[i] - i_deriv) / i_deriv) < accept_diff absolute_ok = np.abs(calculated_derivative[i] - i_deriv) < accept_diff # model0 = self.problem_wrapper.solve(self.fmodel.current_lambdas, quick_run=False, warm_start=False) # error0 = self.get_validate_cost(model0) # self._double_check_derivative_indepth(i, model1, model2, model0, eps) # self._double_check_derivative_indepth(i, model1, model2, None, eps) # assert(relative_ok or absolute_ok) print "calculated_derivative", calculated_derivative print "numerical_derivs", numerical_derivs def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) self.log_file.flush() @staticmethod def _any_model_params_none(model_params): if model_params is None: return True else: return any([m is None for m in model_params])
def run(self, num_runs, log_file=None): self.log_file = log_file # Run spearmint to get next experiment parameters self.fmodel = Fitted_Model(self.num_lambdas) # Find new experiment best_cost = None runtime = 0 for i in range(num_runs): start_time = time.time() self.log("%s: iter %d" % (self.method_label, i)) self.run_spearmint_command(self.result_folder) self.log("Spearmint command completed") with open(self.result_file, 'r') as resfile: newlines = [] for line in resfile.readlines(): values = line.split() if len(values) < 3: continue val = values.pop(0) dur = values.pop(0) lambdas = np.array( [10**float(v) for v in values[:self.num_lambdas]]) if (val == 'P'): # P means pending experiment to run # Run experiment self.log("lambdas %s" % lambdas) model_params = self.problem_wrapper.solve( lambdas, quick_run=True) if model_params is None: current_cost = self.MAX_COST else: current_cost = self.get_validation_cost( model_params) if best_cost is None or best_cost > current_cost: best_cost = current_cost self.fmodel.update(lambdas, model_params, current_cost) self.log("fmodel: %s" % self.fmodel) newlines.append( str(current_cost) + " 0 " + " ".join(values) + "\n") else: # Otherwise these are previous experiment results newlines.append(line) runtime += time.time() - start_time # Don't record time spent on writing files? with open(self.result_file, 'w') as resfile: resfile.writelines(newlines) sys.stdout.flush() self.fmodel.set_runtime(runtime) self.fmodel.set_num_solves(num_runs) self.log("%s: runtime %s" % (self.method_label, runtime)) self.log("fmodel: %s" % self.fmodel) self.log("fmodel best_model_params: %s" % self.fmodel.best_model_params) # VERY IMPORTANT to clean spearmint results self.run_spearmint_clean(self.result_folder)
class Spearmint_Algo: MAX_COST = 1e6 def __init__(self, data, folder_suffix, settings=None): self.data = data self.settings = settings # Call problem wrapper first to determine number of lambdas self._create_problem_wrapper() self._check_make_configs(folder_suffix) self.result_file = "%s/results.dat" % self.result_folder def run(self, num_runs, log_file=None): self.log_file = log_file # Run spearmint to get next experiment parameters self.fmodel = Fitted_Model(self.num_lambdas) # Find new experiment best_cost = None runtime = 0 for i in range(num_runs): start_time = time.time() self.log("%s: iter %d" % (self.method_label, i)) self.run_spearmint_command(self.result_folder) self.log("Spearmint command completed") with open(self.result_file, 'r') as resfile: newlines = [] for line in resfile.readlines(): values = line.split() if len(values) < 3: continue val = values.pop(0) dur = values.pop(0) lambdas = np.array( [10**float(v) for v in values[:self.num_lambdas]]) if (val == 'P'): # P means pending experiment to run # Run experiment self.log("lambdas %s" % lambdas) model_params = self.problem_wrapper.solve( lambdas, quick_run=True) if model_params is None: current_cost = self.MAX_COST else: current_cost = self.get_validation_cost( model_params) if best_cost is None or best_cost > current_cost: best_cost = current_cost self.fmodel.update(lambdas, model_params, current_cost) self.log("fmodel: %s" % self.fmodel) newlines.append( str(current_cost) + " 0 " + " ".join(values) + "\n") else: # Otherwise these are previous experiment results newlines.append(line) runtime += time.time() - start_time # Don't record time spent on writing files? with open(self.result_file, 'w') as resfile: resfile.writelines(newlines) sys.stdout.flush() self.fmodel.set_runtime(runtime) self.fmodel.set_num_solves(num_runs) self.log("%s: runtime %s" % (self.method_label, runtime)) self.log("fmodel: %s" % self.fmodel) self.log("fmodel best_model_params: %s" % self.fmodel.best_model_params) # VERY IMPORTANT to clean spearmint results self.run_spearmint_clean(self.result_folder) def log(self, log_str): if self.log_file is None: print log_str else: self.log_file.write("%s\n" % log_str) def _check_make_configs(self, folder_suffix): self.result_folder = "%s%s" % (self.result_folder_prefix, folder_suffix) if not os.path.exists(self.result_folder): os.makedirs(self.result_folder) config_file_name = "%s/config.json" % self.result_folder if not os.path.exists(config_file_name): with open(config_file_name, 'w') as config_file: config_file.write(self._create_config_string(self.num_lambdas)) @staticmethod @print_time def run_spearmint_command(experiment_folder, use_multiprocessing=True, gridsize=20000): ## Note: not all machines have the capability of running spearmint without crashing. ## Gosset servers will crash! multiprocessing_option = "" if not use_multiprocessing: multiprocessing_option = ",use_multiprocessing=0" options = Namespace(chooser_module="GPEIOptChooser", chooser_args="noiseless=1%s" % multiprocessing_option, grid_size=gridsize, config_file="config.json", results_file="results.dat", grid_seed=1, num_jobs=1, max_finished_jobs=1000) spearmint_lite.main_controller(options, [experiment_folder]) @staticmethod def run_spearmint_clean(experiment_folder): cmd = "rm %s/chooser* %s/results.dat" % (experiment_folder, experiment_folder) os.system(cmd) @staticmethod def _create_config_string(num_lambdas): return '{"lambda": {"name": "lambda","type": "float","size":%d,"min": -6,"max": 2}}' % num_lambdas