class Nelder_Mead_Algo:
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_problem_wrapper()
def run(self, initial_lambdas_set, num_iters=10, log_file=None):
self.log_file = log_file
start = time.time()
total_calls = 0
self.fmodel = Fitted_Model(initial_lambdas_set[0].size)
for initial_lambdas in initial_lambdas_set:
res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter":num_iters})
model_params = self.problem_wrapper.solve(res.x, quick_run=True)
total_calls += res.nfev
self.fmodel.update(res.x, model_params, res.fun)
self.log("fmodel %s" % self.fmodel)
runtime = time.time() - start
self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls))
self.fmodel.set_runtime(runtime)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
def run(self, initial_lambdas_set, num_iters=10, log_file=None):
self.log_file = log_file
start = time.time()
total_calls = 0
self.fmodel = Fitted_Model(initial_lambdas_set[0].size)
for initial_lambdas in initial_lambdas_set:
self.log("initial_lambdas %s" % initial_lambdas)
res = minimize(self.get_validation_cost,
initial_lambdas,
method='nelder-mead',
options={"maxiter": num_iters})
model_params = self.problem_wrapper.solve(res.x, quick_run=True)
total_calls += res.nfev
self.fmodel.update(res.x, model_params, res.fun)
self.log("fmodel %s" % self.fmodel)
runtime = time.time() - start
self.log("%s: best cost %f, lambda %s, total calls %d" %
(self.method_label, self.fmodel.best_cost,
self.fmodel.best_lambdas, total_calls))
self.log("%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_runtime(runtime)
self.fmodel.set_num_solves(total_calls)
def run(self, lambdas1, lambdas2=None, log_file=None):
self.log_file = log_file
start = time.time()
best_cost = self.MAX_COST
# if only one lambda to tune
if lambdas2 is None:
self.fmodel = Fitted_Model(num_lambdas=1)
self.log("%s lambda values: %s" % (self.method_label, lambdas1))
for l1 in lambdas1:
curr_lambdas = np.array([l1])
model_params = self.problem_wrapper.solve(curr_lambdas,
quick_run=True)
cost = self.get_validation_cost(model_params)
self.log("%s: cost %f, lambda: %f, %f" %
(self.method_label, cost, l1))
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f, lambda: %f" %
(self.method_label, best_cost, l1))
self.fmodel.update(curr_lambdas, model_params, cost)
self.log(
"%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_num_solves(len(lambdas1))
else:
self.fmodel = Fitted_Model(num_lambdas=2)
self.log("%s lambda1 values: %s" % (self.method_label, lambdas1))
self.log("%s lambda2 values: %s" % (self.method_label, lambdas2))
# if two lambdas to tune
for l1 in lambdas1:
for l2 in lambdas2:
curr_lambdas = np.array([l1, l2])
model_params = self.problem_wrapper.solve(curr_lambdas,
quick_run=True)
cost = self.get_validation_cost(model_params)
self.log("%s: cost %f, lambdas: %f, %f" %
(self.method_label, cost, l1, l2))
if best_cost > cost:
best_cost = cost
self.log(
"%s: best_validation_error %f, lambdas: %f, %f" %
(self.method_label, best_cost, l1, l2))
self.fmodel.update(curr_lambdas, model_params, cost)
self.log(
"%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2))
runtime = time.time() - start
self.fmodel.set_runtime(runtime)
self.log("%s: best cost %f, lambda %s" %
(self.method_label, best_cost, self.fmodel.current_lambdas))
self.log("%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
class Grid_Search:
MAX_COST = 1e5
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_problem_wrapper()
def run(self, lambdas1, lambdas2=None, log_file=None):
self.log_file = log_file
start = time.time()
best_cost = self.MAX_COST
# if only one lambda to tune
if lambdas2 is None:
self.fmodel = Fitted_Model(num_lambdas=1)
self.log("%s lambda values: %s" % (self.method_label, lambdas1))
for l1 in lambdas1:
curr_lambdas = np.array([l1])
model_params = self.problem_wrapper.solve(curr_lambdas)
cost = self.get_validation_cost(model_params)
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f" % (self.method_label, best_cost))
self.fmodel.update(curr_lambdas, model_params, cost)
self.fmodel.set_num_solves(len(lambdas1))
else:
self.fmodel = Fitted_Model(num_lambdas=2)
self.log("%s lambda1 values: %s" % (self.method_label, lambdas1))
self.log("%s lambda2 values: %s" % (self.method_label, lambdas2))
# if two lambdas to tune
for l1 in lambdas1:
for l2 in lambdas2:
curr_lambdas = np.array([l1,l2])
model_params = self.problem_wrapper.solve(curr_lambdas)
cost = self.get_validation_cost(model_params)
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f" % (self.method_label, best_cost))
self.fmodel.update(curr_lambdas, model_params, cost)
self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2))
runtime = time.time() - start
self.fmodel.set_runtime(runtime)
self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas))
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
def run(self, num_runs, log_file=None):
self.log_file = log_file
# Run spearmint to get next experiment parameters
self.fmodel = Fitted_Model(self.num_lambdas)
# Find new experiment
best_cost = None
runtime = 0
for i in range(num_runs):
start_time = time.time()
self.log("%s: iter %d" % (self.method_label, i))
self.run_spearmint_command(self.result_folder)
self.log("Spearmint command completed")
with open(self.result_file,'r') as resfile:
newlines = []
for line in resfile.readlines():
values = line.split()
if len(values) < 3:
continue
val = values.pop(0)
dur = values.pop(0)
lambdas = np.array([10**float(v) for v in values[:self.num_lambdas]])
if (val == 'P'):
# P means pending experiment to run
# Run experiment
self.log("lambdas %s" % lambdas)
model_params = self.problem_wrapper.solve(lambdas, quick_run=True)
if model_params is None:
current_cost = self.MAX_COST
else:
current_cost = self.get_validation_cost(model_params)
if best_cost is None or best_cost > current_cost:
best_cost = current_cost
self.fmodel.update(lambdas, model_params, current_cost)
self.log("fmodel: %s" % self.fmodel)
newlines.append(str(current_cost) + " 0 "
+ " ".join(values) + "\n")
else:
# Otherwise these are previous experiment results
newlines.append(line)
runtime += time.time() - start_time
# Don't record time spent on writing files?
with open(self.result_file, 'w') as resfile:
resfile.writelines(newlines)
sys.stdout.flush()
self.fmodel.set_runtime(runtime)
self.fmodel.set_num_solves(num_runs)
self.log("%s: runtime %s" % (self.method_label, runtime))
self.log("fmodel: %s" % self.fmodel)
# VERY IMPORTANT to clean spearmint results
self.run_spearmint_clean(self.result_folder)
def run(self, initial_lambda_set, debug=True, log_file=None):
self.log_file = log_file
start_time = time.time()
self.fmodel = Fitted_Model(initial_lambda_set[0].size)
best_cost = None
best_initial_lambdas = None
for initial_lambdas in initial_lambda_set:
self.log("%s: initial_lambdas %s" %
(self.method_label, initial_lambdas))
self._run_lambdas(initial_lambdas, debug=debug)
if best_cost is None or best_cost > self.fmodel.best_cost:
best_cost = self.fmodel.best_cost
best_initial_lambdas = initial_lambdas
self.log("%s: best start lambda %s" %
(self.method_label, best_initial_lambdas))
runtime = time.time() - start_time
self.log("%s: runtime %s" % (self.method_label, runtime))
self.fmodel.set_runtime(runtime)
def run(self, lambdas1, lambdas2=None, log_file=None):
self.log_file = log_file
start = time.time()
best_cost = self.MAX_COST
# if only one lambda to tune
if lambdas2 is None:
self.fmodel = Fitted_Model(num_lambdas=1)
self.log("%s lambda values: %s" % (self.method_label, lambdas1))
for l1 in lambdas1:
curr_lambdas = np.array([l1])
model_params = self.problem_wrapper.solve(curr_lambdas)
cost = self.get_validation_cost(model_params)
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f" % (self.method_label, best_cost))
self.fmodel.update(curr_lambdas, model_params, cost)
self.fmodel.set_num_solves(len(lambdas1))
else:
self.fmodel = Fitted_Model(num_lambdas=2)
self.log("%s lambda1 values: %s" % (self.method_label, lambdas1))
self.log("%s lambda2 values: %s" % (self.method_label, lambdas2))
# if two lambdas to tune
for l1 in lambdas1:
for l2 in lambdas2:
curr_lambdas = np.array([l1,l2])
model_params = self.problem_wrapper.solve(curr_lambdas)
cost = self.get_validation_cost(model_params)
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f" % (self.method_label, best_cost))
self.fmodel.update(curr_lambdas, model_params, cost)
self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2))
runtime = time.time() - start
self.fmodel.set_runtime(runtime)
self.log("%s: best cost %f, lambda %s" % (self.method_label, best_cost, self.fmodel.current_lambdas))
def run(self, initial_lambda_set, debug=True, log_file=None):
self.log_file = log_file
start_time = time.time()
self.fmodel = Fitted_Model(initial_lambda_set[0].size)
best_cost = None
for initial_lambdas in initial_lambda_set:
self._run_lambdas(initial_lambdas, debug=debug) #, max_cost_at_iter=best_cost, check_iter=self.check_iter)
if best_cost < self.fmodel.best_cost:
self.log("%s: best start lambda %s" % (self.method_label, initial_lambdas))
runtime = time.time() - start_time
self.log("%s: runtime %s" % (self.method_label, runtime))
self.fmodel.set_runtime(runtime)
def run(self, initial_lambdas_set, num_iters=10, log_file=None):
self.log_file = log_file
start = time.time()
total_calls = 0
self.fmodel = Fitted_Model(initial_lambdas_set[0].size)
for initial_lambdas in initial_lambdas_set:
res = minimize(self.get_validation_cost, initial_lambdas, method='nelder-mead', options={"maxiter":num_iters})
model_params = self.problem_wrapper.solve(res.x, quick_run=True)
total_calls += res.nfev
self.fmodel.update(res.x, model_params, res.fun)
self.log("fmodel %s" % self.fmodel)
runtime = time.time() - start
self.log("%s: best cost %f, lambda %s, total calls %d" % (self.method_label, self.fmodel.best_cost, self.fmodel.best_lambdas, total_calls))
self.fmodel.set_runtime(runtime)
class Nelder_Mead_Algo:
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_problem_wrapper()
def run(self, initial_lambdas_set, num_iters=10, log_file=None):
self.log_file = log_file
start = time.time()
total_calls = 0
self.fmodel = Fitted_Model(initial_lambdas_set[0].size)
for initial_lambdas in initial_lambdas_set:
self.log("initial_lambdas %s" % initial_lambdas)
res = minimize(self.get_validation_cost,
initial_lambdas,
method='nelder-mead',
options={"maxiter": num_iters})
model_params = self.problem_wrapper.solve(res.x, quick_run=True)
total_calls += res.nfev
self.fmodel.update(res.x, model_params, res.fun)
self.log("fmodel %s" % self.fmodel)
runtime = time.time() - start
self.log("%s: best cost %f, lambda %s, total calls %d" %
(self.method_label, self.fmodel.best_cost,
self.fmodel.best_lambdas, total_calls))
self.log("%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_runtime(runtime)
self.fmodel.set_num_solves(total_calls)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
class Spearmint_Algo:
MAX_COST = 1e6
def __init__(self, data, folder_suffix, settings=None):
self.data = data
self.settings = settings
# Call problem wrapper first to determine number of lambdas
self._create_problem_wrapper()
self._check_make_configs(folder_suffix)
self.result_file = "%s/results.dat" % self.result_folder
def run(self, num_runs, log_file=None):
self.log_file = log_file
# Run spearmint to get next experiment parameters
self.fmodel = Fitted_Model(self.num_lambdas)
# Find new experiment
best_cost = None
runtime = 0
for i in range(num_runs):
start_time = time.time()
self.log("%s: iter %d" % (self.method_label, i))
self.run_spearmint_command(self.result_folder)
self.log("Spearmint command completed")
with open(self.result_file,'r') as resfile:
newlines = []
for line in resfile.readlines():
values = line.split()
if len(values) < 3:
continue
val = values.pop(0)
dur = values.pop(0)
lambdas = np.array([10**float(v) for v in values[:self.num_lambdas]])
if (val == 'P'):
# P means pending experiment to run
# Run experiment
self.log("lambdas %s" % lambdas)
model_params = self.problem_wrapper.solve(lambdas, quick_run=True)
if model_params is None:
current_cost = self.MAX_COST
else:
current_cost = self.get_validation_cost(model_params)
if best_cost is None or best_cost > current_cost:
best_cost = current_cost
self.fmodel.update(lambdas, model_params, current_cost)
self.log("fmodel: %s" % self.fmodel)
newlines.append(str(current_cost) + " 0 "
+ " ".join(values) + "\n")
else:
# Otherwise these are previous experiment results
newlines.append(line)
runtime += time.time() - start_time
# Don't record time spent on writing files?
with open(self.result_file, 'w') as resfile:
resfile.writelines(newlines)
sys.stdout.flush()
self.fmodel.set_runtime(runtime)
self.log("%s: runtime %s" % (self.method_label, runtime))
self.log("fmodel: %s" % self.fmodel)
# VERY IMPORTANT to clean spearmint results
self.run_spearmint_clean(self.result_folder)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
def _check_make_configs(self, folder_suffix):
self.result_folder = "%s%s" % (self.result_folder_prefix, folder_suffix)
if not os.path.exists(self.result_folder):
os.makedirs(self.result_folder)
config_file_name = "%s/config.json" % self.result_folder
if not os.path.exists(config_file_name):
with open(config_file_name, 'w') as config_file:
config_file.write(self._create_config_string(self.num_lambdas))
@staticmethod
def run_spearmint_command(experiment_folder, use_multiprocessing=True, gridsize=20000):
## Note: not all machines have the capability of running spearmint without crashing.
## Gosset servers will crash!
multiprocessing_option = ""
if not use_multiprocessing:
multiprocessing_option = ",use_multiprocessing=0"
options = Namespace(
chooser_module="GPEIOptChooser",
chooser_args="noiseless=1%s" % multiprocessing_option,
grid_size=gridsize,
config_file="config.json",
results_file="results.dat",
grid_seed=1,
num_jobs=1,
max_finished_jobs=1000
)
spearmint_lite.main_controller(options, [experiment_folder])
@staticmethod
def run_spearmint_clean(experiment_folder):
cmd = "rm %s/chooser* %s/results.dat" % (experiment_folder, experiment_folder)
os.system(cmd)
@staticmethod
def _create_config_string(num_lambdas):
return '{"lambda": {"name": "lambda","type": "float","size":%d,"min": -6,"max": 2}}' % num_lambdas
class Grid_Search:
MAX_COST = 1e5
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_problem_wrapper()
def run(self, lambdas1, lambdas2=None, log_file=None):
self.log_file = log_file
start = time.time()
best_cost = self.MAX_COST
# if only one lambda to tune
if lambdas2 is None:
self.fmodel = Fitted_Model(num_lambdas=1)
self.log("%s lambda values: %s" % (self.method_label, lambdas1))
for l1 in lambdas1:
curr_lambdas = np.array([l1])
model_params = self.problem_wrapper.solve(curr_lambdas,
quick_run=True)
cost = self.get_validation_cost(model_params)
self.log("%s: cost %f, lambda: %f, %f" %
(self.method_label, cost, l1))
if best_cost > cost:
best_cost = cost
self.log("%s: best_validation_error %f, lambda: %f" %
(self.method_label, best_cost, l1))
self.fmodel.update(curr_lambdas, model_params, cost)
self.log(
"%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_num_solves(len(lambdas1))
else:
self.fmodel = Fitted_Model(num_lambdas=2)
self.log("%s lambda1 values: %s" % (self.method_label, lambdas1))
self.log("%s lambda2 values: %s" % (self.method_label, lambdas2))
# if two lambdas to tune
for l1 in lambdas1:
for l2 in lambdas2:
curr_lambdas = np.array([l1, l2])
model_params = self.problem_wrapper.solve(curr_lambdas,
quick_run=True)
cost = self.get_validation_cost(model_params)
self.log("%s: cost %f, lambdas: %f, %f" %
(self.method_label, cost, l1, l2))
if best_cost > cost:
best_cost = cost
self.log(
"%s: best_validation_error %f, lambdas: %f, %f" %
(self.method_label, best_cost, l1, l2))
self.fmodel.update(curr_lambdas, model_params, cost)
self.log(
"%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
self.fmodel.set_num_solves(len(lambdas1) * len(lambdas2))
runtime = time.time() - start
self.fmodel.set_runtime(runtime)
self.log("%s: best cost %f, lambda %s" %
(self.method_label, best_cost, self.fmodel.current_lambdas))
self.log("%s: best_model_params %s" %
(self.method_label, self.fmodel.best_model_params))
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
self.log_file.flush()
class Gradient_Descent_Algo:
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_descent_settings()
self._create_problem_wrapper()
self._create_lambda_configs()
def run(self, initial_lambda_set, debug=True, log_file=None):
self.log_file = log_file
start_time = time.time()
self.fmodel = Fitted_Model(initial_lambda_set[0].size)
best_cost = None
best_initial_lambdas = None
for initial_lambdas in initial_lambda_set:
self.log("%s: initial_lambdas %s" %
(self.method_label, initial_lambdas))
self._run_lambdas(initial_lambdas, debug=debug)
if best_cost is None or best_cost > self.fmodel.best_cost:
best_cost = self.fmodel.best_cost
best_initial_lambdas = initial_lambdas
self.log("%s: best start lambda %s" %
(self.method_label, best_initial_lambdas))
runtime = time.time() - start_time
self.log("%s: runtime %s" % (self.method_label, runtime))
self.fmodel.set_runtime(runtime)
def _run_lambdas(self, initial_lambdas, debug=True):
start_history_idx = len(self.fmodel.cost_history)
# warm up the problem
self._solve_wrapper(initial_lambdas, quick_run=True)
# do a real run now
model_params = self._solve_wrapper(initial_lambdas, quick_run=False)
# Check that no model params are None
if self._any_model_params_none(model_params):
self.log("ERROR: No model params fit for initial lambda values")
self.fmodel.update(initial_lambdas, None, None)
return
current_cost = self.get_validate_cost(model_params)
self.fmodel.update(initial_lambdas, model_params, current_cost)
self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost)
self._print_model_details()
step_size = self.step_size_init
for i in range(0, self.num_iters):
lambda_derivatives = self._get_lambda_derivatives_wrapper()
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=True)
self.log(
"potential_lambdas %s, potential_cost %s, curr cost %s" %
(potential_lambdas, potential_cost, self.fmodel.current_cost))
while self._check_should_backtrack(
potential_cost, step_size,
lambda_derivatives) and step_size > self.step_size_min:
if potential_cost is None: # If can't find a solution, shrink faster
step_size *= self.shrink_factor**3
else:
step_size *= self.shrink_factor
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=True)
if potential_cost is not None:
self.log(
"(shrinking) potential_lambdas %s, cost %f, step, %f" %
(potential_lambdas, potential_cost, step_size))
else:
self.log("(shrinking) potential_lambdas None!")
if self.fmodel.current_cost < potential_cost:
self.log("COST IS INCREASING! %f" % potential_cost)
break
else:
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=False)
self.fmodel.update(potential_lambdas, potential_model_params,
potential_cost)
self.log("%s iter: %d step_size %f" %
(self.method_label, i, step_size))
self.log("current model %s" % self.fmodel)
self.log("cost_history %s" %
self.fmodel.cost_history[start_history_idx:])
self.log("current test cost %s" %
self.get_test_cost(self.fmodel.best_model_params))
self._print_model_details()
if self.fmodel.get_cost_diff() < self.decr_enough_threshold:
self.log("decrease amount too small %f" %
self.fmodel.get_cost_diff())
break
if step_size < self.step_size_min:
self.log("STEP SIZE TOO SMALL %f" % step_size)
break
sys.stdout.flush()
self.log("TOTAL ITERS %d" % i)
self.log("full_cost_hist: %s" %
self.fmodel.cost_history[start_history_idx:])
self.log("current_test_cost: %s" %
self.get_test_cost(self.fmodel.best_model_params))
def get_test_cost(self, model):
return None
def _print_model_details(self):
# fill in if you want to print more things
return
def _check_should_backtrack(self, potential_cost, step_size,
lambda_derivatives):
if potential_cost is None:
return True
backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm(
lambda_derivatives)**2
backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw
return potential_cost > backtrack_thres
def _run_potential_lambdas(self,
step_size,
lambda_derivatives,
quick_run=False):
potential_lambdas = self._get_updated_lambdas(step_size,
lambda_derivatives)
try:
potential_model_params = self._solve_wrapper(potential_lambdas,
quick_run=quick_run)
except cvxpy.error.SolverError:
potential_model_params = None
if self._any_model_params_none(potential_model_params):
potential_cost = None
else:
potential_cost = self.get_validate_cost(potential_model_params)
return potential_lambdas, potential_model_params, potential_cost
def _solve_wrapper(self, lambdas, quick_run):
start_solve_time = time.time()
model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run)
if quick_run is False:
self.fmodel.incr_num_solves()
self.log("solve runtime %f" % (time.time() - start_solve_time))
return model_params
def _get_lambda_derivatives_wrapper(self):
start_solve_time = time.time()
lambda_derivatives = self._get_lambda_derivatives()
self.log("lambda_derivatives runtime %f" %
(time.time() - start_solve_time))
self.log("lambda_derivatives %s" % lambda_derivatives)
return lambda_derivatives
def _get_updated_lambdas(self, method_step_size, lambda_derivatives):
current_lambdas = self.fmodel.current_lambdas
new_step_size = method_step_size
if self.use_boundary:
potential_lambdas = current_lambdas - method_step_size * lambda_derivatives
for idx in range(0, current_lambdas.size):
if current_lambdas[idx] > self.lambda_mins[
idx] and potential_lambdas[idx] < self.lambda_mins[idx]:
smaller_step_size = self.boundary_factor * (
current_lambdas[idx] -
self.lambda_mins[idx]) / lambda_derivatives[idx]
new_step_size = min(new_step_size, smaller_step_size)
self.log("USING THE BOUNDARY %f" % new_step_size)
return np.maximum(current_lambdas - new_step_size * lambda_derivatives,
self.lambda_mins)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
self.log_file.flush()
@staticmethod
def _any_model_params_none(model_params):
if model_params is None:
return True
else:
return any([m is None for m in model_params])
class Gradient_Descent_Algo:
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_descent_settings()
self._create_problem_wrapper()
self._create_lambda_configs()
def run(self, initial_lambda_set, debug=True, log_file=None):
self.log_file = log_file
start_time = time.time()
self.fmodel = Fitted_Model(initial_lambda_set[0].size)
best_cost = None
for initial_lambdas in initial_lambda_set:
self._run_lambdas(initial_lambdas, debug=debug) #, max_cost_at_iter=best_cost, check_iter=self.check_iter)
if best_cost < self.fmodel.best_cost:
self.log("%s: best start lambda %s" % (self.method_label, initial_lambdas))
runtime = time.time() - start_time
self.log("%s: runtime %s" % (self.method_label, runtime))
self.fmodel.set_runtime(runtime)
def _run_lambdas(self, initial_lambdas, debug=True): #, max_cost_at_iter=None, check_iter=None):
self.log("%s: initial_lambdas %s" % (self.method_label, initial_lambdas))
start_history_idx = len(self.fmodel.cost_history)
# warm up the problem
self._solve_wrapper(initial_lambdas, quick_run=True)
# do a real run now
model_params = self._solve_wrapper(initial_lambdas, quick_run=False)
# Check that no model params are None
if self._any_model_params_none(model_params):
self.log("ERROR: No model params fit for initial lambda values")
self.fmodel.update(initial_lambdas, None, None)
return
current_cost = self.get_validate_cost(model_params)
self.fmodel.update(initial_lambdas, model_params, current_cost)
self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost)
step_size = self.step_size_init
for i in range(0, self.num_iters):
lambda_derivatives = self._get_lambda_derivatives_wrapper()
if debug:
self._double_check_derivative(lambda_derivatives)
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size,
lambda_derivatives,
quick_run=True
)
# TODO: Do backtracking
while self._check_should_backtrack(potential_cost, step_size, lambda_derivatives) and step_size > self.step_size_min:
if potential_cost is None: # Then cvxpy couldn't find a solution. Shrink faster
step_size *= self.shrink_factor**3
else:
step_size *= self.shrink_factor
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size,
lambda_derivatives,
quick_run=True
)
if potential_cost is not None:
self.log("(shrinking) potential_lambdas %s, cost %f, step, %f" % (potential_lambdas, potential_cost, step_size))
else:
self.log("(shrinking) potential_lambdas None!")
if self.fmodel.current_cost < potential_cost:
self.log("COST IS INCREASING! %f" % potential_cost)
break
else:
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size,
lambda_derivatives,
quick_run=False
)
self.fmodel.update(potential_lambdas, potential_model_params, potential_cost)
self.log("%s iter: %d step_size %f" % (self.method_label, i, step_size))
self.log("current model %s" % self.fmodel)
self.log("cost_history %s" % self.fmodel.cost_history[start_history_idx:])
if self.fmodel.get_cost_diff() < self.decr_enough_threshold:
self.log("decrease amount too small %f" % self.fmodel.get_cost_diff())
break
if step_size < self.step_size_min:
self.log("STEP SIZE TOO SMALL %f" % step_size)
break
# if check_iter is not None and max_cost_at_iter is not None and check_iter == i and max_cost_at_iter < potential_cost:
# self.log("Cost %f higher than threshold %f" % (potential_cost, max_cost_at_iter))
# break
sys.stdout.flush()
self.log("TOTAL ITERS %d" % i)
self.log("%s" % self.fmodel.cost_history[start_history_idx:])
def _check_should_backtrack(self, potential_cost, step_size, lambda_derivatives):
if potential_cost is None:
return True
backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm(lambda_derivatives)**2
backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw
return potential_cost > backtrack_thres
def _run_potential_lambdas(self, step_size, lambda_derivatives, quick_run=False):
potential_lambdas = self._get_updated_lambdas(
step_size,
lambda_derivatives
)
try:
potential_model_params = self._solve_wrapper(potential_lambdas, quick_run=quick_run)
except cvxpy.error.SolverError:
potential_model_params = None
if self._any_model_params_none(potential_model_params):
potential_cost = None
else:
potential_cost = self.get_validate_cost(potential_model_params)
return potential_lambdas, potential_model_params, potential_cost
def _solve_wrapper(self, lambdas, quick_run):
start_solve_time = time.time()
model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run)
self.log("CVX runtime %f" % (time.time() - start_solve_time))
return model_params
def _get_lambda_derivatives_wrapper(self):
start_solve_time = time.time()
lambda_derivatives = self._get_lambda_derivatives()
self.log("lambda_derivatives runtime %f" % (time.time() - start_solve_time))
self.log("lambda_derivatives %s" % lambda_derivatives)
return lambda_derivatives
def _get_updated_lambdas(self, method_step_size, lambda_derivatives):
current_lambdas = self.fmodel.current_lambdas
new_step_size = method_step_size
if self.use_boundary:
potential_lambdas = current_lambdas - method_step_size * lambda_derivatives
for idx in range(0, current_lambdas.size):
if current_lambdas[idx] > self.lambda_mins[idx] and potential_lambdas[idx] < (1 - self.boundary_factor) * current_lambdas[idx]:
smaller_step_size = self.boundary_factor * current_lambdas[idx] / lambda_derivatives[idx]
new_step_size = min(new_step_size, smaller_step_size)
self.log("USING THE BOUNDARY %f" % new_step_size)
return np.maximum(current_lambdas - new_step_size * lambda_derivatives, self.lambda_mins)
def _double_check_derivative(self, calculated_derivative, accept_diff=1e-1, epsilon=1e-5):
# Returns the numeral derivative if you want it
deriv = []
num_lambdas = len(self.fmodel.current_lambdas)
for i in range(num_lambdas):
print "===========CHECK I= %d ===============" % i
# don't allow the discrete derivative perturb too much if the lambda value is low already
eps = min(epsilon, self.fmodel.current_lambdas[i]/100)
reg1 = np.copy(self.fmodel.current_lambdas)
reg1[i] += eps
model1 = self.problem_wrapper.solve(np.array(reg1), quick_run=False)
error1 = self.get_validate_cost(model1)
reg2 = np.copy(self.fmodel.current_lambdas)
reg2[i] -= eps
model2 = self.problem_wrapper.solve(np.array(reg2), quick_run=False)
error2 = self.get_validate_cost(model2)
i_deriv = (error1 - error2)/(epsilon * 2)
# print "numerical sum_dthetas_dlambda", np.sum((model1 - thetas2)/(epsilon * 2), axis=1)
print "calculated_derivative[i]", calculated_derivative[i]
print "numerical deriv", i_deriv
deriv.append(i_deriv)
print "np.abs(calculated_derivative[i] - i_deriv)", np.abs(calculated_derivative[i] - i_deriv)
relative_ok = np.abs((calculated_derivative[i] - i_deriv)/i_deriv) < accept_diff
absolute_ok = np.abs(calculated_derivative[i] - i_deriv) < accept_diff
assert(relative_ok or absolute_ok)
return np.hstack(deriv)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
@staticmethod
def _any_model_params_none(model_params):
if model_params is None:
return True
else:
return any([m is None for m in model_params])
class Gradient_Descent_Algo:
def __init__(self, data, settings=None):
self.data = data
self.settings = settings
self._create_descent_settings()
self._create_problem_wrapper()
self._create_lambda_configs()
def run(self, initial_lambda_set, debug=True, log_file=None):
self.log_file = log_file
start_time = time.time()
self.fmodel = Fitted_Model(initial_lambda_set[0].size)
best_cost = None
best_initial_lambdas = None
for initial_lambdas in initial_lambda_set:
self.log("%s: initial_lambdas %s" %
(self.method_label, initial_lambdas))
self._run_lambdas(
initial_lambdas, debug=debug
) #, max_cost_at_iter=best_cost, check_iter=self.check_iter)
if best_cost is None or best_cost > self.fmodel.best_cost:
best_cost = self.fmodel.best_cost
best_initial_lambdas = initial_lambdas
self.log("%s: best start lambda %s" %
(self.method_label, best_initial_lambdas))
runtime = time.time() - start_time
self.log("%s: runtime %s" % (self.method_label, runtime))
self.fmodel.set_runtime(runtime)
# self.fmodel.set_num_solves(len(self.fmodel.cost_history))
def _check_optimality_conditions(self, model_params, lambdas, thres=1):
return
def _run_lambdas(self,
initial_lambdas,
debug=True): #, max_cost_at_iter=None, check_iter=None):
start_history_idx = len(self.fmodel.cost_history)
# warm up the problem
self._solve_wrapper(initial_lambdas, quick_run=True)
# do a real run now
model_params = self._solve_wrapper(initial_lambdas, quick_run=False)
if debug:
self._check_optimality_conditions(model_params, initial_lambdas)
# Check that no model params are None
if self._any_model_params_none(model_params):
self.log("ERROR: No model params fit for initial lambda values")
self.fmodel.update(initial_lambdas, None, None)
return
current_cost = self.get_validate_cost(model_params)
self.fmodel.update(initial_lambdas, model_params, current_cost)
self.log("self.fmodel.current_cost %f" % self.fmodel.current_cost)
self._print_model_details()
step_size = self.step_size_init
for i in range(0, self.num_iters):
lambda_derivatives = self._get_lambda_derivatives_wrapper()
if debug:
self._double_check_derivative(lambda_derivatives)
1 / 0
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=True)
self.log(
"potential_lambdas %s, potential_cost %s, curr cost %s" %
(potential_lambdas, potential_cost, self.fmodel.current_cost))
while self._check_should_backtrack(
potential_cost, step_size,
lambda_derivatives) and step_size > self.step_size_min:
if potential_cost is None: # Then cvxpy couldn't find a solution. Shrink faster
step_size *= self.shrink_factor**3
else:
step_size *= self.shrink_factor
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=True)
if potential_cost is not None:
self.log(
"(shrinking) potential_lambdas %s, cost %f, step, %f" %
(potential_lambdas, potential_cost, step_size))
else:
self.log("(shrinking) potential_lambdas None!")
if self.fmodel.current_cost < potential_cost:
self.log("COST IS INCREASING! %f" % potential_cost)
break
else:
# Note to self: it is possible that solving to a lower accuracy results in lower validation loss
# but higher accuracy gives higher validation loss.
potential_lambdas, potential_model_params, potential_cost = self._run_potential_lambdas(
step_size, lambda_derivatives, quick_run=False)
if debug:
self._check_optimality_conditions(potential_model_params,
potential_lambdas)
self.fmodel.update(potential_lambdas, potential_model_params,
potential_cost)
self.log("%s iter: %d step_size %f" %
(self.method_label, i, step_size))
self.log("current model %s" % self.fmodel)
self.log("cost_history %s" %
self.fmodel.cost_history[start_history_idx:])
self.log("current test cost %s" %
self.get_test_cost(self.fmodel.best_model_params))
self._print_model_details()
if self.fmodel.get_cost_diff() < self.decr_enough_threshold:
self.log("decrease amount too small %f" %
self.fmodel.get_cost_diff())
break
if step_size < self.step_size_min:
self.log("STEP SIZE TOO SMALL %f" % step_size)
break
# if check_iter is not None and max_cost_at_iter is not None and check_iter == i and max_cost_at_iter < potential_cost:
# self.log("Cost %f higher than threshold %f" % (potential_cost, max_cost_at_iter))
# break
sys.stdout.flush()
self.log("TOTAL ITERS %d" % i)
self.log("full_cost_hist: %s" %
self.fmodel.cost_history[start_history_idx:])
self.log("current_test_cost: %s" %
self.get_test_cost(self.fmodel.best_model_params))
def get_test_cost(self, model):
return None
def _print_model_details(self):
# fill in if you want to print more things
return
def _check_should_backtrack(self, potential_cost, step_size,
lambda_derivatives):
if potential_cost is None:
return True
backtrack_thres_raw = self.fmodel.current_cost - self.backtrack_alpha * step_size * np.linalg.norm(
lambda_derivatives)**2
backtrack_thres = self.fmodel.current_cost if backtrack_thres_raw < 0 else backtrack_thres_raw
return potential_cost > backtrack_thres
def _run_potential_lambdas(self,
step_size,
lambda_derivatives,
quick_run=False):
potential_lambdas = self._get_updated_lambdas(step_size,
lambda_derivatives)
try:
potential_model_params = self._solve_wrapper(potential_lambdas,
quick_run=quick_run)
except cvxpy.error.SolverError:
potential_model_params = None
if self._any_model_params_none(potential_model_params):
potential_cost = None
else:
potential_cost = self.get_validate_cost(potential_model_params)
return potential_lambdas, potential_model_params, potential_cost
def _solve_wrapper(self, lambdas, quick_run):
start_solve_time = time.time()
model_params = self.problem_wrapper.solve(lambdas, quick_run=quick_run)
if quick_run is False:
self.fmodel.incr_num_solves()
self.log("CVX runtime %f" % (time.time() - start_solve_time))
return model_params
def _get_lambda_derivatives_wrapper(self):
start_solve_time = time.time()
lambda_derivatives = self._get_lambda_derivatives()
self.log("lambda_derivatives runtime %f" %
(time.time() - start_solve_time))
self.log("lambda_derivatives %s" % lambda_derivatives)
return lambda_derivatives
def _get_updated_lambdas(self, method_step_size, lambda_derivatives):
current_lambdas = self.fmodel.current_lambdas
new_step_size = method_step_size
if self.use_boundary:
potential_lambdas = current_lambdas - method_step_size * lambda_derivatives
for idx in range(0, current_lambdas.size):
if current_lambdas[idx] > self.lambda_mins[
idx] and potential_lambdas[idx] < self.lambda_mins[idx]:
smaller_step_size = self.boundary_factor * (
current_lambdas[idx] -
self.lambda_mins[idx]) / lambda_derivatives[idx]
new_step_size = min(new_step_size, smaller_step_size)
self.log("USING THE BOUNDARY %f" % new_step_size)
return np.maximum(current_lambdas - new_step_size * lambda_derivatives,
self.lambda_mins)
def _double_check_derivative_indepth(self, i, model1, model2, model0, eps):
# override this function if you want to do more detailed checking of the derivatives
# useful when you want to also double check the derivatives of the model parameters
# wrt to lambda penalty parameters
return
def _double_check_derivative(self,
calculated_derivative,
accept_diff=1e-1,
epsilon=1e-6):
num_lambdas = len(self.fmodel.current_lambdas)
print "self.fmodel.current_lambdas", self.fmodel.current_lambdas
numerical_derivs = []
for i in range(num_lambdas):
print "===========CHECK I= %d ===============" % i
# don't allow the discrete derivative perturb too much if the lambda value is low already
eps = min(epsilon, self.fmodel.current_lambdas[i] / 100)
reg1 = np.copy(self.fmodel.current_lambdas)
reg1[i] += eps
model1 = self.problem_wrapper.solve(np.array(reg1),
quick_run=False,
warm_start=False)
reg2 = np.copy(self.fmodel.current_lambdas)
reg2[i] -= eps
model2 = self.problem_wrapper.solve(np.array(reg2),
quick_run=False,
warm_start=False)
# Calculate derivative of validation error
error1 = self.get_validate_cost(model1)
error2 = self.get_validate_cost(model2)
i_deriv = (error1 - error2) / (eps * 2)
numerical_derivs.append(i_deriv)
print "********** jean calc derivative[i]", calculated_derivative[
i]
print "********** i_deriv", i_deriv
print "np.abs(calculated_derivative[i] - i_deriv)", np.abs(
calculated_derivative[i] - i_deriv)
relative_ok = np.abs(
(calculated_derivative[i] - i_deriv) / i_deriv) < accept_diff
absolute_ok = np.abs(calculated_derivative[i] -
i_deriv) < accept_diff
# model0 = self.problem_wrapper.solve(self.fmodel.current_lambdas, quick_run=False, warm_start=False)
# error0 = self.get_validate_cost(model0)
# self._double_check_derivative_indepth(i, model1, model2, model0, eps)
# self._double_check_derivative_indepth(i, model1, model2, None, eps)
# assert(relative_ok or absolute_ok)
print "calculated_derivative", calculated_derivative
print "numerical_derivs", numerical_derivs
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
self.log_file.flush()
@staticmethod
def _any_model_params_none(model_params):
if model_params is None:
return True
else:
return any([m is None for m in model_params])
def run(self, num_runs, log_file=None):
self.log_file = log_file
# Run spearmint to get next experiment parameters
self.fmodel = Fitted_Model(self.num_lambdas)
# Find new experiment
best_cost = None
runtime = 0
for i in range(num_runs):
start_time = time.time()
self.log("%s: iter %d" % (self.method_label, i))
self.run_spearmint_command(self.result_folder)
self.log("Spearmint command completed")
with open(self.result_file, 'r') as resfile:
newlines = []
for line in resfile.readlines():
values = line.split()
if len(values) < 3:
continue
val = values.pop(0)
dur = values.pop(0)
lambdas = np.array(
[10**float(v) for v in values[:self.num_lambdas]])
if (val == 'P'):
# P means pending experiment to run
# Run experiment
self.log("lambdas %s" % lambdas)
model_params = self.problem_wrapper.solve(
lambdas, quick_run=True)
if model_params is None:
current_cost = self.MAX_COST
else:
current_cost = self.get_validation_cost(
model_params)
if best_cost is None or best_cost > current_cost:
best_cost = current_cost
self.fmodel.update(lambdas, model_params,
current_cost)
self.log("fmodel: %s" % self.fmodel)
newlines.append(
str(current_cost) + " 0 " + " ".join(values) +
"\n")
else:
# Otherwise these are previous experiment results
newlines.append(line)
runtime += time.time() - start_time
# Don't record time spent on writing files?
with open(self.result_file, 'w') as resfile:
resfile.writelines(newlines)
sys.stdout.flush()
self.fmodel.set_runtime(runtime)
self.fmodel.set_num_solves(num_runs)
self.log("%s: runtime %s" % (self.method_label, runtime))
self.log("fmodel: %s" % self.fmodel)
self.log("fmodel best_model_params: %s" %
self.fmodel.best_model_params)
# VERY IMPORTANT to clean spearmint results
self.run_spearmint_clean(self.result_folder)
class Spearmint_Algo:
MAX_COST = 1e6
def __init__(self, data, folder_suffix, settings=None):
self.data = data
self.settings = settings
# Call problem wrapper first to determine number of lambdas
self._create_problem_wrapper()
self._check_make_configs(folder_suffix)
self.result_file = "%s/results.dat" % self.result_folder
def run(self, num_runs, log_file=None):
self.log_file = log_file
# Run spearmint to get next experiment parameters
self.fmodel = Fitted_Model(self.num_lambdas)
# Find new experiment
best_cost = None
runtime = 0
for i in range(num_runs):
start_time = time.time()
self.log("%s: iter %d" % (self.method_label, i))
self.run_spearmint_command(self.result_folder)
self.log("Spearmint command completed")
with open(self.result_file, 'r') as resfile:
newlines = []
for line in resfile.readlines():
values = line.split()
if len(values) < 3:
continue
val = values.pop(0)
dur = values.pop(0)
lambdas = np.array(
[10**float(v) for v in values[:self.num_lambdas]])
if (val == 'P'):
# P means pending experiment to run
# Run experiment
self.log("lambdas %s" % lambdas)
model_params = self.problem_wrapper.solve(
lambdas, quick_run=True)
if model_params is None:
current_cost = self.MAX_COST
else:
current_cost = self.get_validation_cost(
model_params)
if best_cost is None or best_cost > current_cost:
best_cost = current_cost
self.fmodel.update(lambdas, model_params,
current_cost)
self.log("fmodel: %s" % self.fmodel)
newlines.append(
str(current_cost) + " 0 " + " ".join(values) +
"\n")
else:
# Otherwise these are previous experiment results
newlines.append(line)
runtime += time.time() - start_time
# Don't record time spent on writing files?
with open(self.result_file, 'w') as resfile:
resfile.writelines(newlines)
sys.stdout.flush()
self.fmodel.set_runtime(runtime)
self.fmodel.set_num_solves(num_runs)
self.log("%s: runtime %s" % (self.method_label, runtime))
self.log("fmodel: %s" % self.fmodel)
self.log("fmodel best_model_params: %s" %
self.fmodel.best_model_params)
# VERY IMPORTANT to clean spearmint results
self.run_spearmint_clean(self.result_folder)
def log(self, log_str):
if self.log_file is None:
print log_str
else:
self.log_file.write("%s\n" % log_str)
def _check_make_configs(self, folder_suffix):
self.result_folder = "%s%s" % (self.result_folder_prefix,
folder_suffix)
if not os.path.exists(self.result_folder):
os.makedirs(self.result_folder)
config_file_name = "%s/config.json" % self.result_folder
if not os.path.exists(config_file_name):
with open(config_file_name, 'w') as config_file:
config_file.write(self._create_config_string(self.num_lambdas))
@staticmethod
@print_time
def run_spearmint_command(experiment_folder,
use_multiprocessing=True,
gridsize=20000):
## Note: not all machines have the capability of running spearmint without crashing.
## Gosset servers will crash!
multiprocessing_option = ""
if not use_multiprocessing:
multiprocessing_option = ",use_multiprocessing=0"
options = Namespace(chooser_module="GPEIOptChooser",
chooser_args="noiseless=1%s" %
multiprocessing_option,
grid_size=gridsize,
config_file="config.json",
results_file="results.dat",
grid_seed=1,
num_jobs=1,
max_finished_jobs=1000)
spearmint_lite.main_controller(options, [experiment_folder])
@staticmethod
def run_spearmint_clean(experiment_folder):
cmd = "rm %s/chooser* %s/results.dat" % (experiment_folder,
experiment_folder)
os.system(cmd)
@staticmethod
def _create_config_string(num_lambdas):
return '{"lambda": {"name": "lambda","type": "float","size":%d,"min": -6,"max": 2}}' % num_lambdas