def run(X_train, y_train, X_validate, y_validate, group_feature_sizes, pooled=True):
start = time.time()
if pooled:
problem_wrapper = GroupedLassoProblemWrapperSimple(X_train, y_train, group_feature_sizes)
init_regularizations = [1,1]
else:
# Note: this is really slow
init_regularizations = np.ones(len(group_feature_sizes) + 1)
problem_wrapper = GroupedLassoProblemWrapper(X_train, y_train, group_feature_sizes)
def get_validation_cost(regularization):
if np.any(regularization < 0):
return 10000
betas = problem_wrapper.solve(regularization)
validation_cost = testerror_grouped(X_validate, y_validate, betas)
return validation_cost
res = minimize(get_validation_cost, (init_regularizations), method='nelder-mead')
runtime = time.time() - start
best_beta = problem_wrapper.solve(res.x)
return best_beta, runtime
def run(X_train, y_train, X_validate, y_validate, group_feature_sizes):
infty_norm = 0
start_feature = 0
for group_feature_size in group_feature_sizes:
infty_norm = max(infty_norm, np.linalg.norm(X_train[:, start_feature : start_feature + group_feature_size].T * y_train, 2))
start_feature += group_feature_size
num_lambdas = 10
max_power = np.log(infty_norm)
min_power = np.log(LAMBDA_MIN_FACTOR * max_power)
lambda_guesses = np.power(np.e, np.arange(min_power, max_power, (max_power - min_power - 0.01) / (num_lambdas - 1)))
print "gridsearch: lambda_guesses", lambda_guesses
problem_wrapper = GroupedLassoProblemWrapperSimple(X_train, y_train, group_feature_sizes)
best_cost = 1e5
best_betas = []
best_regularization = [lambda_guesses[0], lambda_guesses[0]]
for l1 in lambda_guesses:
for l2 in lambda_guesses:
betas = problem_wrapper.solve([l1, l2], high_accur=False)
current_cost = testerror_grouped(X_validate, y_validate, betas)
if best_cost > current_cost:
best_cost = current_cost
best_betas = betas
best_regularization = [l1, l2]
print "best_cost so far", best_cost, "best_regularization", best_regularization
print "gridsearch: best_validation_error", best_cost
print "gridsearch: best lambdas:", best_regularization
return best_betas, best_cost
def run(X_train, y_train, X_validate, y_validate, group_feature_sizes):
infty_norm = 0
start_feature = 0
for group_feature_size in group_feature_sizes:
infty_norm = max(infty_norm, np.linalg.norm(X_train[:, start_feature : start_feature + group_feature_size].T * y_train, 2))
start_feature += group_feature_size
num_lambdas = 10
max_power = np.log(infty_norm)
min_power = np.log(LAMBDA_MIN_FACTOR * max_power)
lambda_guesses = np.power(np.e, np.arange(min_power, max_power, (max_power - min_power - 0.01) / (num_lambdas - 1)))
print "gridsearch: lambda_guesses", lambda_guesses
problem_wrapper = GroupedLassoProblemWrapperSimple(X_train, y_train, group_feature_sizes)
best_cost = 1e5
best_betas = []
best_regularization = [lambda_guesses[0], lambda_guesses[0]]
for l1 in lambda_guesses:
for l2 in lambda_guesses:
betas = problem_wrapper.solve([l1, l2], high_accur=False)
current_cost = testerror_grouped(X_validate, y_validate, betas)
if best_cost > current_cost:
best_cost = current_cost
best_betas = betas
best_regularization = [l1, l2]
print "best_cost so far", best_cost, "best_regularization", best_regularization
print "gridsearch: best_validation_error", best_cost
print "gridsearch: best lambdas:", best_regularization
return best_betas, best_cost
class SGL_Nelder_Mead_Simple(Nelder_Mead_Algo):
method_label = "SGL_Nelder_Mead_simple"
MAX_COST = 100000
def _create_problem_wrapper(self):
self.problem_wrapper = GroupedLassoProblemWrapperSimple(
self.data.X_train,
self.data.y_train,
self.settings.get_expert_group_sizes()
)
def get_validation_cost(self, lambdas):
# if any are not positive, then just return max value
for l in lambdas:
if l <= 0:
return self.MAX_COST
model_params = self.problem_wrapper.solve(lambdas, quick_run=True)
validation_cost = testerror_grouped(
self.data.X_validate,
self.data.y_validate,
model_params
)
self.log("validation_cost %f" % validation_cost)
return validation_cost
class SGL_Nelder_Mead_Simple(Nelder_Mead_Algo):
method_label = "SGL_Nelder_Mead_simple"
MAX_COST = 100000
def _create_problem_wrapper(self):
self.problem_wrapper = GroupedLassoProblemWrapperSimple(
self.data.X_train, self.data.y_train,
self.settings.get_expert_group_sizes())
def get_validation_cost(self, lambdas):
# if any are not positive, then just return max value
for l in lambdas:
if l <= 0:
return self.MAX_COST
model_params = self.problem_wrapper.solve(lambdas, quick_run=True)
validation_cost = testerror_grouped(self.data.X_validate,
self.data.y_validate, model_params)
self.log("validation_cost %f" % validation_cost)
return validation_cost
def _create_problem_wrapper(self):
self.problem_wrapper = GroupedLassoProblemWrapperSimple(
self.data.X_train,
self.data.y_train,
self.settings.get_expert_group_sizes()
)
def _create_problem_wrapper(self):
self.problem_wrapper = GroupedLassoProblemWrapperSimple(
self.data.X_train, self.data.y_train,
self.settings.get_expert_group_sizes())
def run_nesterov(X_train, y_train, X_validate, y_validate, group_feature_sizes, initial_lambda1=DEFAULT_LAMBDA):
def _get_accelerated_lambdas(curr_lambdas, prev_lambdas, iter_num):
return np.maximum(
curr_lambdas + (iter_num - 2) / (iter_num + 1.0) * (curr_lambdas - prev_lambdas),
np.minimum(curr_lambdas, MIN_LAMBDA)
)
method_label = "HillclimbGroupedLasso Nesterov"
curr_regularizations = np.array([initial_lambda1] * 2)
acc_regularizations = curr_regularizations
problem_wrapper = GroupedLassoProblemWrapperSimple(X_train, y_train, group_feature_sizes)
betas = problem_wrapper.solve(acc_regularizations)
best_cost = testerror_grouped(X_validate, y_validate, betas)
# track progression
cost_path = [best_cost]
method_step_size = STEP_SIZE
lambda_derivatives = _get_lambda_derivatives(X_train, y_train, X_validate, y_validate, betas, acc_regularizations)
prev_regularizations = curr_regularizations
# Perform Nesterov with adaptive restarts
i_max = 3
i_total = 0
while i_max > 2 and i_total < NUMBER_OF_ITERATIONS:
print "restart! with i_max", i_max
# curr_regularizations = acc_regularizations
potential_cost = best_cost * 10
while potential_cost > best_cost and method_step_size > MIN_METHOD_STEP:
pot_regularizations = _get_updated_lambdas(acc_regularizations, method_step_size, lambda_derivatives, use_boundary=True)
pot_betas = problem_wrapper.solve(pot_regularizations)
potential_cost = testerror_grouped(X_validate, y_validate, pot_betas)
if potential_cost > best_cost:
method_step_size *= SHRINK_SHRINK
if potential_cost > best_cost and method_step_size <= MIN_SHRINK:
break
print "Nesterov: found good step size", method_step_size
for i in range(2, NUMBER_OF_ITERATIONS + 1):
i_max = i
i_total += 1
if i != 2:
lambda_derivatives = _get_lambda_derivatives(X_train, y_train, X_validate, y_validate, betas, acc_regularizations)
if np.array_equal(lambda_derivatives, np.array([0] * lambda_derivatives.size)):
print method_label, "derivatives zero. break."
break
curr_regularizations = _get_updated_lambdas(acc_regularizations, method_step_size, lambda_derivatives, use_boundary=True)
pot_acc_regularizations = _get_accelerated_lambdas(curr_regularizations, prev_regularizations, i)
prev_regularizations = curr_regularizations
potential_betas = problem_wrapper.solve(pot_acc_regularizations)
current_cost = testerror_grouped(X_validate, y_validate, potential_betas)
is_decreasing_significantly = best_cost - current_cost > DECREASING_ENOUGH_THRESHOLD
if current_cost < best_cost:
best_cost = current_cost
betas = potential_betas
acc_regularizations = pot_acc_regularizations
cost_path.append(current_cost)
if not is_decreasing_significantly:
if best_cost < current_cost:
print method_label, "Cost going up!", best_cost - current_cost
else:
print method_label, "decreasing too slow", best_cost - current_cost
break
print method_label, "iter", i, "current cost", best_cost, "lambdas:", acc_regularizations
print method_label, "best cost", best_cost, "best lambdas:", acc_regularizations
return betas, cost_path
def run(X_train, y_train, X_validate, y_validate, group_feature_sizes, initial_lambda1=DEFAULT_LAMBDA):
method_step_size = STEP_SIZE
method_label = HC_GROUPED_LASSO_LABEL
curr_regularizations = np.array([initial_lambda1] * 2)
problem_wrapper = GroupedLassoProblemWrapperSimple(X_train, y_train, group_feature_sizes)
betas = problem_wrapper.solve(curr_regularizations)
best_beta = betas
best_cost = testerror_grouped(X_validate, y_validate, betas)
print "first regularization", curr_regularizations, "first cost", best_cost
# track progression
cost_path = [best_cost]
shrink_factor = 1
for i in range(0, NUMBER_OF_ITERATIONS):
lambda_derivatives = _get_lambda_derivatives(X_train, y_train, X_validate, y_validate, betas, curr_regularizations)
# do the gradient descent!
pot_lambdas = _get_updated_lambdas(curr_regularizations, shrink_factor * method_step_size, lambda_derivatives)
# get corresponding beta
pot_betas = problem_wrapper.solve(pot_lambdas)
try:
pot_cost = testerror_grouped(X_validate, y_validate, pot_betas)
except ValueError as e:
print "value error", e
pot_cost = 1e10
while pot_cost > best_cost - BACKTRACK_ALPHA * shrink_factor * method_step_size * np.linalg.norm(lambda_derivatives)**2 and shrink_factor > MIN_SHRINK:
shrink_factor *= SHRINK_SHRINK
pot_lambdas = _get_updated_lambdas(curr_regularizations, shrink_factor * method_step_size, lambda_derivatives)
pot_betas = problem_wrapper.solve(pot_lambdas)
try:
pot_cost = testerror_grouped(X_validate, y_validate, pot_betas)
except ValueError as e:
print "value error", e
pot_cost = 1e10
print "shrink!", shrink_factor, "pot_cost", pot_cost, "pot_lambdas", pot_lambdas
is_decreasing_signficantly = best_cost - pot_cost > DECREASING_ENOUGH_THRESHOLD
betas = pot_betas
curr_regularizations = pot_lambdas
if pot_cost < best_cost:
best_cost = pot_cost
best_beta = betas
if not is_decreasing_signficantly:
print "is_decreasing_signficantly NO!", pot_cost - best_cost
break
if shrink_factor <= MIN_SHRINK:
print method_label, "shrink factor too small"
break
# track progression
cost_path.append(pot_cost)
print method_label, "iter", i, "best cost", best_cost, "lambdas:", curr_regularizations
print method_label, "best cost", best_cost, "lambdas:", curr_regularizations
return best_beta, cost_path
