def train_test_one(X,Y,X_tes,Y_tes,Y_tes_raw, set_id, group_size, lam, query_starts): v_groups, v_costs = yahoo_common.load_group(group_size) groups = v_groups[0] sorted_groups = np.array(sorted(groups)) costs = v_costs[0] (W, optim_info) = spams_train(X, Y, groups, costs, lam) loss = opt.loss(W[:,0], X_tes, Y_tes[:,0]) budget = np.sum(costs[ list(set(sorted_groups[np.nonzero(W)[0]])) ]) ndcg_val = ndcg_overall(X_tes.dot(W[:,0]), Y_tes_raw, query_starts) return loss, budget, ndcg_val, W[:,0]
def train_test_one(X, Y, X_tes, Y_tes, set_id, lam):
groups, costs = grain_common.load_group()
sorted_groups = np.array(sorted(groups))
(W, optim_info) = spams_train(X, Y, groups, costs, lam)
loss = opt.loss(W[:, 0], X_tes, Y_tes)
budget = np.sum(costs[list(set(sorted_groups[np.nonzero(W)[0]]))])
err = np.sum((np.sign(Y_tes) *
(Y_tes - X_tes.dot(W[:, 0]))) > 0.5) / np.float64(len(Y_tes))
np.savez('grain_results/spams_%d_lam_%f.npz' % (set_id, lam),
budget=budget,
loss=loss,
err=err,
models=W[:, 0])
return loss, budget, err, W[:, 0]
def batch_all_old(X,Y, X_tes, Y_tes, set_id, group_size):
nbr_train = X.shape[0]
v_lam = np.array([ 1e-1, 1e-2,
6.7e-3, 3.3e-3, 1e-3,
8e-4, 6e-4, 4e-4, 2e-4, 1e-4,
8e-5, 5e-5, 1e-5 ]) * nbr_train
budget = np.zeros(len(v_lam))
loss = np.zeros(len(v_lam))
sorted_groups = np.array(sorted(groups))
v_W = []
v_optim_info = []
for i, lam in enumerate(v_lam):
(W, optim_info) = spams_train(X, Y, groups, costs, lam)
budget[i] = np.sum(costs[ list(set(sorted_groups[np.nonzero(W)[0]])) ])
loss[i] = opt.loss(W[:,0], X_tes, Y_tes[:,0])
v_W.append(W)
v_optim_info.append(optim_info)
np.savez('yahoo_results/spams_%d_%d.npz' % (set_id, group_size), v_W=v_W, v_optim_info=v_optim_info, budgets=budget, losses=loss)
for method_idx, method in enumerate(methods):
l.append([])
err.append([])
auc.append(0)
costs.append([])
d_omp = d[method]
d_selected = d_omp['selected']
d_w = d_omp['w']
d_cost = d_omp['cost']
for idx, _ in enumerate(d_cost):
selected = d_selected[idx]
w = d_w[idx]
selected_X = X[:, selected]
if selected_X.shape[1] > 0:
l[-1].append(opt.loss(w, selected_X, Y))
err[-1].append(
np.float64(
np.sum((np.sign(Y) * (Y - selected_X.dot(w)) > 0.5))) /
Y.shape[0])
if ignore_cost:
costs[-1].append(
np.sum(group_costs[np.unique(groups[selected])]))
else:
costs[-1].append(d_cost[idx])
auc[-1] += (costs[-1][-1] - costs[-1][-2]) * (l[-1][-2] +
l[-1][-1]) / 2.0
else:
l[-1].append(opt.loss(0, np.zeros(Y.shape[0]), Y))
err[-1].append(1.0)
costs[-1].append(0)