def group_fs(X, y, z1, z2, idx, **kwargs): """ This function implements supervised sparse group feature selection with least square loss, i.e., min_{w} ||Xw-y||_2^2 + z_1||w||_1 + z_2*sum_{i} h_{i}||w_{G_{i}}|| where h_i is the weight for the i-th group Input ----- X: {numpy array}, shape (n_samples, n_features) input data y: {numpy array}, shape (n_samples,) input class labels or regression target z1: {float} regularization parameter of L1 norm for each element z2: {float} regularization parameter of L2 norm for the non-overlapping group idx: {numpy array}, shape (3, n_nodes) 3*nodes matrix, where nodes denotes the number of groups idx[1,:] contains the starting index of a group idx[2,: contains the ending index of a group idx[3,:] contains the corresponding weight (w_{j}) kwargs: {dictionary} verbose: {boolean} True if user want to print out the objective function value in each iteration, false if not Output ------ w: {numpy array}, shape (n_features, ) weight matrix obj: {numpy array}, shape (n_iterations, ) objective function value during iterations value_gamma: {numpy array}, shape (n_iterations, ) suitable step size during iterations Reference --------- Liu, Jun, et al. "Moreau-Yosida Regularization for Grouped Tree Structure Learning." NIPS. 2010. Liu, Jun, et al. "SLEP: Sparse Learning with Efficient Projections." http://www.public.asu.edu/~jye02/Software/SLEP, 2009. """ if 'verbose' not in kwargs: verbose = False else: verbose = kwargs['verbose'] # starting point initialization n_samples, n_features = X.shape # compute X'y Xty = np.dot(np.transpose(X), y) # initialize a starting point w = np.zeros(n_features) # compute Xw = X*w Xw = np.dot(X, w) # starting the main program, the Armijo Goldstein line search scheme + accelerated gradient descent # initialize step size gamma = 1 gamma = 1 # assign wp with w, and Xwp with Xw Xwp = Xw wwp = np.zeros(n_features) alphap = 0 alpha = 1 # indicates whether the gradient step only changes a little flag = False max_iter = 1000 value_gamma = np.zeros(max_iter) obj = np.zeros(max_iter) for iter_step in range(max_iter): # step1: compute search point s based on wp and w (with beta) beta = (alphap - 1) / alpha s = w + beta * wwp # step2: line search for gamma and compute the new approximation solution w Xs = Xw + beta * (Xw - Xwp) # compute X'* Xs XtXs = np.dot(np.transpose(X), Xs) # obtain the gradient g G = XtXs - Xty # copy w and Xw to wp and Xwp wp = w Xwp = Xw while True: # let s walk in a step in the antigradient of s to get v and then do the L1/L2-norm regularized projection v = s - G / gamma # tree overlapping group lasso projection n_nodes = int(idx.shape[1]) idx_tmp = np.zeros((3, n_nodes + 1)) idx_tmp[0:2, :] = np.concatenate( (np.array([[-1], [-1]]), idx[0:2, :]), axis=1) idx_tmp[2, :] = np.concatenate( (np.array([z1 / gamma]), z2 / gamma * idx[2, :]), axis=1) w = tree_lasso_projection(v, n_features, idx_tmp, n_nodes + 1) # the difference between the new approximate solution w and the search point s v = w - s # compute Xw = X*w Xw = np.dot(X, w) Xv = Xw - Xs r_sum = np.inner(v, v) l_sum = np.inner(Xv, Xv) # determine weather the gradient step makes little improvement if r_sum <= 1e-20: flag = True break # the condition is ||Xv||_2^2 <= gamma * ||v||_2^2 if l_sum <= r_sum * gamma: break else: gamma = max(2 * gamma, l_sum / r_sum) value_gamma[iter_step] = gamma # step3: update alpha and alphap, and check weather converge alphap = alpha alpha = (1 + math.sqrt(4 * alpha * alpha + 1)) / 2 wwp = w - wp Xwy = Xw - y # calculate the regularization part idx_tmp = np.zeros((3, n_nodes + 1)) idx_tmp[0:2, :] = np.concatenate((np.array([[-1], [-1]]), idx[0:2, :]), axis=1) idx_tmp[2, :] = np.concatenate((np.array([z1]), z2 * idx[2, :]), axis=1) tree_norm_val = tree_norm(w, n_features, idx_tmp, n_nodes + 1) # function value = loss + regularization obj[iter_step] = np.inner(Xwy, Xwy) / 2 + tree_norm_val if verbose: print('obj at iter ' + str(iter_step + 1) + ': ' + str(obj[iter_step])) if flag is True: break # determine weather converge if iter_step >= 2 and math.fabs(obj[iter_step] - obj[iter_step - 1]) < 1e-3: break return w, obj, value_gamma
def group_fs(X, y, z1, z2, idx, **kwargs): """ This function implements supervised sparse group feature selection with least square loss, i.e., min_{w} ||Xw-y||_2^2 + z_1||w||_1 + z_2*sum_{i} h_{i}||w_{G_{i}}|| where h_i is the weight for the i-th group Input ----- X: {numpy array}, shape (n_samples, n_features) input data y: {numpy array}, shape (n_samples,) input class labels or regression target z1: {float} regularization parameter of L1 norm for each element z2: {float} regularization parameter of L2 norm for the non-overlapping group idx: {numpy array}, shape (3, n_nodes) 3*nodes matrix, where nodes denotes the number of groups idx[1,:] contains the starting index of a group idx[2,: contains the ending index of a group idx[3,:] contains the corresponding weight (w_{j}) kwargs: {dictionary} verbose: {boolean} True if user want to print out the objective function value in each iteration, false if not Output ------ w: {numpy array}, shape (n_features, ) weight matrix obj: {numpy array}, shape (n_iterations, ) objective function value during iterations value_gamma: {numpy array}, shape (n_iterations, ) suitable step size during iterations Reference --------- Liu, Jun, et al. "Moreau-Yosida Regularization for Grouped Tree Structure Learning." NIPS. 2010. Liu, Jun, et al. "SLEP: Sparse Learning with Efficient Projections." http://www.public.asu.edu/~jye02/Software/SLEP, 2009. """ if 'verbose' not in kwargs: verbose = False else: verbose = kwargs['verbose'] # starting point initialization n_samples, n_features = X.shape # compute X'y Xty = np.dot(np.transpose(X), y) # initialize a starting point w = np.zeros(n_features) # compute Xw = X*w Xw = np.dot(X, w) # starting the main program, the Armijo Goldstein line search scheme + accelerated gradient descent # initialize step size gamma = 1 gamma = 1 # assign wp with w, and Xwp with Xw Xwp = Xw wwp = np.zeros(n_features) alphap = 0 alpha = 1 # indicates whether the gradient step only changes a little flag = False max_iter = 1000 value_gamma = np.zeros(max_iter) obj = np.zeros(max_iter) for iter_step in range(max_iter): # step1: compute search point s based on wp and w (with beta) beta = (alphap-1)/alpha s = w + beta*wwp # step2: line search for gamma and compute the new approximation solution w Xs = Xw + beta*(Xw - Xwp) # compute X'* Xs XtXs = np.dot(np.transpose(X), Xs) # obtain the gradient g G = XtXs - Xty # copy w and Xw to wp and Xwp wp = w Xwp = Xw while True: # let s walk in a step in the antigradient of s to get v and then do the L1/L2-norm regularized projection v = s - G/gamma # tree overlapping group lasso projection n_nodes = int(idx.shape[1]) idx_tmp = np.zeros((3, n_nodes+1)) idx_tmp[0:2, :] = np.concatenate((np.array([[-1], [-1]]), idx[0:2, :]), axis=1) idx_tmp[2, :] = np.concatenate((np.array([z1/gamma]), z2/gamma*idx[2, :]), axis=1) w = tree_lasso_projection(v, n_features, idx_tmp, n_nodes+1) # the difference between the new approximate solution w and the search point s v = w - s # compute Xw = X*w Xw = np.dot(X, w) Xv = Xw - Xs r_sum = np.inner(v, v) l_sum = np.inner(Xv, Xv) # determine weather the gradient step makes little improvement if r_sum <= 1e-20: flag = True break # the condition is ||Xv||_2^2 <= gamma * ||v||_2^2 if l_sum <= r_sum*gamma: break else: gamma = max(2*gamma, l_sum/r_sum) value_gamma[iter_step] = gamma # step3: update alpha and alphap, and check weather converge alphap = alpha alpha = (1+math.sqrt(4*alpha*alpha+1))/2 wwp = w - wp Xwy = Xw -y # calculate the regularization part idx_tmp = np.zeros((3, n_nodes+1)) idx_tmp[0:2, :] = np.concatenate((np.array([[-1], [-1]]), idx[0:2, :]), axis=1) idx_tmp[2, :] = np.concatenate((np.array([z1]), z2*idx[2, :]), axis=1) tree_norm_val = tree_norm(w, n_features, idx_tmp, n_nodes+1) # function value = loss + regularization obj[iter_step] = np.inner(Xwy, Xwy)/2 + tree_norm_val if verbose: print('obj at iter ' + str(iter_step+1) + ': ' + str(obj[iter_step])) if flag is True: break # determine weather converge if iter_step >= 2 and math.fabs(obj[iter_step] - obj[iter_step-1]) < 1e-3: break return w, obj, value_gamma
def tree_fs(X, y, z, idx, verbose=False, **kwargs): """ This function implements tree structured group lasso regularization with least square loss, i.e., min_{w} ||Xw-Y||_2^2 + z\sum_{i}\sum_{j} h_{j}^{i}|||w_{G_{j}^{i}}|| where h_{j}^{i} is the weight for the j-th group from the i-th level (the root node is in level 0) Input ----- X: {numpy array}, shape (n_samples, n_features) input data y: {numpy array}, shape (n_samples,) input class labels or regression target z: {float} regularization parameter of L2 norm for the non-overlapping group idx: {numpy array}, shape (3, n_nodes) 3*nodes matrix, where nodes denotes the number of nodes of the tree idx(1,:) contains the starting index idx(2,:) contains the ending index idx(3,:) contains the corresponding weight (w_{j}) kwargs: {dictionary} verbose: {boolean} True if user want to print out the objective function value in each iteration, false if not Output ------ w: {numpy array}, shape (n_features,) weight vector obj: {numpy array}, shape (n_iterations,) objective function value during iterations value_gamma: {numpy array}, shape (n_iterations,) suitable step size during iterations Note for input parameter idx: (1) For idx, if each entry in w is a leaf node of the tree and the weight for this leaf node are the same, then idx[0,0] = -1 and idx[1,0] = -1, idx[2,0] denotes the common weight (2) In idx, the features of the left tree is smaller than the right tree (idx[0,i] is always smaller than idx[1,i]) Reference: Liu, Jun, et al. "Moreau-Yosida Regularization for Grouped Tree Structure Learning." NIPS. 2010. Liu, Jun, et al. "SLEP: Sparse Learning with Efficient Projections." http://www.public.asu.edu/~jye02/Software/SLEP, 2009. """ # starting point initialization n_samples, n_features = X.shape # compute X'y Xty = np.dot(np.transpose(X), y) # initialize a starting point w = np.zeros(n_features) # compute Xw = X*w Xw = np.dot(X, w) # starting the main program, the Armijo Goldstein line search scheme + accelerated gradient descent # initialize step size gamma = 1 gamma = 1 # assign wp with w, and Xwp with Xw Xwp = Xw wwp = np.zeros(n_features) alphap = 0 alpha = 1 # indicates whether the gradient step only changes a little flag = False max_iter = 1000 value_gamma = np.zeros(max_iter) obj = np.zeros(max_iter) for iter_step in range(max_iter): # step1: compute search point s based on wp and w (with beta) beta = old_div((alphap - 1), alpha) s = w + beta * wwp # step2: line search for gamma and compute the new approximation solution w Xs = Xw + beta * (Xw - Xwp) # compute X'* Xs XtXs = np.dot(np.transpose(X), Xs) # obtain the gradient g G = XtXs - Xty # copy w and Xw to wp and Xwp wp = w Xwp = Xw while True: # let s walk in a step in the antigradient of s to get v and then do the L1/L2-norm regularized projection v = s - old_div(G, gamma) # tree overlapping group lasso projection n_nodes = int(idx.shape[1]) idx_tmp = idx.copy() idx_tmp[2, :] = idx[2, :] * z / gamma w = tree_lasso_projection(v, n_features, idx_tmp, n_nodes) # the difference between the new approximate solution w and the search point s v = w - s # compute Xw = X*w Xw = np.dot(X, w) Xv = Xw - Xs r_sum = np.inner(v, v) l_sum = np.inner(Xv, Xv) # determine weather the gradient step makes little improvement if r_sum <= 1e-20: flag = True break # the condition is ||Xv||_2^2 <= gamma * ||v||_2^2 if l_sum <= r_sum * gamma: break else: gamma = max(2 * gamma, old_div(l_sum, r_sum)) value_gamma[iter_step] = gamma # step3: update alpha and alphap, and check weather converge alphap = alpha alpha = old_div((1 + math.sqrt(4 * alpha * alpha + 1)), 2) wwp = w - wp Xwy = Xw - y # calculate the regularization part tree_norm_val = tree_norm(w, n_features, idx, n_nodes) # function value = loss + regularization obj[iter_step] = old_div(np.inner(Xwy, Xwy), 2) + z * tree_norm_val if verbose: print('obj at iter ' + str(iter_step + 1) + ': ' + str(obj[iter_step])) if flag is True: break # determine whether converge if iter_step >= 2 and math.fabs(obj[iter_step] - obj[iter_step - 1]) < 1e-3: break return w, obj, value_gamma