def test_dgapl21l1(): """Test duality gap for L21 + L1 regularization.""" n_orient = 2 M, G, active_set = _generate_tf_data() n_times = M.shape[1] n_sources = G.shape[1] tstep, wsize = 4, 32 n_steps = int(np.ceil(n_times / float(tstep))) n_freqs = wsize // 2 + 1 n_coefs = n_steps * n_freqs phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freqs, n_steps, n_times) for l1_ratio in [0.05, 0.1]: alpha_max = norm_epsilon_inf(G, M, phi, l1_ratio, n_orient) alpha_space = (1. - l1_ratio) * alpha_max alpha_time = l1_ratio * alpha_max Z = np.zeros([n_sources, n_coefs]) shape = (-1, n_steps, n_freqs) # for alpha = alpha_max, Z = 0 is the solution so the dgap is 0 gap = dgap_l21l1(M, G, Z, np.ones(n_sources, dtype=bool), alpha_space, alpha_time, phi, phiT, shape, n_orient, -np.inf)[0] assert_allclose(0., gap) # check that solution for alpha smaller than alpha_max is non 0: X_hat_tf, active_set_hat_tf, E, gap = tf_mixed_norm_solver( M, G, alpha_space / 1.01, alpha_time / 1.01, maxit=200, tol=1e-8, verbose=True, debias=False, n_orient=n_orient, tstep=tstep, wsize=wsize, return_gap=True) # allow possible small numerical errors (negative gap) assert_array_less(-1e-10, gap) assert_array_less(gap, 1e-8) assert_array_less(1, len(active_set_hat_tf)) X_hat_tf, active_set_hat_tf, E, gap = tf_mixed_norm_solver( M, G, alpha_space / 5., alpha_time / 5., maxit=200, tol=1e-8, verbose=True, debias=False, n_orient=n_orient, tstep=tstep, wsize=wsize, return_gap=True) assert_array_less(-1e-10, gap) assert_array_less(gap, 1e-8) assert_array_less(1, len(active_set_hat_tf))
def test_norm_epsilon(): """Test computation of espilon norm on TF coefficients.""" tstep = np.array([2]) wsize = np.array([4]) n_times = 10 n_steps = np.ceil(n_times / tstep.astype(float)).astype(int) n_freqs = wsize // 2 + 1 n_coefs = n_steps * n_freqs phi = _Phi(wsize, tstep, n_coefs, n_times) Y = np.zeros(n_steps * n_freqs) l1_ratio = 0.03 assert_allclose(norm_epsilon(Y, l1_ratio, phi), 0.) Y[0] = 2. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) l1_ratio = 1. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) # dummy value without random: Y = np.arange(n_steps * n_freqs).reshape(-1, ) l1_ratio = 0.0 assert_allclose( norm_epsilon(Y, l1_ratio, phi)**2, stft_norm2(Y.reshape(-1, n_freqs[0], n_steps[0]))) l1_ratio = 0.03 # test that vanilla epsilon norm = weights equal to 1 w_time = np.ones(n_coefs[0]) Y = np.abs(np.random.randn(n_coefs[0])) assert_allclose(norm_epsilon(Y, l1_ratio, phi), norm_epsilon(Y, l1_ratio, phi, w_time=w_time)) # scaling w_time and w_space by the same amount should divide # epsilon norm by the same amount Y = np.arange(n_coefs) + 1 mult = 2. assert_allclose( norm_epsilon(Y, l1_ratio, phi, w_space=1, w_time=np.ones(n_coefs)) / mult, norm_epsilon(Y, l1_ratio, phi, w_space=mult, w_time=mult * np.ones(n_coefs)))
def test_dgapl21l1(): """Test duality gap for L21 + L1 regularization.""" n_orient = 2 M, G, active_set = _generate_tf_data() n_times = M.shape[1] n_sources = G.shape[1] tstep, wsize = np.array([4, 2]), np.array([64, 16]) n_steps = np.ceil(n_times / tstep.astype(float)).astype(int) n_freqs = wsize // 2 + 1 n_coefs = n_steps * n_freqs phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freqs, n_steps, n_times) for l1_ratio in [0.05, 0.1]: alpha_max = norm_epsilon_inf(G, M, phi, l1_ratio, n_orient) alpha_space = (1. - l1_ratio) * alpha_max alpha_time = l1_ratio * alpha_max Z = np.zeros([n_sources, phi.n_coefs.sum()]) # for alpha = alpha_max, Z = 0 is the solution so the dgap is 0 gap = dgap_l21l1(M, G, Z, np.ones(n_sources, dtype=bool), alpha_space, alpha_time, phi, phiT, n_orient, -np.inf)[0] assert_allclose(0., gap) # check that solution for alpha smaller than alpha_max is non 0: X_hat_tf, active_set_hat_tf, E, gap = tf_mixed_norm_solver( M, G, alpha_space / 1.01, alpha_time / 1.01, maxit=200, tol=1e-8, verbose=True, debias=False, n_orient=n_orient, tstep=tstep, wsize=wsize, return_gap=True) # allow possible small numerical errors (negative gap) assert_array_less(-1e-10, gap) assert_array_less(gap, 1e-8) assert_array_less(1, len(active_set_hat_tf)) X_hat_tf, active_set_hat_tf, E, gap = tf_mixed_norm_solver( M, G, alpha_space / 5., alpha_time / 5., maxit=200, tol=1e-8, verbose=True, debias=False, n_orient=n_orient, tstep=tstep, wsize=wsize, return_gap=True) assert_array_less(-1e-10, gap) assert_array_less(gap, 1e-8) assert_array_less(1, len(active_set_hat_tf))
def test_norm_epsilon(): """Test computation of espilon norm on TF coefficients.""" tstep = np.array([2]) wsize = np.array([4]) n_times = 10 n_steps = np.ceil(n_times / tstep.astype(float)).astype(int) n_freqs = wsize // 2 + 1 n_coefs = n_steps * n_freqs phi = _Phi(wsize, tstep, n_coefs) Y = np.zeros(n_steps * n_freqs) l1_ratio = 0.5 assert_allclose(norm_epsilon(Y, l1_ratio, phi), 0.) Y[0] = 2. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) l1_ratio = 1. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) # dummy value without random: Y = np.arange(n_steps * n_freqs).reshape(-1, ) l1_ratio = 0. assert_allclose(norm_epsilon(Y, l1_ratio, phi) ** 2, stft_norm2(Y.reshape(-1, n_freqs[0], n_steps[0])))
def test_norm_epsilon(): """Test computation of espilon norm on TF coefficients.""" tstep = np.array([2]) wsize = np.array([4]) n_times = 10 n_steps = np.ceil(n_times / tstep.astype(float)).astype(int) n_freqs = wsize // 2 + 1 n_coefs = n_steps * n_freqs phi = _Phi(wsize, tstep, n_coefs) Y = np.zeros(n_steps * n_freqs) l1_ratio = 0.5 assert_allclose(norm_epsilon(Y, l1_ratio, phi), 0.) Y[0] = 2. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) l1_ratio = 1. assert_allclose(norm_epsilon(Y, l1_ratio, phi), np.max(Y)) # dummy value without random: Y = np.arange(n_steps * n_freqs).reshape(-1, ) l1_ratio = 0. assert_allclose(norm_epsilon(Y, l1_ratio, phi) ** 2, stft_norm2(Y.reshape(-1, n_freqs[0], n_steps[0])))
snr = Simu_obj['snr'] X = Simu_obj['X'] coef = Simu_obj['coef'] Z_true = Simu_obj['Z_true'] n_channels, n_times = evoked_list[0].data.shape n_trials = len(evoked_list) wsize, tstep = 16,4 n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq p = 2 n_coefs_all = n_coefs*p phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) G = fwd['sol']['data'] n_dipoles = G.shape[1] DipoleGroup = list([label_ind[0], label_ind[1]]) non_label_ind = np.arange(0,n_dipoles,1) for i in range(len(label_ind)): non_label_ind = np.setdiff1d(non_label_ind, label_ind[i]) for i in range(len(non_label_ind)): DipoleGroup.append(np.array([non_label_ind[i]])) true_active_set = np.union1d(label_ind[0], label_ind[1]) DipoleGroupWeight = np.ones(len(DipoleGroup))/np.float(len(DipoleGroup))
def solve_stft_regression_L2_tsparse(M,G_list, G_ind, X, Z0, active_set_z0, active_t_ind_z0, coef_non_zero_mat, wsize=16, tstep = 4, delta = 0, maxit=200, tol = 1e-3,lipschitz_constant = None, Flag_backtrack = True, eta = 1.5, L0 = 1.0, Flag_verbose = False): """ Use the accelerated gradient descent (exactly FISTA without non-smooth penalty) to find the solution given an active set min 1/2||R||_F^2 + delta ||Z||_F^2 Input: M, [n_channels, n_times, n_trials] array of the sensor data G_list, a list of [n_channels, n_dipoles] forward gain matrix G_ind, [n_trial], marks the index of G for this run X, [n_trials, p],the design matrix, it must include an all 1 colume Z0, [n_active_dipoles, p*n_freq * n_active_step] acitve_set_z0, [n_dipoles,] a boolean array, indicating the active set of dipoles active_t_ind_z0, [n_step, ] a boolean array, indicating the active set of time points, the union of all frequencies, columns of X, and dipoles coef_non_zero_mat,[n_active_dipoles,n_coef*pq] boolean matrix, since some active_set_z and active_t_ind_z is a super set of the active set, but we assume that it is the UNION of each coefficient of X and all trials wsize, window size of the STFT tstep, length of the time step delta, the regularization parameter maxit, maximum number of iteration allowed tol, tolerance of the objective function lipschitz_constant, the lipschitz constant No Flag trial by trial is allowed in this version """ n_sensors, n_times, q= M.shape n_dipoles = G_list[0].shape[1] p = X.shape[1] n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq #coef_non_zero_mat_full = np.tile(coef_non_zero_mat,[1,pq]) coef_non_zero_mat_full = coef_non_zero_mat phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) sparse_phi = sparse_Phi(wsize, tstep) sparse_phiT = sparse_PhiT(tstep, n_freq, n_step, n_times) if (active_t_ind_z0.sum()*n_freq*p != Z0.shape[1] or n_dipoles!= len(active_set_z0)): print active_t_ind_z0.sum()*n_freq*p, Z0.shape[1],n_dipoles,len(active_set_z0) raise ValueError("wrong number of dipoles or coefs") if lipschitz_constant is None and not Flag_backtrack: lipschitz_constant = 1.1* (get_lipschitz_const(M,G_list[0],X,phi,phiT, Flag_trial_by_trial = False,n_coefs = n_coefs, tol = 1e-3) +2.0*delta) print "lipschitz_constant = %e" % lipschitz_constant if Flag_backtrack: L = L0 else: L = lipschitz_constant #initialization active_t_ind_z = active_t_ind_z0.copy() active_set_z = active_set_z0.copy() Z = Z0.copy() Z[coef_non_zero_mat_full==0] = 0 Y = Z0.copy() Y[coef_non_zero_mat_full==0] = 0 n_active_dipole = active_set_z.sum() # number of coeficients s n_coefs_z = n_freq*active_t_ind_z.sum() n_coefs_all_active = p*n_coefs_z #==== the main loop ===== tau, tau0 = 1.0, 1.0 obj = np.inf old_obj = np.inf # prepare the M_list from the M n_run = len(np.unique(G_ind)) M_list = list() X_list = list() for run_id in range(n_run): M_list.append(M[:,:,G_ind == run_id]) X_list.append(X[G_ind == run_id, :]) # greadient part one is fixed -G^T( \sum_r M(r)) PhiT gradient_y0 = get_gradient0_tsparse(M_list, G_list, X_list, p, n_run, n_active_dipole, active_set_z, n_times, n_coefs_z, n_coefs_all_active, active_t_ind_z, sparse_phi, sparse_phiT) # iterations, we only need to compute the second part of gradient # +G^T G(\sum_r X_k^(r) \sum_k Z_k X_k^r) PhiT Phi for i in range(maxit): Z0 = Z.copy() gradient_y1 = get_gradient1_tsparse(M_list, G_list, X_list, Y, p, n_run, n_active_dipole, active_set_z, n_times, n_coefs_z, n_coefs_all_active, active_t_ind_z, sparse_phi, sparse_phiT) gradient_y = gradient_y0 + gradient_y1 # compare the gradient(tested) # if False: # gradient_y2 = np.zeros([n_active_dipole, n_coefs_all_active], dtype =np.complex) # R_all_sq = np.zeros([n_sensors, n_times]) # for r in range(q): # # tmp_coef = y(0) + \sum_k y(k)* X(r,k) # tmp_coef = np.zeros([n_active_dipole,n_coefs_z], dtype = np.complex) # for k in range(p): # tmp_coef += Y[:,k*n_coefs_z:(k+1)*n_coefs_z]*X[r,k] # # current residual for this trial # tmpR = np.real(M[:,:,r] - G_list[G_ind[r]][:,active_set_z].dot(sparse_phiT(tmp_coef, active_t_ind_z))) # R_all_sq += np.abs(tmpR)**2 # tmpA = G_list[G_ind[r]][:,active_set_z].T.dot(sparse_phi(tmpR, active_t_ind_z)) # for k in range(p): # gradient_y2[:,k*n_coefs_z:(k+1)*n_coefs_z] += - X[r,k]*tmpA # print "obj = %e" %(0.5* R_all_sq.sum() + delta*np.sum(np.abs(Y)**2)) # #plt.plot(np.real(gradient_y2.ravel()), np.real(gradient_y.ravel()), '.') # #plt.plot(np.real(gradient_y2.ravel()), np.real(gradient_y2.ravel()), 'r') # print np.linalg.norm(gradient_y2-gradient_y)/np.linalg.norm(gradient_y2) # #gradient_y = gradient_y2.copy() # the L2 pentalty gradient_y += 2*delta*Y # compute the variable y- 1/L gradient y gradient_y[coef_non_zero_mat_full==0] =0 # follow the criterion in FISTA f(Y - gradient Y/L) > f(Y)+(0.5/L**2 -1/L ) ||gradient Y||^2 # this is different from the general case f(Y - gradient Y/L) > f(Y)-0.5/L ||gradient Y||^2 Z = Y- (gradient_y/L) objz = f_l2(Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT, delta) if Flag_backtrack: objy = f_l2( Y, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT,delta) # Ryan's slides # https://www.cs.cmu.edu/~ggordon/10725-F12/slides/05-gd-revisited.pdf # page 10 diff_bt = objz-objy+ (0.5/L)* np.sum( np.abs(gradient_y)**2) while diff_bt > 0: L = L*eta Z = Y-(gradient_y/L) objz = f_l2( Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT, delta) diff_bt = objz-objy + (0.5/L)* np.sum( np.abs(gradient_y)**2) ## ==== FISTA step, update the variables ===== tau0 = tau; tau = 0.5*(1+ np.sqrt(4*tau**2+1)) diff = Z-Z0 Y = Z + (tau0 - 1.0)/tau* diff ## ===== compute objective function, check stopping criteria ==== old_obj = obj obj = np.linalg.norm(gradient_y) diff_obj = old_obj-obj if Flag_verbose: print "\n iteration %d" % i print "sum sq gradient = %f" %obj print "diff_obj = %e" % (diff_obj) print "diff = %e" %(np.abs(diff).sum()/np.sum(abs(Y))) print "obj = %f" % objz stop = np.abs(diff).sum()/np.sum(abs(Y)) < tol if stop: print "convergence reached!" break Z = Y.copy() Z[coef_non_zero_mat_full ==0] =0 return Z, objz
def select_delta_stft_regression_cv(M,G_list, G_ind,X,Z00, active_set_z0, active_t_ind_z0, coef_non_zero_mat, delta_seq,cv_partition_ind, wsize=16, tstep = 4, maxit=200, tol = 1e-3, Flag_backtrack = True, L0 = 1.0, eta = 1.5, Flag_verbose = False): ''' Find the best L2 regularization parameter delta by cross validation Note that here, in training, the trial by trial paramter is estimated, but in testing, only the regression coefficients were used. Input: M, [n_channels, n_times, n_trials] array of the sensor data G_list, a list of [n_channels, n_dipoles] forward gain matrix G_ind, [n_trial], marks the index of G for this run X, [n_trials, p],the design matrix, it must include an all 1 colume Z00, [n_active_dipoles, p*n_freq * n_active_step] e.g # initial value Z00 = (np.random.randn(n_true_dipoles, n_coefs_all_active) \ + np.random.randn(n_true_dipoles, n_coefs_all_active)*1j)*1E-15 n_coefs_all_active = active_t_ind_z0.sum()*n_freq*pq acitve_set_z0, [n_dipoles,] a boolean array, indicating the active set of dipoles active_t_ind_z0, [n_step, ] a boolean array, indicating the active set of time points, the union of all frequencies, columns of X, and dipoles coef_non_zero_mat,[n_active_dipoles,n_coefs*pq] boolean matrix, since some active_set_z and active_t_ind_z is a super set of the active set, but we assume that it is the UNION of each coefficient of X and all trials delta_seq, np.array, a sequence of delta to be tested cv_parition_ind, an integer array of which cross-validation group each trial is in wsize, window size of the STFT tstep, length of the time step maxit, maximum number of iteration tol, tolerance, Output: delta_star, the best delta cv_MSE, the cv MSE for all elements in delta_seq ''' n_sensors, n_times, n_trials = M.shape n_dipoles = G_list[0].shape[1] if len(active_set_z0) != n_dipoles: raise ValueError("the number of dipoles does not match") p = X.shape[1] n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) n_fold = len(np.unique(cv_partition_ind)) n_delta = len(delta_seq) # n_true_dipoles = np.sum(active_set_z0) lipschitz_constant0 = get_lipschitz_const(M,G_list[0],X,phi,phiT,n_coefs, tol = 1e-3,Flag_trial_by_trial = False) cv_MSE = np.zeros([n_fold, n_delta]) for j in range(n_fold): # partition test_trials = np.nonzero(cv_partition_ind == j)[0] train_trials = np.nonzero(cv_partition_ind != j)[0] Z0 = Z00 tmp_coef_non_zero_mat = coef_non_zero_mat Mtrain = M[:,:,train_trials] Xtrain = X[train_trials,:] Mtest = M[:,:,test_trials] Xtest = X[test_trials,:] G_ind_train = G_ind[train_trials] G_ind_test = G_ind[test_trials] for i in range(n_delta): tmp_delta = delta_seq[i] # lipschitz constant L = (lipschitz_constant0+ 2*tmp_delta)*1.1 # training Z, _ = solve_stft_regression_L2_tsparse (Mtrain,G_list, G_ind_train, Xtrain, Z0, active_set_z0, active_t_ind_z0, tmp_coef_non_zero_mat, wsize=wsize, tstep =tstep,delta = tmp_delta, maxit=maxit,tol = tol,lipschitz_constant =L, Flag_backtrack = Flag_backtrack, L0 = L0, eta = eta, Flag_verbose = Flag_verbose) # only take the regression coefficients out Z_star = Z[:,0:p*active_t_ind_z0.sum()*n_freq] Z0 = Z.copy() # testing tmp_val, _,_,_ = get_MSE_stft_regresion_tsparse(Mtest,G_list, G_ind_test,Xtest, Z_star, active_set_z0, active_t_ind_z0, wsize=wsize, tstep = tstep) cv_MSE[j,i] = tmp_val # debug #import matplotlib.pyplot as plt #plt.figure() #plt.plot(np.real(Z_star).T) #plt.title(tmp_delta) cv_MSE = np.mean(cv_MSE, axis = 0) best_ind = np.argmin(cv_MSE) delta_star = delta_seq[best_ind] return delta_star, cv_MSE
def solve_stft_regression_tree_group(M,G_list, G_ind,X, alpha,beta, gamma, DipoleGroup,DipoleGroupWeight, Z_ini, active_set_z_ini, n_orient=1, wsize=16, tstep = 4, maxit=200, tol = 1e-3,lipschitz_constant = None, Flag_backtrack = True, eta = 1.5, L0 = 1.0, Flag_verbose = False): """ Input: M, [n_sensors,n_times,n_trials] array of the sensor data G_list, a list of [n_sensors, n_dipoles] forward gain matrix G_ind, [n_trials], index of run number (which G to use) X, [n_trials, p]the design matrix, it must include an all 1 colume alpha, tuning parameter for the regularization beta, the tuning paramter of balance between single frequency- time basis and dipoles gamma, the penalty on the absolute value of entries in Z DipoleGroup, grouping of dipoles, the dipoles in the same ROI are in the same group, the dipoles outside ROIs form one-dipole groups DipleGroupWeight, weights of each dipole group (ROI) Z_ini, [n_dipoles, n_coefs*p], initial value of Z, the ravel order is [n_dioles, p, n_freqs, n_step] active_set_z_ini, [n_dipoles,] boolean, active_set of dipoles n_orient, number of orientations for each dipole note that if n_orient == 3, the DipoleGroup is still on the columns of G, but grouped in the 3-set way. The initial values and initial active set of Z should also be correspondingly correct. wsize, number of frequence tstep, step in time of the stft maxit, maximum number of iteration allowed tol, tolerance of the objective function lipschitz_constant, the lipschitz constant, Flag_backtrack, if True, use backtracking instead of constant stepsize ( the lipschitz constant ) eta, L0, the shrinking parameters and initial 1/stepsize Flag_verbose, if true, print the objective values and difference, else not Output: Z, the solustion, only rows in the active set, note that it is not guaranteed that all rows from the same group will be in the final solutions, if all of the coefficients for that row is zero, it is dropped too. active_set_z, a boolean vector, active dipoles (rows) active_t_ind_z, a boolean vector, active time steps, (time steps) obj, the objective function """ # check the active_set_structure and group structure for n_orient ==3 if n_orient == 3: active_set_mat = active_set_z_ini.copy() active_set_mat = active_set_mat.reshape([-1,n_orient]) any_ind = np.any(active_set_mat,axis =1) all_ind = np.all(active_set_mat,axis =1) if np.sum(np.abs(any_ind-all_ind)) >0: raise ValueError("wrong active set for n_orient = 3") # DipoleGroup must also satisfy the structure for l in range(len(DipoleGroup)): if np.remainder(len(DipoleGroup[l]),n_orient)!=0: raise ValueError("wrong group") tmp_mat = np.reshape(DipoleGroup[l],[-1,n_orient],order = 'C') if np.sum(np.abs(tmp_mat[:,2] - tmp_mat[:,1] - 1)) != 0 \ or np.sum(np.abs(tmp_mat[:,1] - tmp_mat[:,0] - 1)) != 0: raise ValueError("wrong group") n_sensors, n_times, q = M.shape n_dipoles = G_list[0].shape[1] p = X.shape[1] n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq # create the sparse and non sparse version of the STFT, iSTFT phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) # initialization if lipschitz_constant is None and not Flag_backtrack: lipschitz_constant = 1.1* get_lipschitz_const(M,G_list[0],X,phi,phiT, Flag_trial_by_trial = False,n_coefs = n_coefs, tol = 1e-3) print "lipschitz_constant = %e" % lipschitz_constant if Flag_backtrack: L = L0 else: L = lipschitz_constant # indices for the active set is only for rows Z = Z_ini.copy() Y = Z_ini.copy() active_set_z = active_set_z_ini.copy() active_set_y = active_set_z_ini.copy() if Z.shape[0] != active_set_z.sum() or Z.shape[1] != n_coefs*p: raise ValueError('Z0 shape does not match active sets') #==== the main loop ===== tau, tau0 = 1.0, 1.0 obj = np.inf old_obj = np.inf # prepare the M_list from the M n_run = len(np.unique(G_ind)) M_list = list() X_list = list() for run_id in range(n_run): M_list.append(M[:,:,G_ind == run_id]) X_list.append(X[G_ind == run_id, :]) # greadient part one is fixed -G^T( \sum_r M(r)) PhiT # this should be on all dipoles gradient_y0 = get_gradient0(M_list, G_list, X_list, p, n_run, n_dipoles, np.ones(n_dipoles, dtype = np.bool), n_times, n_coefs, n_coefs*p, phi, phiT) # only keep on full matrix #gradient_y = np.zeros([n_dipoles, n_coefs*p],dtype =np.complex) for i in range(maxit): Z0 = Z.copy() active_set_z0 = active_set_z.copy() # this part can be only on the active set # but gradient_y1 should be a full matrix gradient_y1 = get_gradient1_L21(M_list, G_list, X_list, Y, p, n_run, np.int(active_set_y.sum()), active_set_y, n_times, n_coefs, n_coefs*p, phi, phiT) gradient_y = gradient_y0 + gradient_y1 # active_set_z0, active rows/dipoles for z0 # active_t_ind_z0, active columns/time_points for z0 tmp_Y_L_gradient_Y = -gradient_y/L tmp_Y_L_gradient_Y[active_set_y,:] += Y ## ==== ISTA step, get the proximal operator ==== ## Input must be a full matrix, so the active set is set to full Z, active_set_z = prox_tree_hard_coded_full_matrix(tmp_Y_L_gradient_Y, n_coefs, p, alpha/L, beta/L, gamma/L, DipoleGroup, DipoleGroupWeight, n_orient) # check if it is zero solution if not np.any(active_set_z): print "active_set = 0" return 0,0,0,0 objz = f( Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT) # compute Z-Y diff_z, active_set_diff_z = _add_z([Z,-Y], np.vstack([active_set_z, active_set_y])) if Flag_backtrack: objy = f( Y, active_set_y, M, G_list, G_ind, X, n_coefs, q, p, phiT) # compute the criterion for back track: f(z)-f(y) -grad_y.dot(z-y) +0.5*(z-y)**2 # note gradient_y is a full matrix diff_bt = objz-objy- np.sum( np.real(gradient_y[active_set_diff_z])* np.real(diff_z))\ - np.sum(np.imag(gradient_y[active_set_diff_z])* np.imag(diff_z)) \ -0.5*L*np.sum( np.abs(diff_z)**2) while diff_bt > 0: L = L*eta tmp_Y_L_gradient_Y = -gradient_y/L tmp_Y_L_gradient_Y[active_set_y,:] += Y Z, active_set_z = prox_tree_hard_coded_full_matrix(tmp_Y_L_gradient_Y, n_coefs, p, alpha/L, beta/L, gamma/L, DipoleGroup, DipoleGroupWeight, n_orient) if not np.any(active_set_z): print "active_set = 0" return 0,0,0 objz = f( Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT) # Z-Y diff_z, active_set_diff_z = _add_z([Z,-Y], np.vstack([active_set_z, active_set_y])) # the criterion for back track: diff_bt = objz-objy- np.sum( np.real(gradient_y[active_set_diff_z])* np.real(diff_z))\ - np.sum(np.imag(gradient_y[active_set_diff_z])* np.imag(diff_z)) \ -0.5*L*np.sum( np.abs(diff_z)**2) ## ==== FISTA step, update the variables ===== tau0 = tau; tau = 0.5*(1+ np.sqrt(4*tau**2+1)) if np.any(active_set_diff_z) == 0: print "active_set = 0" return 0,0,0 else: # y <- z + (tau0-1)/tau (z-z0) =(tau+tau0-1)/tau z - (tao0-1)/tau z0 Y, active_set_y = _add_z( [(tau+tau0-1)/tau*Z, -(tau0-1)/tau*Z0], np.vstack([active_set_z, active_set_z0])) ## ===== compute objective function, check stopping criteria ==== old_obj = obj full_Z = np.zeros([n_dipoles, n_coefs*p], dtype = np.complex) full_Z[active_set_z, :] = Z obj = objz \ + get_tree_norm_hard_coded(full_Z,n_coefs, p, alpha, beta, gamma, DipoleGroup, DipoleGroupWeight, n_orient) diff_obj = old_obj-obj relative_diff = np.sum(np.abs(diff_z))/np.sum(np.abs(Z)) if Flag_verbose: print "\n iteration %d" % i print "diff_obj = %e" % (diff_obj/obj) print "obj = %e" %obj print "diff = %e" %relative_diff stop = ( relative_diff < tol and np.abs(diff_obj/obj) < tol) if stop: print "convergence reached!" break Z = Y.copy() active_set_z = active_set_y.copy() return Z, active_set_z, obj
active_set = active_set.astype(np.int) n_valid_source = len(active_set) # parameters for STFT-transform : # tstep_phi: time steps of STFT-R # wsize_phi: window size of the STFT-R tstep_phi = 4 wsize_phi = 16 # number of time steps n_step = int(np.ceil(n_times/float(tstep_phi))) # number of frequencies n_freq = wsize_phi// 2+1 # n_coefs, total number of time-frequency components n_tfs = n_step*n_freq #STFT and inverse STFT function from MNE-python phi = _Phi(wsize_phi, tstep_phi, n_tfs) phiT = _PhiT(tstep_phi, n_freq, n_step, n_times) #Z_true: regression coeffients in the time-frequency domain coef_per_ROI = np.ones([n_ROI,n_freq,n_step, p]) Z_true = np.tile(phi(stc0_data),[1,p]) for i in range(len(labels)): Z_true[label_ind[i]] *= np.reshape(np.transpose(coef_per_ROI[i],[2,0,1]),[1,-1]) # create the sensor data and source data for each trial evoked_list = list() stc_data_list = list() label_all = labels[0] for i in range(1,len(labels)): label_all += labels[i]
def compute_dual_gap(M, G_list, G_ind, X, Z, active_set, alpha, beta, gamma, DipoleGroup, DipoleGroupWeight, n_orient, wsize = 16, tstep = 4): """ Compute the duality gap, and check the feasibility of the dual function. Input: M, G_list, G_ind, X, Z, active_set, all the full primal. alpha, beta, gamma, penalty paramters n_orient, number of orientations Output: dict(feasibility_dist = feasibility_dist, gradient = gradient, feasibility_dist_DipoleGroup = feasibility_dist_DipoleGroup ) feasibility_dist, is the major criterion, if it is small enougth, then accept the results gradient, the gradient """ # initialization, only update when seeing greater values if gamma == 0: raise ValueError( "the third level must be penalized!!") n_dipoles = G_list[0].shape[1] n_sensors, n_times, q= M.shape n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) n_trials, p = X.shape n_run = len(np.unique(G_ind)) M_list = list() X_list = list() for run_id in range(n_run): M_list.append(M[:,:,G_ind == run_id]) X_list.append(X[G_ind == run_id, :]) # compute the gradient to check feasibility gradient_y0 = get_gradient0(M_list, G_list, X_list, p, n_run, n_dipoles, np.ones(n_dipoles, dtype = np.bool), n_times, n_coefs, n_coefs*p, phi, phiT) gradient_y1 = get_gradient1_L21(M_list, G_list, X_list, Z, p, n_run, np.int(active_set.sum()), active_set, n_times, n_coefs, n_coefs*p, phi,phiT) gradient = gradient_y0 + gradient_y1 # sanity check # for each dipole, get the maximum abs value among the real and imag parts #max_grad = np.max(np.vstack([np.max(np.abs(np.real(gradient)), axis = 1), # np.max(np.abs(np.imag(gradient)),axis = 1)]), axis = 0) alpha_weight = np.zeros(n_dipoles) for i in range(len(DipoleGroup)): alpha_weight[DipoleGroup[i]] = alpha*DipoleGroupWeight[i] # if max_grad is greater than alpha+beta+gamma, # then the dual variable can not be feasible #if np.any(max_grad > alpha_weight+ beta+gamma): # feasibility_dist = np.inf #else: # # feasibility check. b = A^T u + \sum_g D_g^T v_g # where g is the non zero groups b = gradient.copy() active_set_ind = np.nonzero(active_set)[0] Z_full = np.zeros([n_dipoles, Z.shape[1]],dtype = np.complex) Z_full[active_set,:] = Z # add g in the alpha level, dipole groups # a bool vector showing whether the group is in the active set DipoleGroup_active = np.zeros(len(DipoleGroup), dtype = np.bool) for i in range(len(DipoleGroup)): if np.intersect1d(DipoleGroup[i],active_set_ind).size >0: DipoleGroup_active[i] = True # l2 norm of the group l2_norm_alpha = np.sqrt(np.sum( ( np.abs(Z_full[DipoleGroup[i],:]) )**2) ) # add the sum of the gradient if l2_norm_alpha == 0: raise ValueError("all zero in an active group!") b[DipoleGroup[i],:]+= Z_full[DipoleGroup[i],:]/l2_norm_alpha * alpha_weight[i] # add g in the beta level, same dipole, same stft coef if n_orient == 1: Z_reshape = np.reshape(Z, [active_set.sum(), p, -1]) # active_set.sum() x n_coefs l2_norm_beta = np.sqrt(np.sum( (np.abs(Z_reshape))**2, axis = 1)) # active_set.sum() x n_coefs * p l2_norm_beta_large = np.tile(l2_norm_beta, [1, p]) else: # n_orient == 3 Z_reshape = np.reshape(Z, [active_set.sum()//3,3, p, -1]) l2_norm_beta = np.sqrt( np.sum( np.sum( ( np.abs(Z_reshape) )**2, axis = 2) , axis = 1 ) ) l2_norm_beta_large = np.reshape( np.tile(l2_norm_beta,[ 1, 3*p]) , Z.shape) tmp_add_to_b = np.zeros([active_set.sum(), Z.shape[1]], dtype = np.complex) nonzero_beta = l2_norm_beta_large > 0 tmp_add_to_b[nonzero_beta] = Z[nonzero_beta]/l2_norm_beta_large[nonzero_beta]*beta b[active_set,:] += tmp_add_to_b # add g in the gamma level, each element of the matrix if n_orient == 1: l2_norm_gamma = np.abs(Z) else: # n_orient == 3: Z_reshape = np.reshape(Z, [active_set.sum()//3,3,-1]) l2_norm_gamma = np.sqrt ( np.sum( ( np.abs(Z_reshape) )**2, axis = 1 ) ) l2_norm_gamma = np.reshape( np.tile(l2_norm_gamma, [1,3* Z.shape[1]]), Z.shape) nonzero_gamma = l2_norm_gamma > 0 tmp_add_to_b = np.zeros([active_set.sum(), Z.shape[1]], dtype = np.complex) tmp_add_to_b[nonzero_gamma] = Z[nonzero_gamma]/l2_norm_gamma[nonzero_gamma]*gamma b[active_set,:] += tmp_add_to_b if np.any(np.isnan(b)): raise ValueError("nan found in b!") # use coordinate descent to solve the feasibility problem nonzero_Z = np.abs(Z) >0 feasibility_result = get_feasibility(b, active_set, DipoleGroup, alpha_weight, beta, gamma, n_coefs, p, nonzero_Z, DipoleGroup_active) feasibility_dist = feasibility_result['feasibility_dist'] # the dist in each dipole, squared feasibility_dist_in_alpha_level = feasibility_result['feasibility_dist_in_alpha_level'] feasibility_dist_DipoleGroup = np.zeros(len(DipoleGroup)) for i in range(len(DipoleGroup)): feasibility_dist_DipoleGroup[i] = \ np.sqrt( np.sum(feasibility_dist_in_alpha_level[DipoleGroup[i]]**2)) return dict(feasibility_dist = feasibility_dist, gradient = gradient, b = b, feasibility_dist_DipoleGroup = feasibility_dist_DipoleGroup)
def solve_stft_regression_tree_group_tsparse(M,G_list, G_ind,X, alpha,beta, gamma, DipoleGroup,DipoleGroupWeight, Z_ini, active_set_z_ini, n_orient=1, wsize=16, tstep = 4, maxit=200, tol = 1e-3,lipschitz_constant = None, Flag_backtrack = True, eta = 1.5, L0 = 1.0, Flag_verbose = False): """ Input: M, [n_sensors,n_times,n_trials] array of the sensor data G_list, a list of [n_sensors, n_dipoles] forward gain matrix, for different runs if applicable G_ind, [n_trials], index of run number (which G to use) X, [n_trials, p]the design matrix, it must include an all 1 colume alpha, tuning parameter for the regularization beta, the tuning paramter, for the frequency- time basis gamma, the penalty on the absolute value of entries in Z DipoleGroup, grouping of dipoles, the dipoles in the same ROI are in the same group, the dipoles outside ROIs form one-dipole groups DipleGroupWeight, weights of each dipole group (ROI) Z_ini, [n_dipoles, n_coefs*p], initial value of Z, the ravel order is [n_dioles, p, n_freqs, n_step] active_set_z_ini, [n_dipoles,] boolean, active_set of dipoles n_orient, number of orientations for each dipole note that if n_orient == 3, the DipoleGroup is still on the columns of G, but grouped in the 3-set way. The initial values and initial active set of Z should also be correspondingly correct. wsize, number of frequence tstep, step in time of the stft maxit, maximum number of iteration allowed tol, tolerance of the objective function lipschitz_constant, the lipschitz constant, Flag_backtrack, if True, use backtracking instead of constant stepsize ( the lipschitz constant ) eta, L0, the shrinking parameters and initial 1/stepsize Flag_verbose, if true, print the objective values and difference, else not Output: Z, the solustion, only rows in the active set, note that it is not guaranteed that all rows from the same group will be in the final solutions, if all of the coefficients for that row is zero, it is dropped too. active_set_z, a boolean vector, active dipoles (rows) active_t_ind_z, a boolean vector, active time steps, (time steps) obj, the objective function """ # check the active_set_structure and group structure for n_orient ==3 if n_orient == 3: active_set_mat = active_set_z_ini.copy() active_set_mat = active_set_mat.reshape([-1,n_orient]) any_ind = np.any(active_set_mat,axis =1) all_ind = np.all(active_set_mat,axis =1) if np.sum(np.abs(any_ind-all_ind)) >0: raise ValueError("wrong active set for n_orient = 3") # DipoleGroup must also satisfy the structure for l in range(len(DipoleGroup)): if np.remainder(len(DipoleGroup[l]),n_orient)!=0: raise ValueError("wrong group") tmp_mat = np.reshape(DipoleGroup[l],[-1,n_orient],order = 'C') if np.sum(np.abs(tmp_mat[:,2] - tmp_mat[:,1] - 1)) != 0 \ or np.sum(np.abs(tmp_mat[:,1] - tmp_mat[:,0] - 1)) != 0: raise ValueError("wrong group") n_sensors, n_times, q = M.shape n_dipoles = G_list[0].shape[1] p = X.shape[1] n_step = int(np.ceil(n_times/float(tstep))) n_freq = wsize// 2+1 n_coefs = n_step*n_freq # create the sparse and non sparse version of the STFT, iSTFT phi = _Phi(wsize, tstep, n_coefs) phiT = _PhiT(tstep, n_freq, n_step, n_times) sparse_phi = sparse_Phi(wsize, tstep) sparse_phiT = sparse_PhiT(tstep, n_freq, n_step, n_times) # initialization if lipschitz_constant is None and not Flag_backtrack: lipschitz_constant = 1.1* get_lipschitz_const(M,G_list[0],X,phi,phiT, Flag_trial_by_trial = False,n_coefs = n_coefs, tol = 1e-3) print "lipschitz_constant = %e" % lipschitz_constant if Flag_backtrack: L = L0 else: L = lipschitz_constant # indices for the active set is only for rows Z = Z_ini.copy() Y = Z_ini.copy() active_set_z = active_set_z_ini.copy() active_set_y = active_set_z_ini.copy() # my code has sparse time steps. (sparse_phi and sparse_phiT) # for consistency, I still kept the active_t_ind variables, # but make sure they are all true, all time steps are used. active_t_ind_full = np.ones(n_step, dtype = np.bool) if Z.shape[0] != active_set_z.sum() or Z.shape[1] != n_coefs*p: raise ValueError('Z0 shape does not match active sets') #==== the main loop ===== tau, tau0 = 1.0, 1.0 obj = np.inf old_obj = np.inf # prepare the M_list from the M n_run = len(np.unique(G_ind)) M_list = list() X_list = list() for run_id in range(n_run): M_list.append(M[:,:,G_ind == run_id]) X_list.append(X[G_ind == run_id, :]) # greadient part one is fixed -G^T( \sum_r M(r)) PhiT # this should be on all dipoles gradient_y0 = get_gradient0_tsparse(M_list, G_list, X_list, p, n_run, n_dipoles, np.ones(n_dipoles, dtype = np.bool), n_times, n_coefs, n_coefs*p, active_t_ind_full, sparse_phi, sparse_phiT) # only keep on full matrix #gradient_y = np.zeros([n_dipoles, n_coefs*p],dtype =np.complex) for i in range(maxit): Z0 = Z.copy() active_set_z0 = active_set_z.copy() # this part can be only on the active set # but gradient_y1 should be a full matrix gradient_y1 = get_gradient1_L21_tsparse(M_list, G_list, X_list, Y, p, n_run, np.int(active_set_y.sum()), active_set_y, n_times, n_coefs, n_coefs*p, active_t_ind_full, sparse_phi, sparse_phiT) gradient_y = gradient_y0 + gradient_y1 # active_set_z0, active rows/dipoles for z0 # active_t_ind_z0, active columns/time_points for z0 # verify the gradient with two different computations # if False: # gradient_y20 = np.zeros([n_dipoles, n_coefs*p], dtype = np.complex) # gradient_y21 = np.zeros([n_dipoles, n_coefs*p], dtype = np.complex) # #GTG_active = G.T.dot(G[:,active_set_y]) # Y_reshape = np.reshape(Y[:,0:p*n_coefs],[active_set_y.sum(), p,n_coefs]) # Y_reshape = Y_reshape.swapaxes(1,2) # for run_id in range(n_run): # GTG_active = G_list[run_id].T.dot(G_list[run_id][:,active_set_y]) # for k in range(p): # # first term of the gradient # M_sum = np.sum(M_list[run_id]*X_list[run_id][:,k],axis = 2) # # G^T(\sum_r X_k(r) M(r))\Phi # GTM_sumPhi = G_list[run_id].T.dot(sparse_phi(M_sum,active_t_ind_full)) # # second term of the gradient # sum_X_coef = np.zeros([active_set_y.sum(), n_coefs], dtype = np.complex) # # the first level is n_coefs, second level is p # for r in range(X_list[run_id].shape[0]): # sum_X_coef += np.sum(Y_reshape*X_list[run_id][r,:],axis = 2)*X_list[run_id][r,k] # gradient_y20[:,k*n_coefs:(k+1)*n_coefs] += -GTM_sumPhi # gradient_y21 [:,k*n_coefs:(k+1)*n_coefs] += sparse_phi(sparse_phiT(GTG_active.dot(sum_X_coef), # active_t_ind_full),active_t_ind_full) # #print (np.linalg.norm(gradient_y0-gradient_y20)/np.linalg.norm(gradient_y20)) # #print (np.linalg.norm(gradient_y1-gradient_y21)/np.linalg.norm(gradient_y21)) # gradient_y2 = gradient_y20+gradient_y21 # print "diff gradient=%e" % (np.linalg.norm(gradient_y-gradient_y2)/np.linalg.norm(gradient_y2)) # #plt.plot(np.real(gradient_y2).ravel(),np.real(gradient_y).ravel(), '.' ) # gradient_y = gradient_y2.copy() tmp_Y_L_gradient_Y = -gradient_y/L tmp_Y_L_gradient_Y[active_set_y,:] += Y ## ==== ISTA step, get the proximal operator ==== ## Input must be a full matrix, so the active set is set to full Z, active_set_z = prox_tree_hard_coded_full_matrix(tmp_Y_L_gradient_Y, n_coefs, p, alpha/L, beta/L, gamma/L, DipoleGroup, DipoleGroupWeight, n_orient) # check if it is zero solution if not np.any(active_set_z): print "active_set = 0" return 0,0,0 objz = f( Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT) # compute Z-Y diff_z, active_set_diff_z = _add_z([Z,-Y], np.vstack([active_set_z, active_set_y])) if Flag_backtrack: objy = f( Y, active_set_y, M, G_list, G_ind, X, n_coefs, q, p, phiT) # compute the criterion for back track: f(z)-f(y) -grad_y.dot(z-y) -0.5*(z-y)**2 # note gradient_y is a full matrix # note diff_z is complex # http://www.seas.ucla.edu/~vandenbe/236C/lectures/fgrad.pdf 7-17+ FISTA paper Beck diff_bt = objz-objy- np.sum( np.real(gradient_y[active_set_diff_z])* np.real(diff_z))\ - np.sum(np.imag(gradient_y[active_set_diff_z])* np.imag(diff_z)) \ -0.5*L*np.sum( np.abs(diff_z)**2) while diff_bt > 0: L = L*eta tmp_Y_L_gradient_Y = -gradient_y/L tmp_Y_L_gradient_Y[active_set_y,:] += Y Z, active_set_z = prox_tree_hard_coded_full_matrix(tmp_Y_L_gradient_Y, n_coefs, p, alpha/L, beta/L, gamma/L, DipoleGroup, DipoleGroupWeight, n_orient) if not np.any(active_set_z): print "active_set = 0" return 0,0,0 objz = f( Z, active_set_z, M, G_list, G_ind, X, n_coefs, q, p, phiT) # Z-Y diff_z, active_set_diff_z = _add_z([Z,-Y], np.vstack([active_set_z, active_set_y])) # the criterion for back track: diff_bt = objz-objy- np.sum( np.real(gradient_y[active_set_diff_z])* np.real(diff_z))\ - np.sum(np.imag(gradient_y[active_set_diff_z])* np.imag(diff_z)) \ -0.5*L*np.sum( np.abs(diff_z)**2) ## ==== FISTA step, update the variables ===== tau0 = tau; tau = 0.5*(1+ np.sqrt(4*tau**2+1)) if np.any(active_set_diff_z) == 0: print "active_set = 0" return 0,0,0 else: # y <- z + (tau0-1)/tau (z-z0) =(tau+tau0-1)/tau z - (tao0-1)/tau z0 Y, active_set_y = _add_z( [(tau+tau0-1)/tau*Z, -(tau0-1)/tau*Z0], np.vstack([active_set_z, active_set_z0])) # commented after verification #ind_y = np.nonzero(active_set_y)[0] #Z_larger = np.zeros([active_set_y.sum(), n_coefs*p], dtype = np.complex) #tmp_intersection = np.all(np.vstack([active_set_y, active_set_z]),axis=0) #tmp_ind_inter = np.nonzero(tmp_intersection)[0] #tmp_ind = [k0 for k0 in range(len(ind_y)) if ind_y[k0] in tmp_ind_inter] #Z_larger[tmp_ind,:] = Z # expand Z0 #Z0_larger = np.zeros([active_set_y.sum(), n_coefs*p], dtype = np.complex) #tmp_intersection0 = np.all(np.vstack([active_set_y, active_set_z0]),axis=0) #tmp_ind_inter0 = np.nonzero(tmp_intersection0)[0] #tmp_ind0 = [k0 for k0 in range(len(ind_y)) if ind_y[k0] in tmp_ind_inter0] #Z0_larger[tmp_ind0,:] = Z0 ## update y #diff = Z_larger-Z0_larger #Y1 = Z_larger + (tau0 - 1.0)/tau *diff #print "Y1-Y=%f" %np.linalg.norm(Y1-Y) ## ===== compute objective function, check stopping criteria ==== old_obj = obj full_Z = np.zeros([n_dipoles, n_coefs*p], dtype = np.complex) full_Z[active_set_z, :] = Z # residuals in each trial # if we use the trial by trial model, also update # the gradient of the trial-by-trial coefficients # debug, can be commented soon. #R_all_sq = 0 #for r in range(q): # tmp_coef = np.zeros([active_set_z.sum(),n_coefs], dtype = np.complex) # for k in range(p): # tmp_coef += Z[:,k*n_coefs:(k+1)*n_coefs]*X[r,k] # tmpR = M[:,:,r] - phiT(G_list[G_ind[r]][:,active_set_z].dot(tmp_coef)) # R_all_sq += np.sum(tmpR**2) #print "diff in f(z) = %f" %(objz - 0.5* R_all_sq.sum() ) obj = objz \ + get_tree_norm_hard_coded(full_Z,n_coefs, p, alpha, beta, gamma, DipoleGroup, DipoleGroupWeight, n_orient) diff_obj = old_obj-obj relative_diff = np.sum(np.abs(diff_z))/np.sum(np.abs(Z)) if Flag_verbose: print "\n iteration %d" % i print "diff_obj = %e" % (diff_obj/obj) print "obj = %e" %obj print "diff = %e" %relative_diff stop = ( relative_diff < tol and np.abs(diff_obj/obj) < tol) if stop: print "convergence reached!" break Z = Y.copy() active_set_z = active_set_y.copy() return Z, active_set_z, obj