def test_one_sided(): projSplit = ps.ProjSplitFit() def deriv(x,y): return (x>=y)*(x-y) def val(x,y): return (x>=y)*(x-y)**2 loss = ls.LossPlugIn(deriv) loss = ls.LossPlugIn(deriv,val) m = 20 d = 50 A = normal(0,1,[m,d]) y = normal(0,1,m) projSplit.addData(A,y,loss=loss,intercept=False,normalize=False) projSplit.addRegularizer(L1()) projSplit.run(keepHistory=False,nblocks=10) primTol = projSplit.getPrimalViolation() dualTol = projSplit.getDualViolation() assert primTol <1e-6 assert dualTol <1e-6
def test_linear_op_data_term_wrong(): m = 40 d = 10 if getNewOptVals: A,y = getLSdata(m,d) cache['Awrongdata']=A cache['ywrongdata']=y else: A=cache['Awrongdata'] y=cache['ywrongdata'] projSplit = ps.ProjSplitFit() stepsize = 1e-1 processor = lp.Forward2Fixed(stepsize) gamma = 1e0 projSplit.setDualScaling(gamma) p = 15 d2 = 11 H = np.random.normal(0,1,[d2,p]) try: projSplit.addData(A,y,2,processor,normalize=False,intercept=False, linearOp = aslinearoperator(H)) notExcept = True except: notExcept = False assert notExcept == False
def test_ls_PrimDual(processor, inter, norm, nblk, firsttest): processor.setStep(1e-1) projSplit = ps.ProjSplitFit() m = 10 d = 20 if getNewOptVals and firsttest: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['AprimDual'] = A cache['yprimDual'] = y else: A = cache['AprimDual'] y = cache['yprimDual'] projSplit.setDualScaling(1e-1) projSplit.addData(A, y, 2, processor, intercept=inter, normalize=norm) projSplit.run(maxIterations=None, keepHistory=True, primalTol=1e-3, dualTol=1e-3, nblocks=nblk, historyFreq=1) print("Primal violation = {}".format(projSplit.getPrimalViolation())) print("Dual violation = {}".format(projSplit.getDualViolation())) assert projSplit.getPrimalViolation() < 1e-3 assert projSplit.getDualViolation() < 1e-3
def test_bad_getParams(): projSplit = ps.ProjSplitFit() try: testing = projSplit.numPrimalVars() testing = projSplit.numObservations() testing = projSplit.getObjective() testing = projSplit.getSolution() testing = projSplit.getDualViolation() testing = projSplit.getHistory() testing = projSplit.getPrimalViolation() testing = projSplit.getScale() noExcept = True except: noExcept = False assert noExcept == False
def test_a_reg(builtInReg, cvxf, testNum): projSplit = ps.ProjSplitFit() if getNewOptVals: A = cache.get('Areg') y = cache.get('yreg') if A is None: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['Areg'] = A cache['yreg'] = y else: A = cache['Areg'] y = cache['yreg'] projSplit.addData(A, y, intercept=False, normalize=False, loss=2) projSplit.addRegularizer(builtInReg) projSplit.run(nblocks=5) psval = projSplit.getObjective() if getNewOptVals: opt = cache.get(('optreg', testNum)) if opt is None: cvxf += (1 / (2 * m)) * cvx.sum_squares(A @ xcvx - y) prob = cvx.Problem(cvx.Minimize(cvxf)) prob.solve(verbose=False) opt = prob.value cache[('optreg', testNum)] = opt else: opt = cache[('optreg', testNum)] print(f"psval = {psval}") print(f"CVX opt = {opt}") assert psval - opt < 1e-5
def test_add_linear_ops(): projSplit = ps.ProjSplitFit() m = 10 d = 20 A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) processDummy = ProcessDummy() projSplit.addData(A, y, 2, processDummy) p = 11 H = np.random.normal(0, 1, [p, d]) lam = 0.01 step = 1.0 regObj = L1(lam, step) projSplit.addRegularizer(regObj, linearOp=aslinearoperator(H)) d2 = 9 H = np.random.normal(0, 1, [p, d2]) try: projSplit.addRegularizer(regObj, linearOp=aslinearoperator(H)) == -1 noExcept = True except: noExcept = False assert noExcept == False
def test_ls_Int_Norm(processor, norm, inter, firsttest): processor.setStep(5e-1) projSplit = ps.ProjSplitFit() m = 20 d = 10 print(f"firsttest = {firsttest}") print(f"getNewOptVals={getNewOptVals}") if getNewOptVals and firsttest: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['AlsintNorm'] = A cache['ylsintNorm'] = y else: A = cache['AlsintNorm'] y = cache['ylsintNorm'] gamma = 1e-2 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=norm, intercept=inter) projSplit.run(maxIterations=5000, keepHistory=True, nblocks=10, primalTol=0.0, dualTol=0.0) ps_sol = projSplit.getSolution() ps_sol_simp = projSplit.getSolution(ergodic="simple") ps_sol_weight = projSplit.getSolution(ergodic="weighted") if getNewOptVals: LSval = cache.get((inter, norm, 'lsIntNormOpt')) if LSval is None: if inter: AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A result = np.linalg.lstsq(AwithIntercept, y, rcond=None) xhat = result[0] else: result = np.linalg.lstsq(A, y, rcond=None) xhat = result[0] if norm == False: assert (np.linalg.norm(xhat - ps_sol) < 1e-2) assert (np.linalg.norm(xhat - ps_sol_simp) < 1e-2) assert (np.linalg.norm(xhat - ps_sol_weight) < 1e-2) if inter: LSval = 0.5 * np.linalg.norm(AwithIntercept.dot(xhat) - y, 2)**2 / m else: LSval = 0.5 * np.linalg.norm(A.dot(xhat) - y, 2)**2 / m cache[(inter, norm, 'lsIntNormOpt')] = LSval else: LSval = cache.get((inter, norm, 'lsIntNormOpt')) psSolVal = projSplit.getObjective() assert (abs(psSolVal - LSval) < 1e-2)
def test_add_regularizer(): projSplit = ps.ProjSplitFit() scale = 11.5 regObj = L1(scale) projSplit.addRegularizer(regObj) scale2 = 15.7 regObj.setScaling(scale2) assert (projSplit.allRegularizers[0].getScaling() == scale2)
def test_backward(nblk, inter, norm, processor): m = 80 d = 20 if getNewOptVals: A = cache.get('Aback') y = cache.get('yback') if A is None: A, y = getLSdata(m, d) cache['Aback'] = A cache['yback'] = y else: A = cache.get('Aback') y = cache.get('yback') projSplit = ps.ProjSplitFit() gamma = 1e-3 #if nblk==10: # gamma = 1e3 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=norm, intercept=inter) projSplit.run(maxIterations=5000, keepHistory=True, nblocks=nblk, blockActivation="random", primalTol=1e-7, dualTol=1e-7) #psvals = projSplit.getHistory()[0] #plt.plot(psvals) #plt.show() ps_opt = projSplit.getObjective() print('ps func opt = {}'.format(ps_opt)) if getNewOptVals: LSval = cache.get((inter, 'optback')) if LSval is None: if inter: AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A result = np.linalg.lstsq(AwithIntercept, y, rcond=None) xhat = result[0] LSval = 0.5 * np.linalg.norm(AwithIntercept.dot(xhat) - y, 2)**2 / m else: result = np.linalg.lstsq(A, y, rcond=None) xhat = result[0] LSval = 0.5 * np.linalg.norm(A.dot(xhat) - y, 2)**2 / m cache[(inter, 'optback')] = LSval else: LSval = cache.get((inter, 'optback')) print('LSval = {}'.format(LSval)) assert ps_opt - LSval < 1e-2
def test_all_adds_together(sparse_type): psObj = ps.ProjSplitFit() obs = sparse_type([[1, 2, 3], [4, 5, 6]]) y = [1, 1] H = sparse_type([[1, 1, 7, 8], [7, 9, 7, 8], [4, 4, 3, 4]]) psObj.addData(obs, y, 2, linearOp=H) regObj = regularizers.L1() G = sparse_type([[1, 1, 1, 11], [7, 7, 11, 42]]) psObj.addRegularizer(regObj, linearOp=G)
def test_wrong_obs_matrix(): psObj = ps.ProjSplitFit() obs = [[1, 2, 3], [4, 5, 6]] y = [1, 1] with pytest.raises(Exception): psObj.addData(obs, y, 2) obs = "hello world" with pytest.raises(Exception): psObj.addData(obs, y, 2)
def test_good_embed(): projSplit = ps.ProjSplitFit() m = 10 d = 20 A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) processDummy = ProcessDummy() regObj = L1() projSplit.addData(A, y, 2, processDummy, embed=regObj) assert projSplit.numRegs == 0
def test_L1LR(processor, nrm, inter, testNumber): m = 40 d = 10 if getNewOptVals and (testNumber == 0): A, y = getLRdata(m, d) cache['A'] = A cache['y'] = y else: A = cache['A'] y = cache['y'] projSplit = ps.ProjSplitFit() gamma = 1e0 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 'logistic', processor, normalize=nrm, intercept=inter) lam = 5e-2 step = 1.0 regObj = L1(lam, step) projSplit.addRegularizer(regObj) projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1) ps_val = projSplit.getObjective() if getNewOptVals: opt = cache.get((nrm, inter, 'opt')) if opt is None: if nrm: Anorm = A n = A.shape[0] scaling = np.linalg.norm(Anorm, axis=0) scaling += 1.0 * (scaling < 1e-10) A = np.sqrt(n) * Anorm / scaling if inter: AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A A = AwithIntercept opt, _ = runCVX_LR(A, y, lam, inter) cache[(nrm, inter, 'opt')] = opt else: opt = cache[(nrm, inter, 'opt')] print('cvx opt val = {}'.format(opt)) print('ps opt val = {}'.format(ps_val)) assert abs(ps_val - opt) < 1e-2
def test_getParams(): projSplit = ps.ProjSplitFit() m = 10 d = 20 A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) processDummy = ProcessDummy() projSplit.addData(A, y, 2, processDummy) nvar = projSplit.numPrimalVars() nobs = projSplit.numObservations() assert (nvar == d + 1), "test failed, nvar!=d+1" assert (nobs == m), "test failed, nobs != m"
def test_cyclic(processor, firsttest): processor.setStep(5e-1) projSplit = ps.ProjSplitFit() m = 20 d = 10 if getNewOptVals and firsttest: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['Acyclic'] = A cache['ycyclic'] = y else: A = cache['Acyclic'] y = cache['ycyclic'] if getNewOptVals and firsttest: result = np.linalg.lstsq(A, y, rcond=None) xhat = result[0] LSval = 0.5 * np.linalg.norm(A.dot(xhat) - y, 2)**2 / m cache['optCyclic'] = LSval else: LSval = cache['optCyclic'] gamma = 1e-1 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=False, intercept=False) #projSplit.addRegularizer(regObj) projSplit.run(maxIterations=2000, keepHistory=True, nblocks=5, blockActivation="cyclic") #fps = projSplit.getHistory()[0] #plt.plot(fps) #plt.show() ps_opt = projSplit.getObjective() print("PS opt = {}".format(ps_opt)) print("LS opt = {}".format(LSval)) assert abs(ps_opt - LSval) < 1e-2 for blks in range(2, 7): projSplit.run(maxIterations=2000, keepHistory=True, nblocks=5, blockActivation="cyclic", resetIterate=True, blocksPerIteration=blks) ps_opt = projSplit.getObjective() assert abs(ps_opt - LSval) < 1e-2
def test_other_p(p, process, testNumber): process.setStep(1.0) gamma = 1e0 projSplit = ps.ProjSplitFit(gamma) m = 40 d = 20 if getNewOptVals: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache_otherLosses[(testNumber, 'A')] = A cache_otherLosses[(testNumber, 'y')] = y else: A = cache_otherLosses[(testNumber, 'A')] y = cache_otherLosses[(testNumber, 'y')] projSplit.addData(A, y, p, process, normalize=False, intercept=False) lam = 0.01 regObj = L1(lam, 1.0) projSplit.addRegularizer(regObj) projSplit.run(primalTol=1e-3, dualTol=1e-3, keepHistory=True, nblocks=2, maxIterations=1000, historyFreq=1) ps_val = projSplit.getObjective() #ps_vals = projSplit.getHistory()[0] #plt.plot(ps_vals) #plt.show() if getNewOptVals: x_cvx = cvx.Variable(d) f = (1 / (m * p)) * cvx.pnorm(A @ x_cvx - y, p)**p f += lam * cvx.norm(x_cvx, 1) prob = cvx.Problem(cvx.Minimize(f)) prob.solve(verbose=True) opt = prob.value cache_otherLosses[(testNumber, 'opt')] = opt else: opt = cache_otherLosses[(testNumber, 'opt')] print("ps val = {}".format(ps_val)) print("cvx val = {}".format(opt)) assert abs(ps_val - opt) < 1e-2
def test_lr(processor, norm, inter): processor.setStep(1e0) projSplit = ps.ProjSplitFit() m = 50 d = 10 if getNewOptVals: A = cache.get('Alr') y = cache.get('ylr') if A is None: A = np.random.normal(0, 1, [m, d]) y = 2.0 * (np.random.normal(0, 1, m) > 0) - 1.0 cache['Alr'] = A cache['ylr'] = y else: A = cache.get('Alr') y = cache.get('ylr') lam = 0.0 gamma = 1e-4 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 'logistic', processor, normalize=norm, intercept=inter) projSplit.run(maxIterations=3000, keepHistory=True, nblocks=10) ps_opt_val = projSplit.getObjective() if getNewOptVals: opt = cache.get((inter, 'optlr')) if opt is None: if inter: AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A A = AwithIntercept opt, _ = runCVX_LR(A, y, lam, intercept=inter) cache[(inter, 'optlr')] = opt else: opt = cache.get((inter, 'optlr')) print("ps opt is {}".format(ps_opt_val)) print("cvx opt is {}".format(opt)) assert abs(opt - ps_opt_val) < 1e-2
def test_blockIs1bug(processor): m = 40 d = 10 if getNewOptVals: A = cache.get('AblockBug') y = cache.get('yblockBug') if A is None: A, y = getLSdata(m, d) cache['AblockBug'] = A cache['yblockBug'] = y else: A = cache.get('AblockBug') y = cache.get('yblockBug') projSplit = ps.ProjSplitFit() gamma = 1e1 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=False, intercept=False) projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1, blockActivation="random") ps_opt = projSplit.getObjective() print('ps func opt = {}'.format(ps_opt)) if getNewOptVals: LSval = cache.get('optBug') if LSval is None: result = np.linalg.lstsq(A, y, rcond=None) xhat = result[0] LSval = 0.5 * np.linalg.norm(A.dot(xhat) - y, 2)**2 / m cache['optBug'] = LSval else: LSval = cache.get('optBug') print('LSval = {}'.format(LSval)) assert abs(LSval - ps_opt) < 1e-2
def test_f1backtrack(gf): projSplit = ps.ProjSplitFit() m = 10 d = 20 if getNewOptVals and (gf == 1.0): A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['Af1bt'] = A cache['yf1bt'] = y else: A = cache['Af1bt'] y = cache['yf1bt'] processor = lp.Forward1Backtrack(growFactor=gf, growFreq=10) projSplit.setDualScaling(1e-1) projSplit.addData(A, y, 2, processor, intercept=True, normalize=True) vec = projSplit.getScaling() assert len(vec) == d projSplit.run(maxIterations=None, keepHistory=True, primalTol=1e-3, dualTol=1e-3, nblocks=5) ps_val = projSplit.getObjective() if getNewOptVals and (gf == 1.0): AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A result = np.linalg.lstsq(AwithIntercept, y, rcond=None) xhat = result[0] LSval = 0.5 * np.linalg.norm(AwithIntercept.dot(xhat) - y, 2)**2 / m cache['optf1bt'] = LSval else: LSval = cache['optf1bt'] assert ps_val - LSval < 1e-2
def test_l1_lasso_blocks(processor, testNumber): m = 40 d = 10 if getNewOptVals and (testNumber == 0): A, y = getLSdata(m, d) cache['lassoA'] = A cache['lassoy'] = y else: A = cache['lassoA'] y = cache['lassoy'] projSplit = ps.ProjSplitFit() gamma = 1e0 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=False, intercept=False) lam = 0.01 step = 1.0 regObj = L1(lam, step) projSplit.addRegularizer(regObj) projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1) ps_val = projSplit.getObjective() if getNewOptVals and (testNumber == 0): opt, xopt = runCVX_lasso(A, y, lam) cache['optlasso'] = opt cache['xlasso'] = xopt else: opt = cache['optlasso'] xopt = cache['xlasso'] print('cvx opt val = {}'.format(opt)) print('ps opt val = {}'.format(ps_val)) assert abs(ps_val - opt) < 1e-2 for numBlocks in range(2, 10): projSplit.run(maxIterations=2000, keepHistory=True, nblocks=numBlocks) ps_val = projSplit.getObjective() #print('cvx opt val = {}'.format(opt)) #print('ps opt val = {}'.format(ps_val)) assert abs(ps_val - opt) < 1e-2
def test_linear_op_l1(norm,inter): m = 40 d = 10 p = 15 if getNewOptVals: A = cache.get('AlinL1') y = cache.get('ylinL1') H = cache.get('HlinL1') if A is None: A,y = getLSdata(m,d) H = np.random.normal(0,1,[p,d]) cache['AlinL1']=A cache['ylinL1']=y cache['HlinL1']=H else: A=cache['AlinL1'] y=cache['ylinL1'] H=cache['HlinL1'] projSplit = ps.ProjSplitFit() stepsize = 1e-1 processor = lp.Forward2Fixed(stepsize) gamma = 1e0 projSplit.setDualScaling(gamma) projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter) lam = 0.01 step = 1.0 regObj = L1(lam,step) projSplit.addRegularizer(regObj,linearOp = aslinearoperator(H)) projSplit.run(maxIterations=5000,keepHistory = True, nblocks = 1, primalTol=1e-3,dualTol=1e-3) ps_val = projSplit.getObjective() if getNewOptVals: opt = cache.get((norm,inter,'optlinL1')) if opt is None: (m,d) = A.shape if norm: Anorm = A scaling = np.linalg.norm(Anorm,axis=0) scaling += 1.0*(scaling < 1e-10) Anorm = np.sqrt(m)*Anorm/scaling A = Anorm if inter: AwithIntercept = np.zeros((m,d+1)) AwithIntercept[:,0] = np.ones(m) AwithIntercept[:,1:(d+1)] = A A = AwithIntercept HwithIntercept = np.zeros((p,d+1)) HwithIntercept[:,0] = np.zeros(p) HwithIntercept[:,1:(d+1)] = H H = HwithIntercept x_cvx = cvx.Variable(d+1) else: x_cvx = cvx.Variable(d) f = (1/(2*m))*cvx.sum_squares(A@x_cvx - y) f += lam*cvx.norm(H @ x_cvx,1) prob = cvx.Problem(cvx.Minimize(f)) prob.solve(verbose=True) opt = prob.value cache[(norm,inter,'optlinL1')]=opt else: opt=cache[(norm,inter,'optlinL1')] primViol = projSplit.getPrimalViolation() dualViol = projSplit.getDualViolation() print("primal violation = {}".format(primViol)) print("dual violation = {}".format(dualViol)) print("ps val = {}".format(ps_val)) print("cvx val = {}".format(opt)) assert ps_val - opt < 1e-2
def test_linear_op_data_term(norm,inter,addL1,add2L1,processor,testNumber): m = 40 d = 10 p = 15 d2 = 10 if getNewOptVals and (testNumber==0): A,y = getLSdata(m,d) H = np.random.normal(0,1,[d2,p]) cache['AdataTerm']=A cache['ydataTerm']=y cache['HdataTerm']=H else: A = cache['AdataTerm'] y = cache['ydataTerm'] H = cache['HdataTerm'] projSplit = ps.ProjSplitFit() processor.setStep(5e-1) gamma = 1e0 projSplit.setDualScaling(gamma) projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter, linearOp = aslinearoperator(H)) lam = 0.01 step = 1.0 if addL1: regObj = L1(lam,step) projSplit.addRegularizer(regObj) if add2L1: regObj2 = L1(lam,step) projSplit.addRegularizer(regObj2) projSplit.run(maxIterations=10000,keepHistory = True, nblocks = 3,primalTol=1e-3,dualTol=1e-3) ps_val = projSplit.getObjective() primViol = projSplit.getPrimalViolation() dualViol = projSplit.getDualViolation() print("primal violation = {}".format(primViol)) print("dual violation = {}".format(dualViol)) if getNewOptVals: opt = cache.get((addL1,add2L1,inter,norm,'optdata')) if opt == None: if norm == True: scaling = np.linalg.norm(A,axis=0) scaling += 1.0*(scaling < 1e-10) A = np.sqrt(A.shape[0])*A/scaling if inter == True: AwithIntercept = np.zeros((m,d+1)) AwithIntercept[:,0] = np.ones(m) AwithIntercept[:,1:(d+1)] = A A = AwithIntercept HwithIntercept = np.zeros((d2+1,p+1)) HwithIntercept[:,0] = np.zeros(d2+1) HwithIntercept[0] = np.ones(p+1) HwithIntercept[0,0] = 1.0 HwithIntercept[1:(d2+1),1:(p+1)] = H H = HwithIntercept (m,_) = A.shape if inter: x_cvx = cvx.Variable(p+1) else: x_cvx = cvx.Variable(p) f = (1/(2*m))*cvx.sum_squares(A@H@x_cvx - y) if addL1: f += lam*cvx.norm(x_cvx,1) if add2L1: f += lam*cvx.norm(x_cvx,1) prob = cvx.Problem(cvx.Minimize(f)) prob.solve(verbose=True) opt = prob.value cache[(addL1,add2L1,inter,norm,'optdata')]=opt else: opt=cache[(addL1,add2L1,inter,norm,'optdata')] print("ps opt = {}".format(ps_val)) print("cvx opt = {}".format(opt)) assert(ps_val-opt<1e-2)
def test_multi_linear_op_l1(norm,inter,testNumber,numblocks): m = 40 d = 10 numregs = 5 if getNewOptVals and (testNumber==0): A,y = getLSdata(m,d) cache['AmutliLinL1']=A cache['ymutliLinL1']=y H = [] for i in range(numregs): p = np.random.randint(1,100) H.append(np.random.normal(0,1,[p,d])) cache['HmultiLinL1']=H else: H=cache['HmultiLinL1'] A=cache['AmutliLinL1'] y=cache['ymutliLinL1'] projSplit = ps.ProjSplitFit() stepsize = 1e-1 processor = lp.Forward2Fixed(stepsize) gamma = 1e0 if norm and inter: gamma = 1e2 projSplit.setDualScaling(gamma) projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter) lam = [] for i in range(numregs): lam.append(0.001*(i+1)) step = 1.0 regObj = L1(lam[-1],step) projSplit.addRegularizer(regObj,linearOp = aslinearoperator(H[i])) projSplit.run(maxIterations=5000,keepHistory = True, nblocks = numblocks, primalTol=1e-6,dualTol=1e-6) ps_val = projSplit.getObjective() if getNewOptVals: if norm: Anorm = A m = Anorm.shape[0] scaling = np.linalg.norm(Anorm,axis=0) scaling += 1.0*(scaling < 1e-10) Anorm = np.sqrt(m)*Anorm/scaling A = Anorm if inter: AwithIntercept = np.zeros((m,d+1)) AwithIntercept[:,0] = np.ones(m) AwithIntercept[:,1:(d+1)] = A A = AwithIntercept (m,d) = A.shape x_cvx = cvx.Variable(d) f = (1/(2*m))*cvx.sum_squares(A@x_cvx - y) for i in range(numregs): if inter: f += lam[i]*cvx.norm(H[i] @ x_cvx[1:d],1) else: f += lam[i]*cvx.norm(H[i] @ x_cvx,1) prob = cvx.Problem(cvx.Minimize(f)) prob.solve(verbose=True) opt = prob.value cache[(norm,inter,'opt')]=opt else: opt=cache[(norm,inter,'opt')] print("ps val = {}".format(ps_val)) print("cvx val = {}".format(opt)) assert ps_val - opt < 1e-2
def test_getGamma2(gammain): projSplit = ps.ProjSplitFit(gammain) gamma = projSplit.getDualScaling() assert gamma == gammain, "failed, gamma != gammain"
def test_ls_blocks(processor): processor.setStep(5e-1) projSplit = ps.ProjSplitFit() m = 20 d = 10 if getNewOptVals: A = cache.get('AlsBlocks') y = cache.get('ylsBlocks') if A is None: A = np.random.normal(0, 1, [m, d]) y = np.random.normal(0, 1, m) cache['AlsBlocks'] = A cache['ylsBlocks'] = y else: A = cache.get('AlsBlocks') y = cache.get('ylsBlocks') gamma = 1e-1 projSplit.setDualScaling(gamma) projSplit.addData(A, y, 2, processor, normalize=False) projSplit.run(maxIterations=1500, keepHistory=True, nblocks=10, primalTol=1e-3, dualTol=1e-3) assert projSplit.getObjective() >= 0, "objective is not >= 0" sol = projSplit.getSolution() assert sol.shape == (d + 1, ) if getNewOptVals: LSresid = cache.get('optlsblocks') if LSresid is None: AwithIntercept = np.zeros((m, d + 1)) AwithIntercept[:, 0] = np.ones(m) AwithIntercept[:, 1:(d + 1)] = A result = np.linalg.lstsq(AwithIntercept, y, rcond=None) xhat = result[0] LSresid = 0.5 * np.linalg.norm(AwithIntercept.dot(xhat) - y, 2)**2 / m cache['optlsblocks'] = LSresid else: LSresid = cache.get('optlsblocks') PSresid = projSplit.getObjective() assert abs(LSresid - PSresid) < 1e-2 PSresidErg = projSplit.getObjective(ergodic="simple") assert abs(LSresid - PSresidErg) < 1e-2 PsresidErgWeight = projSplit.getObjective(ergodic="weighted") assert abs(LSresid - PsresidErgWeight) < 1e-2 projSplit.run(maxIterations=1000, keepHistory=True, resetIterate=True, blockActivation="random", nblocks=10) PSrandom = projSplit.getObjective() assert abs(PSresid - PSrandom) < 1e-2 projSplit.run(maxIterations=1000, keepHistory=True, resetIterate=True, blockActivation="cyclic", nblocks=10) PScyclic = projSplit.getObjective() assert abs(PSresid - PScyclic) < 1e-2
def test_bad_gamma(gammain): projSplit = ps.ProjSplitFit(gammain) gamma = projSplit.getDualScaling() assert gamma == 1.0,"failed, gamma != 1.0"
def test_bad_dims(y): psObj = ps.ProjSplitFit() obs = np.array([[1,2,3],[4,5,6]]) with pytest.raises(Exception): psObj.addData(obs,y,2)
print("================") if loss == "log": loss2use = "logistic" backProcess = lp.BackwardLBFGS() else: loss2use = 2 backProcess = lp.BackwardCG() embed = False if run2f: print("2f") gamma2f = gamma2fs[i] t0 = time.time() psObj = ps.ProjSplitFit(gamma2f) if embed: psObj.addData(X, y, loss2use, linearOp=H, normalize=False, process=lp.Forward2Backtrack(), embed=regularizers.L1(scaling=(1 - mu) * lam)) else: psObj.addData(X, y, loss2use, linearOp=H, normalize=False,
iter=maxIter, gamma1=tuned,gamma2=tuned,G=G,Gt=Gt) results[(i,'tseng')] = outtseng.finalFuncVal if False: outfrb = algo.for_reflect_back(theFunc,proxfstar_4_tseng,proxg,theGrad,init,iter=maxIter, gamma0=tuned,gamma1=tuned,G=G,Gt=Gt,verbose=False,getFuncVals=False) results[(i,'frb')] = outfrb.finalFuncVal if loss == "log": loss2use = "logistic" else: loss2use = 2 gamma = 1.0 if True : psObj = ps.ProjSplitFit(gamma) proc = lp.Forward2Backtrack() psObj.addData(X,y,loss2use,linearOp=H,normalize=False,process=proc, embed=regularizers.L1(scaling=(1-mu)*lam)) (nbeta,ngamma) = H.shape shape = (ngamma-1,ngamma) G_for_ps = sl.LinearOperator(shape,matvec=lambda x: x[:-1],rmatvec = lambda x : np.concatenate((x,np.array([0])))) psObj.addRegularizer(regularizers.L1(scaling = mu*lam,step=tuned),linearOp=G_for_ps) psObj.run(nblocks=10,maxIterations=maxIter,verbose=False,keepHistory=False, primalTol=0.0,dualTol=0.0,blockActivation="greedy") results[(i,'ps2fembed_g')] = psObj.getObjective() if False: psObj = ps.ProjSplitFit(gamma) psObj.addData(X,y,loss2use,linearOp=H,normalize=False,process=lp.Forward1Backtrack(),
import sys sys.path.append('../') import numpy as np import projSplitFit as ps ### Basic Setup with a Quadratic Loss ### Test on random data m = 500 d = 1000 np.random.seed(1) A = np.random.normal(0, 1, [m, d]) r = np.random.normal(0, 1, m) projSplit = ps.ProjSplitFit() projSplit.addData(A, r, loss=2, intercept=False) projSplit.run() optimalVal = projSplit.getObjective() z = projSplit.getSolution() print(f"Objective value LS prob = {optimalVal}") ### changing the dual scaling gamma = 1e2 projSplit.setDualScaling(gamma) projSplit.run() ### adding a regularizer from regularizers import L1 lam1 = 0.1 regObj = L1(scaling=lam1) projSplit.addRegularizer(regObj) projSplit.run()