def test_one_sided():
projSplit = ps.ProjSplitFit()
def deriv(x,y):
return (x>=y)*(x-y)
def val(x,y):
return (x>=y)*(x-y)**2
loss = ls.LossPlugIn(deriv)
loss = ls.LossPlugIn(deriv,val)
m = 20
d = 50
A = normal(0,1,[m,d])
y = normal(0,1,m)
projSplit.addData(A,y,loss=loss,intercept=False,normalize=False)
projSplit.addRegularizer(L1())
projSplit.run(keepHistory=False,nblocks=10)
primTol = projSplit.getPrimalViolation()
dualTol = projSplit.getDualViolation()
assert primTol <1e-6
assert dualTol <1e-6
def test_add_linear_ops():
projSplit = ps.ProjSplitFit()
m = 10
d = 20
A = np.random.normal(0, 1, [m, d])
y = np.random.normal(0, 1, m)
processDummy = ProcessDummy()
projSplit.addData(A, y, 2, processDummy)
p = 11
H = np.random.normal(0, 1, [p, d])
lam = 0.01
step = 1.0
regObj = L1(lam, step)
projSplit.addRegularizer(regObj, linearOp=aslinearoperator(H))
d2 = 9
H = np.random.normal(0, 1, [p, d2])
try:
projSplit.addRegularizer(regObj, linearOp=aslinearoperator(H)) == -1
noExcept = True
except:
noExcept = False
assert noExcept == False
def test_add_regularizer():
projSplit = ps.ProjSplitFit()
scale = 11.5
regObj = L1(scale)
projSplit.addRegularizer(regObj)
scale2 = 15.7
regObj.setScaling(scale2)
assert (projSplit.allRegularizers[0].getScaling() == scale2)
def test_good_embed():
projSplit = ps.ProjSplitFit()
m = 10
d = 20
A = np.random.normal(0, 1, [m, d])
y = np.random.normal(0, 1, m)
processDummy = ProcessDummy()
regObj = L1()
projSplit.addData(A, y, 2, processDummy, embed=regObj)
assert projSplit.numRegs == 0
def test_L1LR(processor, nrm, inter, testNumber):
m = 40
d = 10
if getNewOptVals and (testNumber == 0):
A, y = getLRdata(m, d)
cache['A'] = A
cache['y'] = y
else:
A = cache['A']
y = cache['y']
projSplit = ps.ProjSplitFit()
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A,
y,
'logistic',
processor,
normalize=nrm,
intercept=inter)
lam = 5e-2
step = 1.0
regObj = L1(lam, step)
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1)
ps_val = projSplit.getObjective()
if getNewOptVals:
opt = cache.get((nrm, inter, 'opt'))
if opt is None:
if nrm:
Anorm = A
n = A.shape[0]
scaling = np.linalg.norm(Anorm, axis=0)
scaling += 1.0 * (scaling < 1e-10)
A = np.sqrt(n) * Anorm / scaling
if inter:
AwithIntercept = np.zeros((m, d + 1))
AwithIntercept[:, 0] = np.ones(m)
AwithIntercept[:, 1:(d + 1)] = A
A = AwithIntercept
opt, _ = runCVX_LR(A, y, lam, inter)
cache[(nrm, inter, 'opt')] = opt
else:
opt = cache[(nrm, inter, 'opt')]
print('cvx opt val = {}'.format(opt))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
def test_L1():
#projSplit = ps.ProjSplitFit()
scale = 15.0
regObj = L1(scale)
assert regObj.getScaling() == scale
scale = -1.0
regObj = L1(scale)
assert (regObj.getScaling(), regObj.getStep()) == (1.0, 1.0)
regObj = L1(scale)
assert (regObj.getScaling(), regObj.getStep()) == (1.0, 1.0)
scale = 11.5
rho = 3.0
regObj = L1(scale, rho)
lenx = 10
x = np.ones(lenx)
assert regObj.evaluate(x) == lenx * scale
toTest = regObj.getProx(x)
assert toTest.shape == (lenx, )
diff = toTest - np.zeros(lenx)
assert (diff == 0.0).all()
def test_other_p(p, process, testNumber):
process.setStep(1.0)
gamma = 1e0
projSplit = ps.ProjSplitFit(gamma)
m = 40
d = 20
if getNewOptVals:
A = np.random.normal(0, 1, [m, d])
y = np.random.normal(0, 1, m)
cache_otherLosses[(testNumber, 'A')] = A
cache_otherLosses[(testNumber, 'y')] = y
else:
A = cache_otherLosses[(testNumber, 'A')]
y = cache_otherLosses[(testNumber, 'y')]
projSplit.addData(A, y, p, process, normalize=False, intercept=False)
lam = 0.01
regObj = L1(lam, 1.0)
projSplit.addRegularizer(regObj)
projSplit.run(primalTol=1e-3,
dualTol=1e-3,
keepHistory=True,
nblocks=2,
maxIterations=1000,
historyFreq=1)
ps_val = projSplit.getObjective()
#ps_vals = projSplit.getHistory()[0]
#plt.plot(ps_vals)
#plt.show()
if getNewOptVals:
x_cvx = cvx.Variable(d)
f = (1 / (m * p)) * cvx.pnorm(A @ x_cvx - y, p)**p
f += lam * cvx.norm(x_cvx, 1)
prob = cvx.Problem(cvx.Minimize(f))
prob.solve(verbose=True)
opt = prob.value
cache_otherLosses[(testNumber, 'opt')] = opt
else:
opt = cache_otherLosses[(testNumber, 'opt')]
print("ps val = {}".format(ps_val))
print("cvx val = {}".format(opt))
assert abs(ps_val - opt) < 1e-2
def test_l1_lasso_blocks(processor, testNumber):
m = 40
d = 10
if getNewOptVals and (testNumber == 0):
A, y = getLSdata(m, d)
cache['lassoA'] = A
cache['lassoy'] = y
else:
A = cache['lassoA']
y = cache['lassoy']
projSplit = ps.ProjSplitFit()
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A, y, 2, processor, normalize=False, intercept=False)
lam = 0.01
step = 1.0
regObj = L1(lam, step)
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1)
ps_val = projSplit.getObjective()
if getNewOptVals and (testNumber == 0):
opt, xopt = runCVX_lasso(A, y, lam)
cache['optlasso'] = opt
cache['xlasso'] = xopt
else:
opt = cache['optlasso']
xopt = cache['xlasso']
print('cvx opt val = {}'.format(opt))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
for numBlocks in range(2, 10):
projSplit.run(maxIterations=2000, keepHistory=True, nblocks=numBlocks)
ps_val = projSplit.getObjective()
#print('cvx opt val = {}'.format(opt))
#print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
def test_linear_op_l1(norm,inter):
m = 40
d = 10
p = 15
if getNewOptVals:
A = cache.get('AlinL1')
y = cache.get('ylinL1')
H = cache.get('HlinL1')
if A is None:
A,y = getLSdata(m,d)
H = np.random.normal(0,1,[p,d])
cache['AlinL1']=A
cache['ylinL1']=y
cache['HlinL1']=H
else:
A=cache['AlinL1']
y=cache['ylinL1']
H=cache['HlinL1']
projSplit = ps.ProjSplitFit()
stepsize = 1e-1
processor = lp.Forward2Fixed(stepsize)
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter)
lam = 0.01
step = 1.0
regObj = L1(lam,step)
projSplit.addRegularizer(regObj,linearOp = aslinearoperator(H))
projSplit.run(maxIterations=5000,keepHistory = True, nblocks = 1,
primalTol=1e-3,dualTol=1e-3)
ps_val = projSplit.getObjective()
if getNewOptVals:
opt = cache.get((norm,inter,'optlinL1'))
if opt is None:
(m,d) = A.shape
if norm:
Anorm = A
scaling = np.linalg.norm(Anorm,axis=0)
scaling += 1.0*(scaling < 1e-10)
Anorm = np.sqrt(m)*Anorm/scaling
A = Anorm
if inter:
AwithIntercept = np.zeros((m,d+1))
AwithIntercept[:,0] = np.ones(m)
AwithIntercept[:,1:(d+1)] = A
A = AwithIntercept
HwithIntercept = np.zeros((p,d+1))
HwithIntercept[:,0] = np.zeros(p)
HwithIntercept[:,1:(d+1)] = H
H = HwithIntercept
x_cvx = cvx.Variable(d+1)
else:
x_cvx = cvx.Variable(d)
f = (1/(2*m))*cvx.sum_squares(A@x_cvx - y)
f += lam*cvx.norm(H @ x_cvx,1)
prob = cvx.Problem(cvx.Minimize(f))
prob.solve(verbose=True)
opt = prob.value
cache[(norm,inter,'optlinL1')]=opt
else:
opt=cache[(norm,inter,'optlinL1')]
primViol = projSplit.getPrimalViolation()
dualViol = projSplit.getDualViolation()
print("primal violation = {}".format(primViol))
print("dual violation = {}".format(dualViol))
print("ps val = {}".format(ps_val))
print("cvx val = {}".format(opt))
assert ps_val - opt < 1e-2
def test_linear_op_data_term(norm,inter,addL1,add2L1,processor,testNumber):
m = 40
d = 10
p = 15
d2 = 10
if getNewOptVals and (testNumber==0):
A,y = getLSdata(m,d)
H = np.random.normal(0,1,[d2,p])
cache['AdataTerm']=A
cache['ydataTerm']=y
cache['HdataTerm']=H
else:
A = cache['AdataTerm']
y = cache['ydataTerm']
H = cache['HdataTerm']
projSplit = ps.ProjSplitFit()
processor.setStep(5e-1)
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter,
linearOp = aslinearoperator(H))
lam = 0.01
step = 1.0
if addL1:
regObj = L1(lam,step)
projSplit.addRegularizer(regObj)
if add2L1:
regObj2 = L1(lam,step)
projSplit.addRegularizer(regObj2)
projSplit.run(maxIterations=10000,keepHistory = True,
nblocks = 3,primalTol=1e-3,dualTol=1e-3)
ps_val = projSplit.getObjective()
primViol = projSplit.getPrimalViolation()
dualViol = projSplit.getDualViolation()
print("primal violation = {}".format(primViol))
print("dual violation = {}".format(dualViol))
if getNewOptVals:
opt = cache.get((addL1,add2L1,inter,norm,'optdata'))
if opt == None:
if norm == True:
scaling = np.linalg.norm(A,axis=0)
scaling += 1.0*(scaling < 1e-10)
A = np.sqrt(A.shape[0])*A/scaling
if inter == True:
AwithIntercept = np.zeros((m,d+1))
AwithIntercept[:,0] = np.ones(m)
AwithIntercept[:,1:(d+1)] = A
A = AwithIntercept
HwithIntercept = np.zeros((d2+1,p+1))
HwithIntercept[:,0] = np.zeros(d2+1)
HwithIntercept[0] = np.ones(p+1)
HwithIntercept[0,0] = 1.0
HwithIntercept[1:(d2+1),1:(p+1)] = H
H = HwithIntercept
(m,_) = A.shape
if inter:
x_cvx = cvx.Variable(p+1)
else:
x_cvx = cvx.Variable(p)
f = (1/(2*m))*cvx.sum_squares(A@H@x_cvx - y)
if addL1:
f += lam*cvx.norm(x_cvx,1)
if add2L1:
f += lam*cvx.norm(x_cvx,1)
prob = cvx.Problem(cvx.Minimize(f))
prob.solve(verbose=True)
opt = prob.value
cache[(addL1,add2L1,inter,norm,'optdata')]=opt
else:
opt=cache[(addL1,add2L1,inter,norm,'optdata')]
print("ps opt = {}".format(ps_val))
print("cvx opt = {}".format(opt))
assert(ps_val-opt<1e-2)
def test_multi_linear_op_l1(norm,inter,testNumber,numblocks):
m = 40
d = 10
numregs = 5
if getNewOptVals and (testNumber==0):
A,y = getLSdata(m,d)
cache['AmutliLinL1']=A
cache['ymutliLinL1']=y
H = []
for i in range(numregs):
p = np.random.randint(1,100)
H.append(np.random.normal(0,1,[p,d]))
cache['HmultiLinL1']=H
else:
H=cache['HmultiLinL1']
A=cache['AmutliLinL1']
y=cache['ymutliLinL1']
projSplit = ps.ProjSplitFit()
stepsize = 1e-1
processor = lp.Forward2Fixed(stepsize)
gamma = 1e0
if norm and inter:
gamma = 1e2
projSplit.setDualScaling(gamma)
projSplit.addData(A,y,2,processor,normalize=norm,intercept=inter)
lam = []
for i in range(numregs):
lam.append(0.001*(i+1))
step = 1.0
regObj = L1(lam[-1],step)
projSplit.addRegularizer(regObj,linearOp = aslinearoperator(H[i]))
projSplit.run(maxIterations=5000,keepHistory = True, nblocks = numblocks,
primalTol=1e-6,dualTol=1e-6)
ps_val = projSplit.getObjective()
if getNewOptVals:
if norm:
Anorm = A
m = Anorm.shape[0]
scaling = np.linalg.norm(Anorm,axis=0)
scaling += 1.0*(scaling < 1e-10)
Anorm = np.sqrt(m)*Anorm/scaling
A = Anorm
if inter:
AwithIntercept = np.zeros((m,d+1))
AwithIntercept[:,0] = np.ones(m)
AwithIntercept[:,1:(d+1)] = A
A = AwithIntercept
(m,d) = A.shape
x_cvx = cvx.Variable(d)
f = (1/(2*m))*cvx.sum_squares(A@x_cvx - y)
for i in range(numregs):
if inter:
f += lam[i]*cvx.norm(H[i] @ x_cvx[1:d],1)
else:
f += lam[i]*cvx.norm(H[i] @ x_cvx,1)
prob = cvx.Problem(cvx.Minimize(f))
prob.solve(verbose=True)
opt = prob.value
cache[(norm,inter,'opt')]=opt
else:
opt=cache[(norm,inter,'opt')]
print("ps val = {}".format(ps_val))
print("cvx val = {}".format(opt))
assert ps_val - opt < 1e-2
import numpy as np
import projSplitFit as ps
### Basic Setup with a Quadratic Loss
### Test on random data
m = 500
d = 1000
np.random.seed(1)
A = np.random.normal(0, 1, [m, d])
r = np.random.normal(0, 1, m)
projSplit = ps.ProjSplitFit()
projSplit.addData(A, r, loss=2, intercept=False)
projSplit.run()
optimalVal = projSplit.getObjective()
z = projSplit.getSolution()
print(f"Objective value LS prob = {optimalVal}")
### changing the dual scaling
gamma = 1e2
projSplit.setDualScaling(gamma)
projSplit.run()
### adding a regularizer
from regularizers import L1
lam1 = 0.1
regObj = L1(scaling=lam1)
projSplit.addRegularizer(regObj)
projSplit.run()
optimalVal = projSplit.getObjective()
print(f"Objective value L1LS prob = {optimalVal}")
def test_l1_intercept_and_normalize(processor, inter, norm):
m = 40
d = 10
if getNewOptVals:
A = cache.get('Al1intAndNorm')
y = cache.get('yl1intAndNorm')
if A is None:
A, y = getLSdata(m, d)
cache['Al1intAndNorm'] = A
cache['yl1intAndNorm'] = y
else:
A = cache['Al1intAndNorm']
y = cache['yl1intAndNorm']
projSplit = ps.ProjSplitFit()
if inter and norm:
gamma = 1e-2
elif (inter == False) and norm:
gamma = 1e-4
else:
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A, y, 2, processor, normalize=norm, intercept=inter)
lam = 1e-3
step = 1.0
regObj = L1(lam, step)
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=5000,
keepHistory=True,
nblocks=10,
primalTol=1e-3,
dualTol=1e-3)
ps_val = projSplit.getObjective()
primViol = projSplit.getPrimalViolation()
dualViol = projSplit.getDualViolation()
print("primal violation = {}".format(primViol))
print("dual violation = {}".format(dualViol))
if getNewOptVals:
opt = cache.get((inter, norm, 'l1opt'))
if opt is None:
if norm:
Anorm = np.copy(A)
n = A.shape[0]
scaling = np.linalg.norm(Anorm, axis=0)
scaling += 1.0 * (scaling < 1e-10)
Anorm = np.sqrt(n) * Anorm / scaling
else:
Anorm = A
AwithIntercept = np.zeros((m, d + 1))
if inter:
AwithIntercept[:, 0] = np.ones(m)
else:
AwithIntercept[:, 0] = np.zeros(m)
AwithIntercept[:, 1:(d + 1)] = Anorm
opt, _ = runCVX_lasso(AwithIntercept, y, lam, True)
cache[(inter, norm, 'l1opt')] = opt
else:
opt = cache[(inter, norm, 'l1opt')]
print('cvx opt val = {}'.format(opt))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
def test_embedded(processor, testNumber):
m = 40
d = 10
if getNewOptVals and (testNumber == 0):
A, y = getLSdata(m, d)
cache['A_embed'] = A
cache['y_embed'] = y
else:
A = cache['A_embed']
y = cache['y_embed']
projSplit = ps.ProjSplitFit()
gamma = 1e0
projSplit.setDualScaling(gamma)
lam = 0.01
step = 1.0
regObj = L1(lam, step)
projSplit.addData(A,
y,
2,
processor,
normalize=False,
intercept=False,
embed=regObj)
if getNewOptVals and (testNumber == 0):
opt, _ = runCVX_lasso(A, y, lam)
cache['embed_opt1'] = opt
else:
opt = cache['embed_opt1']
for nblocks in range(1, 11, 3):
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=nblocks)
ps_val = projSplit.getObjective()
print('cvx opt val = {}'.format(opt))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=5)
ps_val = projSplit.getObjective()
if getNewOptVals and (testNumber == 0):
opt2, _ = runCVX_lasso(A, y, 2 * lam)
cache['embed_opt2'] = opt2
else:
opt2 = cache['embed_opt2']
print('cvx opt val = {}'.format(opt2))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt2) < 1e-2
projSplit = ps.ProjSplitFit()
stepsize = 1e-1
processor = lp.Forward2Fixed(stepsize)
gamma = 1e-2
projSplit.setDualScaling(gamma)
lam = 0.01
step = 1.0
regObj = L1(lam, step)
projSplit.addData(A,
y,
2,
processor,
normalize=True,
intercept=True,
embed=regObj)
regObj = L1(lam, step)
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=5)
ps_val = projSplit.getObjective()
if getNewOptVals and (testNumber == 0):
AwithIntercept = np.zeros((m, d + 1))
AwithIntercept[:, 0] = np.ones(m)
AwithIntercept[:, 1:(d + 1)] = A
opt3, _ = runCVX_lasso(AwithIntercept, y, 2 * lam, True, True)
cache['embed_opt3'] = opt3
else:
opt3 = cache['embed_opt3']
print('cvx opt val = {}'.format(opt3))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt3) < 1e-2
def test_l1_multi_lasso(processor, testNumber, equalize):
m = 40
d = 10
if getNewOptVals and (testNumber == 0):
A, y = getLSdata(m, d)
cache['Amulti'] = A
cache['ymulti'] = y
else:
A = cache['Amulti']
y = cache['ymulti']
projSplit = ps.ProjSplitFit()
gamma = 1e0
projSplit.setDualScaling(gamma)
projSplit.addData(A, y, 2, processor, normalize=False, intercept=False)
lam = 0.01
step = 1.0
regObj = L1(lam, step)
fac = 5 # add the same regularizer twice, same as using
# it once with twice the parameter
for _ in range(fac):
projSplit.addRegularizer(regObj)
projSplit.run(maxIterations=1000,
keepHistory=True,
nblocks=1,
equalizeStepsizes=equalize)
ps_val = projSplit.getObjective()
if getNewOptVals and (testNumber == 0):
opt, _ = runCVX_lasso(A, y, fac * lam)
cache['opt_multi'] = opt
else:
opt = cache['opt_multi']
print('cvx opt val = {}'.format(opt))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt) < 1e-2
# test with intercept
projSplit.addData(A, y, 2, processor, normalize=False, intercept=True)
projSplit.run(maxIterations=1000, keepHistory=True, nblocks=1)
ps_val = projSplit.getObjective()
if getNewOptVals and (testNumber == 0):
AwithIntercept = np.zeros((m, d + 1))
AwithIntercept[:, 0] = np.ones(m)
AwithIntercept[:, 1:(d + 1)] = A
opt_multi_inter, _ = runCVX_lasso(AwithIntercept, y, fac * lam, True)
cache['opt_multi_inter'] = opt_multi_inter
else:
opt_multi_inter = cache['opt_multi_inter']
#print('cvx opt val = {}'.format(opt))
#print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt_multi_inter) < 1e-2
# test multi-data-blocks
for bblocks in range(2, 11):
projSplit.run(maxIterations=2000, keepHistory=True, nblocks=bblocks)
ps_val = projSplit.getObjective()
print('cvx opt val = {}'.format(opt_multi_inter))
print('ps opt val = {}'.format(ps_val))
assert abs(ps_val - opt_multi_inter) < 1e-2