def profile_cwpath_robust_graphnet():
# get training data and constants
Data = np.load("Data.npz")
X = Data['X'][0:1000,:]
print "Data matrix size:",X.shape
Y = Data['Y'][0:1000]
nx = np.sqrt(X.shape[1])
ny = np.sqrt(X.shape[1])
A = construct_adjacency_list(nx,ny,1)
lam_max = get_lambda_max(X,Y)
tol = 1e-6
initial=None
# choose penalty grid
l1 = np.linspace(4*lam_max, 0.2*lam_max, num=100)
l2 = 100.
l3 = 1000.
delta = 1.0
# setup problem
problemtype = graphnet.RobustGraphNet
problemkey = 'RobustGraphNet'
print "Robust GraphNet with penalties (l1, l2, l3, delta)", l1, l2, l3, delta
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs=initial) #,initial_coefs=np.array([14.]*10))
l.problem.assign_penalty(path_key='l1',l1=l1,l2=l2,l3=l3,delta=delta)
coefficients, residuals = l.fit(tol=tol, initial=initial)
def profile_cwpath_robust_graphnet():
# get training data and constants
Data = np.load("Data.npz")
X = Data["X"][0:1000, :]
print "Data matrix size:", X.shape
Y = Data["Y"][0:1000]
nx = np.sqrt(X.shape[1])
ny = np.sqrt(X.shape[1])
A = construct_adjacency_list(nx, ny, 1)
lam_max = get_lambda_max(X, Y)
tol = 1e-6
initial = None
# choose penalty grid
l1 = np.linspace(4 * lam_max, 0.2 * lam_max, num=100)
l2 = 100.0
l3 = 1000.0
delta = 1.0
# setup problem
problemtype = graphnet.RobustGraphNet
problemkey = "RobustGraphNet"
print "Robust GraphNet with penalties (l1, l2, l3, delta)", l1, l2, l3, delta
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs=initial) # ,initial_coefs=np.array([14.]*10))
l.problem.assign_penalty(path_key="l1", l1=l1, l2=l2, l3=l3, delta=delta)
coefficients, residuals = l.fit(tol=tol, initial=initial)
def test_graphnet(X,Y,G=None,l1=500.,l2=-999.0,l3=-999.0,delta=-999.0,svmdelta=-999.0,initial=None,adaptive=False,svm=False,scipy_compare=True,tol=1e-5):
tic = time.clock()
# Cases set based on parameters and robust/adaptive/svm flags
if l2 != -999.0 or l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if G is None:
nx = 60
ny = 60
A, Afull = construct_adjacency_list(nx,ny,1,return_full=True)
# A, Afull = gen_adj(X.shape[1])
else:
A = G.copy()
if delta != -999.0:
if svmdelta != -999.0:
print "-------------------------------------------HUBER SVM---------------------------------------------------"
problemtype = "HuberSVMGraphNet"
problemkey = "HuberSVMGraphNet"
print "HuberSVM GraphNet with penalties (l1,l2,l3,delta):", l1, l2, l3, delta
Y = 2*np.round(np.random.uniform(0,1,len(Y)))-1
l = cwpath.CoordWise((X, Y, A), graphnet.GraphSVM) #, initial_coefs=10.*np.array(range(11)*1))
l.problem.assign_penalty(path_key='l1',l1=l1,l2=l2,l3=l3,delta=delta)
else:
print "----------------------------------------ROBUST GRAPHNET------------------------------------------------"
problemtype = graphnet.RobustGraphNet
problemkey = 'RobustGraphNet'
print "Robust GraphNet with penalties (l1, l2, l3, delta)", l1, l2, l3, delta
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs = initial) #,initial_coefs=np.array([14.]*10))
l.problem.assign_penalty(path_key='l1',l1=l1,l2=l2,l3=l3,delta=delta)
else:
print "-------------------------------------------GRAPHNET---------------------------------------------------"
problemtype = graphnet.NaiveGraphNet
problemkey = 'NaiveGraphNet'
print "Testing GraphNet with penalties (l1,l2,l3):", l1,l2,l3
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs = initial)
l.problem.assign_penalty(path_key='l1',l1=l1, l2=l2, l3=l3)
else:
print "-------------------------------------------ELASTIC NET---------------------------------------------------"
problemtype = graphnet.NaiveENet
problemkey = 'NaiveENet'
print "Testing ENET with penalties (l1,l2):", l1, l2
l = cwpath.CoordWise((X, Y), problemtype, initial_coefs = initial) #, initial_coefs = np.array([4.]*10))
l.problem.assign_penalty(path_key='l1',l1=l1, l2=l2)
else:
print "-------------------------------------------LASSO---------------------------------------------------"
problemtype = graphnet.Lasso
problemkey = 'Lasso'
print "Testing LASSO with penalty:", l1
l = cwpath.CoordWise((X, Y), problemtype, initial_coefs = initial) #, initial_coefs= np.array([7.]*10))
l.problem.assign_penalty(path_key='l1',l1=l1)
# fit and get results
coefficients, residuals = l.fit(tol=tol, initial=initial)
print "\t---> Fitting GraphNet problem with coordinate descent took:", time.clock()-tic, "seconds."
if adaptive:
tic = time.clock()
l1weights = 1./beta
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs = initial)
l.problem.assign_penalty(l1=l1,l2=l2,l3=l3,delta=delta,l1weights=l1weights,newl1=l1)
adaptive_coefficients, adaptive_residuals = l.fit(tol=tol, initial=initial)
print "\t---> Fitting Adaptive GraphNet problem with coordinate descent took:", time.clock()-tic, "seconds."
# if compare to scipy flag is set,
# compare the above result with the same problem
# solved using a built in scipy solver (fmin_powell).
if scipy_compare:
print "\t---> Fitting with scipy for comparison..."
tic = time.clock()
l1 = l1[-1] # choose only last l1 value
beta = coefficients[-1] # coordinate-wise coefficients
if l2 != -999.0 or l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if delta != -999.0:
if adaptive:
if svmdelta != -999.0:
# HuberSVM Graphnet
Xp2 = np.hstack([np.ones(X.shape[0])[:,np.newaxis],X])
def f(beta):
ind = range(1,len(beta))
return huber_svm_error(beta,Y,Xp2,delta).sum() + np.fabs(beta[ind]).sum()*l1 + l2 * np.linalg.norm(beta[ind])**2/2 + l3 * np.dot(beta[ind], np.dot(Afull, beta[ind]))/2
else:
# Robust Adaptive Graphnet
def f(beta):
return huber(Y - np.dot(X, beta),delta).sum()/2 + l1*np.dot(np.fabs(beta),l1weights) + l2*np.linalg.norm(beta)**2/2 + l3*np.dot(beta, np.dot(Afull, beta))/2
else:
# Robust Graphnet
def f(beta):
try:
return huber(Y - np.dot(X, beta.T),delta).sum()/2 + np.fabs(beta).sum()*l1 + l2 * np.linalg.norm(beta)**2/2 + l3 * np.dot(beta, np.dot(Afull, beta.T))/2
except:
return huber(Y - np.dot(X, beta),delta).sum()/2 + np.fabs(beta).sum()*l1 + l2 * np.linalg.norm(beta)**2/2 + l3 * np.dot(beta, np.dot(Afull, beta).T)/2
else:
# Graphnet
def f(beta):
return np.linalg.norm(Y - np.dot(X, beta))**2/2 + np.fabs(beta).sum()*l1 + l2 * np.linalg.norm(beta)**2/2 + l3 * np.dot(beta, np.dot(Afull, beta))/2
else:
# Elastic Net
def f(beta):
return np.linalg.norm(Y - np.dot(X, beta))**2/2 + np.fabs(beta).sum()*l1 + l2 * np.linalg.norm(beta)**2/2
else:
# Lasso
def f(beta):
return np.linalg.norm(Y - np.dot(X, beta))**2/2 + np.fabs(beta).sum()*l1
# optimize
if problemkey == 'HuberSVMGraphNet':
v = scipy.optimize.fmin_powell(f, np.zeros(Xp2.shape[1]), ftol=1.0e-14, xtol=1.0e-14, maxfun=100000)
else:
v = scipy.optimize.fmin_powell(f, np.zeros(X.shape[1]), ftol=1.0e-10, xtol=1.0e-10,maxfun=100000)
v = np.asarray(v)
print "\t---> Fitting GraphNet with scipy took:", time.clock()-tic, "seconds."
# print np.round(100*v)/100,'\n', np.round(100*beta)/100
assert_true(np.fabs(f(v) - f(beta)) / np.fabs(f(v) + f(beta)) < tol)
if np.linalg.norm(v) > 1e-8:
assert_true(np.linalg.norm(v - beta) / np.linalg.norm(v) < tol)
else:
assert_true(np.linalg.norm(beta) < 1e-8)
print "\t---> Coordinate-wise and Scipy optimization agree!"
return (coefficients, residuals), problemkey
def test_graphnet(X,
Y,
G=None,
l1=500.,
l2=-999.0,
l3=-999.0,
delta=-999.0,
svmdelta=-999.0,
initial=None,
adaptive=False,
svm=False,
scipy_compare=True,
tol=1e-5):
tic = time.clock()
# Cases set based on parameters and robust/adaptive/svm flags
if l2 != -999.0 or l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if G is None:
nx = 60
ny = 60
A, Afull = construct_adjacency_list(nx,
ny,
1,
return_full=True)
# A, Afull = gen_adj(X.shape[1])
else:
A = G.copy()
if delta != -999.0:
if svmdelta != -999.0:
print "-------------------------------------------HUBER SVM---------------------------------------------------"
problemtype = "HuberSVMGraphNet"
problemkey = "HuberSVMGraphNet"
print "HuberSVM GraphNet with penalties (l1,l2,l3,delta):", l1, l2, l3, delta
Y = 2 * np.round(np.random.uniform(0, 1, len(Y))) - 1
l = cwpath.CoordWise(
(X, Y, A), graphnet.GraphSVM
) #, initial_coefs=10.*np.array(range(11)*1))
l.problem.assign_penalty(path_key='l1',
l1=l1,
l2=l2,
l3=l3,
delta=delta)
else:
print "----------------------------------------ROBUST GRAPHNET------------------------------------------------"
problemtype = graphnet.RobustGraphNet
problemkey = 'RobustGraphNet'
print "Robust GraphNet with penalties (l1, l2, l3, delta)", l1, l2, l3, delta
l = cwpath.CoordWise((X, Y, A),
problemtype,
initial_coefs=initial
) #,initial_coefs=np.array([14.]*10))
l.problem.assign_penalty(path_key='l1',
l1=l1,
l2=l2,
l3=l3,
delta=delta)
else:
print "-------------------------------------------GRAPHNET---------------------------------------------------"
problemtype = graphnet.NaiveGraphNet
problemkey = 'NaiveGraphNet'
print "Testing GraphNet with penalties (l1,l2,l3):", l1, l2, l3
l = cwpath.CoordWise((X, Y, A),
problemtype,
initial_coefs=initial)
l.problem.assign_penalty(path_key='l1', l1=l1, l2=l2, l3=l3)
else:
print "-------------------------------------------ELASTIC NET---------------------------------------------------"
problemtype = graphnet.NaiveENet
problemkey = 'NaiveENet'
print "Testing ENET with penalties (l1,l2):", l1, l2
l = cwpath.CoordWise(
(X, Y), problemtype,
initial_coefs=initial) #, initial_coefs = np.array([4.]*10))
l.problem.assign_penalty(path_key='l1', l1=l1, l2=l2)
else:
print "-------------------------------------------LASSO---------------------------------------------------"
problemtype = graphnet.Lasso
problemkey = 'Lasso'
print "Testing LASSO with penalty:", l1
l = cwpath.CoordWise(
(X, Y), problemtype,
initial_coefs=initial) #, initial_coefs= np.array([7.]*10))
l.problem.assign_penalty(path_key='l1', l1=l1)
# fit and get results
coefficients, residuals = l.fit(tol=tol, initial=initial)
print "\t---> Fitting GraphNet problem with coordinate descent took:", time.clock(
) - tic, "seconds."
if adaptive:
betas = coefficients
tic = time.clock()
eps = 1e-5
l1weights = 1. / (beta + eps)
l = cwpath.CoordWise((X, Y, A), problemtype, initial_coefs=initial)
l.problem.assign_penalty(l1=l1,
l2=l2,
l3=l3,
delta=delta,
l1weights=l1weights,
newl1=l1)
adaptive_coefficients, adaptive_residuals = l.fit(tol=tol,
initial=initial)
print "\t---> Fitting Adaptive GraphNet problem with coordinate descent took:", time.clock(
) - tic, "seconds."
# if compare to scipy flag is set,
# compare the above result with the same problem
# solved using a built in scipy solver (fmin_powell).
if scipy_compare:
print "\t---> Fitting with scipy for comparison..."
tic = time.clock()
l1 = l1[-1] # choose only last l1 value
beta = coefficients[-1] # coordinate-wise coefficients
if l2 != -999.0 or l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if l3 != -999.0 or delta != -999.0 or svmdelta != -999.0:
if delta != -999.0:
if adaptive:
if svmdelta != -999.0:
# HuberSVM Graphnet
Xp2 = np.hstack(
[np.ones(X.shape[0])[:, np.newaxis], X])
def f(beta):
ind = range(1, len(beta))
return huber_svm_error(
beta, Y,
Xp2, delta).sum() + np.fabs(beta[ind]).sum(
) * l1 + l2 * np.linalg.norm(
beta[ind])**2 / 2 + l3 * np.dot(
beta[ind], np.dot(
Afull, beta[ind])) / 2
else:
# Robust Adaptive Graphnet
def f(beta):
return huber(Y - np.dot(
X, beta), delta).sum() / 2 + l1 * np.dot(
np.fabs(beta),
l1weights) + l2 * np.linalg.norm(
beta)**2 / 2 + l3 * np.dot(
beta, np.dot(Afull, beta)) / 2
else:
# Robust Graphnet
def f(beta):
try:
return huber(Y - np.dot(X, beta.T), delta).sum(
) / 2 + np.fabs(
beta).sum() * l1 + l2 * np.linalg.norm(
beta)**2 / 2 + l3 * np.dot(
beta, np.dot(Afull, beta.T)) / 2
except:
return huber(Y - np.dot(
X, beta), delta).sum() / 2 + np.fabs(
beta).sum() * l1 + l2 * np.linalg.norm(
beta)**2 / 2 + l3 * np.dot(
beta,
np.dot(Afull, beta).T) / 2
else:
# Graphnet
def f(beta):
return np.linalg.norm(
Y - np.dot(X, beta))**2 / 2 + np.fabs(
beta).sum() * l1 + l2 * np.linalg.norm(
beta)**2 / 2 + l3 * np.dot(
beta, np.dot(Afull, beta)) / 2
else:
# Elastic Net
def f(beta):
return np.linalg.norm(
Y - np.dot(X, beta))**2 / 2 + np.fabs(
beta).sum() * l1 + l2 * np.linalg.norm(beta)**2 / 2
else:
# Lasso
def f(beta):
return np.linalg.norm(Y - np.dot(X, beta))**2 / 2 + np.fabs(
beta).sum() * l1
# optimize
if problemkey == 'HuberSVMGraphNet':
v = scipy.optimize.fmin_powell(f,
np.zeros(Xp2.shape[1]),
ftol=1.0e-14,
xtol=1.0e-14,
maxfun=100000)
else:
v = scipy.optimize.fmin_powell(f,
np.zeros(X.shape[1]),
ftol=1.0e-10,
xtol=1.0e-10,
maxfun=100000)
v = np.asarray(v)
print "\t---> Fitting GraphNet with scipy took:", time.clock(
) - tic, "seconds."
# print np.round(100*v)/100,'\n', np.round(100*beta)/100
assert_true(np.fabs(f(v) - f(beta)) / np.fabs(f(v) + f(beta)) < tol)
if np.linalg.norm(v) > 1e-8:
assert_true(np.linalg.norm(v - beta) / np.linalg.norm(v) < tol)
else:
assert_true(np.linalg.norm(beta) < 1e-8)
print "\t---> Coordinate-wise and Scipy optimization agree!"
return (coefficients, residuals), problemkey