def line_search_armijo(f,
xk,
pk,
gfk,
old_fval,
args=(),
c1=1e-4,
alpha0=0.99):
"""
Armijo linesearch function that works with matrices
find an approximate minimum of f(xk+alpha*pk) that satifies the
armijo conditions.
Parameters
----------
f : function
loss function
xk : np.ndarray
initial position
pk : np.ndarray
descent direction
gfk : np.ndarray
gradient of f at xk
old_fval : float
loss value at xk
args : tuple, optional
arguments given to f
c1 : float, optional
c1 const in armijo rule (>0)
alpha0 : float, optional
initial step (>0)
Returns
-------
alpha : float
step that satisfy armijo conditions
fc : int
nb of function call
fa : float
loss value at step alpha
"""
xk = np.atleast_1d(xk)
fc = [0]
def phi(alpha1):
fc[0] += 1
return f(xk + alpha1 * pk, *args)
if old_fval is None:
phi0 = phi(0.)
else:
phi0 = old_fval
derphi0 = np.sum(pk * gfk) # Quickfix for matrices
alpha, phi1 = scalar_search_armijo(phi,
phi0,
derphi0,
c1=c1,
alpha0=alpha0)
return alpha, fc[0], phi1
def computeTransportLaplacianSymmetric_fw_sinkhorn(distances, Ss, St, xs, xt, reg=1e-9, regls=0, reglt=0, nbitermax=400,
thr_stop=1e-8, **kwargs):
distribS = np.ones((xs.shape[0], 1)) / xs.shape[0]
distribS = distribS.ravel()
distribT = np.ones((xt.shape[0], 1)) / xt.shape[0]
distribT = distribT.ravel()
Ls = get_laplacian(Ss)
Lt = get_laplacian(St)
maxdist = np.max(distances)
regmax = 300. / maxdist
reg0 = regmax * (1 - exp(-reg / regmax))
transp = ot.sinkhorn(distribS, distribT, distances, reg)
niter = 1
while True:
old_transp = transp.copy()
G = np.asarray(regls * get_gradient1(Ls, xt, old_transp) + reglt * get_gradient2(Lt, xs, old_transp))
transp0 = ot.sinkhorn(distribS, distribT, distances + G, reg)
E = transp0 - old_transp
# do a line search for best tau
def f(tau):
T = (1 - tau) * old_transp + tau * transp0
# print np.sum(T*distances),-1./reg0*np.sum(T*np.log(T)),regls*quadloss1(T,Ls,xt),reglt*quadloss2(T,Lt,xs)
return np.sum(T * distances) + 1. / reg0 * np.sum(T * np.log(T)) + \
regls * quadloss1(T, Ls, xt) + reglt * quadloss2(T, Lt, xs)
# compute f'(0)
res = regls * (np.trace(np.dot(xt.T, np.dot(E.T, np.dot(Ls, np.dot(old_transp, xt))))) + \
np.trace(np.dot(xt.T, np.dot(old_transp.T, np.dot(Ls, np.dot(E, xt)))))) \
+ reglt * (np.trace(np.dot(xs.T, np.dot(E, np.dot(Lt, np.dot(old_transp.T, xs))))) + \
np.trace(np.dot(xs.T, np.dot(old_transp, np.dot(Lt, np.dot(E.T, xs))))))
# derphi_zero = np.sum(E*distances) - np.sum(1+E*np.log(old_transp))/reg + res
derphi_zero = np.sum(E * distances) + np.sum(E * (1 + np.log(old_transp))) / reg0 + res
tau, cost = ln.scalar_search_armijo(f, f(0), derphi_zero, alpha0=0.99)
if tau is None:
break
transp = (1 - tau) * old_transp + tau * transp0
err = np.sum(np.fabs(E))
if niter >= nbitermax or err < thr_stop:
break
niter += 1
if niter % 100 == 0:
print('{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19)
print('{:5d}|{:8e}|'.format(niter, err))
return transp
def line_search_armijo(f, xk, pk, gfk, old_fval,
args=(), c1=1e-4, alpha0=0.99):
"""
Armijo linesearch function that works with matrices
find an approximate minimum of f(xk+alpha*pk) that satifies the
armijo conditions.
Parameters
----------
f : function
loss function
xk : np.ndarray
initial position
pk : np.ndarray
descent direction
gfk : np.ndarray
gradient of f at xk
old_fval : float
loss value at xk
args : tuple, optional
arguments given to f
c1 : float, optional
c1 const in armijo rule (>0)
alpha0 : float, optional
initial step (>0)
Returns
-------
alpha : float
step that satisfy armijo conditions
fc : int
nb of function call
fa : float
loss value at step alpha
"""
xk = np.atleast_1d(xk)
fc = [0]
def phi(alpha1):
fc[0] += 1
return f(xk + alpha1 * pk, *args)
if old_fval is None:
phi0 = phi(0.)
else:
phi0 = old_fval
derphi0 = np.sum(pk * gfk) # Quickfix for matrices
alpha, phi1 = scalar_search_armijo(
phi, phi0, derphi0, c1=c1, alpha0=alpha0)
return alpha, fc[0], phi1
def test_armijo_terminate_1(self):
# Armijo should evaluate the function only once if the trial step
# is already suitable
count = [0]
def phi(s):
count[0] += 1
return -s + 0.01*s**2
s, phi1 = ls.scalar_search_armijo(phi, phi(0), -1, alpha0=1)
assert_equal(s, 1)
assert_equal(count[0], 2)
assert_armijo(s, phi)
def test_armijo_terminate_1(self):
# Armijo should evaluate the function only once if the trial step
# is already suitable
count = [0]
def phi(s):
count[0] += 1
return -s + 0.01*s**2
s, phi1 = ls.scalar_search_armijo(phi, phi(0), -1, alpha0=1)
assert_equal(s, 1)
assert_equal(count[0], 2)
assert_armijo(s, phi)
def line_search_armijo(self, f, xk, pk, gfk, transport_map, old_fval=None):
""" Addapted from POT
"""
fc = [0]
def phi(alpha1):
fc[0] += 1
return to_np(f.cost(transport_map, xk + alpha1 * pk))
phi0 = phi(0.) if old_fval is None else to_np(old_fval)
derphi0 = to_np((pk * gfk).sum())
alpha, phi1 = scalar_search_armijo(phi,
phi0,
derphi0,
c1=self.c1,
alpha0=self.alpha0)
return alpha, fc[0], phi1
def armijo_backtracking(
phi,
derphi,
phi0=None,
derphi0=None,
old_phi0=None,
c1=1e-4,
amin=0,
amax=None,
**kwargs,
):
"""Scalar line search method to find step size satisfying Armijo conditions.
Parameters
----------
c1 : float, optional
Parameter for Armijo condition rule.
amax, amin : float, optional
Maxmimum and minimum step size
"""
# TODO: Allow different schemes to choose initial step size
if old_phi0 is not None and derphi0 != 0:
alpha0 = 1.01 * 2 * (phi0 - old_phi0) / derphi0
else:
alpha0 = 1.0
if alpha0 <= 0:
alpha0 = 1.0
if amax is not None:
alpha0 = min(alpha0, amax)
step_size, _ = scalar_search_armijo(phi,
phi0,
derphi0,
c1=c1,
alpha0=alpha0,
amin=amin)
return step_size
def test_scalar_search_armijo(self):
for name, phi, derphi, old_phi0 in self.scalar_iter():
s, phi1 = ls.scalar_search_armijo(phi, phi(0), derphi(0))
assert_equal(phi1, phi(s), name)
assert_armijo(s, phi, err_msg="%s %g" % (name, old_phi0))
def test_scalar_search_armijo(self):
for name, phi, derphi, old_phi0 in self.scalar_iter():
s, phi1 = ls.scalar_search_armijo(phi, phi(0), derphi(0))
assert_fp_equal(phi1, phi(s), name)
assert_armijo(s, phi, err_msg="%s %g" % (name, old_phi0))
Dfp = Df(xk)
fx = f(xk)
while (f(xk + alpha * pk) > fx + c * alpha * np.linalg.norm(Dfp)):
alpha = rho * alpha
return alpha
if __name__ == "__main__":
#print("booyeah")
#f =lambda x: np.exp(x)-4*x
#print(golden_section(f, 0,3))
#print(opt.golden(f,brack = (0,3), tol = 0.001))
#df = lambda x : 2*x + 5*np.cos(5*x)
#d2f = lambda x : 2 - 25*np.sin(5*x)
#print(opt.newton(df, x0 = 0, fprime = d2f ,tol = 1e-10,maxiter = 500))
#print(newton1d(df,d2f,-1.3))
#df = lambda x: 2*x + np.cos(x) + 10*np.cos(10*x)
#print(secant1d(df,0,1))
#print(opt.newton(df, x0=1, tol=1e-10, maxiter=500))
f = lambda x: x[0]**2 + x[1]**2 + x[2]**2
Df = lambda x: np.array([2 * x[0], 2 * x[1], 2 * x[2]])
x = anp.array([150., .03, 40.])
p = anp.array([-.5, -100., -4.5])
phi = lambda alpha: f(x + alpha * p)
dphi = grad(phi)
alpha1, _ = linesearch.scalar_search_armijo(phi, phi(0.), dphi(0.))
print(alpha1)
print(backtracking(f, Df, x, p))
def computeTransportLaplacian_CGS(distribS,
LabelsS,
distribT,
distances,
xs,
xt,
reg=1e-9,
regls=0,
reglt=0,
nbitermax=10,
thr_stop=1e-8,
**kwargs):
Ss = get_sim(xs, 'knnclass', nn=7, labels=LabelsS)
St = get_sim(xt, 'knn', nn=7)
Ls = get_laplacian(Ss)
Lt = get_laplacian(St)
maxdist = np.max(distances)
regmax = 300. / maxdist
reg0 = regmax * (1 - np.exp(-reg / regmax))
transp = computeTransportSinkhorn(distribS, distribT, distances, reg,
maxdist)
niter = 1
while True:
old_transp = transp.copy()
G = regls * get_gradient1(Ls, xt, old_transp) + reglt * get_gradient2(
Lt, xs, old_transp)
transp0 = computeTransportSinkhorn(distribS, distribT, distances + G,
reg, maxdist)
E = transp0 - old_transp
# do a line search for best tau
def f(tau):
T = (1 - tau) * old_transp + tau * transp0
return np.sum(T * distances) + 1. / reg0 * np.sum(
T * np.log(T)) + regls * quadloss1(
T, Ls, xt) + reglt * quadloss2(T, Lt, xs)
# compute f'(0)
res = regls*(np.trace(np.dot(xt.T,np.dot(E.T,np.dot(Ls,np.dot(old_transp,xt)))))+\
np.trace(np.dot(xt.T,np.dot(old_transp.T,np.dot(Ls,np.dot(E,xt))))))\
+reglt*(np.trace(np.dot(xs.T,np.dot(E,np.dot(Lt,np.dot(old_transp.T,xs)))))+\
np.trace(np.dot(xs.T,np.dot(old_transp,np.dot(Lt,np.dot(E.T,xs))))))
derphi_zero = np.sum(
E * distances) + np.sum(E * (1 + np.log(old_transp))) / reg0 + res
tau, cost = ln.scalar_search_armijo(f, f(0), derphi_zero, alpha0=0.99)
if tau is None:
break
transp = (1 - tau) * old_transp + tau * transp0
if niter >= nbitermax or np.sum(np.fabs(E)) < thr_stop:
break
niter += 1
return transp
def computeTransportL1L2_CGS(distribS,
LabelsS,
distribT,
M,
reg,
eta=0.1,
nbitermax=10,
thr_stop=1e-8,
**kwargs):
Nini = len(distribS)
Nfin = len(distribT)
W = np.zeros(M.shape)
maxdist = np.max(M)
distances = M
lstlab = np.unique(LabelsS)
regmax = 300. / maxdist
reg0 = regmax * (1 - np.exp(-reg / regmax))
transp = computeTransportSinkhorn(distribS, distribT, distances, reg,
maxdist)
niter = 1
while True:
old_transp = transp.copy()
W = get_W_L1L2(old_transp, LabelsS, lstlab)
G = eta * W
transp0 = computeTransportSinkhorn(distribS, distribT, distances + G,
reg, maxdist)
deltatransp = transp0 - old_transp
# do a line search for best tau
def f(tau):
T = old_transp + tau * deltatransp
return np.sum(T * distances) + 1. / reg0 * np.sum(
T * np.log(T)) + eta * loss_L1L2(T, LabelsS, lstlab)
# compute f'(0)
res = 0
for i in range(transp.shape[1]):
for lab in lstlab:
temp1 = old_transp[LabelsS == lab, i]
temp2 = deltatransp[LabelsS == lab, i]
res += np.dot(temp1, temp2) / np.linalg.norm(temp1)
derphi_zero = np.sum(deltatransp * distances) + np.sum(
deltatransp * (1 + np.log(old_transp))) / reg0 + eta * res
tau, cost = ln.scalar_search_armijo(f, f(0), derphi_zero, alpha0=0.99)
if tau is None:
break
transp = (1 - tau) * old_transp + tau * transp0
if niter >= nbitermax or np.sum(np.fabs(deltatransp)) < thr_stop:
break
niter += 1
#print 'nbiter=',niter
return transp