def __init__(self,
Domain,
P=IdentityProjection(),
Delta0=1e-2,
GrowthLimit=2,
MinStep=-1e10,
MaxStep=1e10):
self.F = Domain.F
self.Proj = P
self.Proj_Norm = IdentityProjection()
self.StorageSize = 1
self.TempStorage = {}
self.Delta0 = Delta0
self.GrowthLimit = GrowthLimit
self.MinStep = MinStep
self.MaxStep = MaxStep
def __init__(self,
Domain,
P=IdentityProjection(),
Delta0=1e-2,
GrowthLimit=2,
MinStep=-1e10,
MaxStep=1e10,
NTopLEs=None):
self.F = Domain.F
try:
self.Jv = partial(Jv, Jac=Domain.Jac)
except AttributeError:
self.Jv = partial(Jv_num, F=self.F)
self.Proj = P
self.StorageSize = 1
self.TempStorage = {}
self.Delta0 = Delta0
self.GrowthLimit = GrowthLimit
self.MinStep = MinStep
self.MaxStep = MaxStep
self.NTopLEs = NTopLEs
def __init__(self, Domain, P=IdentityProjection(), Delta0=1e-4,
GrowthLimit=2, MinStep=-1e10, MaxStep=1e10):
self.F = Domain.F
self.Proj = P
self.StorageSize = 1
self.TempStorage = {}
self.Delta0 = Delta0
self.GrowthLimit = GrowthLimit
self.MinStep = MinStep
self.MaxStep = MaxStep
self.BT = np.array([
[1./5.,0.,0.,0.,0.,0.],
[3./40.,9./40.,0.,0.,0.,0.],
[3./10.,-9./10.,6./5.,0.,0.,0.],
[-11./54.,5./2.,-70./27.,35./27.,0.,0.],
[1631./55296.,175./512.,575./13824.,44275./110592.,253./4096.,0.],
[37./378.,0.,250./621.,125./594.,0.,512./1771.],
[2825./27648.,0.,18575./48384.,13525./55296.,277./14336.,0.25]
])
def __init__(self, Domain, P=IdentityProjection()):
self.F = Domain.F
self.Proj = P
self.StorageSize = 1
self.TempStorage = {}
def __init__(self, Domain, P=IdentityProjection(), FixStep=False):
self.F = Domain.F
self.Proj = P
self.FixStep = FixStep
self.StorageSize = 1
self.TempStorage = {}
def __init__(self,Domain,P=IdentityProjection(),eps=1e-8):
self.F = Domain.F
self.Proj = P
self.StorageSize = 1
self.TempStorage = {}
self.eps = eps
self.factor = 1.
def __init__(self,
Domain,
P=IdentityProjection(),
Delta0=1e-4,
GrowthLimit=2,
MinStep=-1e10,
MaxStep=1e10,
NTopLEs=None):
self.F = Domain.F
try:
self.Jv = partial(Jv, Jac=Domain.Jac)
except AttributeError:
self.Jv = partial(Jv_num, F=self.F)
self.Proj = P
self.StorageSize = 1
self.TempStorage = {}
self.Delta0 = Delta0
self.GrowthLimit = GrowthLimit
self.MinStep = MinStep
self.MaxStep = MaxStep
self.NTopLEs = NTopLEs
self.BT = np.array(
[[1. / 5., 0., 0., 0., 0., 0.],
[3. / 40., 9. / 40., 0., 0., 0., 0.],
[3. / 10., -9. / 10., 6. / 5., 0., 0., 0.],
[-11. / 54., 5. / 2., -70. / 27., 35. / 27., 0., 0.],
[
1631. / 55296., 175. / 512., 575. / 13824., 44275. / 110592.,
253. / 4096., 0.
], [37. / 378., 0., 250. / 621., 125. / 594., 0., 512. / 1771.],
[
2825. / 27648., 0., 18575. / 48384., 13525. / 55296.,
277. / 14336., 0.25
]])
def __init__(self,
Domain,
P=IdentityProjection(),
FixStep=True,
factor=0.5):
self.F = Domain.F
self.Jac = Domain.J
self.Proj = P
self.FixStep = FixStep
self.StorageSize = 1
self.TempStorage = {}
self.factor = factor
def Demo():
#__SPHERE__##################################################
# Define Dimension and Domain
Domain = Sphere(Dim=100)
# Set Method
Method = HeunEuler(Domain=Domain, P=IdentityProjection(), Delta0=1e-2)
# Set Options
Init = Initialization(Step=-1e-1)
Term = Termination(MaxIter=1000, Tols=[[Domain.f_Error, 1e-3]])
Repo = Reporting(Requests=[Domain.f_Error])
Misc = Miscellaneous()
Options = DescentOptions(Init, Term, Repo, Misc)
# Initialize Starting Point
Start = 100 * np.ones(Domain.Dim)
# Print Stats
PrintSimStats(Domain, Method, Options)
# Start Solver
tic = time.time()
SPHERE_Results = Solve(Start, Method, Domain, Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options, SPHERE_Results, Method, toc)
#__KOJIMA-SHINDO__##################################################
# Define Dimension and Domain
Domain = KojimaShindo()
# Set Method
Method = HeunEuler(Domain=Domain, P=EntropicProjection(), Delta0=1e-1)
# Set Options
Init = Initialization(Step=-1e-1)
Term = Termination(MaxIter=1000, Tols=[[Domain.gap_simplex, 1e-3]])
Repo = Reporting(Requests=[Domain.gap_simplex])
Misc = Miscellaneous()
Options = DescentOptions(Init, Term, Repo, Misc)
# Initialize Starting Point
Start = np.ones(Domain.Dim) / np.double(Domain.Dim)
# Print Stats
PrintSimStats(Domain, Method, Options)
# Start Solver
tic = time.time()
KS_Results = Solve(Start, Method, Domain, Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options, KS_Results, Method, toc)
#__WATSON__##################################################
trials = xrange(10)
WAT_Results = [[] for i in trials]
for p in trials:
#Define Dimension and Domain
Domain = Watson(Pos=p)
# Set Method
Method = HeunEuler(Domain=Domain, P=EntropicProjection(), Delta0=1e-1)
# Set Options
Init = Initialization(Step=-1e-1)
Term = Termination(MaxIter=1000, Tols=[[Domain.gap_simplex, 1e-3]])
Repo = Reporting(Requests=[Domain.gap_simplex])
Misc = Miscellaneous()
Options = DescentOptions(Init, Term, Repo, Misc)
#Initialize Starting Point
Start = np.ones(Domain.Dim) / np.double(Domain.Dim)
# Print Stats
PrintSimStats(Domain, Method, Options)
tic = time.time()
WAT_Results[p] = Solve(Start, Method, Domain, Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options, WAT_Results[p], Method, toc)
#__SUN__##################################################
trials = xrange(8000, 10000 + 1, 2000)
Sun_Results = [[] for i in trials]
for n in trials:
#Define Dimension and Domain
Domain = Sun(Dim=n)
# Set Method
Method = HeunEuler(Domain=Domain, P=EntropicProjection(), Delta0=1e-1)
# Set Options
Init = Initialization(Step=-1e-1)
Term = Termination(MaxIter=1000, Tols=[[Domain.gap_simplex, 1e-3]])
Repo = Reporting(Requests=[Domain.gap_simplex])
Misc = Miscellaneous()
Options = DescentOptions(Init, Term, Repo, Misc)
#Initialize Starting Point
Start = np.ones(Domain.Dim) / np.double(Domain.Dim)
# Print Stats
PrintSimStats(Domain, Method, Options)
tic = time.time()
ind = n / 2000 - 4
Sun_Results[ind] = Solve(Start, Method, Domain, Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options, Sun_Results[ind], Method, toc)
class HeunEuler_AdaGrad_PhaseSpace(Solver):
def __init__(self,
Domain,
P=IdentityProjection(),
Delta0=1e-2,
GrowthLimit=2,
MinStep=-1e10,
MaxStep=1e10):
self.F = Domain.F
self.Proj = P
self.Proj_Norm = IdentityProjection()
self.StorageSize = 1
self.TempStorage = {}
self.Delta0 = Delta0
self.GrowthLimit = GrowthLimit
self.MinStep = MinStep
self.MaxStep = MaxStep
def InitTempStorage(self, Start, Domain, Options):
Step = Options.Init.Step
F_Start = self.F(Start)
F_Start_Norm = np.abs(F_Start)
self.TempStorage['Data'] = self.StorageSize * [Start]
self.TempStorage[self.F] = self.StorageSize * [np.sign(F_Start)]
self.TempStorage['Norms'] = self.StorageSize * [F_Start_Norm]
self.TempStorage['F_Norms'] = self.StorageSize * [
F_Start_Norm / abs(Step)
]
self.TempStorage['Step'] = self.StorageSize * [Step]
self.TempStorage['F Evaluations'] = self.StorageSize * [1]
self.TempStorage['Projections'] = self.StorageSize * [0]
self.TempStorage['Time'] = self.StorageSize * [0.]
return self.TempStorage
# BookKeeping(self,TempData) defined in super class 'Solver'
def PhaseSpaceMultiplier(self, Tl, Tr):
delta = 1e-15
psi = .7
Bmin = .01
Bmax = .1
alpha1 = 5
if Tr > delta:
r = Tl / Tr
elif Tl <= delta:
r = Bmax
else:
r = psi
if r <= Bmin:
mult = alpha1
elif Bmin < r and r <= Bmax:
mult = (alpha1 * (Bmax - r) + (r - Bmin)) / (Bmax - Bmin)
elif Bmax < r and r <= psi:
mult = ((psi - r) + .5 * (r - Bmax)) / (psi - Bmax)
else:
mult = .5
return mult
def Update(self, Record):
# Retrieve Necessary Data
Step = Record.TempStorage['Step'][-1]
Time = Record.TempStorage['Time'][-1]
Data = Record.TempStorage['Data'][-1]
Norm = Record.TempStorage['Norms'][-1]
Fs_Data = np.zeros((2, Data.shape[0]), dtype=Data.dtype)
Fs_Data[0, :] = Record.TempStorage[self.F][-1]
Fs_Norm = np.zeros((2, Data.shape[0]), dtype=Data.dtype)
Fs_Norm[0, :] = Record.TempStorage['F_Norms'][-1]
print(Norm)
# Initialize Storage
TempData = {}
# Perform Update
direction_data = Fs_Data[0, :]
direction_norm = Fs_Norm[0, :]
_NewData = self.Proj.P(Data, Step, direction_data)
_NewNorm = self.Proj_Norm.P(Norm, abs(Step), direction_norm)
F_NewData = self.F(_NewData)
Fs_Data[1, :] = F_NewData / _NewNorm
Fs_Norm[1, :] = (np.abs(F_NewData) - _NewNorm) / (Time + 2 * abs(Step))
direction_data = 0.5 * np.sum(Fs_Data, axis=0)
direction_norm = 0.5 * np.sum(Fs_Norm, axis=0)
NewData = self.Proj.P(Data, Step, direction_data)
NewNorm = self.Proj_Norm.P(Norm, abs(Step), direction_norm)
# Compute Delta + Traditional Stepsize
Delta_data = max(abs(NewData - _NewData))
Delta_norm = max(abs(NewNorm - _NewNorm))
Delta = max(Delta_data, Delta_norm)
if Delta == 0.:
growth_est = self.GrowthLimit
else:
growth_est = (self.Delta0 / Delta)**0.5
# Compute Tl & Tr + Phase Space Stepsize
F_NewData = self.F(NewData)
NewF_data = F_NewData / NewNorm
NewF_norm = (np.abs(F_NewData) - NewNorm) / (Time + 2 * abs(Step))
Tl_data = max(abs(direction_data - .5 * (NewF_data + Fs_Data[0])))
Tl_norm = max(abs(direction_norm - .5 * (NewF_norm + Fs_Norm[0])))
Tl = max(Tl_data, Tl_norm)
Tr_data = max(abs(NewF_data + Fs_Data[0]))
Tr_norm = max(abs(NewF_norm + Fs_Norm[0]))
Tr = .5 * max(Tr_data, Tr_norm)
growth_ps = self.PhaseSpaceMultiplier(Tl, Tr)
# Conservative Adjustment
growth = min(growth_est, growth_ps)
# Adjust Stepsize
Step = np.clip(growth * Step, self.MinStep, self.MaxStep)
# Store Data
TempData['Data'] = NewData
TempData[self.F] = NewF_data
TempData['Norms'] = NewNorm
TempData['F_Norms'] = NewF_norm
TempData['Step'] = Step
TempData['Time'] = Time + abs(Step)
TempData['F Evaluations'] = 2 + self.TempStorage['F Evaluations'][-1]
TempData['Projections'] = 2 + self.TempStorage['Projections'][-1]
self.BookKeeping(TempData)
return self.TempStorage
def Demo():
#__SPHERE__##################################################
# Define Dimension and Domain
Domain = MonotoneCycle()
# Set Method
MethodEuler = Euler(Domain=Domain,P=IdentityProjection(),FixStep=True)
MethodEG = EG(Domain=Domain,P=IdentityProjection(),FixStep=True)
# Set Options
Init = Initialization(Step=-1e-1)
Term = Termination(MaxIter=100)
Repo = Reporting(Requests=['Data'])
Misc = Miscellaneous()
Options = DescentOptions(Init,Term,Repo,Misc)
# Initialize Starting Point
Start = np.ones(Domain.Dim)
# Print Stats
PrintSimStats(Domain,MethodEuler,Options)
# Start Solver
tic = time.time()
Euler_Results = Solve(Start,MethodEuler,Domain,Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options,Euler_Results,MethodEuler,toc)
# Print Stats
PrintSimStats(Domain,MethodEG,Options)
# Start Solver
tic = time.time()
EG_Results = Solve(Start,MethodEG,Domain,Options)
toc = time.time() - tic
# Print Results
PrintSimResults(Options,EG_Results,MethodEG,toc)
data_Euler = ListONP2NP(Euler_Results.PermStorage['Data'])
data_EG = ListONP2NP(EG_Results.PermStorage['Data'])
X, Y = np.meshgrid(np.arange(-2.5, 2.5, .2), np.arange(-2.5, 2.5, .2))
U = np.zeros_like(X)
V = np.zeros_like(Y)
for i in range(X.shape[0]):
for j in range(X.shape[1]):
vec = -Domain.F([X[i,j],Y[i,j]])
U[i,j] = vec[0]
V[i,j] = vec[1]
# plt.figure()
# plt.title('Arrows scale with plot width, not view')
# Q = plt.quiver(X, Y, U, V, units='width')
# qk = plt.quiverkey(Q, 0.9, 0.9, 2, r'$2 \frac{m}{s}$', labelpos='E',
# coordinates='figure')
fig = plt.figure()
ax = fig.add_axes([0.1, 0.1, 0.6, 0.75])
plt.title("Extragradient vs Simultaneous Gradient Descent")
Q = plt.quiver(X[::3, ::3], Y[::3, ::3], U[::3, ::3], V[::3, ::3],
pivot='mid', units='inches')
vec_Euler = -Domain.F(data_Euler[-1,:])
vec_EG = -Domain.F(data_EG[-1,:])
print(data_Euler[-1])
print(vec_Euler)
plt.quiver(data_Euler[-1][0], data_Euler[-1][1], vec_Euler[0], vec_Euler[1], pivot='mid', units='inches', color='b',
headwidth=5)
plt.quiver(data_EG[-1][0], data_EG[-1][1], vec_EG[0], vec_EG[1], pivot='mid', units='inches', color='r',
headwidth=5)
plt.scatter(X[::3, ::3], Y[::3, ::3], color='gray', s=5)
plt.plot(data_Euler[:,0],data_Euler[:,1],'b',linewidth=5,label='Simultaneous\nGradient Descent')
plt.plot(data_EG[:,0],data_EG[:,1],'r',linewidth=5,label='Extragradient')
plt.plot([0],[0],linestyle="None",marker=(5,1,0),markersize=20,color='gold',label='Equilibrium')
plt.axis([-2.5,2.5,-2.5,2.5])
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), borderaxespad=0.5)
# plt.ion()
# plt.show()
plt.savefig('EGvsEuler.png')
