def rdfds(func,
initial_x,
L,
m,
t,
maximum_iterations=1000,
direction_generator=None,
newton_eps=1e-5):
'''
m: batch size when computing gradient.
t: smoothing parameter when computing gradient.
'''
x = np.matrix(initial_x)
n = x.shape[0]
# initialization
xs = []
xs.append(x.copy())
runtimes = []
start_time = time.time()
runtimes.append(time.time() - start_time)
if not direction_generator:
direction_generator = sphere_point_generator(n)
for k in range(maximum_iterations):
alpha = 1 / (48 * n * L)
gradient = approximate_gradient(func, x, direction_generator, t, m)
#newton_f = lambda z, order: V_function(alpha, gradient, x, z, order)
#x, newton_values, _, _ = newton( newton_f, x, newton_eps, 100, backtracking_line_search)
x = mirror_descent(x, gradient, alpha * n)
xs.append(x.copy())
runtimes.append(time.time() - start_time)
return x, xs, runtimes, "RDFDS"
def ardfds_e(func,
noisy_func,
initial_x,
L,
m,
t,
f_star,
filename,
maximum_iterations=1000,
direction_generator=None,
sigma=0.0001,
stepsize_constant=1):
x = initial_x.copy()
y = x.copy()
z = x.copy()
n = len(x)
# initialization
# new
residuals_func = np.zeros((maximum_iterations + 1)) * 1.0
start_residual_func = func(x) - f_star
residuals_func[0] = 1.0
runtimes = np.zeros((maximum_iterations + 1)) * 1.0
start_time = time.time()
runtimes[0] = (time.time() - start_time)
if not direction_generator:
direction_generator = sphere_point_generator(n)
for k in range(maximum_iterations):
alpha = (k + 2) / (96 * n * n * L)
tau = 2 / (k + 2)
x = tau * z + (1 - tau) * y
xi = norm(scale=sigma).rvs()
gradient = approximate_gradient(noisy_func, x, xi, direction_generator,
t, m)
y = x - 1 / (2 * L) * gradient
z = mirror_descent(z, gradient, alpha * n * stepsize_constant)
residuals_func[k] = ((func(y) - f_star) * 1.0 / start_residual_func)
runtimes[k] = (time.time() - start_time)
pickle.dump(
runtimes,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it_' + str(stepsize_constant) +
'stepsz' + "_runtimes_ardfds_e.txt", 'wb'))
pickle.dump(
residuals_func,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it_' + str(stepsize_constant) +
'stepsz' + "_func_ardfds_e.txt", 'wb'))
return y
def rsgf(func,
noisy_func,
initial_x,
L,
m,
mu,
f_star,
filename,
maximum_iterations=1000,
initial_stepsize=1,
direction_generator=None,
sigma=0.0001):
x = initial_x.copy()
n = len(x)
residuals_func = np.zeros((maximum_iterations + 1)) * 1.0
start_residual_func = func(x) - f_star
residuals_func[0] = 1.0
runtimes = np.zeros((maximum_iterations + 1)) * 1.0
start_time = time.time()
runtimes[0] = (time.time() - start_time)
if not direction_generator:
direction_generator = sphere_point_generator(n)
h = 1 / np.sqrt(n + 4) * min(
1 / (4 * L * np.sqrt(n + 4)),
initial_stepsize / np.sqrt(maximum_iterations))
for k in range(maximum_iterations):
xi = norm(scale=sigma).rvs()
gradient = approximate_gradient(noisy_func, x, xi, direction_generator,
mu, 1)
x = x - h * gradient
residuals_func[k] = ((func(x) - f_star) * 1.0 / start_residual_func)
runtimes[k] = (time.time() - start_time)
pickle.dump(
runtimes,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it' + "_runtimes_rsgf.txt", 'wb'))
pickle.dump(
residuals_func,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it' + "_func_rsgf.txt", 'wb'))
return x
def rdfds_ne(func,
noisy_func,
initial_x,
L,
m,
t,
f_star,
filename,
maximum_iterations=1000,
direction_generator=None,
sigma=0.0001,
stepsize_constant=1):
x = initial_x.copy()
n = len(x)
residuals_func = np.zeros((maximum_iterations + 1)) * 1.0
start_residual_func = func(x) - f_star
residuals_func[0] = 1.0
runtimes = np.zeros((maximum_iterations + 1)) * 1.0
start_time = time.time()
runtimes[0] = (time.time() - start_time)
if not direction_generator:
direction_generator = sphere_point_generator(n)
rho_n = (16.0 * np.log(n) - 8.0) / n
for k in range(maximum_iterations):
alpha = 1 / (48 * n * rho_n * L)
xi = norm(scale=sigma).rvs()
gradient = approximate_gradient(noisy_func, x, xi, direction_generator,
t, m)
x = mirror_descent_ne(x, gradient, alpha * n * stepsize_constant)
residuals_func[k] = ((func(x) - f_star) * 1.0 / start_residual_func)
runtimes[k] = (time.time() - start_time)
pickle.dump(
runtimes,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it_' + str(stepsize_constant) +
'stepsz' + "_runtimes_rdfds_ne.txt", 'wb'))
pickle.dump(
residuals_func,
open(
"dump/" + filename + '_' + str(len(x)) + 'dim_' +
str(maximum_iterations) + 'it_' + str(stepsize_constant) +
'stepsz' + "_func_rdfds_ne.txt", 'wb'))
return x
def ardfds(func,
initial_x,
L,
m,
t,
maximum_iterations=1000,
direction_generator=None,
newton_eps=1e-5):
'''
m: batch size when computing gradient.
t: smoothing parameter when computing gradient.
'''
x = np.matrix(initial_x)
y = x.copy()
z = x.copy()
n = x.shape[0]
# initialization
xs = []
xs.append(x.copy())
runtimes = []
start_time = time.time()
runtimes.append(time.time() - start_time)
if not direction_generator:
direction_generator = sphere_point_generator(n)
for k in range(maximum_iterations):
alpha = (k + 2) / (96 * n * n * L)
tau = 2 / (k + 2)
gradient = approximate_gradient(func, x, direction_generator, t, m)
x = tau * z + (1 - tau) * y
y = x - 1 / (2 * L) * gradient
z = mirror_descent(z, gradient, alpha * n)
#newton_f = lambda x, order: V_function(alpha, gradient, z, x, order)
#z, newton_values, _, _ = newton( newton_f, x, newton_eps, 100, backtracking_line_search)
xs.append(y.copy())
runtimes.append(time.time() - start_time)
return y, xs, runtimes, "ARDFDS"
# the vector of linear coefficient of the quadratic function
b = np.matrix('0; 0')
#func = lambda x, order=0: quadratic(H, b,x, order)
func = lambda x, order=0: nesterov_function(x, order)
noisy_func = lambda x, n, explicit_noise=None: noisy_function(
func, x, noise_G, n, explicit_noise=explicit_noise, noise_mode="multiply")
initial_x = np.matrix('10;10')
N = 4000
t = 5e-3
t1 = 5e-3
m = 100
n = initial_x.shape[0]
L = compute_L(func, n, 100)
#L = 1
print(L)
direction_G1 = sphere_point_generator(n)
direction_G2 = gaussian_point_generator(n)
x0, xs0, _, _ = alg.rdfds(noisy_func,
initial_x,
L,
m,
t,
N,
direction_generator=direction_G1)
'''
x1, xs1,time2 = alg.rsgf(
noisy_func,
initial_x,
L,
m,
t1,