def W_optimize_Gaussian(db):
y_tilde = create_y_tilde(db)
db['y_tilde'] = y_tilde
#print 'y tilde \n', y_tilde
previous_gw = np.ones((db['N'], db['N']))
w_converged = False
last_W = None
for m in range(db['q']):
db['W_matrix'][:, m] = get_orthogonal_vector(db, m, db['W_matrix'][:,
m])
counter = 0
new_alpha = 0.5
while not w_converged:
w_l = db['W_matrix'][:, m]
#i_values = np.random.permutation( np.array(range(db['N'])) )
#i_values = i_values[0:db['SGD_size']]
#j_values = np.random.permutation( np.array(range(db['N'])) )
#j_values = j_values[0:db['SGD_size']]
i_values = [0]
j_values = [1, 2]
wolfe_alpha = obtain_Wolfe_alpha(db, y_tilde, w_l, previous_gw,
i_values, j_values, m)
def W_optimize_Gaussian(db):
iv = np.arange(db['N'])
jv = np.arange(db['N'])
sdg = SDG(db, iv, jv)
sdg.y_tilde = create_y_tilde(db)
db['W_matrix'] = sdg.run()
def W_optimize_Gaussian(db):
iv = np.arange(db['N'])
jv = np.arange(db['N'])
dg = direct_GD(db, iv, jv)
dg.y_tilde = create_y_tilde(db)
db['W_matrix'] = dg.run()
def W_optimize_Gaussian(db):
iv = np.arange(db['N'])
jv = np.arange(db['N'])
db['lowest_cost'] = float("inf")
db['lowest_gradient'] = float("inf")
sdg = SDG(db, iv, jv)
sdg.y_tilde = create_y_tilde(db)
db['W_matrix'] = sdg.run()
def W_optimize_Gaussian(db):
y_tilde = create_y_tilde(db)
previous_gw = np.ones((db['N'], db['N']))
w_converged = False
last_W = None
for m in range(db['q']):
db['W_matrix'][:, m] = get_orthogonal_vector(db, m, db['W_matrix'][:,
m])
counter = 0
while not w_converged:
w_l = db['W_matrix'][:, m]
alpha = obtain_Wolfe_alpha(db, y_tilde, w_l, new_direction)
def W_optimize_Gaussian(db):
y_tilde = create_y_tilde(db)
db['y_tilde'] = y_tilde
db['Z_matrix'] = db['W_matrix']
db['L1'] = np.eye(db['q'])
db['L2'] = np.eye(db['q'], db['d'])
db['L'] = np.append(db['L1'], db['L2'].T, axis=0)
use_all_data = True
#use_all_data = False
if use_all_data:
iv = np.array(range(db['N']))
jv = iv
else:
i_values = np.random.permutation(np.array(range(db['N'])))
iv = i_values[0:db['SGD_size']]
j_values = np.random.permutation(np.array(range(db['N'])))
jv = j_values[0:db['SGD_size']]
eSolver = exponential_solver(db, iv, jv, y_tilde)
optimize_result = eSolver.run()
def W_optimize_polynomial(db):
y_tilde = create_y_tilde(db)
#print 'y tilde \n', y_tilde
previous_gw = np.zeros((db['N'],db['N']))
w_converged = False
last_W = None
for m in range(db['q']):
db['W_matrix'][:,m] = get_orthogonal_vector(db, m, db['W_matrix'][:,m])
counter = 0
new_alpha = 0.5
while not w_converged:
w_l = db['W_matrix'][:,m]
i_values = np.random.permutation( np.array(range(db['N'])) )
i_values = i_values[0:db['SGD_size']]
j_values = np.random.permutation( np.array(range(db['N'])) )
j_values = j_values[0:db['SGD_size']]
#i_values = [0,1]
#j_values = [0,1]
[update_direction, db['updated_magnitude']] = Stochastic_W_gradient(db, y_tilde, previous_gw, w_l, i_values, j_values)
update_direction = get_orthogonal_vector(db, m+1, update_direction) # m+1 is to also remove the current dimension
new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction
[tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values)
#om = objective_magnitude
#print m, db['updated_magnitude'], new_mag, w_l, new_alpha
#import pdb; pdb.set_trace()
while new_mag < db['updated_magnitude']:
new_alpha = new_alpha * 0.8
if new_alpha > 0.0001 :
new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction
[tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values)
#import pdb; pdb.set_trace()
else:
new_alpha = 0
break
if type(last_W) != type(None):
relative_magnitude = np.linalg.norm(new_W)
distance_change_since_last_update = np.linalg.norm(last_W - new_W)
if (distance_change_since_last_update/relative_magnitude) < 0.001 :
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except: pass
#print new_W
last_W = new_W/np.linalg.norm(new_W)
db['W_matrix'][:,m] = last_W
counter += 1
if counter > db['maximum_W_update_count']:
print '\nExit due to maximum update reached'
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except: pass
w_converged = False
previous_gw = update_previous_gw(db, last_W, previous_gw)
db['W_matrix'] = db['W_matrix'][:,0:db['q']]
def W_optimize_Gaussian(db):
y_tilde = create_y_tilde(db)
#print 'y tilde \n', y_tilde
previous_gw = np.ones((db['N'],db['N']))
w_converged = False
last_W = None
for m in range(db['q']):
db['W_matrix'][:,m] = get_orthogonal_vector(db, m, db['W_matrix'][:,m])
counter = 0
new_alpha = 0.5
while not w_converged:
w_l = db['W_matrix'][:,m]
[update_direction, db['updated_magnitude']] = get_W_gradient(db, y_tilde, previous_gw, w_l)
update_direction = get_orthogonal_vector(db, m+1, update_direction) # m+1 is to also remove the current dimension
#new_alpha = get_alpha_passing_wolfe(db, y_tilde, w_l, update_direction)
new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction
#import pdb; pdb.set_trace()
[tmp_dir, new_mag] = get_W_gradient(db, y_tilde, previous_gw, new_W)
#print db['updated_magnitude'], new_mag, new_alpha
while new_mag < db['updated_magnitude']:
new_alpha = new_alpha * 0.8
if new_alpha > 0.00001 :
new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction
[tmp_dir, new_mag] = get_W_gradient(db, y_tilde, previous_gw, new_W)
#import pdb; pdb.set_trace()
else:
new_alpha = 0
break
if type(last_W) != type(None):
relative_magnitude = np.linalg.norm(new_W)
distance_change_since_last_update = np.linalg.norm(last_W - new_W)
if (distance_change_since_last_update/relative_magnitude) < 0.001 :
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except: pass
#print new_W
last_W = new_W/np.linalg.norm(new_W)
db['W_matrix'][:,m] = last_W
counter += 1
if counter > db['maximum_W_update_count']:
print '\nExit due to maximum update reached'
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except: pass
w_converged = False
previous_gw = update_previous_gw(db, last_W, previous_gw)
#print previous_gw
db['W_matrix'] = db['W_matrix'][:,0:db['q']]
def W_optimize_Gaussian(db):
y_tilde = create_y_tilde(db)
db['y_tilde'] = y_tilde
previous_gw = np.ones((db['N'], db['N']))
w_converged = False
last_W = None
for m in range(db['q']):
#import pdb; pdb.set_trace()
try:
db['W_matrix'][:,
m] = get_orthogonal_vector(db, m, db['W_matrix'][:,
m])
except:
import pdb
pdb.set_trace()
counter = 0
new_alpha = 1
#print 'Starting a new dimension'
while not w_converged:
w_l = db['W_matrix'][:, m]
i_values = np.random.permutation(np.array(range(db['N'])))
i_values = i_values[0:db['N']] #SGD_size
j_values = np.random.permutation(np.array(range(db['N'])))
j_values = j_values[0:db['N']]
[update_direction, db['updated_magnitude']
] = Stochastic_W_gradient(db, y_tilde, previous_gw, w_l, i_values,
j_values)
update_direction = get_orthogonal_vector(
db, m + 1, update_direction
) # m+1 is to also remove the current dimension
#new_alpha = 1 # this is with back trace
new_W = np.sqrt(1 - new_alpha *
new_alpha) * w_l + new_alpha * update_direction
#import pdb; pdb.set_trace()
[tmp_dir,
new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W,
i_values, j_values)
if db['run_debug_1']:
print 'Previous mag : ', db[
'updated_magnitude'], 'New mag : ', new_mag, new_alpha
while new_mag < db['updated_magnitude']:
new_alpha = new_alpha * 0.4
#print 'Alpha : ', new_alpha
if new_alpha > 0.00001:
new_W = np.sqrt(1 - new_alpha * new_alpha
) * w_l + new_alpha * update_direction
[tmp_dir,
new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw,
new_W, i_values,
j_values)
#import pdb; pdb.set_trace()
else:
new_alpha = 0
break
if db['run_debug_1']:
print 'magnitude , counter , time since start: ', new_mag, counter, time.time(
) - db['start_time']
else:
pass
#time_since_start = time.time() - db['start_time']
#sys.stdout.write("\b\b\b\b\b\b\b\b\b\b")
#sys.stdout.write("\b\b\b\b\b\b\b\b\b\b")
#sys.stdout.write("\b\b\b\b\b\b\b\b\b\b")
#sys.stdout.write("\b\b\b\b\b\b\b\b\b\b")
#sys.stdout.write(" Time since start : %f, loop counter : %d." % (time_since_start, counter))
#sys.stdout.flush()
if type(last_W) != type(None):
relative_magnitude = np.linalg.norm(new_W)
distance_change_since_last_update = np.linalg.norm(last_W -
new_W)
if db['run_debug_1']:
print "\t\t\texit condition : ", distance_change_since_last_update / relative_magnitude
if (distance_change_since_last_update /
relative_magnitude) < 0.001:
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except:
pass
last_W = new_W / np.linalg.norm(new_W)
db['W_matrix'][:, m] = last_W
counter += 1
if counter > db['maximum_W_update_count']:
if db['run_debug_1']:
print '\nExit due to maximum update reached'
w_converged = True
try:
db.pop('previous_wolfe_direction')
db.pop('previous_wolfe_magnitude')
except:
pass
w_converged = False
previous_gw = update_previous_gw(db, last_W, previous_gw)
db['W_matrix'] = db['W_matrix'][:, 0:db['q']]