def W_optimize_Gaussian(db): y_tilde = create_y_tilde(db) db['y_tilde'] = y_tilde #print 'y tilde \n', y_tilde previous_gw = np.ones((db['N'], db['N'])) w_converged = False last_W = None for m in range(db['q']): db['W_matrix'][:, m] = get_orthogonal_vector(db, m, db['W_matrix'][:, m]) counter = 0 new_alpha = 0.5 while not w_converged: w_l = db['W_matrix'][:, m] #i_values = np.random.permutation( np.array(range(db['N'])) ) #i_values = i_values[0:db['SGD_size']] #j_values = np.random.permutation( np.array(range(db['N'])) ) #j_values = j_values[0:db['SGD_size']] i_values = [0] j_values = [1, 2] wolfe_alpha = obtain_Wolfe_alpha(db, y_tilde, w_l, previous_gw, i_values, j_values, m)
def W_optimize_Gaussian(db): iv = np.arange(db['N']) jv = np.arange(db['N']) sdg = SDG(db, iv, jv) sdg.y_tilde = create_y_tilde(db) db['W_matrix'] = sdg.run()
def W_optimize_Gaussian(db): iv = np.arange(db['N']) jv = np.arange(db['N']) dg = direct_GD(db, iv, jv) dg.y_tilde = create_y_tilde(db) db['W_matrix'] = dg.run()
def W_optimize_Gaussian(db): iv = np.arange(db['N']) jv = np.arange(db['N']) db['lowest_cost'] = float("inf") db['lowest_gradient'] = float("inf") sdg = SDG(db, iv, jv) sdg.y_tilde = create_y_tilde(db) db['W_matrix'] = sdg.run()
def W_optimize_Gaussian(db): y_tilde = create_y_tilde(db) previous_gw = np.ones((db['N'], db['N'])) w_converged = False last_W = None for m in range(db['q']): db['W_matrix'][:, m] = get_orthogonal_vector(db, m, db['W_matrix'][:, m]) counter = 0 while not w_converged: w_l = db['W_matrix'][:, m] alpha = obtain_Wolfe_alpha(db, y_tilde, w_l, new_direction)
def W_optimize_Gaussian(db): y_tilde = create_y_tilde(db) db['y_tilde'] = y_tilde db['Z_matrix'] = db['W_matrix'] db['L1'] = np.eye(db['q']) db['L2'] = np.eye(db['q'], db['d']) db['L'] = np.append(db['L1'], db['L2'].T, axis=0) use_all_data = True #use_all_data = False if use_all_data: iv = np.array(range(db['N'])) jv = iv else: i_values = np.random.permutation(np.array(range(db['N']))) iv = i_values[0:db['SGD_size']] j_values = np.random.permutation(np.array(range(db['N']))) jv = j_values[0:db['SGD_size']] eSolver = exponential_solver(db, iv, jv, y_tilde) optimize_result = eSolver.run()
def W_optimize_polynomial(db): y_tilde = create_y_tilde(db) #print 'y tilde \n', y_tilde previous_gw = np.zeros((db['N'],db['N'])) w_converged = False last_W = None for m in range(db['q']): db['W_matrix'][:,m] = get_orthogonal_vector(db, m, db['W_matrix'][:,m]) counter = 0 new_alpha = 0.5 while not w_converged: w_l = db['W_matrix'][:,m] i_values = np.random.permutation( np.array(range(db['N'])) ) i_values = i_values[0:db['SGD_size']] j_values = np.random.permutation( np.array(range(db['N'])) ) j_values = j_values[0:db['SGD_size']] #i_values = [0,1] #j_values = [0,1] [update_direction, db['updated_magnitude']] = Stochastic_W_gradient(db, y_tilde, previous_gw, w_l, i_values, j_values) update_direction = get_orthogonal_vector(db, m+1, update_direction) # m+1 is to also remove the current dimension new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction [tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values) #om = objective_magnitude #print m, db['updated_magnitude'], new_mag, w_l, new_alpha #import pdb; pdb.set_trace() while new_mag < db['updated_magnitude']: new_alpha = new_alpha * 0.8 if new_alpha > 0.0001 : new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction [tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values) #import pdb; pdb.set_trace() else: new_alpha = 0 break if type(last_W) != type(None): relative_magnitude = np.linalg.norm(new_W) distance_change_since_last_update = np.linalg.norm(last_W - new_W) if (distance_change_since_last_update/relative_magnitude) < 0.001 : w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass #print new_W last_W = new_W/np.linalg.norm(new_W) db['W_matrix'][:,m] = last_W counter += 1 if counter > db['maximum_W_update_count']: print '\nExit due to maximum update reached' w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass w_converged = False previous_gw = update_previous_gw(db, last_W, previous_gw) db['W_matrix'] = db['W_matrix'][:,0:db['q']]
def W_optimize_Gaussian(db): y_tilde = create_y_tilde(db) #print 'y tilde \n', y_tilde previous_gw = np.ones((db['N'],db['N'])) w_converged = False last_W = None for m in range(db['q']): db['W_matrix'][:,m] = get_orthogonal_vector(db, m, db['W_matrix'][:,m]) counter = 0 new_alpha = 0.5 while not w_converged: w_l = db['W_matrix'][:,m] [update_direction, db['updated_magnitude']] = get_W_gradient(db, y_tilde, previous_gw, w_l) update_direction = get_orthogonal_vector(db, m+1, update_direction) # m+1 is to also remove the current dimension #new_alpha = get_alpha_passing_wolfe(db, y_tilde, w_l, update_direction) new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction #import pdb; pdb.set_trace() [tmp_dir, new_mag] = get_W_gradient(db, y_tilde, previous_gw, new_W) #print db['updated_magnitude'], new_mag, new_alpha while new_mag < db['updated_magnitude']: new_alpha = new_alpha * 0.8 if new_alpha > 0.00001 : new_W = np.sqrt(1-new_alpha*new_alpha)*w_l + new_alpha*update_direction [tmp_dir, new_mag] = get_W_gradient(db, y_tilde, previous_gw, new_W) #import pdb; pdb.set_trace() else: new_alpha = 0 break if type(last_W) != type(None): relative_magnitude = np.linalg.norm(new_W) distance_change_since_last_update = np.linalg.norm(last_W - new_W) if (distance_change_since_last_update/relative_magnitude) < 0.001 : w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass #print new_W last_W = new_W/np.linalg.norm(new_W) db['W_matrix'][:,m] = last_W counter += 1 if counter > db['maximum_W_update_count']: print '\nExit due to maximum update reached' w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass w_converged = False previous_gw = update_previous_gw(db, last_W, previous_gw) #print previous_gw db['W_matrix'] = db['W_matrix'][:,0:db['q']]
def W_optimize_Gaussian(db): y_tilde = create_y_tilde(db) db['y_tilde'] = y_tilde previous_gw = np.ones((db['N'], db['N'])) w_converged = False last_W = None for m in range(db['q']): #import pdb; pdb.set_trace() try: db['W_matrix'][:, m] = get_orthogonal_vector(db, m, db['W_matrix'][:, m]) except: import pdb pdb.set_trace() counter = 0 new_alpha = 1 #print 'Starting a new dimension' while not w_converged: w_l = db['W_matrix'][:, m] i_values = np.random.permutation(np.array(range(db['N']))) i_values = i_values[0:db['N']] #SGD_size j_values = np.random.permutation(np.array(range(db['N']))) j_values = j_values[0:db['N']] [update_direction, db['updated_magnitude'] ] = Stochastic_W_gradient(db, y_tilde, previous_gw, w_l, i_values, j_values) update_direction = get_orthogonal_vector( db, m + 1, update_direction ) # m+1 is to also remove the current dimension #new_alpha = 1 # this is with back trace new_W = np.sqrt(1 - new_alpha * new_alpha) * w_l + new_alpha * update_direction #import pdb; pdb.set_trace() [tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values) if db['run_debug_1']: print 'Previous mag : ', db[ 'updated_magnitude'], 'New mag : ', new_mag, new_alpha while new_mag < db['updated_magnitude']: new_alpha = new_alpha * 0.4 #print 'Alpha : ', new_alpha if new_alpha > 0.00001: new_W = np.sqrt(1 - new_alpha * new_alpha ) * w_l + new_alpha * update_direction [tmp_dir, new_mag] = Stochastic_W_gradient(db, y_tilde, previous_gw, new_W, i_values, j_values) #import pdb; pdb.set_trace() else: new_alpha = 0 break if db['run_debug_1']: print 'magnitude , counter , time since start: ', new_mag, counter, time.time( ) - db['start_time'] else: pass #time_since_start = time.time() - db['start_time'] #sys.stdout.write("\b\b\b\b\b\b\b\b\b\b") #sys.stdout.write("\b\b\b\b\b\b\b\b\b\b") #sys.stdout.write("\b\b\b\b\b\b\b\b\b\b") #sys.stdout.write("\b\b\b\b\b\b\b\b\b\b") #sys.stdout.write(" Time since start : %f, loop counter : %d." % (time_since_start, counter)) #sys.stdout.flush() if type(last_W) != type(None): relative_magnitude = np.linalg.norm(new_W) distance_change_since_last_update = np.linalg.norm(last_W - new_W) if db['run_debug_1']: print "\t\t\texit condition : ", distance_change_since_last_update / relative_magnitude if (distance_change_since_last_update / relative_magnitude) < 0.001: w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass last_W = new_W / np.linalg.norm(new_W) db['W_matrix'][:, m] = last_W counter += 1 if counter > db['maximum_W_update_count']: if db['run_debug_1']: print '\nExit due to maximum update reached' w_converged = True try: db.pop('previous_wolfe_direction') db.pop('previous_wolfe_magnitude') except: pass w_converged = False previous_gw = update_previous_gw(db, last_W, previous_gw) db['W_matrix'] = db['W_matrix'][:, 0:db['q']]