def display_results_task_f_4D_simple_ReLu_BT(): get_k = lambda a: 7*a + 4*3*2*a**2 shallow = lambda k: 4*k+k+k bt = lambda f: 2*f+f +2*f*(2*f)+ 2*(2*f) get_f = lambda a: 3*2*a # get_errors_from = mtf.get_errors_based_on_train_error #get_errors_from = mtf.get_errors_based_on_validation_error decider = ns.Namespace(get_errors_from=get_errors_from) task_name = 'task_f_4D_simple_ReLu_BT' experiment_name = mtf.get_experiment_folder(task_name) (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) var_Y_test = np.var(Y_test) print('var_Y_test: ', var_Y_test) plt.subplot(111) path_to_experiments = '../../%s/task_September_23_NN_xavier_relu'%experiment_name expts_best_results = mtf.get_best_results_for_experiments(path_to_experiments,decider,verbose=False) sorted_units, sorted_train_errors, sorted_validation_errors, sorted_test_errors = mtf.get_errors_for_display(expts_best_results) nb_params_shallow = [ shallow(nb_units) for nb_units in sorted_units ] print('nb_params_shallow = ', nb_params_shallow) print('sorted_train_errors = ', sorted_train_errors) print('sorted_test_errors = ', sorted_test_errors) #krls.plot_values(nb_params_shallow,sorted_train_errors,xlabel='number of parameters',y_label='squared error (l2 loss)',label='Train errors for Shallow Neural Net (NN)',markersize=3,colour='b') #krls.plot_values(nb_params_shallow,sorted_test_errors,xlabel='number of parameters',y_label='squared error (l2 loss)',label='Test error for Shallow NN',markersize=3,colour='b') duration_type = 'minutes' duration_type = 'hours' expts_best_results = mtf.get_all_simulation_results(path_to_experiments,decider,verbose=False) sorted_units, sorted_duration_means, sorted_duration_stds = mtf.get_duration_of_experiments(all_results=expts_best_results, duration_type=duration_type) nb_params_shallow = [ shallow(nb_units) for nb_units in sorted_units ] krls.plot_errors_and_bars(nb_params_shallow,sorted_duration_means,sorted_duration_stds,xlabel='number of parameters',y_label='squared error (l2 loss)',label='Duration for experiments '+duration_type,markersize=3,colour='b') path_to_experiments = '../../%s/task_September_23_BTHL_xavier_relu'%experiment_name expts_best_results = mtf.get_best_results_for_experiments(path_to_experiments,decider,verbose=False) sorted_units, sorted_train_errors, sorted_validation_errors, sorted_test_errors = mtf.get_errors_for_display(expts_best_results) nb_params_bt = [ bt(nb_units) for nb_units in sorted_units ] print('nb_params_bt = ', nb_params_bt) print('sorted_train_errors = ', sorted_train_errors) print('sorted_test_errors = ', sorted_test_errors) #krls.plot_values(nb_params_bt,sorted_train_errors,xlabel='number of parameters',y_label='squared error (l2 loss)',label='Train errors for Binary Tree (BT) Neural Net',markersize=3,colour='c') #krls.plot_values(nb_params_bt,sorted_test_errors,xlabel='number of parameters',y_label='squared error (l2 loss)',label='Test errors for Binary Tree (BT) Neural Net',markersize=3,colour='c') # plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.) #plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) #plt.legend() plt.show()
def main(argv): (_, task_name, result_loc, nb_inits, nb_rbf_shapes, units) = argv nb_inits, nb_rbf_shapes = int(nb_inits), int(nb_rbf_shapes) units_list = units.split(',') nb_centers_list = [ int(a) for a in units_list ] print 'task_name ', task_name (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) replace = False # with or without replacement stddevs = np.linspace(start=0.1, stop=3, num=nb_rbf_shapes) print 'number of RBF stddev tried:', len(stddevs) print 'start stddevs: ', stddevs mdl_best_params, mdl_mean_params, errors_best, errors_stats, reconstructions_best, reconstructions_mean = mtf.evalaute_models(data, stddevs, nb_centers_list, replace=False, nb_inits=nb_inits) (C_hat_bests, centers_bests, best_stddevs) = mdl_best_params print 'best_stddevs: ',best_stddevs (train_errors_bests, _, test_errors_bests) = errors_best (train_errors_means,_,test_errors_means, train_error_stds,_,test_error_stds) = errors_stats (Y_pred_train_best, _, Y_pred_test_best) = reconstructions_best # plot errors print 'plotting errors' plt1d.figure(n=1) plt1d.plot_errors(nb_centers_list, train_errors_bests,label='train_Errors_best', markersize=3,colour='b') plt1d.plot_errors(nb_centers_list, test_errors_bests,label='test_Errors_best', markersize=3,colour='r') plt1d.plot_errors_and_bars(nb_centers_list, train_errors_means, train_error_stds, label='train_Errors_average', markersize=3,colour='b') plt1d.plot_errors_and_bars(nb_centers_list, test_errors_means, test_error_stds, label='test_Errors_average', markersize=3,colour='r') # get things to reconstruct if task_name == 'qianli_func': print 'plotting reconstruct for task %s'%task_name pass elif task_name == 'f_2D_task2': print 'plotting reconstruct for task %s'%task_name print 'HERE' pass # plot show plt1d.show() #result_loc = './tmp_test_experiments/tmp_krls_workspace' mtf.save_workspace(filename=result_loc,names_of_spaces_to_save=dir(),dict_of_values_to_save=locals()) print '\a' #makes beep
def main(argv): print 'argv: ', argv (_, task_name, folder, experiment_name, nb_inits, nb_rbf_shapes, units) = argv nb_inits, nb_rbf_shapes = int(nb_inits), int(nb_rbf_shapes) units_list = units.split(',') nb_centers_list = [ int(a) for a in units_list ] date = datetime.date.today().strftime("%B %d").replace (" ", "_") print 'task_name ', task_name (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) print X_train.shape replace = False # with or without replacement stddevs = np.linspace(start=0.1, stop=1250, num=nb_rbf_shapes) print 'number of RBF stddev tried:', len(stddevs) print 'start stddevs: ', stddevs mdl_best_params, mdl_mean_params, errors_best, errors_stats, reconstructions_best, reconstructions_mean = mtf.evalaute_models(data, stddevs, nb_centers_list, replace=False, nb_inits=nb_inits) (C_hat_bests, centers_bests, best_stddevs) = mdl_best_params print 'best_stddevs: ',best_stddevs (train_errors_bests, cv_errors_bests, test_errors_bests) = errors_best (train_errors_means,cv_errors_means,test_errors_means, train_error_stds,cv_error_stds,test_error_stds) = errors_stats (train_errors_bests, cv_errors_bests, test_errors_bests) = [ list(err_list) for err_list in errors_best] (train_errors_means,cv_errors_means,test_errors_means, train_error_stds,cv_error_stds,test_error_stds) = [ list(err_list) for err_list in errors_stats] #(Y_pred_train_best, _, Y_pred_test_best) = reconstructions_best print 'train_errors_means: ', train_errors_means dir_path = '../../om_krls/%s_%s'%(date,experiment_name) mtf.make_and_check_dir(dir_path) # save rbf rbf_params_loc = dir_path+'/rbf_params_%s_%s'%(date,experiment_name) np.savez(rbf_params_loc,C_hat_bests=C_hat_bests,centers_bests=centers_bests,best_stddevs=best_stddevs,units_list=np.array(nb_centers_list)) # save errors result_loc = dir_path+'/results_json_%s_%s'%(date,experiment_name) results = {'nb_centers_list':nb_centers_list} mtf.load_results_dic(results,train_errors_bests=train_errors_bests,cv_errors_bests=cv_errors_bests,test_errors_bests=test_errors_bests, train_errors_means=train_errors_means,cv_errors_means=cv_errors_means,test_errors_means=test_errors_means, train_error_stds=train_error_stds,cv_error_stds=cv_error_stds,test_error_stds=test_error_stds) with open(result_loc, 'w+') as f_json: json.dump(results,f_json,sort_keys=True, indent=2, separators=(',', ': '))
def main3(agv): (_, task_name, result_loc, nb_inits, nb_rbf_shapes, units) = argv nb_inits, nb_rbf_shapes = int(nb_inits), int(nb_rbf_shapes) units_list = units.split(',') nb_centers_list = [ int(a) for a in units_list ] print 'task_name ', task_name (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) replace = False # with or without replacement stddevs = np.linspace(start=0.1, stop=3, num=nb_rbf_shapes) print 'number of RBF stddev tried:', len(stddevs) print 'start stddevs: ', stddevs mdl_best_params, mdl_mean_params, errors_best, errors_stats, reconstructions_best, reconstructions_mean = mtf.evalaute_models(data, stddevs, nb_centers_list, replace=False, nb_inits=nb_inits) (C_hat_bests, centers_bests, best_stddevs) = mdl_best_params print 'best_stddevs: ',best_stddevs (train_errors_bests, _, test_errors_bests) = errors_best (train_errors_means,_,test_errors_means, train_error_stds,_,test_error_stds) = errors_stats #(Y_pred_train_best, _, Y_pred_test_best) = reconstructions_best #mtf.save_workspace(filename=result_loc,names_of_spaces_to_save=dir(),dict_of_values_to_save=locals()) print '\a' #makes beep
import pdb import tensorflow as tf import numpy as np import sklearn from sklearn.decomposition import PCA from numpy import linalg as LA import my_tf_pkg as mtf from keras.datasets import mnist task_name='task_MNIST_flat_auto_encoder' print '----====> TASK NAME: %s' % task_name (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) #X_train = np.vstack((X_train,X_cv)) # (x_train, _), (x_test, _) = mnist.load_data() # x_train = x_train.astype('float32') / 255. # x_test = x_test.astype('float32') / 255. # x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:]))) # x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:]))) # print x_train.shape # print x_test.shape #print 'MNIST keras fingerprint: ', np.sum(x_train) #print 'MNIST keras fingerprint: ', np.sum(X_train)+np.sum(X_train) def get_reconstruction(X_train,k): pca = PCA(n_components=k) pca = pca.fit(X_train)
def main_gpu(arg): print('Running main') print('--==>', dict(arg) ) arg.act_name = arg.act.__name__ results = {'train_errors':[], 'cv_errors':[],'test_errors':[]} path, errors_pretty, mdl_dir, json_file = set_experiment_folders(arg) set_tensorboard(arg) ## Data sets and task print( '----====> TASK NAME: %s' % arg.data_file_name ) (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(arg) N_frac = arg.N_frac X_train, Y_train, X_cv, Y_cv, X_test, Y_test = X_train[:N_frac,:], Y_train[:N_frac,:], X_cv[:N_frac,:], Y_cv[:N_frac,:], X_test[:N_frac,:], Y_test[:N_frac,:] if arg.data_normalize == 'normalize_input': X_train, X_cv, X_test = preprocessing.scale(X_train), preprocessing.scale(X_cv), preprocessing.scale(X_test) (N_train,D) = X_train.shape (N_test,D_out) = Y_test.shape print( '(N_train,D) = ', (N_train,D) ) print( '(N_test,D_out) = ', (N_test,D_out) ) ## phase_train = tf.placeholder(tf.bool, name='phase_train') if arg.bn else None arg.steps = arg.get_steps(arg) arg.M = arg.get_batch_size(arg) arg.log_learning_rate = arg.get_log_learning_rate(arg) arg.starter_learning_rate = arg.get_start_learning_rate(arg) print( '++> starter_learning_rate ', arg.starter_learning_rate ) ## decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) arg.decay_rate = arg.get_decay_rate(arg) arg.decay_steps = arg.get_decay_steps(arg) if arg.optimization_alg == 'GD': pass elif arg.optimization_alg=='Momentum': arg.use_nesterov = arg.get_use_nesterov() arg.momentum = arg.get_momentum(arg) print('arg.use_nesterov', arg.use_nesterov) print('arg.momentum', arg.momentum) elif arg.optimization_alg == 'Adadelta': arg.rho = arg.get_rho(arg) print('arg.rho', arg.rho) elif arg.optimization_alg == 'Adagrad': #only has learning rate pass elif arg.optimization_alg == 'Adam': arg.beta1 = arg.get_beta1(arg) arg.beta2 = arg.get_beta2(arg) print('arg.beta1', arg.beta1) print('arg.beta2', arg.beta2) elif arg.optimization_alg == 'RMSProp': arg.decay = arg.get_decay(arg) arg.momentum = arg.get_momentum(arg) print('arg.decay', arg.decay) print('arg.momentum', arg.momentum) else: pass ############################## # if data_file_name == 'task_MNIST_flat_auto_encoder': # PCA_errors = {12:24.8254684915, 48:9.60052317906, 96:4.72118325768} # if len(units_list) == 1: # k = units_list[0] # else: # k = units_list[0] * len(units_list) # if not k in PCA_errors.keys(): # print( 'COMPUTING PCA... k = ', k) # X_reconstruct_pca, _, _ = mtf. get_reconstruction_pca(X_train,k=units_list[0]) # pca_error = mtf.report_l2_loss(Y=X_train,Y_pred=X_reconstruct_pca) # PCA_errors[k] = pca_error # else: # pca_error = PCA_errors[k] # print( '*************> PCA error: ', pca_error) # else: # pca_error = None # rbf_error = None # # hbf1_error = None # if model == 'hbf': # #error, Y_pred, Kern, C, subsampled_data_points = report_RBF_error_from_data(X_train, dims, stddev) # if len(units_list) > 1: # k = units_list[0]*len(units_list) # print( 'RBF units = ', k) # nb_units = [None, k] # rbf_error, _, _, _, _ = mtf.report_RBF_error_from_data(X_train, X_train, nb_units, S_init[1]) # print( rbf_error) # hbf1={12:26.7595} # if k in hbf1.keys(): # hbf1_error = hbf1[k] # else: # nb_units = dims # rbf_error, _, _, _, _ = mtf.report_RBF_error_from_data(X_train, X_train, nb_units, S_init[1]) ## pca_error = None rbf_error = None hbf1_error = None ## Make Model if arg.mdl == 'standard_nn': arg.dims = [D]+arg.units+[D_out] arg.mu_init_list = arg.get_W_mu_init(arg) arg.std_init_list = arg.get_W_std_init(arg) arg.b_init = arg.get_b_init(arg) float_type = tf.float64 x = tf.placeholder(float_type, shape=[None, D], name='x-input') # M x D nb_layers = len(arg.dims)-1 nb_hidden_layers = nb_layers-1 (inits_C,inits_W,inits_b) = mtf.get_initilizations_standard_NN(init_type=arg.init_type,dims=arg.dims,mu=arg.mu_init_list,std=arg.std_init_list,b_init=arg.b_init, X_train=X_train, Y_train=Y_train) with tf.name_scope("standardNN") as scope: mdl = mtf.build_standard_NN(arg, x,arg.dims,(None,inits_W,inits_b),phase_train,arg.trainable_bn) mdl = mtf.get_summation_layer(l=str(nb_layers),x=mdl,init=inits_C[0]) inits_S = inits_b elif arg.mdl == 'hbf': arg.dims = [D]+arg.units+[D_out] trainable_S = True if (arg.trainable_S=='train_S') else False arg.b_init = arg.get_b_init(arg) arg.S_init = arg.b_init float_type = tf.float64 #arg.mu , arg.std = arg.get_W_mu_init(arg), arg.get_W_std_init(arg) x = tf.placeholder(float_type, shape=[None, D], name='x-input') # M x D (inits_C,inits_W,inits_S,rbf_error) = mtf.get_initilizations_HBF(init_type=arg.init_type,dims=arg.dims,mu=arg.mu,std=arg.std,b_init=arg.b_init,S_init=arg.S_init, X_train=X_train, Y_train=Y_train, train_S_type=arg.train_S_type) #print(inits_W) nb_layers = len(arg.dims)-1 nb_hidden_layers = nb_layers-1 with tf.name_scope("HBF") as scope: mdl = mtf.build_HBF2(x,arg.dims,(inits_C,inits_W,inits_S),phase_train,arg.trainable_bn,trainable_S) mdl = mtf.get_summation_layer(l=str(nb_layers),x=mdl,init=inits_C[0]) elif arg.mdl == 'binary_tree_4D': pass elif arg.mdl == 'binary_tree_4D_conv_hidden_layer': print( 'binary_tree_4D' ) inits_S = None pca_error, rbf_error = None, None float_type = tf.float32 # Data sizes needed for reshaping N_cv, N_test = X_cv.shape[0], X_test.shape[0] # reshape data sets X_train, X_cv, X_test = X_train.reshape(N_train,1,D,1), X_cv.reshape(N_cv,1,D,1), X_test.reshape(N_test,1,D,1) x = tf.placeholder(float_type, shape=[None,1,D,1], name='x-input') # arg.stride_convd1, arg.filter_size = 2, 2 #fixed for Binary Tree BT #arg.mean, arg.stddev = arg.get_W_mu_init(arg), arg.get_W_std_init(arg) with tf.name_scope("build_binary_model") as scope: mdl = mtf.build_binary_tree_4D_hidden_layer(x,arg,phase_train=phase_train) arg.dims = [D]+[arg.nb_filters]+[arg.nb_final_hidden_units]+[D_out] elif arg.mdl == 'binary_tree_4D_conv_hidden_layer_automatic': print( arg.scope_name ) inits_S = None pca_error, rbf_error = None, None float_type = tf.float32 # N_cv, N_test = X_cv.shape[0], X_test.shape[0] X_train, X_cv, X_test = X_train.reshape(N_train,1,D,1), X_cv.reshape(N_cv,1,D,1), X_test.reshape(N_test,1,D,1) x = tf.placeholder(tf.float32, shape=[None,1,D,1], name='x-input') #[M, 1, D, 1] # with tf.name_scope("mdl"+arg.scope_name) as scope: mdl = mtf.bt_mdl_conv(arg,x) arg.dims = [D]+arg.F[1:]+[D_out] elif arg.mdl == 'binary_tree_8D_conv_hidden_layer': print( arg.scope_name ) inits_S = None pca_error, rbf_error = None, None float_type = tf.float32 # N_cv, N_test = X_cv.shape[0], X_test.shape[0] X_train, X_cv, X_test = X_train.reshape(N_train,1,D,1), X_cv.reshape(N_cv,1,D,1), X_test.reshape(N_test,1,D,1) x = tf.placeholder(tf.float32, shape=[None,1,D,1], name='x-input') #[M, 1, D, 1] # with tf.name_scope("mdl"+arg.scope_name) as scope: mdl = mtf.bt_mdl_conv(arg,x) arg.dims = [D]+arg.F[1:]+[D_out] ## Output and Loss y = mdl y_ = tf.placeholder(float_type, shape=[None, D_out]) # (M x D) with tf.name_scope("L2_loss") as scope: l2_loss = tf.reduce_sum( tf.reduce_mean(tf.square(y_-y), 0) ) #l2_loss = (2.0/N_train)*tf.nn.l2_loss(y_-y) #l2_loss = tf.reduce_mean(tf.square(y_-y)) nb_params = count_number_trainable_params(y) results['nb_params'] = nb_params print( '---> nb_params ', nb_params ) ## with tf.name_scope("train") as scope: # If the argument staircase is True, then global_step / decay_steps is an integer division and the decayed earning rate follows a staircase function. ## decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(learning_rate=arg.starter_learning_rate, global_step=global_step,decay_steps=arg.decay_steps, decay_rate=arg.decay_rate, staircase=arg.staircase) # Passing global_step to minimize() will increment it at each step. if arg.optimization_alg == 'GD': opt = tf.train.GradientDescentOptimizer(learning_rate) elif arg.optimization_alg == 'Momentum': opt = tf.train.MomentumOptimizer(learning_rate=learning_rate,momentum=arg.momentum,use_nesterov=arg.use_nesterov) elif arg.optimization_alg == 'Adadelta': opt = tf.train.AdadeltaOptimizer(learning_rate=learning_rate, rho=arg.rho, epsilon=1e-08, use_locking=False, name='Adadelta') elif arg.optimization_alg == 'Adam': opt = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=arg.beta1, beta2=arg.beta2, epsilon=1e-08, name='Adam') elif arg.optimization_alg == 'Adagrad': opt = tf.train.AdagradOptimizer(learning_rate) elif arg.optimization_alg == 'RMSProp': opt = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=arg.decay, momentum=arg.momentum, epsilon=1e-10, name='RMSProp') ## TODO if arg.re_train == 're_train' and arg.data_file_name == 'hrushikesh': print( 'data_file_name: ', data_file_name) print( 're_train: ', re_train) var_list = [v for v in tf.all_variables() if v.name == 'C:0'] #train_step = opt.minimize(l2_loss, var_list=var_list) else: train_step = opt.minimize(l2_loss, global_step=global_step) ## with tf.name_scope('learning_rate'): learning_rate_scalar_summary = tf.scalar_summary("learning_rate", learning_rate) with tf.name_scope("l2_loss") as scope: ls_scalar_summary = tf.scalar_summary("l2_loss", l2_loss) if arg.data_file_name == 'task_MNIST_flat_auto_encoder': with tf.name_scope('input_reshape'): x_image = tf.to_float(x, name='ToFloat') image_shaped_input_x = tf.reshape(x_image, [-1, 28, 28, 1]) # tf.image_summary(tag, tensor, max_images=3, collections=None, name=None) tf.image_summary('input', image_shaped_input_x, 10) with tf.name_scope('reconstruct'): y_image = tf.to_float(y, name='ToFloat') image_shaped_input_y = tf.reshape(x_image, [-1, 28, 28, 1]) # tf.image_summary(tag, tensor, max_images=3, collections=None, name=None) tf.image_summary('reconstruct', image_shaped_input_y, 10) def register_all_variables_and_grads(y): all_vars = tf.all_variables() grad_vars = opt.compute_gradients(y,all_vars) #[ (gradient,variable) ] for (dldw,v) in grad_vars: if dldw != None: prefix_name = 'derivative_'+v.name suffix_text = 'dJd'+v.name #mtf.put_summaries(var=tf.sqrt( tf.reduce_sum(tf.square(dldw)) ),prefix_name=prefix_name,suffix_text=suffix_text) mtf.put_summaries(var=tf.abs(dldw),prefix_name=prefix_name,suffix_text='_abs_'+suffix_text) tf.histogram_summary('hist'+prefix_name, dldw) register_all_variables_and_grads(y) ## TRAIN if phase_train is not None: #DO BN feed_dict_train = {x:X_train, y_:Y_train, phase_train: False} feed_dict_cv = {x:X_cv, y_:Y_cv, phase_train: False} feed_dict_test = {x:X_test, y_:Y_test, phase_train: False} else: #Don't do BN feed_dict_train = {x:X_train, y_:Y_train} feed_dict_cv = {x:X_cv, y_:Y_cv} feed_dict_test = {x:X_test, y_:Y_test} def get_batch_feed(X, Y, M, phase_train): mini_batch_indices = np.random.randint(M,size=M) Xminibatch = X[mini_batch_indices,:] # ( M x D^(0) ) Yminibatch = Y[mini_batch_indices,:] # ( M x D^(L) ) if phase_train is not None: #DO BN feed_dict = {x: Xminibatch, y_: Yminibatch, phase_train: True} else: #Don't do BN feed_dict = {x: Xminibatch, y_: Yminibatch} return feed_dict def print_messages(*args): for i, msg in enumerate(args): print('>%s'%msg, flush=True) if arg.use_tensorboard: if tf.gfile.Exists('/tmp/mdl_logs'): tf.gfile.DeleteRecursively('/tmp/mdl_logs') tf.gfile.MakeDirs('/tmp/mdl_logs') tf.add_check_numerics_ops() # Add ops to save and restore all the variables. if arg.mdl_save: saver = tf.train.Saver(max_to_keep=arg.max_to_keep) start_time = time.time() print() #file_for_error = './ray_error_file.txt' if arg.save_config_args: arg_dict = dict(arg).copy() arg_dict = get_remove_functions_from_dict(arg_dict) pickle.dump( arg_dict, open( "pickle-slurm-%s_%s.p"%(arg.slurm_jobid,arg.slurm_array_task_id) , "wb" ) ) #with open('json-slurm-%s_%s.json', 'w+') as f_json: # json.dump(results,f_json,sort_keys=True, indent=2, separators=(',', ': ')) with open(path+errors_pretty, 'w+') as f_err_msgs: #with open(file_for_error, 'w+') as f_err_msgs: #with tf.Session() as sess: sess = tf.Session() ## prepare writers and fetches if arg.use_tensorboard: merged = tf.merge_all_summaries() #writer = tf.train.SummaryWriter(tensorboard_data_dump, sess.graph) train_writer = tf.train.SummaryWriter(arg.tensorboard_data_dump_train, sess.graph) test_writer = tf.train.SummaryWriter(arg.tensorboard_data_dump_test, sess.graph) ## fetches_train = [merged, l2_loss] fetches_cv = l2_loss fetches_test = [merged, l2_loss] else: fetches_train = l2_loss fetches_cv = l2_loss fetches_test = l2_loss sess.run( tf.initialize_all_variables() ) for i in range(arg.steps): ## Create fake data for y = W.x + b where W = 2, b = 0 #(batch_xs, batch_ys) = get_batch_feed(X_train, Y_train, M, phase_train) feed_dict_batch = get_batch_feed(X_train, Y_train, arg.M, phase_train) ## Train if i%arg.report_error_freq == 0: if arg.use_tensorboard: (summary_str_train,train_error) = sess.run(fetches=fetches_train, feed_dict=feed_dict_train) cv_error = sess.run(fetches=fetches_cv, feed_dict=feed_dict_cv) (summary_str_test,test_error) = sess.run(fetches=fetches_test, feed_dict=feed_dict_test) train_writer.add_summary(summary_str_train, i) test_writer.add_summary(summary_str_test, i) else: train_error = sess.run(fetches=fetches_train, feed_dict=feed_dict_train) cv_error = sess.run(fetches=fetches_cv, feed_dict=feed_dict_cv) test_error = sess.run(fetches=fetches_test, feed_dict=feed_dict_test) current_learning_rate = sess.run(fetches=learning_rate) loss_msg = "=> Mdl*%s*-units%s, task: %s, step %d/%d, train err %g, cv err: %g test err %g"%(arg.mdl,arg.dims,arg.data_file_name,i,arg.steps,train_error,cv_error,test_error) mdl_info_msg = "Act: %s, Opt:%s, BN %s, BN_trainable: %s After%d/%d iteration,Init: %s, current_learning_rate %s, M %s, decay_rate %s, decay_steps %s, nb_params %s" % (arg.act.__name__,arg.optimization_alg,arg.bn,arg.trainable_bn,i,arg.steps,arg.init_type,current_learning_rate,arg.M,arg.decay_rate,arg.decay_steps,nb_params) errors_to_beat = 'BEAT: hbf1_error: %s RBF error: %s PCA error: %s '%(hbf1_error, rbf_error,pca_error) print_messages(loss_msg, mdl_info_msg, errors_to_beat) print('S: ', inits_S, flush=True) print() # store results results['train_errors'].append( float(train_error) ) results['cv_errors'].append( float(cv_error) ) results['test_errors'].append( float(test_error) ) # write errors to pretty print f_err_msgs.write(loss_msg+'\n') f_err_msgs.write(mdl_info_msg+'\n') if any_is_NaN(train_error,cv_error,test_error): # if its a nan make sure to stop script print('nan_found') break if arg.mdl_save: save_path = saver.save(sess, path+mdl_dir+'/model.ckpt',global_step=i) if arg.use_tensorboard: sess.run(fetches=[merged,train_step], feed_dict=feed_dict_batch) #sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) else: sess.run(fetches=train_step, feed_dict=feed_dict_batch) #sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) sess.close() _, best_train, best_cv, best_test = arg.get_errors_from(results) results['best_train'], results['best_cv'], results['best_test'] = best_train, best_cv, best_test print('End of main') git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']) results['git_hash'] = str(git_hash) #results['tf_rand_seed'] = tf_rand_seed # seconds = (time.time() - start_time) minutes = seconds/ 60 hours = minutes/ 60 print("--- %s seconds ---" % seconds ) print("--- %s minutes ---" % minutes ) print("--- %s hours ---" % hours ) ## dump results to JSON results['seconds'] = seconds results['minutes'] = minutes results['hours'] = hours #print results #results['arg'] = arg arg_dict = dict(arg) arg_dict = get_remove_functions_from_dict(arg_dict) results['arg_dict'] = arg_dict with open(path+json_file, 'w+') as f_json: print('Writing Json') print('path+json_file', path+json_file) json.dump(results,f_json,indent=2, separators=(',', ': ')) print( '\a') #makes beep #print(results) print('get_errors_from: ', arg.get_errors_from.__name__) print('best results: train, cv, test: ', best_train, best_cv, best_test )
from sklearn.metrics.pairwise import euclidean_distances import numpy as np import my_tf_pkg as mtf def get_pairwise_norm_squared(X, Y): return mtf.euclidean_distances(X=X, Y=Y, squared=True) task_name = 'hrushikesh' X_train, Y_train, X_cv, Y_cv, X_test, Y_test = mtf.get_data(task_name) print 'X_train.shape', X_train.shape print 'X_cv.shape', X_cv.shape print 'X_test.shape', X_test.shape pairwise_norm_squared = get_pairwise_norm_squared(X=X_train, Y=X_train) print 'min', np.amin(pairwise_norm_squared) print 'max', np.amax(pairwise_norm_squared) print 'mean', np.mean(pairwise_norm_squared) print 'std', np.std(pairwise_norm_squared)
def main(arg): results = {'train_errors':[], 'cv_errors':[],'test_errors':[]} path, errors_pretty, mdl_dir, json_file = set_experiment_folders(arg) # try to make directory, if it exists do NOP mtf.make_and_check_dir(path=path) mtf.make_and_check_dir(path=path+mdl_dir) # JSON results structure #results_dic = mtf.fill_results_dic_with_np_seed(np_rnd_seed=np.random.get_state(), results=results) ## Data sets and task print( '----====> TASK NAME: %s' % arg.task_name ) (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(arg.task_name) if arg.data_normalize == 'normalize_input': X_train, X_cv, X_test = preprocessing.scale(X_train), preprocessing.scale(X_cv), preprocessing.scale(X_test) (N_train,D) = X_train.shape (N_test,D_out) = Y_test.shape print( '(N_train,D) = ', (N_train,D) ) print( '(N_test,D_out) = ', (N_test,D_out) ) ## units_list = arg.units_list dims = [D]+arg.units_list+[D_out] mu = arg.W_mu_init(dims, arg) std = arg.W_std_init(dims, arg) b_init = arg.b_init() phase_train = tf.placeholder(tf.bool, name='phase_train') if arg.bn else None steps = np.random.randint(low=arg.steps_low ,high=arg.steps_high) M = np.random.randint(low=arg.M_low , high=arg.M_high) arg.M = M log_learning_rate = np.random.uniform(low=arg.low_log_const_learning_rate, high=arg.high_log_const_learning_rate) starter_learning_rate = 10**log_learning_rate print( '++> starter_learning_rate ', starter_learning_rate ) ## decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) decay_rate = np.random.uniform(low=arg.decay_rate_low, high=arg.decay_rate_high) arg.decay_steps_high decay_steps = np.random.randint(low=arg.decay_steps_low(arg), high=arg.decay_steps_high(arg) ) staircase = arg.staircase if optimization_alg == 'GD': pass elif optimization_alg=='Momentum': use_nesterov = arg.use_nesterov momentum=np.random.uniform(low=0.1, high=0.99) results['momentum']=float(momentum) elif optimization_alg == 'Adadelta': rho=np.random.uniform(low=0.4, high=0.99) results['rho']=float(rho) elif optimization_alg == 'Adagrad': #only has learning rate pass elif optimization_alg == 'Adam': beta1 = arg.get_beta1(arg) beta2 = arg.get_beta2(arg) results['beta1']=float(beta1) results['beta2']=float(beta2) elif optimization_alg == 'RMSProp': decay = np.random.uniform(low=arg.decay_loc,high=arg.decay_high) momentum = np.random.uniform(low=arg.momentum_low,high=arg.momentum_high) results['decay']=float(decay) results['momentum']=float(momentum) else: pass results['range_learning_rate'] = [low_const_learning_rate, high_const_learning_rate] ############################## if task_name == 'task_MNIST_flat_auto_encoder': PCA_errors = {12:24.8254684915, 48:9.60052317906, 96:4.72118325768} if len(units_list) == 1: k = units_list[0] else: k = units_list[0] * len(units_list) if not k in PCA_errors.keys(): print( 'COMPUTING PCA... k = ', k) X_reconstruct_pca, _, _ = mtf. get_reconstruction_pca(X_train,k=units_list[0]) pca_error = mtf.report_l2_loss(Y=X_train,Y_pred=X_reconstruct_pca) PCA_errors[k] = pca_error else: pca_error = PCA_errors[k] print( '*************> PCA error: ', pca_error) else: pca_error = None rbf_error = None hbf1_error = None if model == 'hbf': #error, Y_pred, Kern, C, subsampled_data_points = report_RBF_error_from_data(X_train, dims, stddev) if len(units_list) > 1: k = units_list[0]*len(units_list) print( 'RBF units = ', k) nb_units = [None, k] rbf_error, _, _, _, _ = mtf.report_RBF_error_from_data(X_train, X_train, nb_units, S_init[1]) print( rbf_error) hbf1={12:26.7595} if k in hbf1.keys(): hbf1_error = hbf1[k] else: nb_units = dims rbf_error, _, _, _, _ = mtf.report_RBF_error_from_data(X_train, X_train, nb_units, S_init[1]) S_init = b_init ## ## Make Model nb_layers = len(dims)-1 nb_hidden_layers = nb_layers-1 print( '-----> Running model: %s. (nb_hidden_layers = %d, nb_layers = %d)' % (model,nb_hidden_layers,nb_layers) ) print( '-----> Units: %s)' % (dims) ) if model == 'standard_nn': rbf_error = None #tensorboard_data_dump = '/tmp/standard_nn_logs' float_type = tf.float64 x = tf.placeholder(float_type, shape=[None, D], name='x-input') # M x D (inits_C,inits_W,inits_b) = mtf.get_initilizations_standard_NN(init_type=init_type,dims=dims,mu=mu,std=std,b_init=b_init,S_init=S_init, X_train=X_train, Y_train=Y_train) with tf.name_scope("standardNN") as scope: mdl = mtf.build_standard_NN(x,dims,(inits_C,inits_W,inits_b),phase_train,trainable_bn) mdl = mtf.get_summation_layer(l=str(nb_layers),x=mdl,init=inits_C[0]) inits_S = inits_b elif model == 'hbf': trainable_S = True if (arg.trainable_S=='train_S') else False #tensorboard_data_dump = '/tmp/hbf_logs' float_type = tf.float64 x = tf.placeholder(float_type, shape=[None, D], name='x-input') # M x D (inits_C,inits_W,inits_S,rbf_error) = mtf.get_initilizations_HBF(init_type=init_type,dims=dims,mu=mu,std=std,b_init=b_init,S_init=S_init, X_train=X_train, Y_train=Y_train, train_S_type=train_S_type) print(inits_W) with tf.name_scope("HBF") as scope: mdl = mtf.build_HBF2(x,dims,(inits_C,inits_W,inits_S),phase_train,trainable_bn,trainable_S) mdl = mtf.get_summation_layer(l=str(nb_layers),x=mdl,init=inits_C[0]) elif model == 'binary_tree_4D_conv': print( 'binary_tree_4D') #tensorboard_data_dump = '/tmp/hbf_logs' inits_S = None pca_error = None rbf_error = None float_type = tf.float32 # things that need reshaping N_cv = X_cv.shape[0] N_test = X_test.shape[0] # X_train = X_train.reshape(N_train,1,D,1) #Y_train = Y_train.reshape(N_train,1,D,1) X_cv = X_cv.reshape(N_cv,1,D,1) #Y_cv = Y_cv.reshape(N_cv,1,D,1) X_test = X_test.reshape(N_test,1,D,1) #Y_test = Y_test.reshape(N_test,1,D,1) x = tf.placeholder(float_type, shape=[None,1,D,1], name='x-input') # filter_size = 2 #fixed for Binary Tree BT #nb_filters = nb_filters mean, stddev = bn_tree_init_stats stddev = float( np.random.uniform(low=0.001, high=stddev) ) print( 'stddev', stddev) x = tf.placeholder(float_type, shape=[None,1,D,1], name='x-input') with tf.name_scope("build_binary_model") as scope: mdl = mtf.build_binary_tree(x,filter_size,nb_filters,mean,stddev,stride_convd1=2,phase_train=phase_train,trainable_bn=trainable_bn) # dims = [D]+[nb_filters]+[D_out] results['nb_filters'] = nb_filters elif model == 'binary_tree_D8': #tensorboard_data_dump = '/tmp/hbf_logs' inits_S = None pca_error = None rbf_error = None float_type = tf.float32 # things that need reshaping N_cv = X_cv.shape[0] N_test = X_test.shape[0] # X_train = X_train.reshape(N_train,1,D,1) #Y_train = Y_train.reshape(N_train,1,D,1) X_cv = X_cv.reshape(N_cv,1,D,1) #Y_cv = Y_cv.reshape(N_cv,1,D,1) X_test = X_test.reshape(N_test,1,D,1) #Y_test = Y_test.reshape(N_test,1,D,1) x = tf.placeholder(float_type, shape=[None,1,D,1], name='x-input') # filter_size = 2 #fixed for Binary Tree BT nb_filters1,nb_filters2 = nb_filters mean1,stddev1,mean2,stddev2,mean3,stddev3 = bn_tree_init_stats x = tf.placeholder(float_type, shape=[None,1,D,1], name='x-input') with tf.name_scope("binary_tree_D8") as scope: mdl = mtf.build_binary_tree_8D(x,nb_filters1,nb_filters2,mean1,stddev1,mean2,stddev2,mean3,stddev3,stride_conv1=2) # dims = [D]+nb_filters+[D_out] results['nb_filters'] = nb_filters ## Output and Loss y = mdl y_ = tf.placeholder(float_type, shape=[None, D_out]) # (M x D) with tf.name_scope("L2_loss") as scope: l2_loss = tf.reduce_sum( tf.reduce_mean(tf.square(y_-y), 0) ) #l2_loss = (2.0/N_train)*tf.nn.l2_loss(y_-y) #l2_loss = tf.reduce_mean(tf.square(y_-y)) ## with tf.name_scope("train") as scope: # starter_learning_rate = 0.0000001 # decay_rate = 0.9 # decay_steps = 100 # staircase = True # decay_steps = 10000000 # staircase = False # decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(learning_rate=starter_learning_rate, global_step=global_step,decay_steps=decay_steps, decay_rate=decay_rate, staircase=staircase) # Passing global_step to minimize() will increment it at each step. if optimization_alg == 'GD': opt = tf.train.GradientDescentOptimizer(learning_rate) elif optimization_alg == 'Momentum': opt = tf.train.MomentumOptimizer(learning_rate=learning_rate,momentum=momentum,use_nesterov=use_nesterov) elif optimization_alg == 'Adadelta': tf.train.AdadeltaOptimizer(learning_rate=learning_rate, rho=rho, epsilon=1e-08, use_locking=False, name='Adadelta') elif optimization_alg == 'Adam': opt = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=beta1, beta2=beta2, epsilon=1e-08, name='Adam') elif optimization_alg == 'Adagrad': opt = tf.train.AdagradOptimizer(learning_rate) elif optimization_alg == 'RMSProp': opt = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=decay, momentum=momentum, epsilon=1e-10, name='RMSProp') ## if re_train == 're_train' and task_name == 'hrushikesh': print( 'task_name: ', task_name) print( 're_train: ', re_train) var_list = [v for v in tf.all_variables() if v.name == 'C:0'] #train_step = opt.minimize(l2_loss, var_list=var_list) else: train_step = opt.minimize(l2_loss, global_step=global_step) ## with tf.name_scope("l2_loss") as scope: ls_scalar_summary = tf.scalar_summary("l2_loss", l2_loss) if task_name == 'task_MNIST_flat_auto_encoder': with tf.name_scope('input_reshape'): x_image = tf.to_float(x, name='ToFloat') image_shaped_input_x = tf.reshape(x_image, [-1, 28, 28, 1]) # tf.image_summary(tag, tensor, max_images=3, collections=None, name=None) tf.image_summary('input', image_shaped_input_x, 10) with tf.name_scope('reconstruct'): y_image = tf.to_float(y, name='ToFloat') image_shaped_input_y = tf.reshape(x_image, [-1, 28, 28, 1]) # tf.image_summary(tag, tensor, max_images=3, collections=None, name=None) tf.image_summary('reconstruct', image_shaped_input_y, 10) def register_all_variables_and_grads(y): all_vars = tf.all_variables() grad_vars = opt.compute_gradients(y,all_vars) #[ (gradient,variable) ] for (dldw,v) in grad_vars: if dldw != None: prefix_name = 'derivative_'+v.name suffix_text = 'dJd'+v.name #mtf.put_summaries(var=tf.sqrt( tf.reduce_sum(tf.square(dldw)) ),prefix_name=prefix_name,suffix_text=suffix_text) mtf.put_summaries(var=tf.abs(dldw),prefix_name=prefix_name,suffix_text='_abs_'+suffix_text) tf.histogram_summary('hist'+prefix_name, dldw) register_all_variables_and_grads(y) ## TRAIN if phase_train is not None: #DO BN feed_dict_train = {x:X_train, y_:Y_train, phase_train: False} feed_dict_cv = {x:X_cv, y_:Y_cv, phase_train: False} feed_dict_test = {x:X_test, y_:Y_test, phase_train: False} else: #Don't do BN feed_dict_train = {x:X_train, y_:Y_train} feed_dict_cv = {x:X_cv, y_:Y_cv} feed_dict_test = {x:X_test, y_:Y_test} def get_batch_feed(X, Y, M, phase_train): mini_batch_indices = np.random.randint(M,size=M) Xminibatch = X[mini_batch_indices,:] # ( M x D^(0) ) Yminibatch = Y[mini_batch_indices,:] # ( M x D^(L) ) if phase_train is not None: #DO BN feed_dict = {x: Xminibatch, y_: Yminibatch, phase_train: True} else: #Don't do BN feed_dict = {x: Xminibatch, y_: Yminibatch} return feed_dict def print_messages(*args): for i, msg in enumerate(args): print('>',msg) if use_tensorboard: if tf.gfile.Exists('/tmp/mdl_logs'): tf.gfile.DeleteRecursively('/tmp/mdl_logs') tf.gfile.MakeDirs('/tmp/mdl_logs') tf.add_check_numerics_ops() # Add ops to save and restore all the variables. if mdl_save: saver = tf.train.Saver(max_to_keep=max_to_keep) start_time = time.time() file_for_error = './ray_error_file.txt' #with open(path+errors_pretty, 'w+') as f_err_msgs: with open(file_for_error, 'w+') as f_err_msgs: with tf.Session() as sess: ## prepare writers and fetches if use_tensorboard: merged = tf.merge_all_summaries() #writer = tf.train.SummaryWriter(tensorboard_data_dump, sess.graph) train_writer = tf.train.SummaryWriter(tensorboard_data_dump_train, sess.graph) test_writer = tf.train.SummaryWriter(tensorboard_data_dump_test, sess.graph) ## fetches_train = [merged, l2_loss] fetches_cv = l2_loss fetches_test = [merged, l2_loss] else: fetches_train = l2_loss fetches_cv = l2_loss fetches_test = l2_loss sess.run( tf.initialize_all_variables() ) for i in range(steps): ## Create fake data for y = W.x + b where W = 2, b = 0 #(batch_xs, batch_ys) = get_batch_feed(X_train, Y_train, M, phase_train) feed_dict_batch = get_batch_feed(X_train, Y_train, M, phase_train) ## Train if i%report_error_freq == 0: if use_tensorboard: (summary_str_train,train_error) = sess.run(fetches=fetches_train, feed_dict=feed_dict_train) cv_error = sess.run(fetches=fetches_cv, feed_dict=feed_dict_cv) (summary_str_test,test_error) = sess.run(fetches=fetches_test, feed_dict=feed_dict_test) train_writer.add_summary(summary_str_train, i) test_writer.add_summary(summary_str_test, i) else: train_error = sess.run(fetches=fetches_train, feed_dict=feed_dict_train) cv_error = sess.run(fetches=fetches_cv, feed_dict=feed_dict_cv) test_error = sess.run(fetches=fetches_test, feed_dict=feed_dict_test) loss_msg = "Mdl*%s%s*-units%s, task: %s, step %d/%d, train err %g, cv err: %g test err %g"%(model,nb_hidden_layers,dims,task_name,i,steps,train_error,cv_error,test_error) mdl_info_msg = "Opt:%s, BN %s, BN_trainable: %s After%d/%d iteration,Init: %s" % (optimization_alg,bn,trainable_bn,i,steps,init_type) errors_to_beat = 'BEAT: hbf1_error: %s RBF error: %s PCA error: %s '%(hbf1_error, rbf_error,pca_error) print_messages(loss_msg, mdl_info_msg, errors_to_beat) #sys.stdout.flush() loss_msg+="\n" mdl_info_msg+="\n" errors_to_beat+="\n" print( 'S: ', inits_S) # store results #print type(train_error) results['train_errors'].append( float(train_error) ) #print type(cv_error) results['cv_errors'].append( float(cv_error) ) #print type(test_error) results['test_errors'].append( float(test_error) ) # write errors to pretty print f_err_msgs.write(loss_msg) f_err_msgs.write(mdl_info_msg) # save mdl if mdl_save: save_path = saver.save(sess, path+mdl_dir+'/model.ckpt',global_step=i) if use_tensorboard: sess.run(fetches=[merged,train_step], feed_dict=feed_dict_batch) #sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) else: sess.run(fetches=train_step, feed_dict=feed_dict_batch) #sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']) mtf.load_results_dic(results,git_hash=git_hash,dims=dims,mu=mu,std=std,init_constant=init_constant,b_init=b_init,S_init=S_init,\ init_type=init_type,model=model,bn=bn,path=path,\ tensorboard_data_dump_test=tensorboard_data_dump_test,tensorboard_data_dump_train=tensorboard_data_dump_train,\ report_error_freq=report_error_freq,steps=steps,M=M,optimization_alg=optimization_alg,\ starter_learning_rate=starter_learning_rate,decay_rate=decay_rate,staircase=staircase) ## results['job_name'] = job_name results['slurm_jobid'] = slurm_jobid results['slurm_array_task_id'] = slurm_array_task_id #results['tf_rand_seed'] = tf_rand_seed results['date'] = date results['bn'] = bn results['trainable_bn'] = trainable_bn seconds = (time.time() - start_time) minutes = seconds/ 60 hours = minutes/ 60 print("--- %s seconds ---" % seconds ) print("--- %s minutes ---" % minutes ) print("--- %s hours ---" % hours ) ## dump results to JSON results['seconds'] = seconds results['minutes'] = minutes results['hours'] = hours #print results with open(path+json_file, 'w+') as f_json: json.dump(results,f_json,sort_keys=True, indent=2, separators=(',', ': ')) print( '\a') #makes beep
def main(argv): (_, task_name, result_loc, nb_inits, nb_rbf_shapes, units) = argv nb_inits, nb_rbf_shapes = int(nb_inits), int(nb_rbf_shapes) units_list = units.split(',') nb_centers_list = [ int(a) for a in units_list ] print 'task_name ', task_name #pdb.set_trace() #(X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data_from_file(file_name='./f_1d_cos_no_noise_data.npz') #task_name = 'qianli_func' #task_name = 'hrushikesh' #task_name = 'f_2D_task2' (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) replace = False # with or without replacement #nb_rbf_shapes = 2 #<-- stddevs = np.linspace(start=0.1, stop=3, num=nb_rbf_shapes) print 'number of RBF stddev tried:', len(stddevs) print 'start stddevs: ', stddevs #nb_centers_list = [3, 6, 9, 12, 16, 24, 30, 39, 48, 55] #nb_centers_list = [2, 4, 6, 8, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] #nb_centers_list = [2, 4] #nb_centers_list = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] #centers_to_reconstruct_index = [1, 3, 5, 7, 9] #centers_to_reconstruct_index = [1, 4, 7] # corresponds to centers 4, 12, 18 #colours = ['g','r','c','m','y'] #nb_inits = 2 #<-- mdl_best_params, mdl_mean_params, errors_best, errors_stats, reconstructions_best, reconstructions_mean = evalaute_models(data, stddevs, nb_centers_list, replace=False, nb_inits=nb_inits) (C_hat_bests, centers_bests, best_stddevs) = mdl_best_params print 'best_stddevs: ',best_stddevs (train_errors_bests, _, test_errors_bests) = errors_best (train_errors_means,_,test_errors_means, train_error_stds,_,test_error_stds) = errors_stats (Y_pred_train_best, _, Y_pred_test_best) = reconstructions_best # plot errors print 'plotting errors' plt.figure(3) plot_errors(nb_centers_list, train_errors_bests,label='train_Errors_best', markersize=3,colour='b') plot_errors(nb_centers_list, test_errors_bests,label='test_Errors_best', markersize=3,colour='r') plot_errors_and_bars(nb_centers_list, train_errors_means, train_error_stds, label='train_Errors_average', markersize=3,colour='b') plot_errors_and_bars(nb_centers_list, test_errors_means, test_error_stds, label='test_Errors_average', markersize=3,colour='r') # get things to reconstruct if task_name == 'qianli_func': plot_one_func(fig_num=1, X_original=X_train,Y_original=Y_train, markersize=3, title_name='Reconstruction') print 'plotting reconstruct for task %s'%task_name nb_centers_reconstruct = [nb_centers_list[i] for i in centers_to_reconstruct_index] rbf_predictions_reconstruct_train = [Y_pred_train_best[i] for i in centers_to_reconstruct_index] rbf_predictions_reconstruct_test = [Y_pred_test_best[i] for i in centers_to_reconstruct_index] colours = colours[0:len(centers_to_reconstruct_index)] #colours = [colours[i] for i in centers_to_reconstruct_index] # plot reconstructions print 'plotting reconstructions' plot_reconstruction(fig_num=1, X_original=X_train,Y_original=Y_train, nb_centers=nb_centers_reconstruct, \ rbf_predictions=rbf_predictions_reconstruct_train, colours=colours, markersize=3,title_name='Reconstruction_train') plot_reconstruction(fig_num=2, X_original=X_test,Y_original=Y_test, nb_centers=nb_centers_reconstruct, \ rbf_predictions=rbf_predictions_reconstruct_test, colours=colours, markersize=3,title_name='Reconstruction_test') elif task_name == 'f_2D_task2': print 'HERE' pass print 'plotting reconstruct for task %s'%task_name # plot show plt.legend() plt.show() #result_loc = './tmp_test_experiments/tmp_krls_workspace' mtf.save_workspace(filename=result_loc,names_of_spaces_to_save=dir(),dict_of_values_to_save=locals()) print '\a' #makes beep
import numpy as np import json #from sklearn.cross_validation import train_test_split from tensorflow.examples.tutorials.mnist import input_data import my_tf_pkg as mtf import namespaces as ns import pdb #task_name = 'task_MNIST_flat_auto_encoder' #task_name = 'task_f_4D_conv_2nd' arg = ns.Namespace() arg.data_dirpath = './data/' arg.data_file_name = 'f_8D_conv_cos_poly1_poly1' X_train, Y_train, X_cv, Y_cv, X_test, Y_test = mtf.get_data(arg) print('max: ', np.max(Y_train)) print('min: ', np.min(Y_train)) print('mean: ',np.mean(Y_train)) print('std: ',np.std(Y_train))
def main(argv): (_, task_name, prefix, experiment_name, nb_inits, nb_rbf_shapes, units) = argv nb_inits, nb_rbf_shapes = int(nb_inits), int(nb_rbf_shapes) units_list = units.split(',') nb_centers_list = [int(a) for a in units_list] date = datetime.date.today().strftime("%B %d").replace(" ", "_") print 'task_name ', task_name (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) = mtf.get_data(task_name) data = (X_train, Y_train, X_cv, Y_cv, X_test, Y_test) print X_train.shape replace = False # with or without replacement stddevs = np.linspace(start=0.01, stop=10, num=nb_rbf_shapes) print 'number of RBF stddev tried:', len(stddevs) print 'start stddevs: ', stddevs mdl_best_params, mdl_mean_params, errors_best, errors_stats, reconstructions_best, reconstructions_mean = mtf.evalaute_models( data, stddevs, nb_centers_list, replace=False, nb_inits=nb_inits) (C_hat_bests, centers_bests, best_stddevs) = mdl_best_params print 'best_stddevs: ', best_stddevs (train_errors_bests, cv_errors_bests, test_errors_bests) = errors_best (train_errors_means, cv_errors_means, test_errors_means, train_error_stds, cv_error_stds, test_error_stds) = errors_stats (train_errors_bests, cv_errors_bests, test_errors_bests) = [list(err_list) for err_list in errors_best] (train_errors_means, cv_errors_means, test_errors_means, train_error_stds, cv_error_stds, test_error_stds) = [list(err_list) for err_list in errors_stats] #(Y_pred_train_best, _, Y_pred_test_best) = reconstructions_best print 'train_errors_means: ', train_errors_means dir_path = './%s_experiments/%s_%s' % (prefix, date, experiment_name) mtf.make_and_check_dir(dir_path) # save rbf rbf_params_loc = dir_path + '/rbf_params_%s_%s' % (date, experiment_name) np.savez(rbf_params_loc, C_hat_bests=C_hat_bests, centers_bests=centers_bests, best_stddevs=best_stddevs, units_list=np.array(nb_centers_list)) # save errors result_loc = dir_path + '/results_json_%s_%s' % (date, experiment_name) results = {'nb_centers_list': nb_centers_list} mtf.load_results_dic(results, train_errors_bests=train_errors_bests, cv_errors_bests=cv_errors_bests, test_errors_bests=test_errors_bests, train_errors_means=train_errors_means, cv_errors_means=cv_errors_means, test_errors_means=test_errors_means, train_error_stds=train_error_stds, cv_error_stds=cv_error_stds, test_error_stds=test_error_stds) with open(result_loc, 'w+') as f_json: json.dump(results, f_json, sort_keys=True, indent=2, separators=(',', ': '))
def main_hp(arg): ''' executes the current hp (hyper param) slurm array task. Usually means that it has to either continue training a model that wasn't finished training or start one from scratch. note: ''' #arg.rand_x = int.from_bytes(os.urandom(4), sys.byteorder) if arg.rand_x != None: np.random.seed(arg.rand_x) random.seed(arg.rand_x) tf.set_random_seed( arg.rand_x ) # force to flushing to output as default print = print_func_flush_false if arg.slurm_array_task_id == '1': #arg.display_training = True print = print_func_flush_true if arg.flush: print = print_func_flush_true print(print) print('>>> arg.restore = ', arg.restore) arg.date = datetime.date.today().strftime("%B %d").replace (" ", "_") # tensorboard if arg.use_tb: if tf.gfile.Exists(arg.tb_data_dump): tf.gfile.DeleteRecursively(arg.tb_data_dump) tf.gfile.MakeDirs(arg.tb_data_dump) # current_job_mdl_folder = 'job_mdl_folder_%s/'%arg.job_name arg.path_to_hp = arg.get_path_root(arg)+current_job_mdl_folder arg.path_to_ckpt = arg.get_path_root_ckpts(arg)+current_job_mdl_folder arg.hp_folder_for_ckpt = 'hp_stid_%s/'%str(arg.slurm_array_task_id) ### get folder structure for experiment mtf.make_and_check_dir(path=arg.get_path_root(arg)+current_job_mdl_folder) mtf.make_and_check_dir(path=arg.get_path_root_ckpts(arg)+current_job_mdl_folder) # #errors_pretty = '/errors_file_%s_slurm_sj%s.txt'%(arg.date,arg.slurm_array_task_id) arg.json_hp_filename = 'json_hp_stid%s'%(arg.slurm_array_task_id) arg.csv_errors_filename = 'csv_errors_slurm_array_id%s'%(arg.slurm_array_task_id) ## # if arg.restore: TODO # arg = restore_hps(arg) # arg.float_type = tf.float32 ## get data set X_train, Y_train, X_cv, Y_cv, X_test, Y_test = mtf.get_data(arg,arg.N_frac) print( '(N_train,D) = (%d,%d) \n (N_test,D_out) = (%d,%d) ' % (arg.N_train,arg.D, arg.N_test,arg.D_out) ) ## if (preprocess_data, then preprocess) else (do nothing to the data) if arg.type_preprocess_data: X_train, Y_train, X_cv, Y_cv, X_test, Y_test = preprocess_data(arg, X_train, Y_train, X_cv, Y_cv, X_test, Y_test) #### build graph graph = tf.Graph() with graph.as_default(): ### get mdl x = tf.placeholder(arg.float_type, arg.get_x_shape(arg), name='x-input') y_ = tf.placeholder(arg.float_type, arg.get_y_shape(arg) ) phase_train = tf.placeholder(tf.bool, name='phase_train') # phase_train = tf.placeholder(tf.bool, name='phase_train') if arg.bn else None arg.phase_train = phase_train y = get_mdl(arg,x) ### get loss and accuracy loss, accuracy = get_accuracy_loss(arg,x,y,y_) ### get optimizer variables opt = get_optimizer(arg) train_step = opt.minimize(loss, global_step=arg.global_step) # step for optimizer (useful for ckpts) step, nb_iterations = tf.Variable(0, name='step'), tf.Variable(arg.nb_steps, name='nb_iterations') batch_size = tf.Variable(arg.batch_size, name='batch_size') # saves everything that was saved in the session saver = tf.train.Saver() #### run session arg.save_ckpt_freq = arg.get_save_ckpt_freq(arg) start_time = time.time() #pdb.set_trace() with tf.Session(graph=graph) as sess: with open(arg.path_to_hp+arg.csv_errors_filename,mode='a') as errors_csv_f: # a option: Opens a file for appending. The file pointer is at the end of the file if the file exists. That is, the file is in the append mode. If the file does not exist, it creates a new file for writing. #writer = csv.Writer(errors_csv_f) writer = csv.DictWriter(errors_csv_f,['train_error', 'cv_error', 'test_error','train_acc','cv_acc','test_acc']) # if (there is a restore ckpt mdl restore it) else (create a structure to save ckpt files) if arg.restore: arg.restore = False # after the model has been restored, we continue normal until all hp's are finished saver.restore(sess=sess, save_path=arg.save_path_to_ckpt2restore) # e.g. saver.restore(sess=sess, save_path='./tmp/my-model') arg = restore_hps(arg) print('arg ', arg) #print('arg.save_path_to_ckpt2restore: ',arg.save_path_to_ckpt2restore) print('restored model trained up to, STEP: ', step.eval()) print('restored model, ACCURACY:', sess.run(fetches=accuracy, feed_dict={x: X_train, y_: Y_train, phase_train: False})) else: # NOT Restore # not restored, so its a virgin run from scratch for this hp deleteContent(pfile=errors_csv_f) # since its a virgin run we writer.writeheader() # save_hps(arg) # save current hyper params if arg.save_checkpoints or arg.save_last_mdl: mtf.make_and_check_dir(path=arg.path_to_ckpt+arg.hp_folder_for_ckpt) # creates ./all_ckpts/exp_task_name/mdl_nn10/hp_stid_N sess.run(tf.global_variables_initializer()) start_iteration = step.eval() # last iteration trained is the first iteration for this model batch_size_eval = batch_size.eval() # Setup TensorBoard (or not) if arg.use_tb: merged = tf.summary.merge_all() train_writer, cv_writer, test_writer = tf.summary.FileWriter(arg.tb_data_dump+'/train',graph), tf.summary.FileWriter(arg.tb_data_dump+'/cv',graph), tf.summary.FileWriter(arg.tb_data_dump+'/test',graph) fetches_loss, fetches_acc = maps.NamedDict({'loss':loss, 'merged':merged}), maps.NamedDict({'acc':accuracy, 'merged':merged}) else: fetches_loss, fetches_acc = maps.NamedDict({'loss':loss}), maps.NamedDict({'acc':accuracy}) # train train_accs = [] train_errs = [] #Y_train_c, Y_cv_c, Y_test_c = mtf.radamacher_to_one_hot(Y_train), mtf.radamacher_to_one_hot(Y_cv), mtf.radamacher_to_one_hot(Y_test) for i in range(start_iteration,nb_iterations.eval()): batch_xs, batch_ys = get_batch_feed(X_train, Y_train, batch_size.eval()) sess.run(fetches=train_step, feed_dict={x: batch_xs, y_: batch_ys}) #pdb.set_trace() if i % arg.report_error_freq == 0: sess.run(step.assign(i)) # train evaluate #pdb.set_trace() fetched_loss_train = sess.run(fetches=fetches_loss, feed_dict={x: X_train, y_: Y_train}) # fltr = fetches_loss_train fetched_loss_train.acc = -1 #pdb.set_trace() if arg.classification: fetched_acc_train = sess.run(fetches=fetches_acc, feed_dict={x: X_train, y_: Y_train}) # fatr = fetches_acc_train # cv, test evaluate if arg.collect_generalization: fetched_loss_cv = sess.run(fetches=fetches_loss, feed_dict={x: X_cv, y_: Y_cv}) fetched_loss_test = sess.run(fetches=fetches_loss, feed_dict={x: X_test, y_: Y_test}) fetched_loss_cv.acc, fetched_loss_test= -1, -1 if arg.classification: cv_acc = sess.run(fetches=fetches_acc, feed_dict={x: X_cv, y_: Y_cv}) test_acc = sess.run(fetches=fetches_acc, feed_dict={x: X_test, y_: Y_test}) # set variables train_error, train_acc = fetched_loss_train.loss, fetched_acc_train.acc if arg.collect_generalization: cv_error, test_error = fetched_loss_cv.loss, fetched_loss_test.loss cv_acc, test_acc = fetched_acc_cv.acc, fetched_acc_test.acc # display training stuff if arg.display_training: #y_pred_val = sess.run(fetches=tf.nn.softmax(y), feed_dict={x: X_train, y_: Y_train}) #print('y_val: %s \n Y_train: %s '%(y_pred_val, Y_train)) print('step %d, train error: %s | train acc %s | batch_size(step.eval(),arg.batch_size): %s,%s log_learning_rate: %s | mdl %s '%(i,train_error,train_acc,batch_size_eval,arg.batch_size,arg.log_learning_rate,arg.mdl) ) # write files if not arg.collect_generalization: dict_errs = {'train_error':train_error,'train_acc':train_acc} else: dict_errs = {'train_error':train_error,'cv_error':cv_error,'test_error':test_error,'train_acc':train_acc,'cv_error':cv_error,'test_error':test_error} writer.writerow(dict_errs) # append quick checks train_accs.append(train_acc) train_errs.append(train_error) # write tensorboard stats if arg.use_tb: train_writer.add_summary(fetched_loss_train.merged, i), train_writer.add_summary(fetched_acc_train.merged, i) if arg.collect_generalization: cv_writer.add_summary(fetched_loss_cv.merged, i), cv_writer.add_summary(fetched_acc_cv.merged, i) test_writer.add_summary(fetched_loss_test.merged, i), test_writer.add_summary(fetched_acc_test.merged, i) # save checkpoint if arg.save_checkpoints: if i % arg.save_ckpt_freq == 0: #print('>>>>>>>>>>CKPT',i,arg.save_ckpt_freq) saver.save(sess=sess,save_path=arg.path_to_ckpt+arg.hp_folder_for_ckpt+arg.prefix_ckpt) # save last model if arg.save_last_mdl: saver.save(sess=sess,save_path=arg.path_to_ckpt+arg.hp_folder_for_ckpt+arg.prefix_ckpt) # evaluate #print('Final Test Acc/error: ', sess.run(fetches=accuracy, feed_dict={x: X_test, y_: Y_test})) print('Final -> | train error %s | train: %s'% (min(train_errs), max(train_accs)) ) seconds = (time.time() - start_time) minutes = seconds/ 60 hours = minutes/ 60 print("--- %s seconds ---" % seconds ) print("--- %s minutes ---" % minutes ) print("--- %s hours ---" % hours )