def train_hopt(max_evals, csv_file_path): # Create/reinit csv file open(csv_file_path, 'w').close() logger_hopt.info((u'WITH HYPERPARAMETER OPTIMIZATION').encode('utf8')) logger_hopt.info((u'**** Model : ' + Model_class.name()).encode('utf8')) logger_hopt.info((u'**** Optimization technic : ' + Optimization_method.name()).encode('utf8')) logger_hopt.info((u'**** Temporal granularity : ' + temporal_granularity).encode('utf8')) if binary_unit: logger_hopt.info((u'**** Binary unit (intensity discarded)').encode('utf8')) else: logger_hopt.info((u'**** Real valued unit (intensity taken into consideration)').encode('utf8')) logger_hopt.info((u'**** Quantization : ' + str(quantization)).encode('utf8')) # Define hyper-parameter search space for the model # Those are given by the static methods get_param_dico and get_hp_space model_space = Model_class.get_hp_space() optim_space = Optimization_method.get_hp_space() space = {'model': model_space, 'optim': optim_space} # Get the headers (i.e. list of hyperparameters tuned for printing and # save purposes) header = model_space.keys() + optim_space.keys() + ['accuracy'] global run_counter run_counter = 0 def run_wrapper(params): global run_counter run_counter += 1 logger_hopt.info(('\n').encode('utf8')) logger_hopt.info((u'#'*40).encode('utf8')) logger_hopt.info((u'# Config : {}'.format(run_counter)).encode('utf8')) # Build model and optim dico ################ model_param = params['model'] optim_param = params['optim'] ############################################# # Weights plotted and stored in a folder #### # Same for generated midi sequences ######### weights_folder = result_folder + '/' + str(run_counter) + '/' + 'weights' if not os.path.isdir(weights_folder): os.makedirs(weights_folder) generated_folder = result_folder + '/' + str(run_counter) + '/generated_sequences' if not os.path.isdir(generated_folder): os.makedirs(generated_folder) model_folder = result_folder + '/' + str(run_counter) + '/model' if not os.path.isdir(model_folder): os.makedirs(model_folder) ############################################# # Load data ################################# time_load_0 = time.time() piano_train, orchestra_train, train_index, \ piano_valid, orchestra_valid, valid_index, \ piano_test, orchestra_test, test_index, generation_index \ = load_data(LOCAL_SCRATCH + '/Data', model_param['temporal_order'], model_param['batch_size'], binary_unit=binary_unit, skip_sample=1, logger_load=logger_load) time_load_1 = time.time() logger_load.info('TTT : Loading data took {} seconds'.format(time_load_1-time_load_0)) ############## # visualize_mat(piano_train.get_value(), 'DEBUG', 'piano_train') # visualize_mat(orchestra_train.get_value(), 'DEBUG', 'orchestra_train') # visualize_mat(piano_test.get_value(), 'DEBUG', 'piano_test') # visualize_mat(orchestra_test.get_value(), 'DEBUG', 'orchestra_test') # visualize_mat(piano_valid.get_value(), 'DEBUG', 'piano_valid') # visualize_mat(orchestra_valid.get_value(), 'DEBUG', 'orchestra_valid') ############## # For large datasets # http://deeplearning.net/software/theano/tutorial/aliasing.html # use borrow=True (avoid copying the whole matrix) ? # Load as much as the GPU can handle, train then load other # part of the dataset using shared_variable.set_value(new_value) ############################################# # Train ##################################### time_train_0 = time.time() model, dico_res = train(piano_train, orchestra_train, train_index, piano_valid, orchestra_valid, valid_index, model_param, optim_param, max_iter, weights_folder) time_train_1 = time.time() logger_train.info('TTT : Training data took {} seconds'.format(time_train_1-time_train_0)) error = -dico_res['accuracy'] # Search for a min ############################################# # Generate ################################## time_generate_0 = time.time() generate(model, piano_test, orchestra_test, generation_index, generation_length, seed_size, quantization_write, generated_folder, logger_generate) time_generate_1 = time.time() logger_generate.info('TTT : Generating data took {} seconds'.format(time_generate_1-time_generate_0)) ############################################# # Save ###################################### save_model_file = open(model_folder + '/model.pkl', 'wb') pickle.dump(model, save_model_file, protocol=pickle.HIGHEST_PROTOCOL) ############################################# # log logger_hopt.info((u'# Accuracy : {}'.format(dico_res['accuracy'])).encode('utf8')) logger_hopt.info((u'###################\n').encode('utf8')) # Write the result in result.csv with open(csv_file_path, 'ab') as csvfile: writer = csv.DictWriter(csvfile, delimiter=',', fieldnames=header) writer.writerow(dico_res) return error # Calling get_param_dico with None return an empty dictionary, # Useful to get the header of hparam with open(csv_file_path, 'ab') as csvfile: # Write headers if they don't already exist writerHead = csv.writer(csvfile, delimiter=',') writerHead.writerow(header) best = fmin(run_wrapper, space, algo=tpe.suggest, max_evals=max_evals) return best
elif sys.argv[2] == 'adam_L2': from acidano.utils.optim import adam_L2 as Optimization_method elif sys.argv[2] == 'rmsprop': from acidano.utils.optim import rmsprop as Optimization_method elif sys.argv[2] == 'sgd_nesterov': from acidano.utils.optim import sgd_nesterov as Optimization_method else: raise ValueError(sys.argv[2] + " is not an optimization method") ############################################################ # System paths ############################################################ logging.info('System paths') SOURCE_DIR = os.getcwd() result_folder = SOURCE_DIR + u'/../Results/' + unit_type + '/' + script_param['temporal_granularity'] + '/' +\ 'quantization_' + str(script_param['quantization']) + '/' + Optimization_method.name() + '/' + Model_class.name() # Check if the result folder exists if not os.path.isdir(result_folder): os.makedirs(result_folder) # Count number of files in the folder and name this hparam run by the number hparam_run_counter = 0 while(os.path.isdir(result_folder + '/hrun_' + str(hparam_run_counter))): hparam_run_counter += 1 result_folder = result_folder + '/hrun_' + str(hparam_run_counter) os.mkdir(result_folder) script_param['result_folder'] = result_folder # Data : .pkl files data_folder = SOURCE_DIR + '/../Data'
else: raise ValueError(sys.argv[2] + " is not an optimization method") # System paths MAIN_DIR = os.getcwd().decode('utf8') + u'/' if 'LSCRATCH' in os.environ.keys(): # We are on Guillimin LOCAL_SCRATCH = os.environ['LSCRATCH'] else: LOCAL_SCRATCH = '..' data_folder = LOCAL_SCRATCH + u'/Data' if ONLY_BUILD_DB: result_folder = MAIN_DIR + u'../Results/' + 'Only_build_db' else: result_folder = MAIN_DIR + u'../Results/' + temporal_granularity + '/' + Optimization_method.name() + '/' + Model_class.name() result_file = result_folder + u'/hopt_results.csv' if ONLY_BUILD_DB: log_file_path = result_folder + '/build_db.log' else: log_file_path = result_folder + '/' + Model_class.name() + u'.log' # Fixed hyper parameter max_evals = 50 # number of hyper-parameter configurations evaluated max_iter = 100 # nb max of iterations when training 1 configuration of hparams # Config is set now, no need to modify source below for standard use # Validation validation_order = 5 initial_derivative_length = 20 check_derivative_length = 5