def train_hopt(max_evals, csv_file_path):
# Create/reinit csv file
open(csv_file_path, 'w').close()
logger_hopt.info((u'WITH HYPERPARAMETER OPTIMIZATION').encode('utf8'))
logger_hopt.info((u'**** Model : ' + Model_class.name()).encode('utf8'))
logger_hopt.info((u'**** Optimization technic : ' + Optimization_method.name()).encode('utf8'))
logger_hopt.info((u'**** Temporal granularity : ' + temporal_granularity).encode('utf8'))
if binary_unit:
logger_hopt.info((u'**** Binary unit (intensity discarded)').encode('utf8'))
else:
logger_hopt.info((u'**** Real valued unit (intensity taken into consideration)').encode('utf8'))
logger_hopt.info((u'**** Quantization : ' + str(quantization)).encode('utf8'))
# Define hyper-parameter search space for the model
# Those are given by the static methods get_param_dico and get_hp_space
model_space = Model_class.get_hp_space()
optim_space = Optimization_method.get_hp_space()
space = {'model': model_space, 'optim': optim_space}
# Get the headers (i.e. list of hyperparameters tuned for printing and
# save purposes)
header = model_space.keys() + optim_space.keys() + ['accuracy']
global run_counter
run_counter = 0
def run_wrapper(params):
global run_counter
run_counter += 1
logger_hopt.info(('\n').encode('utf8'))
logger_hopt.info((u'#'*40).encode('utf8'))
logger_hopt.info((u'# Config : {}'.format(run_counter)).encode('utf8'))
# Build model and optim dico ################
model_param = params['model']
optim_param = params['optim']
#############################################
# Weights plotted and stored in a folder ####
# Same for generated midi sequences #########
weights_folder = result_folder + '/' + str(run_counter) + '/' + 'weights'
if not os.path.isdir(weights_folder):
os.makedirs(weights_folder)
generated_folder = result_folder + '/' + str(run_counter) + '/generated_sequences'
if not os.path.isdir(generated_folder):
os.makedirs(generated_folder)
model_folder = result_folder + '/' + str(run_counter) + '/model'
if not os.path.isdir(model_folder):
os.makedirs(model_folder)
#############################################
# Load data #################################
time_load_0 = time.time()
piano_train, orchestra_train, train_index, \
piano_valid, orchestra_valid, valid_index, \
piano_test, orchestra_test, test_index, generation_index \
= load_data(LOCAL_SCRATCH + '/Data',
model_param['temporal_order'],
model_param['batch_size'],
binary_unit=binary_unit,
skip_sample=1,
logger_load=logger_load)
time_load_1 = time.time()
logger_load.info('TTT : Loading data took {} seconds'.format(time_load_1-time_load_0))
##############
# visualize_mat(piano_train.get_value(), 'DEBUG', 'piano_train')
# visualize_mat(orchestra_train.get_value(), 'DEBUG', 'orchestra_train')
# visualize_mat(piano_test.get_value(), 'DEBUG', 'piano_test')
# visualize_mat(orchestra_test.get_value(), 'DEBUG', 'orchestra_test')
# visualize_mat(piano_valid.get_value(), 'DEBUG', 'piano_valid')
# visualize_mat(orchestra_valid.get_value(), 'DEBUG', 'orchestra_valid')
##############
# For large datasets
# http://deeplearning.net/software/theano/tutorial/aliasing.html
# use borrow=True (avoid copying the whole matrix) ?
# Load as much as the GPU can handle, train then load other
# part of the dataset using shared_variable.set_value(new_value)
#############################################
# Train #####################################
time_train_0 = time.time()
model, dico_res = train(piano_train, orchestra_train, train_index,
piano_valid, orchestra_valid, valid_index,
model_param, optim_param, max_iter, weights_folder)
time_train_1 = time.time()
logger_train.info('TTT : Training data took {} seconds'.format(time_train_1-time_train_0))
error = -dico_res['accuracy'] # Search for a min
#############################################
# Generate ##################################
time_generate_0 = time.time()
generate(model,
piano_test, orchestra_test, generation_index,
generation_length, seed_size, quantization_write,
generated_folder, logger_generate)
time_generate_1 = time.time()
logger_generate.info('TTT : Generating data took {} seconds'.format(time_generate_1-time_generate_0))
#############################################
# Save ######################################
save_model_file = open(model_folder + '/model.pkl', 'wb')
pickle.dump(model, save_model_file, protocol=pickle.HIGHEST_PROTOCOL)
#############################################
# log
logger_hopt.info((u'# Accuracy : {}'.format(dico_res['accuracy'])).encode('utf8'))
logger_hopt.info((u'###################\n').encode('utf8'))
# Write the result in result.csv
with open(csv_file_path, 'ab') as csvfile:
writer = csv.DictWriter(csvfile, delimiter=',', fieldnames=header)
writer.writerow(dico_res)
return error
# Calling get_param_dico with None return an empty dictionary,
# Useful to get the header of hparam
with open(csv_file_path, 'ab') as csvfile:
# Write headers if they don't already exist
writerHead = csv.writer(csvfile, delimiter=',')
writerHead.writerow(header)
best = fmin(run_wrapper, space, algo=tpe.suggest, max_evals=max_evals)
return best
elif sys.argv[2] == 'adam_L2':
from acidano.utils.optim import adam_L2 as Optimization_method
elif sys.argv[2] == 'rmsprop':
from acidano.utils.optim import rmsprop as Optimization_method
elif sys.argv[2] == 'sgd_nesterov':
from acidano.utils.optim import sgd_nesterov as Optimization_method
else:
raise ValueError(sys.argv[2] + " is not an optimization method")
############################################################
# System paths
############################################################
logging.info('System paths')
SOURCE_DIR = os.getcwd()
result_folder = SOURCE_DIR + u'/../Results/' + unit_type + '/' + script_param['temporal_granularity'] + '/' +\
'quantization_' + str(script_param['quantization']) + '/' + Optimization_method.name() + '/' + Model_class.name()
# Check if the result folder exists
if not os.path.isdir(result_folder):
os.makedirs(result_folder)
# Count number of files in the folder and name this hparam run by the number
hparam_run_counter = 0
while(os.path.isdir(result_folder + '/hrun_' + str(hparam_run_counter))):
hparam_run_counter += 1
result_folder = result_folder + '/hrun_' + str(hparam_run_counter)
os.mkdir(result_folder)
script_param['result_folder'] = result_folder
# Data : .pkl files
data_folder = SOURCE_DIR + '/../Data'
else:
raise ValueError(sys.argv[2] + " is not an optimization method")
# System paths
MAIN_DIR = os.getcwd().decode('utf8') + u'/'
if 'LSCRATCH' in os.environ.keys():
# We are on Guillimin
LOCAL_SCRATCH = os.environ['LSCRATCH']
else:
LOCAL_SCRATCH = '..'
data_folder = LOCAL_SCRATCH + u'/Data'
if ONLY_BUILD_DB:
result_folder = MAIN_DIR + u'../Results/' + 'Only_build_db'
else:
result_folder = MAIN_DIR + u'../Results/' + temporal_granularity + '/' + Optimization_method.name() + '/' + Model_class.name()
result_file = result_folder + u'/hopt_results.csv'
if ONLY_BUILD_DB:
log_file_path = result_folder + '/build_db.log'
else:
log_file_path = result_folder + '/' + Model_class.name() + u'.log'
# Fixed hyper parameter
max_evals = 50 # number of hyper-parameter configurations evaluated
max_iter = 100 # nb max of iterations when training 1 configuration of hparams
# Config is set now, no need to modify source below for standard use
# Validation
validation_order = 5
initial_derivative_length = 20
check_derivative_length = 5