def train(model, train_splits_batches, valid_splits_batches,
test_splits_batches, normalizer, model_params, parameters,
config_folder, start_time_train, logger_train):
DEBUG = parameters["debug"]
SUMMARIZE = DEBUG["summarize"]
# Build DEBUG dict
if DEBUG["save_measures"]:
DEBUG["save_measures"] = config_folder + "/save_measures"
# Time information used
time_limit = parameters[
'walltime'] * 3600 - 30 * 60 # walltime - 30 minutes in seconds
# Reset graph before starting training
tf.reset_default_graph()
###### PETIT TEST VALIDATION
# Use same validation and train set
# piano_valid, orch_valid, valid_index = piano_train, orch_train, train_index
which_trainer = model.trainer()
# Save it for generation. SO UGLY
with open(os.path.join(config_folder, 'which_trainer'), 'w') as ff:
ff.write(which_trainer)
trainer = import_trainer(which_trainer, model_params, parameters)
# Flag to know if the model has to be trained or not
model_optimize = model.optimize()
############################################################
# Display informations about the models
num_parameters = model_statistics.count_parameters(tf.get_default_graph())
logger_train.info(
'** Num trainable parameters : {}'.format(num_parameters))
with open(os.path.join(config_folder, 'num_parameters.txt'), 'w') as ff:
ff.write("{:d}".format(num_parameters))
############################################################
# Training
logger_train.info("#" * 60)
logger_train.info("#### Training")
epoch = 0
OVERFITTING = False
TIME_LIMIT = False
# Train error
loss_tab = np.zeros(max(1, parameters['max_iter']))
# Select criteria
overfitting_measure = parameters["overfitting_measure"]
save_measures = parameters['save_measures']
# Short-term validation error
valid_tabs = {
'loss': np.zeros(max(1, parameters['max_iter'])),
'accuracy': np.zeros(max(1, parameters['max_iter'])),
'precision': np.zeros(max(1, parameters['max_iter'])),
'recall': np.zeros(max(1, parameters['max_iter'])),
'true_accuracy': np.zeros(max(1, parameters['max_iter'])),
'f_score': np.zeros(max(1, parameters['max_iter'])),
'Xent': np.zeros(max(1, parameters['max_iter']))
}
# Best epoch for each measure
best_epoch = {
'loss': 0,
'accuracy': 0,
'precision': 0,
'recall': 0,
'true_accuracy': 0,
'f_score': 0,
'Xent': 0
}
# Sampled preds measures
valid_tabs_sampled = {
'loss': np.zeros(max(1, parameters['max_iter'])),
'accuracy': np.zeros(max(1, parameters['max_iter'])),
'precision': np.zeros(max(1, parameters['max_iter'])),
'recall': np.zeros(max(1, parameters['max_iter'])),
'true_accuracy': np.zeros(max(1, parameters['max_iter'])),
'f_score': np.zeros(max(1, parameters['max_iter'])),
'Xent': np.zeros(max(1, parameters['max_iter']))
}
# Long-term validation error
valid_tabs_LR = {
'loss': np.zeros(max(1, parameters['max_iter'])),
'accuracy': np.zeros(max(1, parameters['max_iter'])),
'precision': np.zeros(max(1, parameters['max_iter'])),
'recall': np.zeros(max(1, parameters['max_iter'])),
'true_accuracy': np.zeros(max(1, parameters['max_iter'])),
'f_score': np.zeros(max(1, parameters['max_iter'])),
'Xent': np.zeros(max(1, parameters['max_iter']))
}
# Best epoch for each measure
best_epoch_LR = {
'loss': 0,
'accuracy': 0,
'precision': 0,
'recall': 0,
'true_accuracy': 0,
'f_score': 0,
'Xent': 0
}
### Timing file
# open('timing', 'w').close()
if parameters['memory_gpu']:
# This apparently does not work
configSession = tf.ConfigProto()
configSession.gpu_options.per_process_gpu_memory_fraction = parameters[
'memory_gpu']
else:
configSession = None
with tf.Session(config=configSession) as sess:
# Only for models with shared weights
model.init_weights()
##############################
# Create PH and nodes
if parameters['pretrained_model'] is None:
logger_train.info((u'#### Graph'))
start_time_building_graph = time.time()
trainer.build_variables_nodes(model, parameters)
trainer.build_preds_nodes(model)
trainer.build_loss_nodes(model, parameters)
trainer.build_train_step_node(model, config.optimizer())
trainer.save_nodes(model)
time_building_graph = time.time() - start_time_building_graph
logger_train.info("TTT : Building the graph took {0:.2f}s".format(
time_building_graph))
else:
logger_train.info((u'#### Graph'))
start_time_building_graph = time.time()
trainer.load_pretrained_model(parameters['pretrained_model'])
time_building_graph = time.time() - start_time_building_graph
logger_train.info(
"TTT : Loading pretrained model took {0:.2f}s".format(
time_building_graph))
if SUMMARIZE:
tf.summary.scalar('loss', trainer.loss)
##############################
summarize_dict = {}
if SUMMARIZE:
merged_node = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(config_folder + '/summary',
sess.graph)
train_writer.add_graph(tf.get_default_graph())
else:
merged_node = None
summarize_dict['bool'] = SUMMARIZE
summarize_dict['merged_node'] = merged_node
if model.is_keras():
K.set_session(sess)
# Initialize weights
if parameters['pretrained_model']:
trainer.saver.restore(sess,
parameters['pretrained_model'] + '/model')
else:
sess.run(tf.global_variables_initializer())
# if DEBUG:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
N_matrix_files = len(train_splits_batches)
#######################################
# Load first matrix
#######################################
load_data_start = time.time()
pool = ThreadPool(processes=1)
async_train = pool.apply_async(
async_load_mat,
(normalizer, train_splits_batches[0]['chunks_folders'], parameters,
trainer.np_type))
matrices_from_thread = async_train.get()
load_data_time = time.time() - load_data_start
logger_train.info("Load the first matrix time : " +
str(load_data_time))
# For dumb baseline models like random or repeat which don't need training step optimization
if model_optimize == False:
# WARNING : first validation matrix is not necessarily the same as the first train matrix
async_test = pool.apply_async(
async_load_mat,
(normalizer, test_splits_batches[0]['chunks_folders'],
parameters, trainer.np_type))
init_matrices_test = async_test.get()
test_results, test_results_sampled, test_long_range_results, _, _ = validate(
trainer, sess, init_matrices_test, test_splits_batches,
normalizer, parameters, logger_train)
training_utils.mean_and_store_results(test_results, valid_tabs, 0)
training_utils.mean_and_store_results(test_results_sampled,
valid_tabs_sampled, 0)
training_utils.mean_and_store_results(test_long_range_results,
valid_tabs_LR, 0)
test_tab = {}
test_tab_sampled = {}
test_tab_LR = {}
training_utils.mean_and_store_results(test_results, test_tab, None)
training_utils.mean_and_store_results(test_results_sampled,
test_tab_sampled, None)
training_utils.mean_and_store_results(test_long_range_results,
test_tab_LR, None)
accuracy_training = np.zeros((100))
return accuracy_training, training_utils.remove_tail_training_curves(valid_tabs, 1), test_tab, best_epoch, \
training_utils.remove_tail_training_curves(valid_tabs_sampled, 1), test_tab_sampled, best_epoch, \
training_utils.remove_tail_training_curves(valid_tabs_LR, 1), test_tab_LR, best_epoch_LR
accuracy_training = []
# Training iteration
while (not OVERFITTING and not TIME_LIMIT
and epoch != parameters['max_iter']):
start_time_epoch = time.time()
trainer.DEBUG['epoch'] = epoch
train_cost_epoch = []
sparse_loss_epoch = []
train_time = time.time()
this_batch_accuracy = []
for file_ind_CURRENT in range(N_matrix_files):
#######################################
# Get indices and matrices to load
#######################################
# We train on the current matrix
train_index = train_splits_batches[file_ind_CURRENT]['batches']
# But load the one next one
file_ind_NEXT = (file_ind_CURRENT + 1) % N_matrix_files
next_chunks = train_splits_batches[file_ind_NEXT][
'chunks_folders']
#######################################
# Load matrix thread
#######################################
async_train = pool.apply_async(
async_load_mat,
(normalizer, next_chunks, parameters, trainer.np_type))
piano_input, orch_transformed, duration_piano, mask_orch = matrices_from_thread
#######################################
# Train
#######################################
for batch_index in train_index:
loss_batch, preds_batch, debug_outputs, summary = trainer.training_step(
sess, batch_index, piano_input, orch_transformed,
duration_piano, mask_orch, summarize_dict)
orch_t = orch_transformed[batch_index]
accuracy_batch = measure.accuracy_measure(
orch_t, preds_batch)
this_batch_accuracy.extend(accuracy_batch)
# Keep track of cost
train_cost_epoch.append(loss_batch)
sparse_loss_batch = debug_outputs["sparse_loss_batch"]
sparse_loss_epoch.append(sparse_loss_batch)
#######################################
# New matrices from thread
#######################################
del (matrices_from_thread)
matrices_from_thread = async_train.get()
this_batch_accuracy_mean = -100 * np.mean(this_batch_accuracy)
accuracy_training.append(this_batch_accuracy_mean)
train_time = time.time() - train_time
logger_train.info("Training time : {}".format(train_time))
###
# DEBUG
if trainer.DEBUG["plot_weights"]:
# weight_folder=config_folder+"/weights/"+str(epoch)
weight_folder = config_folder + "/weights"
plot_weights.plot_weights(sess, weight_folder)
#
###
# WARNING : first validation matrix is not necessarily the same as the first train matrix
# So now that it's here, parallelization is absolutely useless....
async_valid = pool.apply_async(
async_load_mat,
(normalizer, valid_splits_batches[0]['chunks_folders'],
parameters, trainer.np_type))
if SUMMARIZE:
if (epoch < 5) or (epoch % 10 == 0):
# Note that summarize here only look at the variables after the last batch of the epoch
# If you want to look at all the batches, include it in
train_writer.add_summary(summary, epoch)
mean_loss = np.mean(train_cost_epoch)
loss_tab[epoch] = mean_loss
#######################################
# Validate
#######################################
valid_time = time.time()
init_matrices_validation = async_valid.get()
# Create DEBUG folders
if trainer.DEBUG["plot_nade_ordering_preds"]:
AAA = config_folder + "/DEBUG/preds_nade/" + str(epoch)
trainer.DEBUG["plot_nade_ordering_preds"] = AAA
os.makedirs(AAA)
if trainer.DEBUG["save_accuracy_along_sampling"]:
AAA = config_folder + "/DEBUG/accuracy_along_sampling/" + str(
epoch)
trainer.DEBUG["save_accuracy_along_sampling"] = AAA
os.makedirs(AAA)
if trainer.DEBUG["salience_embedding"]:
AAA = config_folder + "/DEBUG/salience_embeddings/" + str(
epoch)
trainer.DEBUG["salience_embedding"] = AAA
os.makedirs(AAA)
valid_results, valid_results_sampled, valid_long_range_results, preds_val, truth_val = \
validate(trainer, sess,
init_matrices_validation, valid_splits_batches,
normalizer, parameters,
logger_train)
valid_time = time.time() - valid_time
logger_train.info("Validation time : {}".format(valid_time))
training_utils.mean_and_store_results(valid_results, valid_tabs,
epoch)
training_utils.mean_and_store_results(valid_results_sampled,
valid_tabs_sampled, epoch)
training_utils.mean_and_store_results(valid_long_range_results,
valid_tabs_LR, epoch)
end_time_epoch = time.time()
#######################################
# Overfitting ?
if epoch >= parameters['min_number_iteration']:
# Choose short/long range and the measure
OVERFITTING = early_stopping.up_criterion(
valid_tabs[overfitting_measure], epoch,
parameters["number_strips"],
parameters["validation_order"])
if not OVERFITTING:
# Also check for NaN
OVERFITTING = early_stopping.check_for_nan(valid_tabs,
save_measures,
max_nan=3)
#######################################
#######################################
# Monitor time (guillimin walltime)
if (time.time() - start_time_train) > time_limit:
TIME_LIMIT = True
#######################################
#######################################
# Log training
#######################################
logger_train.info(
"############################################################")
logger_train.info(
'Epoch : {} , Training loss : {} , Validation loss : {} \n \
Validation accuracy : {:.3f} ; {:.3f} \n \
Precision : {:.3f} ; {:.3f} \n \
Recall : {:.3f} ; {:.3f} \n \
Xent : {:.6f} ; {:.3f} \n \
Sparse_loss : {:.3f}'.format(epoch, mean_loss, valid_tabs['loss'][epoch],
valid_tabs['accuracy'][epoch],
valid_tabs_sampled['accuracy'][epoch],
valid_tabs['precision'][epoch],
valid_tabs_sampled['precision'][epoch],
valid_tabs['recall'][epoch],
valid_tabs_sampled['recall'][epoch],
valid_tabs['Xent'][epoch],
valid_tabs_sampled['Xent'][epoch],
np.mean(sparse_loss_epoch)))
logger_train.info('Time : {}'.format(end_time_epoch -
start_time_epoch))
#######################################
# Best model ?
# Xent criterion
start_time_saving = time.time()
for measure_name, measure_curve in valid_tabs.items():
best_measure_so_far = measure_curve[best_epoch[measure_name]]
measure_for_this_epoch = measure_curve[epoch]
if (measure_for_this_epoch <= best_measure_so_far) or (epoch
== 0):
if measure_name in save_measures:
trainer.saver.save(
sess, config_folder + "/model_" + measure_name +
"/model")
best_epoch[measure_name] = epoch
#######################################
# DEBUG
# Save numpy arrays of measures values
if trainer.DEBUG["save_measures"]:
if os.path.isdir(trainer.DEBUG["save_measures"]):
shutil.rmtree(trainer.DEBUG["save_measures"])
os.makedirs(trainer.DEBUG["save_measures"])
for measure_name, measure_tab in valid_results.items():
np.save(
os.path.join(trainer.DEBUG["save_measures"],
measure_name + '.npy'),
measure_tab[:2000])
np.save(
os.path.join(trainer.DEBUG["save_measures"],
'preds.npy'),
np.asarray(preds_val[:2000]))
np.save(
os.path.join(trainer.DEBUG["save_measures"],
'truth.npy'),
np.asarray(truth_val[:2000]))
#######################################
end_time_saving = time.time()
logger_train.info('Saving time : {:.3f}'.format(end_time_saving -
start_time_saving))
#######################################
if OVERFITTING:
logger_train.info('OVERFITTING !!')
if TIME_LIMIT:
logger_train.info('TIME OUT !!')
#######################################
# Epoch +1
#######################################
epoch += 1
#######################################
# Test
#######################################
test_time = time.time()
async_test = pool.apply_async(
async_load_mat,
(normalizer, test_splits_batches[0]['chunks_folders'], parameters,
trainer.np_type))
init_matrices_test = async_test.get()
test_results, test_results_sampled, test_long_range_results, preds_test, truth_test = \
validate(trainer, sess,
init_matrices_test, test_splits_batches,
normalizer, parameters,
logger_train)
test_time = time.time() - test_time
logger_train.info("Test time : {}".format(test_time))
test_tab = {}
test_tab_sampled = {}
test_tab_LR = {}
training_utils.mean_and_store_results(test_results, test_tab, None)
training_utils.mean_and_store_results(test_results_sampled,
test_tab_sampled, None)
training_utils.mean_and_store_results(test_long_range_results,
test_tab_LR, None)
logger_train.info(
"############################################################")
logger_train.info("""## Test Scores
Loss : {}
Validation accuracy : {:.3f} %, precision : {:.3f} %, recall : {:.3f} %
True_accuracy : {:.3f} %, f_score : {:.3f} %, Xent : {:.6f}""".format(
test_tab['loss'], test_tab['accuracy'], test_tab['precision'],
test_tab['recall'], test_tab['true_accuracy'], test_tab['f_score'],
test_tab['Xent']))
logger_train.info('Time : {}'.format(test_time))
#######################################
# Close workers' pool
#######################################
pool.close()
pool.join()
return accuracy_training, training_utils.remove_tail_training_curves(valid_tabs, epoch), test_tab, best_epoch, \
training_utils.remove_tail_training_curves(valid_tabs_sampled, epoch), test_tab_sampled, best_epoch,\
training_utils.remove_tail_training_curves(valid_tabs_LR, epoch), test_tab_LR, best_epoch_LR
# bias=[v.eval() for v in tf.global_variables() if v.name == "top_layer_prediction/orch_pred/bias:0"][0]
# kernel=[v.eval() for v in tf.global_variables() if v.name == "top_layer_prediction/orch_pred/kernel:0"][0]
def orderless_NADE_generation(self, sess, feed_dict, orch_t):
# Pre-compute the context embedding
# which will be the same for all the orderings
context_embedding = sess.run(self.context_embedding_out, feed_dict)
# Generate the orderings in parallel -> duplicate the embedding and orch_t matrices along batch dim
batch_size, orch_dim = orch_t.shape
# Start with an orchestra prediction and mask equal to zero
orch_t_reshape = np.concatenate(
[orch_t for _ in range(self.num_ordering)], axis=0)
orch_pred = np.zeros_like(orch_t_reshape)
mask = np.zeros_like(orch_t_reshape)
context_embedding_reshape = np.concatenate(
[context_embedding for _ in range(self.num_ordering)], axis=0)
# Three nodes to feed now: orch_pred, context_embedding, and mask
feed_dict_known_context = {}
feed_dict_known_context[self.orch_t_ph] = orch_t_reshape
feed_dict_known_context[
self.context_embedding_in] = context_embedding_reshape
# Build the orderings (use the same ordering for all elems in batch)
orderings = []
for ordering_ind in range(self.num_ordering):
# This ordering
ordering = list(range(orch_dim))
random.shuffle(ordering)
orderings.append(ordering)
if self.DEBUG["save_accuracy_along_sampling"]:
accuracy_along_sampling = []
# Loop over the length of the orderings
for d in range(orch_dim):
# Generate step
feed_dict_known_context[self.orch_pred] = orch_pred
feed_dict_known_context[self.mask_input] = mask
loss_batch, preds_batch = sess.run([self.loss_val, self.preds_gen],
feed_dict_known_context)
##############################
##############################
# DEBUG
# Observe the evolution of the accuracy along the sampling process
if self.DEBUG["save_accuracy_along_sampling"]:
accuracy_batch = np.mean(
accuracy_measure(orch_t_reshape, preds_batch))
accuracy_along_sampling.append(accuracy_batch)
# Plot the predictions
if self.DEBUG["plot_nade_ordering_preds"] and (
self.DEBUG["batch_counter"]
== (self.DEBUG["num_batch"] - 1)):
for ordering_ind in range(self.num_ordering):
batch_begin = batch_size * ordering_ind
batch_end = batch_size * (ordering_ind + 1)
np.save(
self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) +
'_' + str(ordering_ind) + '.npy',
preds_batch[batch_begin:batch_end, :])
mean_pred_batch = self.mean_parallel_prediction(
batch_size, preds_batch)
np.save(
self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) +
'_mean.npy', mean_pred_batch)
##############################
##############################
# Update matrices
for ordering_ind in range(self.num_ordering):
batch_begin = batch_size * ordering_ind
batch_end = batch_size * (ordering_ind + 1)
mask[batch_begin:batch_end, orderings[ordering_ind][d]] = 1
##################################################
# Mean-field or sampling ? Sampling because need binary values to move along the Gibbs/NADE process
orch_pred[batch_begin:batch_end,
orderings[ordering_ind][d]] = np.random.binomial(
1, preds_batch[batch_begin:batch_end,
orderings[ordering_ind][d]])
##################################################
# Now continue Gibbs sampling
while (d < orch_dim * self.gibbs_sampling_factor):
pitch_resampled = random.randint(0, orch_dim - 1)
d += 1
##############################
# Randomly set one value in the mask to zero
mask[:, pitch_resampled] = 0
##############################
##############################
feed_dict_known_context[self.orch_pred] = orch_pred
feed_dict_known_context[self.mask_input] = mask
##############################
##############################
loss_batch, preds_batch = sess.run([self.loss_val, self.preds_gen],
feed_dict_known_context)
##############################
##############################
##############################
# DEBUG
# Observe the evolution of the accuracy along the sampling process
if self.DEBUG["save_accuracy_along_sampling"]:
accuracy_batch = np.mean(
accuracy_measure(orch_t_reshape, preds_batch))
accuracy_along_sampling.append(accuracy_batch)
# Plot the predictions
if self.DEBUG["plot_nade_ordering_preds"] and (
self.DEBUG["batch_counter"]
== (self.DEBUG["num_batch"] - 1)):
for ordering_ind in range(self.num_ordering):
batch_begin = batch_size * ordering_ind
batch_end = batch_size * (ordering_ind + 1)
np.save(
self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) +
'_' + str(ordering_ind) + '.npy',
preds_batch[batch_begin:batch_end, :])
mean_pred_batch = self.mean_parallel_prediction(
batch_size, preds_batch)
np.save(
self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) +
'_mean.npy', mean_pred_batch)
##############################
##############################
##############################
# Write back the mask to 1
mask[:, pitch_resampled] = 1
# Resample
orch_pred[:, pitch_resampled] = np.random.binomial(
1, preds_batch[:, pitch_resampled])
##############################
if self.DEBUG["plot_nade_ordering_preds"] and (
self.DEBUG["batch_counter"] == (self.DEBUG["num_batch"] - 1)):
np.save(self.DEBUG["plot_nade_ordering_preds"] + '/truth.npy',
orch_t)
# Save accuracy_along_sampling
if self.DEBUG["save_accuracy_along_sampling"]:
save_file_path = self.DEBUG[
"save_accuracy_along_sampling"] + '/' + str(
self.DEBUG["batch_counter"]) + '.txt'
with open(save_file_path, 'w') as thefile:
for item in accuracy_along_sampling:
thefile.write("{:.4f}\n".format(100 * item))
return orch_pred, loss_batch
def validate(trainer, sess, init_matrices_validation, valid_splits_batches,
normalizer, parameters, logger, DEBUG):
temporal_order = trainer.temporal_order
accuracy = []
precision = []
recall = []
val_loss = []
true_accuracy = []
f_score = []
Xent = []
if DEBUG["save_measures"]:
preds = []
truth = []
else:
preds = None
truth = None
accuracy_long_range = []
precision_long_range = []
recall_long_range = []
val_loss_long_range = []
true_accuracy_long_range = []
f_score_long_range = []
Xent_long_range = []
N_matrix_files = len(valid_splits_batches)
pool = ThreadPool(processes=1)
matrices_from_thread = init_matrices_validation
for file_ind_CURRENT in range(N_matrix_files):
#######################################
# Get indices and matrices to load
#######################################
# We train on the current matrix
valid_index = valid_splits_batches[file_ind_CURRENT]['batches']
valid_long_range_index = valid_splits_batches[file_ind_CURRENT][
'batches_lr']
# But load the one next one
file_ind_NEXT = (file_ind_CURRENT + 1) % N_matrix_files
next_chunks = valid_splits_batches[file_ind_NEXT]['chunks_folders']
#######################################
# Load matrix thread
#######################################
async_valid = pool.apply_async(async_load_mat,
(normalizer, next_chunks, parameters))
piano_input, orch_input, duration_piano, mask_orch = matrices_from_thread
#######################################
# Loop for short-term validation
#######################################
for batch_counter, batch_index in enumerate(valid_index):
if DEBUG["plot_nade_ordering_preds"] and (batch_counter
== len(valid_index) - 1):
if os.path.isdir(DEBUG["plot_nade_ordering_preds"]):
shutil.rmtree(DEBUG["plot_nade_ordering_preds"])
os.makedirs(DEBUG["plot_nade_ordering_preds"])
loss_batch, preds_batch, orch_t = trainer.valid_step(
sess,
batch_index,
piano_input,
orch_input,
duration_piano,
mask_orch,
PLOTING_FOLDER=DEBUG["plot_nade_ordering_preds"])
else:
loss_batch, preds_batch, orch_t = trainer.valid_step(
sess,
batch_index,
piano_input,
orch_input,
duration_piano,
mask_orch,
PLOTING_FOLDER=None)
Xent_batch = binary_cross_entropy(orch_t, preds_batch)
accuracy_batch = accuracy_measure(orch_t, preds_batch)
precision_batch = precision_measure(orch_t, preds_batch)
recall_batch = recall_measure(orch_t, preds_batch)
true_accuracy_batch = true_accuracy_measure(orch_t, preds_batch)
f_score_batch = f_measure(orch_t, preds_batch)
val_loss.extend(loss_batch)
accuracy.extend(accuracy_batch)
precision.extend(precision_batch)
recall.extend(recall_batch)
true_accuracy.extend(true_accuracy_batch)
f_score.extend(f_score_batch)
Xent.extend(Xent_batch)
if DEBUG["save_measures"]:
# No need to store all the training points
preds.extend(preds_batch)
truth.extend(orch_t)
#######################################
# Loop for long-term validation
# The task is filling a gap of size parameters["long_range"]
# So the algo is given :
# orch[0:temporal_order]
# orch[temporal_order+parameters["long_range"]:
# (2*temporal_order)+parameters["long_range"]]
# And must fill :
# orch[temporal_order:
# temporal_order+parameters["long_range"]]
#######################################
for batch_index in valid_long_range_index:
# Init
# Extract from piano and orchestra the matrices required for the task
seq_len = (temporal_order - 1) * 2 + parameters["long_range"]
piano_dim = piano_input.shape[1]
orch_dim = orch_input.shape[1]
piano_extracted = np.zeros((len(batch_index), seq_len, piano_dim))
orch_extracted = np.zeros((len(batch_index), seq_len, orch_dim))
if parameters["duration_piano"]:
duration_piano_extracted = np.zeros(
(len(batch_index), seq_len))
else:
duration_piano_extracted = None
orch_gen = np.zeros((len(batch_index), seq_len, orch_dim))
for ind_b, this_batch_ind in enumerate(batch_index):
start_ind = this_batch_ind - temporal_order + 1
end_ind = start_ind + seq_len
piano_extracted[ind_b] = piano_input[start_ind:end_ind, :]
orch_extracted[ind_b] = orch_input[start_ind:end_ind, :]
if parameters["duration_piano"]:
duration_piano_extracted[ind_b] = duration_piano[
start_ind:end_ind]
# We know the past orchestration at the beginning...
orch_gen[:, :temporal_order -
1, :] = orch_extracted[:, :temporal_order - 1, :]
# and the future orchestration at the end
orch_gen[:, -temporal_order +
1:, :] = orch_extracted[:, -temporal_order + 1:, :]
# check we didn't gave the correct information
assert orch_gen[:, temporal_order - 1:(
temporal_order - 1) + parameters["long_range"], :].sum(
) == 0, "The gap to fill in orch_gen contains values !"
for t in range(temporal_order - 1,
temporal_order - 1 + parameters["long_range"]):
loss_batch, preds_batch, orch_t = trainer.valid_long_range_step(
sess, t, piano_extracted, orch_extracted, orch_gen,
duration_piano_extracted)
prediction_sampled = np.random.binomial(1, preds_batch)
orch_gen[:, t, :] = prediction_sampled
# Compute performances measures
Xent_batch = binary_cross_entropy(orch_t, preds_batch)
accuracy_batch = accuracy_measure(orch_t, preds_batch)
precision_batch = precision_measure(orch_t, preds_batch)
recall_batch = recall_measure(orch_t, preds_batch)
true_accuracy_batch = true_accuracy_measure(
orch_t, preds_batch)
f_score_batch = f_measure(orch_t, preds_batch)
val_loss_long_range.extend(loss_batch)
accuracy_long_range.extend(accuracy_batch)
precision_long_range.extend(precision_batch)
recall_long_range.extend(recall_batch)
true_accuracy_long_range.extend(true_accuracy_batch)
f_score_long_range.extend(f_score_batch)
Xent_long_range.extend(Xent_batch)
del (matrices_from_thread)
matrices_from_thread = async_valid.get()
pool.close()
pool.join()
valid_results = {
'accuracy': np.asarray(accuracy),
'precision': np.asarray(precision),
'recall': np.asarray(recall),
'loss': np.asarray(val_loss),
'true_accuracy': np.asarray(true_accuracy),
'f_score': np.asarray(f_score),
'Xent': np.asarray(Xent)
}
valid_long_range_results = {
'accuracy': np.asarray(accuracy_long_range),
'precision': np.asarray(precision_long_range),
'recall': np.asarray(recall_long_range),
'loss': np.asarray(val_loss_long_range),
'true_accuracy': np.asarray(true_accuracy_long_range),
'f_score': np.asarray(f_score_long_range),
'Xent': np.asarray(Xent_long_range)
}
return valid_results, valid_long_range_results, preds, truth
def accuracy_and_binary_Xent(context, batches, plot_folder, N_example):
"""Used to compare what is considered good or bad for the accuracy and the neg-ll. Both values are computed over the whole test set.
After normalization (mean and std), we plot the most representative examples of all the possible combinations of bad, average and good for neg-ll and acc
Parameters
----------
sess
tensor flow session
temporal_order
temporal_order of the model, i.e. how far in the past/future is the model allowed to look
batches
a list containing lists of indices. Each list of indices is a mini-batch
piano
matrix containing the piano score
orch
matrix containing the orch score
inputs_ph
lists of placeholders which are inputs of the graph
orch_t_ph
placeholder for the true orchestral frame
preds
placeholder for the prediction of the network
keras_learning_phase
placeholder for a flag used by Keras
Returns
-------
None
"""
# Clean plot directory
if os.path.isdir(plot_folder):
shutil.rmtree(plot_folder)
os.makedirs(plot_folder)
# Unpack context variables
sess = context['sess']
temporal_order = context['temporal_order']
piano = context['piano']
orch = context['orch']
duration_piano = context['duration_piano']
mask_orch = context["mask_orch"]
inputs_ph = context['inputs_ph']
orch_t_ph = context['orch_t_ph']
preds = context['preds']
keras_learning_phase = context['keras_learning_phase']
piano_t_ph, piano_past_ph, piano_future_ph, orch_past_ph, orch_future_ph = inputs_ph
all_preds = []
all_truth = []
# Get all the predictions for the whole set
for batch_index in batches:
# Build batch
piano_t, piano_past, piano_future, orch_past, orch_future, orch_t = build_batch(
batch_index, piano, orch, duration_piano, mask_orch,
len(batch_index), temporal_order)
# Input nodes
feed_dict = {
piano_t_ph: piano_t,
piano_past_ph: piano_past,
piano_future_ph: piano_future,
orch_past_ph: orch_past,
orch_future_ph: orch_future,
orch_t_ph: orch_t,
keras_learning_phase: 0
}
# Compute validation loss
preds_batch = sess.run(preds, feed_dict)
all_preds.extend(preds_batch)
all_truth.extend(orch_t)
# Cast as np array
preds_mat = np.asarray(all_preds)
truth_mat = np.asarray(all_truth)
# Compute neg-ll (use keras function for coherency with validation step)
truth_mat_ph = tf.placeholder(tf.float32,
shape=(truth_mat.shape),
name="truth_mat")
preds_mat_ph = tf.placeholder(tf.float32,
shape=(preds_mat.shape),
name="preds_mat")
neg_ll_node = keras.losses.binary_crossentropy(truth_mat_ph, preds_mat_ph)
neg_ll = sess.run(neg_ll_node, {
preds_mat_ph: preds_mat,
truth_mat_ph: truth_mat
})
# Compute acc
acc = 100 * accuracy_measure(truth_mat, preds_mat)
# Normalize
def normalize(matrix):
mean = np.mean(matrix)
std = np.std(matrix)
norm_mat = (matrix - mean) / std
return norm_mat, mean, std
neg_ll_norm, neg_ll_mean, neg_ll_std = normalize(neg_ll)
acc_norm, acc_mean, acc_std = normalize(acc)
# Rank perf
arg_neg_ll = np.argsort(neg_ll_norm)
arg_acc = np.argsort(
-acc_norm) # minus to have the best at the first index
num_index = len(arg_neg_ll)
# Plots
def plot(ind, this_folder):
temp_mat = np.stack((truth_mat[ind], preds_mat[ind]))
visualize_mat_proba(
temp_mat, this_folder,
"acc_" + str(acc[ind]) + "_nll_" + str(neg_ll[ind]))
# Reste juste à parser les arg_sorted pour avoir les bons et mauvais exemples
for index in range(N_example):
good_index = index
bad_index = -index - 1
average_index = (num_index / 2) + (1 - 2 * (index % 2)) * index
# Bad Xent
bad_xent_folder = os.path.join(plot_folder, "bad_Xent_" + str(index))
plot(arg_neg_ll[bad_index], bad_xent_folder)
# Average Xent
plot(arg_neg_ll[average_index],
os.path.join(plot_folder, "average_Xent_" + str(index)))
# Good Xent
plot(arg_neg_ll[good_index],
os.path.join(plot_folder, "good_Xent_" + str(index)))
# Bad acc
plot(arg_acc[bad_index],
os.path.join(plot_folder, "bad_acc_" + str(index)))
# Average acc
plot(arg_acc[average_index],
os.path.join(plot_folder, "average_acc_" + str(index)))
# Good acc
plot(arg_acc[good_index],
os.path.join(plot_folder, "good_acc_" + str(index)))
# Write the statistics
with open(os.path.join(plot_folder, "statistics.txt"), "wb") as f:
f.write("Accuracy mean : " + str(acc_mean) + "\n")
f.write("Accuracy std : " + str(acc_std) + "\n")
f.write("Xent mean : " + str(neg_ll_mean) + "\n")
f.write("Xent std : " + str(neg_ll_std) + "\n")
return
def compare_Xent_acc_corresponding_preds(context, batches, save_folder):
"""Save te matrices for the accuracy score and Xent score and the corresponding ground-truth and predictions
"""
# Clean plot directory
if os.path.isdir(save_folder):
shutil.rmtree(save_folder)
os.makedirs(save_folder)
# Unpack context variables
sess = context['sess']
temporal_order = context['temporal_order']
piano = context['piano']
orch = context['orch']
duration_piano = context["duration_piano"]
mask_orch = context["mask_orch"]
inputs_ph = context['inputs_ph']
orch_t_ph = context['orch_t_ph']
preds = context['preds']
keras_learning_phase = context['keras_learning_phase']
piano_t_ph, piano_past_ph, piano_future_ph, orch_past_ph, orch_future_ph = inputs_ph
all_preds = []
all_truth = []
# Get all the predictions for the whole set
for batch_index in batches:
# Build batch
piano_t, piano_past, piano_future, orch_past, orch_future, orch_t = build_batch(batch_index, piano, orch, duration_piano, mask_orch, len(batch_index), temporal_order)
# Input nodes
feed_dict = {piano_t_ph: piano_t,
piano_past_ph: piano_past,
piano_future_ph: piano_future,
orch_past_ph: orch_past,
orch_future_ph: orch_future,
orch_t_ph: orch_t,
keras_learning_phase: 0}
# Compute validation loss
preds_batch = sess.run(preds, feed_dict)
all_preds.extend(preds_batch)
all_truth.extend(orch_t)
# Cast as np array
preds_mat = np.asarray(all_preds)
truth_mat = np.asarray(all_truth)
# Compute neg-ll (use keras function for coherency with validation step)
truth_mat_ph = tf.placeholder(tf.float32, shape=(truth_mat.shape), name="truth_mat")
preds_mat_ph = tf.placeholder(tf.float32, shape=(preds_mat.shape), name="preds_mat")
neg_ll_node = keras.losses.binary_crossentropy(truth_mat_ph, preds_mat_ph)
neg_ll = sess.run(neg_ll_node, {preds_mat_ph: preds_mat, truth_mat_ph: truth_mat})
# Compute acc
acc = 100 * accuracy_measure(truth_mat, preds_mat)
np.save(os.path.join(save_folder, 'accuracy.npy'), acc)
np.save(os.path.join(save_folder, 'Xent.npy'), 100 * neg_ll)
np.save(os.path.join(save_folder, 'preds.npy'), preds_mat)
np.save(os.path.join(save_folder, 'truth.npy'), truth_mat)
return
def orderless_NADE_generation(self, sess, feed_dict, orch_t, instrumentation=None):
# instrumentation takes the form of a binary mask
# Pre-compute the context embedding
# which will be the same for all the orderings
context_embedding = sess.run(self.context_embedding_out, feed_dict)
# Generate the orderings in parallel -> duplicate the embedding and orch_t matrices along batch dim
_, orch_dim = orch_t.shape
# Start with an orchestra prediction and mask equal to zero
orch_t_reshape = np.concatenate([orch_t for _ in range(self.num_ordering)], axis=0)
orch_pred = np.zeros_like(orch_t_reshape)
mask = np.zeros_like(orch_t_reshape)
context_embedding_reshape = np.concatenate([context_embedding for _ in range(self.num_ordering)], axis=0)
# Three nodes to feed now: orch_pred, context_embedding, and mask
feed_dict_known_context = {}
feed_dict_known_context[self.orch_t_ph] = orch_t_reshape
feed_dict_known_context[self.context_embedding_in] = context_embedding_reshape
# Indices to sample given piano
piano_t = feed_dict[self.piano_t_ph]
ind_played_piano = np.where(piano_t > 0)[1]
pitches_played_piano = set(self.pitch_piano[ind_played_piano])
indices_to_sample = []
for pitch_played_piano in pitches_played_piano:
temp = np.where(self.pitch_orch==pitch_played_piano)[0]
indices_to_sample.extend(temp.tolist())
# Indices to sample given instrumentation
indices_to_sample = set(indices_to_sample)
if instrumentation is not None:
indices_to_sample_instru = set(np.where(instrumentation>0)[0].tolist())
indices_to_sample = indices_to_sample.intersection(indices_to_sample_instru)
indices_NOT_sample = set(range(orch_dim)) - indices_to_sample
indices_to_sample = list(indices_to_sample)
indices_NOT_sample = list(indices_NOT_sample)
# Build the orderings (use the same ordering for all elems in batch)
orderings = []
for ordering_ind in range(self.num_ordering):
# This ordering
random.shuffle(indices_to_sample)
orderings.append(indices_NOT_sample + indices_to_sample)
# set not_sample value to 1 in mask
for ind_order in indices_NOT_sample:
mask[:, ind_order] = 1
if self.DEBUG["save_accuracy_along_sampling"]:
accuracy_along_sampling = []
# Loop over the length of the orderings
for d in range(len(indices_NOT_sample), orch_dim):
# Generate step
feed_dict_known_context[self.orch_pred] = orch_pred
feed_dict_known_context[self.mask_input] = mask
loss_batch, preds_batch = sess.run([self.loss_val, self.preds_gen], feed_dict_known_context)
# Update matrices
for ordering_ind in range(self.num_ordering):
mask[ordering_ind, orderings[ordering_ind][d]] = 1
##################################################
# Do we sample or not ??????
orch_pred[ordering_ind, orderings[ordering_ind][d]] = np.random.binomial(1, preds_batch[ordering_ind, orderings[ordering_ind][d]])
##################################################
##############################
##############################
# DEBUG
# Observe the evolution of the accuracy along the sampling process
if self.DEBUG["save_accuracy_along_sampling"]:
accuracy_batch = np.mean(accuracy_measure(orch_t_reshape, orch_pred))
accuracy_along_sampling.append(accuracy_batch)
# Plot the predictions
if self.DEBUG["plot_nade_ordering_preds"] and (self.DEBUG["batch_counter"]==(self.DEBUG["num_batch"]-1)):
for ordering_ind in range(self.num_ordering):
np.save(self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) + '_' + str(ordering_ind) + '.npy', orch_pred[ordering_ind,:])
mean_pred_batch = self.mean_parallel_prediction(orch_pred)
np.save(self.DEBUG["plot_nade_ordering_preds"] + '/' + str(d) + '_mean.npy', mean_pred_batch)
##############################
##############################
if self.DEBUG["plot_nade_ordering_preds"] and (self.DEBUG["batch_counter"]==(self.DEBUG["num_batch"]-1)):
np.save(self.DEBUG["plot_nade_ordering_preds"] + '/truth.npy', orch_t)
# Save accuracy_along_sampling
if self.DEBUG["save_accuracy_along_sampling"]:
save_file_path = self.DEBUG["save_accuracy_along_sampling"] + '/' + str(self.DEBUG["batch_counter"]) + '.txt'
with open(save_file_path, 'w') as thefile:
for item in accuracy_along_sampling:
thefile.write("{:.4f}\n".format(100*item))
return orch_pred, loss_batch