def save(self, path):
"""
:param path:
:return:
"""
path = EPath(path)
# shutil.rmtree(path.as_posix(), ignore_errors=True)
path.mkdir(exist_ok=True, parents=True)
with open(path / 'MyNN.p', 'wb') as dump_file:
pickle.dump(
dict(model_id=self.model_id,
input_param=self.input_param,
opt_param=self.opt_param,
step_length=self.step_length,
model_options=self.model_options,
loss_options=self.loss_options,
mono=self.mono), dump_file)
self.save_weights(path=path)
self.save_checkpoint_weights(path=path)
summary.summarize(
# Function parameters
path=path,
title='My NN',
# Summary parameters
model_id=self.model_id,
**self.model_options,
**self.opt_param,
**self.loss_options)
def save_model(self, path=None):
"""
:param path: path were to save the model, if None it will be at self.saved_model_path
:return:
"""
# Where to save the model
path_to_save = self.saved_model_path if path is None else EPath(path)
# Clean the folder if already in use
shutil.rmtree(path_to_save.as_posix(),
ignore_errors=True) # Delete it if it exists
path_to_save.mkdir(parents=True,
exist_ok=True) # Creation of this folder
self.keras_nn.save(str(path_to_save / 'MyNN'))
with open(str(path_to_save / 'infos.p'), 'wb') as dump_file:
pickle.dump(
dict(
# For new_nn_model
model_id=self.model_id,
work_on=self.work_on,
input_param=self.input_param,
i=self.full_name_i,
name=self.name,
# data
instruments=self.instruments,
notes_range=self.notes_range,
mono=self.mono,
data_transformed_path=self.data_transformed_path,
predict_offset=self.predict_offset,
# logistic
epochs=self.total_epochs,
# Don't really need this anymore
full_name=self.full_name),
dump_file)
self.keras_nn.save_tensorboard_plots(path_to_save / 'plots')
self.keras_nn.save_tensorboard(path_to_save / 'tensorboard')
summary.summarize(
# Function params
path=path_to_save,
title=self.full_name,
# Summary params
**self.summary_dict)
print(colored(f'Model saved in {path_to_save}', 'green'))
return path_to_save
def main(args):
"""
Entry point
"""
# -------------------- Setup --------------------
# ----- pc -----
if args.pc:
# args.data = 'lmd_matched_mini'
data_path = os.path.join('../Dataset', args.data)
else:
data_path = os.path.join('../../../../../../storage1/valentin',
args.data)
data_transformed_path = data_path + '_transformed'
if not args.no_transposed:
data_transformed_path += 'Transposed'
if args.mono:
data_transformed_path += 'Mono'
# Choose GPU
if not args.pc:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
if args.seq2np:
KerasNeuralNetwork.slow_down_cpu(nb_inter=args.nb_inter_threads,
nb_intra=args.nb_intra_threads)
# get x and y if args.seq2np
if args.seq2np:
x_dict, y_dict = {}, {}
from_checkpoint = args.from_checkpoint is not None
saved_checkpoint_folder = None
id = None
if from_checkpoint:
saved_checkpoint_folder = BO.save.get_folder_path(
args.from_checkpoint) / 'checkpoint'
id = args.from_checkpoint if args.in_place else None
folder_path = BO.save.get_folder_path(id=id, name=args.bo_name)
folder_path.mkdir(parents=True,
exist_ok=args.in_place) # We want it to act as a token
checkpoint_folder = folder_path / 'checkpoint'
checkpoint_folder.mkdir(exist_ok=args.in_place)
# -------------------- Setup Bayesian Optimization --------------------
saved_checkpoint, dimensions = None, None
global best_accuracy
global iteration
global max_iterations
if not from_checkpoint:
best_accuracy = 0
iteration = 0
else:
saved_checkpoint, dimensions = BO.load.from_checkpoint(
saved_checkpoint_folder)
best_accuracy = -saved_checkpoint.fun # Because skopt minimizes it
iteration = saved_checkpoint.func_vals.size
print('Checkpoint loaded from',
colored(f'{saved_checkpoint_folder.parent}', 'green'),
'save in place:', colored(f'{args.in_place}', 'yellow'))
max_iterations = args.n_calls + iteration
dimensions = create_dimensions(args) if dimensions is None else dimensions
def create_model(lr, optimizer, decay, dropout_d, dropout_c, dropout_r,
sampling, kld, all_sequence, model_name, model_param,
nb_steps, kld_annealing_start, kld_annealing_stop,
kld_sum, loss_name, l_scale, l_rhythm,
take_all_step_rhythm, sah, l_semitone, l_tone, l_tritone,
rpoe, prior_expert):
"""
Creates a model from all the inputs == dimensions of the bayesian optimization
"""
midi_generator = MidiGenerator(name=args.name)
midi_generator.load_data(data_transformed_path=data_transformed_path,
verbose=0)
model_id = f'{model_name},{model_param},{nb_steps}'
opt_params = dict(lr=lr,
name=optimizer,
decay_drop=float(args.decay_drop),
epoch_drop=float(args.epochs_drop),
decay=decay)
model_options = dict(dropout_d=dropout_d,
dropout_c=dropout_c,
dropout_r=dropout_r,
all_sequence=all_sequence,
lstm_state=args.lstm_state,
sampling=sampling,
kld=kld,
kld_annealing_start=kld_annealing_start,
kld_annealing_stop=kld_annealing_stop,
kld_sum=kld_sum,
sah=sah,
rpoe=rpoe,
prior_expert=prior_expert)
loss_options = dict(loss_name=loss_name,
l_scale=l_scale,
l_rhythm=l_rhythm,
take_all_step_rhythm=take_all_step_rhythm,
l_semitone=l_semitone,
l_tone=l_tone,
l_tritone=l_tritone,
use_binary=args.use_binary)
midi_generator.new_nn_model(model_id=model_id,
opt_param=opt_params,
work_on=args.work_on,
use_binary=args.use_binary,
model_options=model_options,
loss_options=loss_options,
print_model=False,
predict_offset=args.predict_offset)
return midi_generator
def fitness(l):
"""
From all the inputs == dimensions of the bayesian optimization, it creates a model, train it, add return
its negative accuracy (skopt works by minimizing a function)
"""
global iteration
global max_iterations
iteration += 1
# ---------------------------------------- Get the variables ----------------------------------------
lr = dimensions.get_value_param('lr', l)
optimizer = dimensions.get_value_param('optimizer', l)
decay = dimensions.get_value_param('decay', l)
dropout_d = dimensions.get_value_param('dropout_d', l)
dropout_c = dimensions.get_value_param('dropout_c', l)
dropout_r = dimensions.get_value_param('dropout_r', l)
sampling = dimensions.get_value_param('sampling', l)
kld = dimensions.get_value_param('kld', l)
all_sequence = dimensions.get_value_param('all_sequence', l)
model_name = dimensions.get_value_param('model_name', l)
model_param = dimensions.get_value_param('model_param', l)
nb_steps = dimensions.get_value_param('nb_steps', l)
kld_annealing_start = dimensions.get_value_param(
'kld_annealing_start', l)
kld_annealing_stop = dimensions.get_value_param(
'kld_annealing_stop', l)
kld_sum = dimensions.get_value_param('kld_sum', l)
loss_name = dimensions.get_value_param('loss_name', l)
l_scale = dimensions.get_value_param('l_scale', l)
l_rhythm = dimensions.get_value_param('l_rhythm', l)
take_all_step_rhythm = dimensions.get_value_param(
'take_all_step_rhythm', l)
sah = dimensions.get_value_param('sah', l)
l_semitone = dimensions.get_value_param('l_semitone', l)
l_tone = dimensions.get_value_param('l_tone', l)
l_tritone = dimensions.get_value_param('l_tritone', l)
rpoe = dimensions.get_value_param('rpoe', l)
prior_expert = dimensions.get_value_param('prior_expert', l)
# ------------------------------ Print the information to the user ------------------------------
s = 'Iteration ' + colored(f'{iteration}/{max_iterations}', 'yellow')
model_id = f'{model_name},{model_param},{nb_steps}'
s += str_hp_to_print('model', model_id, first_printed=False)
s += str_hp_to_print('lr', lr, exp_format=True)
s += str_hp_to_print('opt', optimizer)
s += str_hp_to_print('decay', decay, exp_format=True)
s += str_hp_to_print('dropout_d', dropout_d, exp_format=True)
s += str_hp_to_print('dropout_c', dropout_c, exp_format=True)
s += str_hp_to_print('dropout_r', dropout_r, exp_format=True)
s += str_hp_to_print('sampling', sampling)
s += str_hp_to_print('kld', kld)
s += str_hp_to_print('all_sequence', all_sequence)
s += str_hp_to_print('kld_annealing_start',
kld_annealing_start,
exp_format=True)
s += str_hp_to_print('kld_annealing_stop',
kld_annealing_stop,
exp_format=True)
s += str_hp_to_print('kld_sum', kld_sum)
s += str_hp_to_print('loss_name', loss_name)
s += str_hp_to_print('l_scale', l_scale, exp_format=True)
s += str_hp_to_print('l_rhythm', l_rhythm, exp_format=True)
s += str_hp_to_print('take_all_step_rhythm', take_all_step_rhythm)
s += str_hp_to_print('sah', sah)
s += str_hp_to_print('l_semitone', l_semitone, exp_format=True)
s += str_hp_to_print('l_tone', l_tone, exp_format=True)
s += str_hp_to_print('l_tritone', l_tritone, exp_format=True)
s += str_hp_to_print('rpoe', rpoe)
s += str_hp_to_print('prior_expert', prior_expert)
print(s)
# -------------------- Create the model --------------------
midi_generator = create_model(
lr=lr,
optimizer=optimizer,
decay=decay,
dropout_d=dropout_d,
dropout_c=dropout_c,
dropout_r=dropout_r,
sampling=sampling,
kld=kld,
all_sequence=all_sequence,
model_name=model_name,
model_param=model_param,
nb_steps=nb_steps,
kld_annealing_start=kld_annealing_start,
kld_annealing_stop=kld_annealing_stop,
kld_sum=kld_sum,
loss_name=loss_name,
l_scale=l_scale,
l_rhythm=l_rhythm,
take_all_step_rhythm=take_all_step_rhythm,
sah=sah,
l_semitone=l_semitone,
l_tone=l_tone,
l_tritone=l_tritone,
rpoe=rpoe,
prior_expert=prior_expert)
# -------------------- Train the model --------------------
best_accuracy_callback = Callbacks.BestAccuracy()
if args.seq2np:
# get x and y
if nb_steps not in x_dict or nb_steps not in y_dict:
midi_generator.get_sequence(
path=midi_generator.data_transformed_path.as_posix(),
nb_steps=midi_generator.nb_steps,
batch_size=args.batch,
work_on=midi_generator.work_on)
x, y = Sequences.sequence_to_numpy(midi_generator.sequence)
x_dict[nb_steps] = x
y_dict[nb_steps] = y
del x, y
# Train
history = midi_generator.keras_nn.train(
epochs=args.epochs,
x=x_dict[nb_steps],
y=y_dict[nb_steps],
verbose=1,
validation=args.validation,
callbacks=[best_accuracy_callback],
batch_size=args.batch)
else:
history = midi_generator.train(epochs=args.epochs,
batch=args.batch,
callbacks=[best_accuracy_callback],
verbose=1,
validation=args.validation,
fast_sequence=args.fast_seq,
memory_sequence=args.memory_seq)
accuracy = best_accuracy_callback.best_acc
global best_accuracy
if accuracy > best_accuracy:
best_accuracy = accuracy
# Print accuracy to the user
print(f'Accuracy:', colored(f'{accuracy:.2%}', 'cyan'),
'- Best Accuracy for now:',
colored(f'{best_accuracy:.2%}', 'white', 'on_blue'), '\n')
midi_generator.keras_nn.clear_session()
del midi_generator
del history
gc.collect()
'''
# To do quick tests
res = lr * decay / dropout ** (2 + nb_steps)
print(res)
return res
'''
# Return negative accuracy because skopt works by minimizing functions
return -accuracy
# ------------------------------------------------------------
# Actually run the Bayesian search
# ------------------------------------------------------------
dimensions.save(checkpoint_folder / 'dimensions.p')
with open(checkpoint_folder / 'args.p', 'wb') as dump_file:
pickle.dump(dict(args=args), dump_file)
summary.summarize(
# Function parameters
path=folder_path,
title='Args of bayesian optimization',
# Summary params
**vars(args))
checkpoint_callback = skopt.callbacks.CheckpointSaver(
checkpoint_path=(checkpoint_folder / 'search_result.pkl').path,
store_objective=False)
n_random_starts = 10
if args.pc and not args.no_pc_arg:
n_random_starts = 1
if from_checkpoint:
n_random_starts = 0
search_result = gp_minimize(func=fitness,
dimensions=dimensions.dimensions,
acq_func='EI',
n_calls=args.n_calls,
x0=dimensions.default_dim,
callback=[checkpoint_callback],
n_random_starts=n_random_starts)
BO.save.save_search_result(search_result=search_result,
dimensions=dimensions,
folder_path=folder_path)
cprint('---------- Done ----------', 'grey', 'on_green')
def main(args):
"""
Entry point
"""
args, data_path, data_transformed_path = check_args(args)
# --------------------------------------------------
# ----- All the paths -----
dataset_p = data_transformed_path / 'dataset.p' # Pickle file with the information of the data set kept
infos_dataset_p = EPath(
data_transformed_path
) / 'infos_dataset.p' # pickle file with the information of the dataset (smaller file)
all_midi_paths_dataset = midi.open.all_midi_files(data_path.as_posix(),
False)
# --------------------------------------------------
# Compute dataset
# --------------------------------------------------
npy_path = data_transformed_path / 'npy'
npy_path.mkdir(parents=True, exist_ok=True)
all_shapes = []
# ----- Actually compute the datas -----
print('----- Compute the data in', colored(data_path, 'grey', 'on_white'),
'-----')
print('Number of files : ', colored(len(all_midi_paths_dataset),
'magenta'))
bach_string = ''
if args.bach:
bach_string = colored('(Bach)', 'magenta')
print('Instruments :', colored(args.instruments, 'magenta'), bach_string,
'-- Notes range :', colored(args.notes_range, 'magenta'))
matrix_of_all_midis = []
all_midi_paths = []
# All Midi have to be in same shape.
bar = progressbar.ProgressBar(maxval=len(all_midi_paths_dataset),
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
i = 0
all_shapes_npy = []
for index, single_midi_path in enumerate(all_midi_paths_dataset):
bar.update(index)
# ---------- Get the matrix ----------
if args.bach:
matrix_of_single_midi = midi.open.midi_to_matrix_bach(
single_midi_path,
length=args.length,
notes_range=args.notes_range,
transpose=not args.no_transpose
) # (nb_instruments, 88, nb_steps, 2)
else:
matrix_of_single_midi = midi.open.midi_to_matrix(
single_midi_path,
args.instruments,
length=args.length,
notes_range=args.notes_range,
transpose=not args.no_transpose
) # (nb_instruments, 88, nb_steps, 2)
if matrix_of_single_midi is None: # It means an error happened
continue
if args.mono:
matrix_of_single_midi = midi.open.to_mono_matrix(
matrix_of_single_midi)
matrix_of_single_midi = np.transpose(
matrix_of_single_midi,
(2, 0, 1, 3)) # (length, nb_instruments, 88, 2)
# ---------- Add the matrix ----------
all_midi_paths.append(single_midi_path)
matrix_of_all_midis.append(matrix_of_single_midi)
# print('shape of the matrix : {0}'.format(matrix_of_single_midi.shape))
all_shapes_npy.append(matrix_of_single_midi.shape)
i += 1
# ---------- Save it ----------
if i % g.midi.nb_files_per_npy == 0: # Save 1 npy file with 100 songs in it
np.save(
str(npy_path /
'{0}.npy'.format(int(i / g.midi.nb_files_per_npy) - 1)), {
'list': matrix_of_all_midis,
'shapes': all_shapes_npy
})
all_shapes.append(all_shapes_npy)
all_shapes_npy = []
matrix_of_all_midis = []
# ---------- If we didn't save at the end ----------
if len(all_shapes_npy) > 0: # If some songs are missing
np.save(
str(npy_path / '{0}.npy'.format(int(i / g.midi.nb_files_per_npy))),
{
'list': matrix_of_all_midis,
'shapes': all_shapes_npy
})
all_shapes.append(all_shapes_npy)
# ---------- Save the path of all the midis ----------
with open(dataset_p, 'wb') as dump_file:
pickle.dump({'Midi': all_midi_paths}, dump_file)
bar.finish()
# Now all_midi_paths is defined and we don't need all_midi_paths_dataset anymore
nb_valid_files = len(all_midi_paths)
# ---------- Save all the information of the dataset ----------
with open(infos_dataset_p, 'wb') as dump_file:
# Save the information of the data in a smaller file (without all the big array)
pickle.dump(
{
'nb_files': nb_valid_files,
'instruments': args.instruments,
'nb_instruments': len(args.instruments),
'all_shapes': all_shapes,
'input_size': all_shapes[0][0][2], # The number of notes
'notes_range': args.notes_range,
'mono': args.mono,
'nb_files_per_npy': g.midi.nb_files_per_npy,
'transposed': not args.no_transpose
},
dump_file)
summary.summarize(
# Function params
path=data_transformed_path,
title=args.data,
# Summary params
nb_files=nb_valid_files,
nb_instruments=len(args.instruments),
instruments=args.instruments,
input_size=all_shapes[0][0][2],
notes_range=args.notes_range)
print('Number of songs :', colored('{0}'.format(nb_valid_files), 'blue'))
print(colored('---------- Done ----------', 'grey', 'on_green'))
def redo_song_replicate(self,
song_number=None,
instrument_order=None,
no_duration=False,
save_images=True,
noise=0):
"""
:param instrument_order: The order of the instruments to remplace
:param song_number: The number of the song in the dataset
:param no_duration:
:param save_images:
:param noise:
:return:
"""
self.sequence = Sequences.KerasSequence(
path=self.data_transformed_path,
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on,
noise=noise,
replicate=True)
song_number = np.random.randint(
self.sequence.nb_songs) if song_number is None else song_number
instrument_order = np.random.permutation(
self.nb_instruments
) if instrument_order is None else instrument_order
all_arrays = []
# Construct the truth array
x, y = self.sequence.get_all_song(song_number=song_number,
in_batch_format=False)
# x: (nb_instruments, batch=1, nb_steps, step_size, input_size, channels)]
# y: (nb_instruments, batch=1, nb_steps, step_size, input_size, channels)]
# x and y are the same except that in x, there is some noise
truth = x
# truth: (nb_instruments, batch=1, len_song, step_size, input_size, channels
# We then take a number of step which is a multiple of self.nb_steps
indices = range(truth.shape[2] // self.nb_steps * self.nb_steps)
truth = np.take(truth, axis=2, indices=indices)
length = truth.shape[2]
all_arrays.append(truth)
cprint('Start redoing song (replicate) ...', 'blue')
bar = progressbar.ProgressBar(maxval=length * self.nb_instruments,
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
for instrument in range(len(instrument_order)):
# We replace the instruments one by one
instrument_to_remove = instrument_order[instrument]
generated = []
for step in range(0, length, self.nb_steps):
inputs = np.take(all_arrays[-1],
axis=2,
indices=range(step, step + self.nb_steps))
# inputs = (nb_instruments, batch=1, nb_steps, step_size, input_size, channels)]
mask = np.ones(
(1, self.nb_instruments,
self.nb_steps)) # (batch=1, nb_instruments, nb_steps)
# Remove the instrument from the input
mask[:, instrument_to_remove] = 0
inputs[instrument_to_remove] = 0
preds = np.asarray(
self.keras_nn.generate(input=list(inputs) +
[mask])).astype('float64')
# preds: (nb_instruments, batch=1, nb_steps, step_size, input_size, channels)
preds = midi.create.normalize_activation(
preds, mono=self.mono, use_binary=self.use_binary)
outputs = np.copy(inputs)
outputs[instrument_to_remove] = preds[instrument_to_remove]
generated.append(outputs)
bar.update(instrument * length + step)
all_arrays.append(np.concatenate(generated, axis=2))
# all_arrays: List(nb_instruments + 1)[(nb_instruments, batch=1, nb_steps, step_size, input_size, channels)]
bar.finish()
self.save_midis_path.mkdir(exist_ok=True, parents=True)
generated_midi = [
self.reshape_generated_array(arr) for arr in all_arrays
]
# Save the truth
accuracies, accuracies_inst = self.compute_generated_array(
generated_array=generated_midi[0],
folder_path=self.save_midis_path,
name='redo_song_replicate_truth',
no_duration=no_duration,
save_images=save_images,
replicate=True)
accuracies, accuracies_inst = [accuracies], [accuracies_inst]
for inst in range(self.nb_instruments):
acc, acc_inst = self.compute_generated_array(
generated_array=generated_midi[inst + 1],
folder_path=self.save_midis_path,
name=
f'redo_song_replicate_{inst}_(inst_{instrument_order[inst]})',
no_duration=no_duration,
array_truth=generated_midi[0],
save_images=save_images,
save_truth=False,
replicate=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
if self.batch is not None:
self.sequence.batch_size = self.batch
self.save_generated_arrays_cross_images(
generated_arrays=generated_midi,
folder_path=self.save_midis_path,
name='redo_song_all',
replicate=True,
titles=['Truth'] + [
f'Iteration {i}: change inst {instrument_order[i]}'
for i in range(self.nb_instruments)
],
subtitles=[
f'Acc: {accuracies[i]}, Acc inst: {accuracies_inst[i]}'
for i in range(self.nb_instruments + 1)
])
summary.summarize(
# Function params
path=self.save_midis_path,
title=self.full_name,
file_name='redo_song_replicate_summary.txt',
# Summary params
song_number=song_number,
instrument_order=instrument_order,
no_duration=no_duration,
noise=noise,
# Generic summary
**self.summary_dict)
cprint('Done redo song replicate', 'green')
def replicate_from_data(self,
length=None,
save_images=False,
no_duration=False,
verbose=1,
noise=0):
"""
Generate Midi file from the seed and the trained model
:param length: Length of th generation
:param save_images: To save the pianoroll of the generation (.jpg images)
:param no_duration: if True : all notes will be the shortest length possible
:param verbose: Level of verbose
:param noise:
:return:
"""
# ---------- Verify the inputs ----------
# ----- Create the seed -----
if self.data_transformed_path is None:
raise Exception('Some data need to be loaded before replicating')
self.sequence = Sequences.AllInstSequence.replicate(
path=str(self.data_transformed_path),
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on,
noise=noise)
nb_instruments = self.sequence.nb_instruments
# -- Length --
length = length if length is not None else min(20, len(self.sequence))
# --- Done Verifying the inputs ---
self.save_midis_path.mkdir(parents=True, exist_ok=True)
shape_with_no_step = list(np.array(self.sequence[0][0]).shape)
shape_with_no_step[2] = 0
shape_with_no_step = tuple(shape_with_no_step)
generated = np.zeros(
shape=shape_with_no_step
) # (nb_instruments, batch=1, nb_steps=0, step_size, inputs_size, 2)
truth = np.zeros(
shape=shape_with_no_step
) # (nb_instruments, batch=1, nb_steps=0, step_size, inputs_size, 2)
mask = self.get_mask(nb_instruments)
cprint('Start replicating ...', 'blue')
bar = progressbar.ProgressBar(maxval=length,
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
for l in range(0, length, self.nb_steps):
x, y = self.sequence[l]
y = np.asarray(y)
preds = self.keras_nn.generate(
input=x +
mask) # (nb_instruments, batch=1 , nb_steps=1, length, 88, 2)
preds = np.asarray(preds).astype(
'float64') # (nb_instruments, 1, 1, step_size, input_size, 2)
if len(preds.shape
) == 5: # Only one instrument : output of nn not a list
preds = preds[np.newaxis]
if len(y.shape) == 5:
y = np.expand_dims(y, axis=0)
next_array = midi.create.normalize_activation(
preds, mono=self.mono,
use_binary=self.use_binary) # Normalize the activation part
generated = np.concatenate(
(generated, next_array),
axis=2) # (nb_instruments, 1, nb_steps, length, 88, 2)
truth = np.concatenate(
(truth, y), axis=2
) # (nb_instruments, 1, nb_steps, step_length, size, channels)
bar.update(l + 1)
bar.finish()
generated_midi_final = self.reshape_generated_array(generated)
truth_final = self.reshape_generated_array(truth)
self.compute_generated_array(generated_array=generated_midi_final,
folder_path=self.save_midis_path,
name=f'replicated',
no_duration=no_duration,
array_truth=truth_final,
verbose=verbose,
save_images=save_images,
save_truth=True,
replicate=True)
if self.batch is not None:
self.sequence.batch_size = self.batch
summary.summarize(
# Function params
path=self.save_midis_path,
title=self.full_name,
file_name='replicate_summary.txt',
# Summary params
length=length,
no_duration=no_duration,
noise=noise,
# Generic Summary
**self.summary_dict)
cprint('Done replicating', 'green')
def replicate_fill(self,
max_length=None,
no_duration=False,
verbose=1,
save_images=True,
noise=0):
"""
:param max_length:
:param no_duration:
:param verbose:
:param save_images:
:param noise:
:return:
"""
max_length = max_length if max_length is not None else 300 / self.step_length
if self.data_transformed_path is None:
raise Exception('Some data need to be loaded before replicating')
self.sequence = Sequences.KerasSequence(path=str(
self.data_transformed_path),
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on,
noise=noise,
replicate=True)
max_length = int(min(max_length, len(self.sequence)))
nb_instruments = self.sequence.nb_instruments
self.save_midis_path.mkdir(parents=True, exist_ok=True)
shape_with_no_step = list(np.array(self.sequence[0][0]).shape)
shape_with_no_step[2] = 0
shape_with_no_step = tuple(shape_with_no_step)
generated_list = [
np.zeros(
shape=shape_with_no_step
) # (nb_instruments, batch=1, nb_steps=0, step_size, inputs_size, 2)
for inst in range(nb_instruments)
]
truth = np.zeros(
shape=shape_with_no_step
) # (nb_instruments, batch=1, nb_steps=0, step_size, inputs_size, 2)
mask = np.ones((nb_instruments, nb_instruments, self.nb_steps))
for inst in range(nb_instruments):
mask[inst, inst] = 0
cprint('Start replicating (fill) ...', 'blue')
bar = progressbar.ProgressBar(maxval=max_length,
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
for l in range(0, max_length, self.nb_steps):
x, y = self.sequence[l]
x_missing_inst_list = []
for inst in range(nb_instruments):
x_missing_inst = np.copy(x)
x_missing_inst[
inst] = 0 # (nb_instruments, batch, nb_steps, step_length, size, channels)
x_missing_inst_list.append(x_missing_inst)
nn_input = np.concatenate(
tuple(x_missing_inst_list), axis=1
) # (nb_instruments, batch=nb_instruments, nb_steps, step_length, size, channels)
preds = self.keras_nn.generate(
input=list(nn_input) +
[mask
]) # (nb_instruments, batch=1 , nb_steps=1, length, 88, 2)
preds = np.asarray(preds).astype(
'float64') # (nb_instruments, 1, 1, step_size, input_size, 2)
if len(preds.shape
) == 5: # Only one instrument : output of nn not a list
preds = preds[np.newaxis]
if len(y.shape) == 5:
y = np.expand_dims(y, axis=0)
preds = midi.create.normalize_activation(
preds, mono=self.mono,
use_binary=self.use_binary) # Normalize the activation part
for inst in range(nb_instruments):
p = np.copy(
y
) # (nb_instruments, batch=1, nb_steps, step_length, size, channels)
p[inst] = np.take(preds, axis=1, indices=[inst])[inst]
generated_list[inst] = np.concatenate(
(generated_list[inst], p), axis=2
) # (nb_instruments, batch=1, nb_steps, step_length, size, channels)
truth = np.concatenate(
(truth, np.asarray(y)), axis=2
) # (nb_instruments, 1, nb_steps, step_length, size, channels)
bar.update(l + 1)
bar.finish()
generated_midi_final_list = [
self.reshape_generated_array(generated_list[inst])
for inst in range(nb_instruments)
]
truth_final = self.reshape_generated_array(truth)
accuracies, accuracies_inst = self.compute_generated_array(
generated_array=truth_final,
folder_path=self.save_midis_path,
name='replicated_fill_truth',
no_duration=no_duration,
verbose=verbose,
save_images=save_images,
replicate=True)
accuracies, accuracies_inst = [accuracies], [accuracies_inst]
for inst in range(nb_instruments):
acc, acc_inst = self.compute_generated_array(
generated_array=generated_midi_final_list[inst],
folder_path=self.save_midis_path,
name=f'replicated_fill_{inst}',
no_duration=no_duration,
array_truth=truth_final,
verbose=verbose,
save_images=save_images,
save_truth=False,
replicate=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
if self.batch is not None:
self.sequence.batch_size = self.batch
# Save the image of all in a subplot to allow easier comparaisons
self.save_generated_arrays_cross_images(
generated_arrays=[truth_final] + generated_midi_final_list,
folder_path=self.save_midis_path,
name=f'replicated_fill_all',
replicate=True,
titles=['Truth'] +
[f'Fill Inst {i}' for i in range(nb_instruments)],
subtitles=[
f'Acc: {accuracies_inst[i][int(max(0, i - 1))]}'
for i in range(nb_instruments + 1)
] # Truth is in it
)
# Save the summary of the generation
summary.summarize(
# Function params
path=self.save_midis_path,
title=self.full_name,
file_name='replicate_fill_summary.txt',
# Summary params
length=max_length,
no_duration=no_duration,
noise=noise,
# Generic Summary
**self.summary_dict)
cprint('Done replicating (fill)', 'green')
def generate_from_data(self,
nb_seeds=10,
length=None,
new_save_path=None,
save_images=False,
no_duration=False,
verbose=1):
"""
Generate Midi file from the seed and the trained model
:param nb_seeds: number of seeds for the generation
:param length: Length of th generation
:param new_save_path:
:param save_images: To save the pianoroll of the generation (.jpg images)
:param no_duration: if True : all notes will be the shortest length possible
:param verbose: Level of verbose
:return:
"""
# ---------- Verify the inputs ----------
# ----- Create the seed -----
if self.data_transformed_path is None and self.data_test_transformed_path is None:
raise Exception('Some data need to be loaded before generating')
path = self.data_transformed_path if self.data_test_transformed_path is None else self.data_test_transformed_path
self.sequence_test = Sequences.AllInstSequence(path=path,
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on)
seeds = []
seeds_indexes = random.sample(range(len(self.sequence_test)), nb_seeds)
for s in range(nb_seeds):
seeds.append(
np.array(self.sequence_test[seeds_indexes[s]][0])
) # (nb_instruments, 1, nb_steps, step_size, input_size, channels)
seeds = np.concatenate(
seeds, axis=1
) # (nb_instruments, batch, nb_steps, step_size, input_size, channels)
# -- Length --
length = length if length is not None else 10
# --- Done Verifying the inputs ---
self.save_midis_path.mkdir(parents=True, exist_ok=True)
cprint('Start generating from data ...', 'blue')
generated_midi_final = self.generate_from_array(
array=seeds,
length=length,
) # (batch, nb_instruments, size, length, channels)
for s in range(nb_seeds):
self.compute_generated_array(
generated_array=generated_midi_final[s],
folder_path=self.save_midis_path,
name=f'generated_{s}',
no_duration=no_duration,
verbose=verbose,
save_images=save_images)
if self.batch is not None:
self.sequence.batch_size = self.batch
summary.summarize(
# Function params
path=self.save_midis_path,
title=self.full_name,
file_name='generate_summary.txt',
# Summary params
lenght=length,
no_duration=no_duration,
# Generic Summary
**self.summary_dict)
cprint('Done generating', 'green')
def compare_generation(self,
max_length=None,
no_duration=False,
verbose=1):
"""
:return:
"""
# -------------------- Find informations --------------------
if self.data_transformed_path is None and self.data_test_transformed_path is None:
raise Exception(
'Some data need to be loaded before comparing the generation')
path = self.data_transformed_path if self.data_test_transformed_path is None else self.data_test_transformed_path
self.sequence = Sequences.AllInstSequence(path=path,
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on,
noise=0)
max_length = len(self.sequence) if max_length is None else min(
max_length, len(self.sequence))
max_length = min(max_length, 10)
# -------------------- Construct seeds --------------------
generated = np.array(
self.sequence[0][0]
) # (nb_instrument, 1, nb_steps, step_size, input_size, 2) (1=batch)
generated_helped = np.copy(
generated) # Each step will take the truth as an input
generated_truth = np.copy(generated) # The truth
mask = self.get_mask(self.sequence.nb_instruments, batch_size=2)
# -------------------- Generation --------------------
cprint('Start comparing generation ...', 'blue')
bar = progressbar.ProgressBar(maxval=max_length,
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
for l in range(max_length):
ms_input, ms_output = self.sequence[l]
sample = np.concatenate(
(generated[:, :, l:], np.array(ms_input)), axis=1
) # (nb_instruments, 2, nb_steps, step_size, input_size, 2)
# Generation
preds = self.keras_nn.generate(input=list(sample) + mask)
# Reshape
preds = np.asarray(preds).astype(
'float64'
) # (nb_instruments, batch=2, nb_steps=1, length, 88, 2)
preds_truth = np.array(
ms_output) # (nb_instruments, 1, 1, step_size, input_size, 2)
# if only one instrument
if len(preds.shape
) == 5: # Only one instrument : output of nn not a list
preds = np.expand_dims(preds, axis=0)
if len(preds_truth.shape
) == 5: # Only one instrument : output of nn not a list
preds_truth = np.expand_dims(preds_truth)
preds = midi.create.normalize_activation(
preds, mono=self.mono,
use_binary=self.use_binary) # Normalize the activation part
preds_helped = preds[:,
[1]] # (nb_instruments, 1, 1, length, 88, 2)
preds = preds[:, [0]]
# Concatenation
generated = np.concatenate(
(generated, preds),
axis=2) # (nb_instruments, 1, nb_steps, length, 88, 2)
generated_helped = np.concatenate(
(generated_helped, preds_helped),
axis=2) # (nb_instruments, 1, nb_steps, length, 88, 2)
generated_truth = np.concatenate((generated_truth, preds_truth),
axis=2)
bar.update(l + 1)
bar.finish()
# -------------------- Compute notes list --------------------
# Generated
generated_midi_final = self.reshape_generated_array(generated)
# Helped
generated_midi_final_helped = self.reshape_generated_array(
generated_helped)
# Truth
generated_midi_final_truth = self.reshape_generated_array(
generated_truth)
# ---------- find the name for the midi_file ----------
self.save_midis_path.mkdir(parents=True, exist_ok=True)
accuracies, accuracies_inst = [], []
# Generated
acc, acc_inst = self.compute_generated_array(
generated_array=generated_midi_final,
folder_path=self.save_midis_path,
name='compare_generation_alone',
no_duration=no_duration,
array_truth=generated_midi_final_truth,
verbose=verbose,
save_truth=False,
save_images=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
# Helped
acc, acc_inst = self.compute_generated_array(
generated_array=generated_midi_final_helped,
folder_path=self.save_midis_path,
name='compare_generation_helped',
no_duration=no_duration,
array_truth=generated_midi_final_truth,
verbose=verbose,
save_truth=False,
save_images=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
# Truth
self.compute_generated_array(
generated_array=generated_midi_final_truth,
folder_path=self.save_midis_path,
name='compare_generation_truth',
no_duration=no_duration,
array_truth=None,
verbose=verbose,
save_truth=False,
save_images=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
# Save the image of all in a subplot to allow easier comparaisons
self.save_generated_arrays_cross_images(
generated_arrays=[
generated_midi_final_truth, generated_midi_final_helped,
generated_midi_final
],
folder_path=self.save_midis_path,
name=f'compare_generation_all',
replicate=False,
titles=['Truth', 'Helped', 'Alone'],
subtitles=[
'Acc : 1',
f'Acc: {accuracies[1]:.3}, Acc_inst: [{", ".join([f"{a:.3}" for a in accuracies_inst[1]])}]',
f'Acc: {accuracies[0]:.3}, Acc_inst: [{", ".join([f"{a:.3}" for a in accuracies_inst[0]])}]'
] # Truth is in it
)
# ----- Summarize the generation -----
# Creation of the summary .txt file
summary.summarize(
# Function parameters
path=self.save_midis_path,
title=self.full_name,
file_name='compare_generation_summary.txt',
# Summary paramters,
length=max_length,
no_duration=no_duration,
generated_accuracy=accuracies[0],
generated_accuracies=accuracies_inst[0],
helped_accuracy=accuracies[1],
helped_accuracies=accuracies_inst[1],
# Generic Summary
**self.summary_dict)
cprint('Done comparing generation', 'green')
def generate_fill(self, max_length=None, no_duration=False, verbose=1):
"""
:param max_length:
:param no_duration:
:param verbose:
:return:
"""
# ----- Parameters -----
max_length = 300 / self.step_length if max_length is None else max_length
# ----- Variables -----
if self.data_transformed_path is None and self.data_test_transformed_path is None:
raise Exception(
'Some data need to be loaded before comparing the generation')
path = self.data_transformed_path if self.data_test_transformed_path is None else self.data_test_transformed_path
sequence = Sequences.KerasSequence(
path=path,
nb_steps=self.nb_steps,
batch_size=1,
work_on=self.work_on
) # Return array instead of list (for instruments)
max_length = int(min(max_length, len(sequence)))
nb_instruments = sequence.nb_instruments
# ----- Seeds -----
truth = sequence[0][0]
filled_list = [np.copy(truth) for inst in range(nb_instruments)]
mask = np.ones((nb_instruments, nb_instruments, self.nb_steps))
for inst in range(nb_instruments):
filled_list[inst][inst] = 0
mask[inst, inst] = 0
# ----- Generation -----
cprint('Start generating (fill) ...', 'blue')
bar = progressbar.ProgressBar(maxval=max_length,
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
bar.start() # To see it working
for l in range(max_length):
s_input, s_output = sequence[l]
to_fill_list = [np.copy(s_input) for inst in range(nb_instruments)]
for inst in range(nb_instruments):
to_fill_list[inst][inst] = 0
nn_input = np.concatenate(
tuple(to_fill_list), axis=1
) # (nb_instruments, batch=nb_instruments, nb_steps, step_size, input_size, channels)
preds = self.keras_nn.generate(input=list(nn_input) + [mask])
preds = np.asarray(preds).astype(
'float64'
) # (nb_instruments, bath=nb_instruments, nb_steps=1, step_size, input_size, channels)
if len(preds.shape
) == 5: # Only one instrument : output of nn not a list
preds = np.expand_dims(preds, axis=0)
if len(s_output.shape
) == 5: # Only one instrument : output of nn not a list
s_output = np.expand_dims(s_output)
preds = midi.create.normalize_activation(
preds, mono=self.mono, use_binary=self.use_binary)
truth = np.concatenate((truth, s_output), axis=2)
for inst in range(nb_instruments):
p = np.copy(s_output)
p[inst] = np.take(preds, axis=1, indices=[inst])[inst]
filled_list[inst] = np.concatenate(
(filled_list[inst], p), axis=2
) # (nb_instruments, batch=1, nb_steps, step_size, input_size, channels)
bar.update(l + 1)
bar.finish()
# -------------------- Compute notes list --------------------
# ----- Reshape -----
truth = self.reshape_generated_array(truth)
for inst in range(nb_instruments):
filled_list[inst] = self.reshape_generated_array(filled_list[inst])
self.save_midis_path.mkdir(parents=True, exist_ok=True)
accuracies, accuracies_inst = self.compute_generated_array(
generated_array=truth,
folder_path=self.save_midis_path,
name='generated_fill_truth',
no_duration=no_duration,
verbose=verbose,
save_images=True)
accuracies, accuracies_inst = [accuracies], [accuracies_inst]
for inst in range(nb_instruments):
acc, acc_inst = self.compute_generated_array(
generated_array=filled_list[inst],
folder_path=self.save_midis_path,
name=f'generated_fill_{inst}',
no_duration=no_duration,
array_truth=truth,
verbose=verbose,
save_truth=False,
save_images=True)
accuracies.append(acc)
accuracies_inst.append(acc_inst)
# Save the image of all in a subplot to allow easier comparaisons
self.save_generated_arrays_cross_images(
generated_arrays=[truth] + filled_list,
folder_path=self.save_midis_path,
name=f'generated_fill_all',
replicate=False,
titles=['Truth'] +
[f'Fill Inst {i}' for i in range(nb_instruments)],
subtitles=[
f'Acc: {accuracies_inst[i][int(max(0, i - 1))]}'
for i in range(nb_instruments + 1)
] # Truth is in it
)
# Save the summary of the generation
summary.summarize(
# Function parameters
path=self.save_midis_path,
title=self.full_name,
file_name='generate_fill_summary.txt',
# Summary parameters
length=max_length,
no_duration=no_duration,
# Generic Summary
**self.summary_dict)
cprint('Done generating (fill)', 'green')
def generate_from_noise(self,
nb_seeds=10,
length=None,
new_save_path=None,
save_images=False,
no_duration=False,
verbose=1):
"""
Generate Midi file from the seed and the trained model
:param nb_seeds: number of seeds for the generation
:param length: Length of th generation
:param new_save_path:
:param save_images: To save the pianoroll of the generation (.jpg images)
:param no_duration: if True : all notes will be the shortest length possible
:param verbose: Level of verbose
:return:
"""
# ---------- Verify the inputs ----------
# ----- Create the seed -----
random_type = ['gaussian', 'softmax', 'binary']
for t in random_type:
if t == 'gaussian':
seeds = np.random.normal(
loc=0.5,
scale=1.0,
size=(self.nb_instruments, nb_seeds, self.nb_steps,
self.step_length, self.input_size, self.nb_channels))
else:
seeds = np.zeros(
(self.nb_instruments, nb_seeds, self.nb_steps,
self.step_length, self.input_size, self.nb_channels))
for inst in range(self.nb_instruments):
for s in range(nb_seeds):
for step in range(self.nb_steps):
for i in range(self.step_length):
if t == 'softmax':
note = np.random.randint(self.input_size)
else:
bin = np.random.randint(2)
if bin == 0:
note = self.input_size - 1
else:
note = np.random.randint(
self.input_size - 1)
seeds[inst, s, step, i, note] = 1
# -- Length --
length = length if length is not None else 10
# --- Done Verifying the inputs ---
self.save_midis_path.mkdir(parents=True, exist_ok=True)
cprint(f'Start generating from {t} noise ...', 'blue')
generated_midi_final = self.generate_from_array(
array=seeds,
length=length,
) # (batch, nb_instruments, size, length, channels)
for s in range(nb_seeds):
self.compute_generated_array(
generated_array=generated_midi_final[s],
folder_path=self.save_midis_path,
name=f'generated_noise_{t}_{s}',
no_duration=no_duration,
verbose=verbose,
save_images=save_images)
if self.batch is not None:
self.sequence.batch_size = self.batch
summary.summarize(
# Function params
path=self.save_midis_path,
title=self.full_name,
file_name='generate_noise_summary.txt',
# Summary params
lenght=length,
no_duration=no_duration,
# Generic Summary
**self.summary_dict)
cprint('Done generating from noise', 'green')