def main(args):
# To have colors from the first script
# ----------------------------------------
# Check that all the folder are available
# ----------------------------------------
if not args.in_place:
if args.from_checkpoint is not None:
if len(args.from_checkpoint.split('-')) == 1:
name = 'bayesian_opt'
else:
name = f'bayesian_opt_{"_".join([str(s) for s in args.from_checkpoint.split("-")[:-1]])}'
i_start = int(str(args.from_checkpoint).split('-')[-1]) + 1
else:
i_start = 0
print('bo name', args.bo_name)
name = 'bayesian_opt' if args.bo_name is None else f'bayesian_opt_{args.bo_name}'
cprint('This script will need the next paths to be available:', 'red')
for i in range(i_start, i_start + args.nscripts):
path = EPath('hp_search', name + f'_{i}')
cprint('path ' + path.as_posix(), 'yellow')
if path.exists():
raise FileExistsError(f'The folder "{path}" already exists')
# ----------------------------------------
# For the first Bayesian Optimization run, we just copy all the arguments except n-script
# ----------------------------------------
print(f'Script 1/{args.nscripts}')
s = 'python bayesian-opt.py'
for k, value in vars(args).items():
if k not in ['nscripts', 'from_checkpoint', 'bo_name']:
if k not in [
'in_place', 'no_eager', 'pc', 'no_pc_arg', 'debug',
'seq2np', 'fast_seq', 'memory_seq', 'mono',
'from_checkpoint'
]:
s += f' --{k.replace("_", "-")} {value}'
elif value:
s += f' --{k.replace("_", "-")}'
if args.from_checkpoint is not None:
s += f' --from-checkpoint {args.from_checkpoint}'
if args.bo_name is not None:
s += f' --bo-name {args.bo_name}'
os.system(s)
# ----------------------------------------
# For all the others
# ----------------------------------------
if args.from_checkpoint:
cp_id = args.from_checkpoint.split('-')
id_saved_folder_base = '' if len(cp_id) == 1 else cp_id[0]
elif args.bo_name:
id_saved_folder_base = args.bo_name + '-'
else:
id_saved_folder_base = ''
i_cp = int(args.from_checkpoint.split('-')
[-1]) if args.from_checkpoint is not None else 0
if not args.in_place and args.from_checkpoint is not None:
i_cp += 1
for script in range(1, args.nscripts):
print(f'Script {script + 1}/{args.nscripts}')
s = 'python bayesian-opt.py'
# We only need to say the number of call and the check point
# And the if in_place, we keep it
s += f' --n-calls {args.n_calls}'
s += f' --from-checkpoint {id_saved_folder_base}{i_cp}'
if args.in_place:
s += ' --in-place'
else:
i_cp += 1
os.system(s)
class LossHistory(KerasCallback):
def __init__(self):
super().__init__()
self.logs = [
] # logs = {'loss': 4.495124205389112, 'Output_0_loss' : 2.400269569744329,
# 'Output_1_loss': 2.094854634212782, 'Output_0_acc_act': 0.9934636604502837,
# 'Output_0_mae_dur': 0.2902308425676854, 'Output_1_acc_act': 0.9946330100062025,
# 'Output_1_mae_dur': 0.25196381778232657}
self.current_logs = None
self.paths = [] # Where are stored the saved_models
self.hparams = [] # the hyper parameters of the model
self.best_index = None
i = 0
while EPath('tests_hp/Summary_test_{0}.txt'.format(i)).exists():
i += 1
self.path = EPath('tests_hp/Summary_test_{0}.txt'.format(i))
EPath('tests_hp').mkdir(parents=True, exist_ok=True)
with open(self.path.as_posix(), 'w') as f:
f.write('\n')
def on_train_begin(self, logs={}):
self.current_logs = None
def on_epoch_end(self, epoch, logs={}):
self.current_logs = logs
def on_train_end(self, logs=None):
self.logs.append(self.current_logs)
# Update the best index
if len(self.logs) == 1:
self.best_index = 0
elif LossHistory.better_than(self.logs[-1],
self.logs[self.best_index]):
self.best_index = len(self.logs) - 1
# ------ Personal methods ------
def find_best_index(self):
best_index = 0
for i in range(1, len(self.logs)):
if LossHistory.better_than(self.logs[best_index], self.logs[i]):
best_index = i
self.best_index = best_index
return self.best_index
@staticmethod
def better_than(d1, d2):
# Global loss
if d1['loss'] != d2['loss']:
return d1['loss'] < d2['loss']
# Square of single loss
l1, l2 = 0, 0
i = 0
key = 'Output_{0}_loss'.format(i)
while key in d1 and key in d2:
l1 += d1[key]**2
l2 += d2[key]**2
i += 1
key = 'Output_{0}_loss'.format(i)
if l1 != l2:
return l1 < l2
# Accuracy
a1, a2 = 0, 0
i = 0
key = 'Output_{0}_acc_act'.format(i)
while key in d1 and key in d2:
a1 += d1[key]
a2 += d2[key]
i += 1
key = 'Output_{0}_acc_act'.format(i)
return a1 >= a2
def update_summary(self, i=None):
summary.update_summary_loss_history(self.path.as_posix(),
self.logs[-1], self.paths[-1],
self.hparams[-1], i)
def update_best_summary(self):
summary.update_best_summary_loss_history(self.path.as_posix(),
self.logs[self.best_index],
self.paths[self.best_index],
self.hparams[self.best_index],
self.best_index)
def main(args):
"""
Entry point
"""
if args.pc:
data_path = EPath(os.path.join('../Dataset', args.data))
else:
data_path = EPath(
os.path.join('../../../../../../storage1/valentin', args.data))
data_checked_path = EPath(data_path.as_posix() + '_checked')
if data_checked_path.exists(): # Delete the folder of the transformed data
shutil.rmtree(data_checked_path.as_posix())
shutil.copytree(src=data_path.as_posix(), dst=data_checked_path)
# Stat
max_notes_range, min_notes_range = 0, int(args.notes_range[1]) - int(
args.notes_range[0])
nb_correct_files = 0
nb_measures = 0
print(
'\t',
colored('Check_dataset with instruments : ', 'cyan', 'on_white') +
colored('{0}'.format(args.instruments), 'magenta', 'on_white'), '\n')
all_midi_paths = Midi.open.all_midi_files(data_checked_path.as_posix(),
False)
nb_files = len(all_midi_paths)
print('note_range:', colored(args.notes_range, 'magenta'))
for i in range(nb_files):
midi_path = EPath(all_midi_paths[i])
checked_file_name = EPath(
midi_path.parent, midi_path.stem + '_checked' + midi_path.suffix)
checked_file_name_image = EPath(midi_path.parent,
midi_path.stem + '_checked.jpg')
print(
colored(
"-- {0}/{1} ----- : ----- Checking {2} ----------".format(
i + 1, nb_files, midi_path), 'white', 'on_blue'))
if args.bach:
matrix_midi = Midi.open.midi_to_matrix_bach(
filename=midi_path.as_posix(),
print_instruments=True,
notes_range=args.notes_range,
transpose=not args.no_transpose)
else:
matrix_midi = Midi.open.midi_to_matrix(
filename=midi_path.as_posix(),
instruments=args.instruments,
print_instruments=True,
notes_range=args.notes_range,
) # (nb_args.instruments, 128, nb_steps, 2)
if matrix_midi is None:
continue
if args.mono:
matrix_midi = Midi.open.to_mono_matrix(matrix_midi)
# Update stats
matrix_bound_min, matrix_bound_max = Midi.common.range_notes_in_matrix(
matrix_midi, mono=args.mono)
min_notes_range = min(min_notes_range, matrix_bound_min)
max_notes_range = max(max_notes_range, matrix_bound_max)
nb_correct_files += 1
nb_measures += Midi.common.nb_measures(matrix_midi)
# matrix_midi = np.transpose(matrix_midi, , 3))
output_notes = Midi.create.matrix_to_midi(matrix_midi,
instruments=args.instruments,
notes_range=args.notes_range,
mono=args.mono)
Midi.create.save_midi(output_notes, args.instruments,
checked_file_name.as_posix())
Images.pianoroll.save_array_as_pianoroll(array=matrix_midi,
folder_path=midi_path.parent,
name=midi_path.stem +
'_checked',
seed_length=0,
mono=args.mono or args.bach)
min_notes_range += args.notes_range[0]
max_notes_range += args.notes_range[0]
print('----------', colored('Checking is done', 'white', 'on_green'),
'----------')
print('Number of correct files :', colored(nb_correct_files, 'magenta'),
'-- nb measures :', colored(nb_measures, 'magenta'))
print('Range of notes :', colored(min_notes_range, 'magenta'), ':',
colored(max_notes_range, 'magenta'))