]
config_folder_bds = [
'/fast-1/leo/automatic_arrangement/Saved_models/LSTM_bd/' + str(e)
for e in range(10)
]
config_folder_corrs = [
'/fast-1/leo/automatic_arrangement/Saved_models/LSTM_corr/' + str(e)
for e in range(10)
]
for config_folder_fd, config_folder_bd, config_folder_corr in zip(
config_folder_fds, config_folder_bds, config_folder_corrs):
score_sources = []
with open(config_folder_fd + '/test_names.txt', 'r') as ff:
for row in ff:
score_sources.append(config.database_root() + "/" +
row.strip('\n'))
logger = logging.getLogger('worker')
logger.setLevel(logging.INFO)
for initialization_type in ["zeros", "random", "constant"]:
for score_source in score_sources:
generate_midi(config_folder_fd,
config_folder_bd,
config_folder_corr,
score_source,
save_folder,
initialization_type=initialization_type,
number_of_version=3,
duration_gen=100,
num_pass_correct=5,
logger_generate=logger)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Set up
# NOTE : can't do data augmentation with K-folds, or it would require to build K times the database
# because train is data augmented but not test and validate
temporal_granularity = 'event_level'
quantization = 8
binary_piano = True
binary_orch = True
# Database have to be built jointly so that the ranges match
database_orchestration = config.database_root()
database_arrangement = config.database_pretraining_root()
##############################
# Subsets
subset_A = [
database_orchestration + "/liszt_classical_archives",
]
subset_B = [
database_orchestration + "/bouliane",
database_orchestration + "/hand_picked_Spotify",
database_orchestration + "/imslp"
]
subset_C = [
def submit_job(config_folder_fold, parameters, model_params, dimensions,
K_fold, train_names, valid_names, test_names, save_bool,
generate_bool, local, logger):
context_folder = config_folder_fold + '/context'
os.mkdir(context_folder)
# Build track path generation
track_paths_generation = [
config.database_root() + '/' + e for e in test_names
]
if len(track_paths_generation) > 10:
track_paths_generation = track_paths_generation[:10]
# Save all the arguments of the wrapper script
pkl.dump(parameters, open(context_folder + "/parameters.pkl", 'wb'))
pkl.dump(model_params, open(context_folder + '/model_params.pkl', 'wb'))
pkl.dump(dimensions, open(context_folder + '/dimensions.pkl', 'wb'))
pkl.dump(K_fold, open(context_folder + '/K_fold.pkl', 'wb'))
pkl.dump(test_names, open(context_folder + '/test_names.pkl', 'wb'))
pkl.dump(track_paths_generation,
open(context_folder + '/track_paths_generation.pkl', 'wb'))
pkl.dump(save_bool, open(context_folder + '/save_bool.pkl', 'wb'))
pkl.dump(generate_bool, open(context_folder + '/generate_bool.pkl', 'wb'))
# Write filenames of this split
with open(os.path.join(config_folder_fold, "train_names.txt"), "w") as f:
for filename in train_names:
f.write(filename + "\n")
with open(os.path.join(config_folder_fold, "test_names.txt"), "w") as f:
for filename in test_names:
f.write(filename + "\n")
with open(os.path.join(config_folder_fold, "valid_names.txt"), "w") as f:
for filename in valid_names:
f.write(filename + "\n")
if local:
# subprocess.check_output('python train_wrapper.py ' + config_folder_fold, shell=True)
train_wrapper.train_wrapper(parameters, model_params, dimensions,
config_folder_fold, K_fold,
track_paths_generation, save_bool,
generate_bool, logger)
job_id = -1
else:
# Write pbs script
file_pbs = context_folder + '/submit.pbs'
split_config_folder_fold = re.split('/', config_folder_fold)
script_name = split_config_folder_fold[
-4] + "__" + split_config_folder_fold[
-3] + "__" + split_config_folder_fold[
-2] + "__" + split_config_folder_fold[-1]
text_pbs = """#!/bin/bash
#PBS -j oe
#PBS -N job_outputs/""" + script_name + """
#PBS -l nodes=1:ppn=2:gpus=1
#PBS -l walltime=""" + str(parameters['walltime']) + """:00:00
source /software/soft.computecanada.ca.sh
module load python/3.6
source ~/Virtualenvs/tf_3/bin/activate
SRC=/home/crestel/acids/team/leopold/lop/Source/LOP/Scripts
cd $SRC
python train_wrapper.py '""" + config_folder_fold + "'"
with open(file_pbs, 'w') as f:
f.write(text_pbs)
job_id = subprocess.check_output('qsub ' + file_pbs, shell=True)
return job_id
continue
split_name = re.split("/", new_name_piano)
folder_name = "Piano_files_for_embeddings/" + split_name[-3]
file_name = split_name[-1] + '.mid'
if not os.path.isdir(folder_name):
os.makedirs(folder_name)
write_midi(new_pr_piano, 1000, folder_name + '/' + file_name, tempo=80)
return
if __name__ == '__main__':
temporal_granularity='event_level'
quantization=8
pretraining_bool=True
# Database have to be built jointly so that the ranges match
DATABASE_PATH = config.database_root()
DATABASE_PATH_PRETRAINING = config.database_pretraining_root()
DATABASE_NAMES = [
DATABASE_PATH + "/bouliane",
DATABASE_PATH + "/hand_picked_Spotify",
DATABASE_PATH + "/liszt_classical_archives",
DATABASE_PATH + "/imslp"
# DATABASE_PATH_PRETRAINING + "/OpenMusicScores",
# DATABASE_PATH_PRETRAINING + "/Kunstderfuge",
# DATABASE_PATH_PRETRAINING + "/Musicalion",
# DATABASE_PATH_PRETRAINING + "/Mutopia"
]
DATABASE_NAMES_PRETRAINING = [
# DATABASE_PATH_PRETRAINING + "/OpenMusicScores",
def no_valid_tracks():
no_valid_tracks = [
# Too good
os.path.join(config.database_root(), "hand_picked_Spotify/40"),
os.path.join(config.database_root(), "hand_picked_Spotify/45"),
os.path.join(config.database_root(), "imslp/21"),
os.path.join(config.database_root(), "imslp/43"),
os.path.join(config.database_root(), "imslp/20"),
os.path.join(config.database_root(), "imslp/44"),
os.path.join(config.database_root(), "imslp/22"),
os.path.join(config.database_root(), "imslp/12"),
os.path.join(config.database_root(), "imslp/14"),
os.path.join(config.database_root(), "imslp/62"),
os.path.join(config.database_root(), "imslp/68"),
os.path.join(config.database_root(), "imslp/39"),
os.path.join(config.database_root(), "imslp/15"),
os.path.join(config.database_root(), "imslp/26"),
os.path.join(config.database_root(), "imslp/71"),
os.path.join(config.database_root(), "imslp/3"),
os.path.join(config.database_root(), "imslp/78"),
os.path.join(config.database_root(), "imslp/11"),
os.path.join(config.database_root(), "imslp/86"),
os.path.join(config.database_root(), "imslp/16"),
os.path.join(config.database_root(), "imslp/25"),
os.path.join(config.database_root(), "imslp/56"),
os.path.join(config.database_root(), "imslp/77"),
os.path.join(config.database_root(), "imslp/5"),
os.path.join(config.database_root(), "imslp/23"),
os.path.join(config.database_root(), "imslp/45"),
os.path.join(config.database_root(), "imslp/50"),
os.path.join(config.database_root(), "imslp/64"),
os.path.join(config.database_root(), "debug/1"),
os.path.join(config.database_root(), "debug/2"),
]
# All IMSLP files
# imslp_files = glob.glob(config.database_root() + '/imslp/[0-9]*')
# training_avoid += imslp_files
tracks_with_too_few_instruments = []
# with open(config.data_root() + "/few_instrument_files_pretraining.txt", 'rb') as ff:
# for line in ff:
# tracks_with_too_few_instruments.append(os.path.join(config.database_pretraining_root(), line.rstrip("\n")))
with open(config.data_root() + "/few_instrument_files.txt", 'r') as ff:
for line in ff:
tracks_with_too_few_instruments.append(
os.path.join(config.database_root(), line.rstrip("\n")))
return no_valid_tracks + tracks_with_too_few_instruments
