] config_folder_bds = [ '/fast-1/leo/automatic_arrangement/Saved_models/LSTM_bd/' + str(e) for e in range(10) ] config_folder_corrs = [ '/fast-1/leo/automatic_arrangement/Saved_models/LSTM_corr/' + str(e) for e in range(10) ] for config_folder_fd, config_folder_bd, config_folder_corr in zip( config_folder_fds, config_folder_bds, config_folder_corrs): score_sources = [] with open(config_folder_fd + '/test_names.txt', 'r') as ff: for row in ff: score_sources.append(config.database_root() + "/" + row.strip('\n')) logger = logging.getLogger('worker') logger.setLevel(logging.INFO) for initialization_type in ["zeros", "random", "constant"]: for score_source in score_sources: generate_midi(config_folder_fd, config_folder_bd, config_folder_corr, score_source, save_folder, initialization_type=initialization_type, number_of_version=3, duration_gen=100, num_pass_correct=5, logger_generate=logger)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) # Set up # NOTE : can't do data augmentation with K-folds, or it would require to build K times the database # because train is data augmented but not test and validate temporal_granularity = 'event_level' quantization = 8 binary_piano = True binary_orch = True # Database have to be built jointly so that the ranges match database_orchestration = config.database_root() database_arrangement = config.database_pretraining_root() ############################## # Subsets subset_A = [ database_orchestration + "/liszt_classical_archives", ] subset_B = [ database_orchestration + "/bouliane", database_orchestration + "/hand_picked_Spotify", database_orchestration + "/imslp" ] subset_C = [
def submit_job(config_folder_fold, parameters, model_params, dimensions, K_fold, train_names, valid_names, test_names, save_bool, generate_bool, local, logger): context_folder = config_folder_fold + '/context' os.mkdir(context_folder) # Build track path generation track_paths_generation = [ config.database_root() + '/' + e for e in test_names ] if len(track_paths_generation) > 10: track_paths_generation = track_paths_generation[:10] # Save all the arguments of the wrapper script pkl.dump(parameters, open(context_folder + "/parameters.pkl", 'wb')) pkl.dump(model_params, open(context_folder + '/model_params.pkl', 'wb')) pkl.dump(dimensions, open(context_folder + '/dimensions.pkl', 'wb')) pkl.dump(K_fold, open(context_folder + '/K_fold.pkl', 'wb')) pkl.dump(test_names, open(context_folder + '/test_names.pkl', 'wb')) pkl.dump(track_paths_generation, open(context_folder + '/track_paths_generation.pkl', 'wb')) pkl.dump(save_bool, open(context_folder + '/save_bool.pkl', 'wb')) pkl.dump(generate_bool, open(context_folder + '/generate_bool.pkl', 'wb')) # Write filenames of this split with open(os.path.join(config_folder_fold, "train_names.txt"), "w") as f: for filename in train_names: f.write(filename + "\n") with open(os.path.join(config_folder_fold, "test_names.txt"), "w") as f: for filename in test_names: f.write(filename + "\n") with open(os.path.join(config_folder_fold, "valid_names.txt"), "w") as f: for filename in valid_names: f.write(filename + "\n") if local: # subprocess.check_output('python train_wrapper.py ' + config_folder_fold, shell=True) train_wrapper.train_wrapper(parameters, model_params, dimensions, config_folder_fold, K_fold, track_paths_generation, save_bool, generate_bool, logger) job_id = -1 else: # Write pbs script file_pbs = context_folder + '/submit.pbs' split_config_folder_fold = re.split('/', config_folder_fold) script_name = split_config_folder_fold[ -4] + "__" + split_config_folder_fold[ -3] + "__" + split_config_folder_fold[ -2] + "__" + split_config_folder_fold[-1] text_pbs = """#!/bin/bash #PBS -j oe #PBS -N job_outputs/""" + script_name + """ #PBS -l nodes=1:ppn=2:gpus=1 #PBS -l walltime=""" + str(parameters['walltime']) + """:00:00 source /software/soft.computecanada.ca.sh module load python/3.6 source ~/Virtualenvs/tf_3/bin/activate SRC=/home/crestel/acids/team/leopold/lop/Source/LOP/Scripts cd $SRC python train_wrapper.py '""" + config_folder_fold + "'" with open(file_pbs, 'w') as f: f.write(text_pbs) job_id = subprocess.check_output('qsub ' + file_pbs, shell=True) return job_id
continue split_name = re.split("/", new_name_piano) folder_name = "Piano_files_for_embeddings/" + split_name[-3] file_name = split_name[-1] + '.mid' if not os.path.isdir(folder_name): os.makedirs(folder_name) write_midi(new_pr_piano, 1000, folder_name + '/' + file_name, tempo=80) return if __name__ == '__main__': temporal_granularity='event_level' quantization=8 pretraining_bool=True # Database have to be built jointly so that the ranges match DATABASE_PATH = config.database_root() DATABASE_PATH_PRETRAINING = config.database_pretraining_root() DATABASE_NAMES = [ DATABASE_PATH + "/bouliane", DATABASE_PATH + "/hand_picked_Spotify", DATABASE_PATH + "/liszt_classical_archives", DATABASE_PATH + "/imslp" # DATABASE_PATH_PRETRAINING + "/OpenMusicScores", # DATABASE_PATH_PRETRAINING + "/Kunstderfuge", # DATABASE_PATH_PRETRAINING + "/Musicalion", # DATABASE_PATH_PRETRAINING + "/Mutopia" ] DATABASE_NAMES_PRETRAINING = [ # DATABASE_PATH_PRETRAINING + "/OpenMusicScores",
def no_valid_tracks(): no_valid_tracks = [ # Too good os.path.join(config.database_root(), "hand_picked_Spotify/40"), os.path.join(config.database_root(), "hand_picked_Spotify/45"), os.path.join(config.database_root(), "imslp/21"), os.path.join(config.database_root(), "imslp/43"), os.path.join(config.database_root(), "imslp/20"), os.path.join(config.database_root(), "imslp/44"), os.path.join(config.database_root(), "imslp/22"), os.path.join(config.database_root(), "imslp/12"), os.path.join(config.database_root(), "imslp/14"), os.path.join(config.database_root(), "imslp/62"), os.path.join(config.database_root(), "imslp/68"), os.path.join(config.database_root(), "imslp/39"), os.path.join(config.database_root(), "imslp/15"), os.path.join(config.database_root(), "imslp/26"), os.path.join(config.database_root(), "imslp/71"), os.path.join(config.database_root(), "imslp/3"), os.path.join(config.database_root(), "imslp/78"), os.path.join(config.database_root(), "imslp/11"), os.path.join(config.database_root(), "imslp/86"), os.path.join(config.database_root(), "imslp/16"), os.path.join(config.database_root(), "imslp/25"), os.path.join(config.database_root(), "imslp/56"), os.path.join(config.database_root(), "imslp/77"), os.path.join(config.database_root(), "imslp/5"), os.path.join(config.database_root(), "imslp/23"), os.path.join(config.database_root(), "imslp/45"), os.path.join(config.database_root(), "imslp/50"), os.path.join(config.database_root(), "imslp/64"), os.path.join(config.database_root(), "debug/1"), os.path.join(config.database_root(), "debug/2"), ] # All IMSLP files # imslp_files = glob.glob(config.database_root() + '/imslp/[0-9]*') # training_avoid += imslp_files tracks_with_too_few_instruments = [] # with open(config.data_root() + "/few_instrument_files_pretraining.txt", 'rb') as ff: # for line in ff: # tracks_with_too_few_instruments.append(os.path.join(config.database_pretraining_root(), line.rstrip("\n"))) with open(config.data_root() + "/few_instrument_files.txt", 'r') as ff: for line in ff: tracks_with_too_few_instruments.append( os.path.join(config.database_root(), line.rstrip("\n"))) return no_valid_tracks + tracks_with_too_few_instruments