def __build_folds(self, total_number_folds, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool): # Load files lists t_dict_A = pkl.load(open(self.database_path + '/train_only_A.pkl', 'rb')) tvt_dict_A = pkl.load(open(self.database_path + '/train_and_valid_A.pkl', 'rb')) t_dict_B = pkl.load(open(self.database_path + '/train_only_B.pkl', 'rb')) tv_dict_B = pkl.load(open(self.database_path + '/train_and_valid_B.pkl', 'rb')) if total_number_folds == -1: total_number_folds = len(tvt_dict_A.keys()) folds_A = [] train_names_A = [] valid_names_A = [] test_names_A = [] # Pretraining is done one time on one single fold (8/1/1 split -> number of k-folds = 10) folds_B, train_names_B, valid_names_B, test_names_B = build_one_fold(0, 10, t_dict_B, tv_dict_B, {}, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) # Build the list of split_matrices for current_fold_number in range(total_number_folds): one_fold, this_train_names, this_valid_names, this_test_names = build_one_fold(current_fold_number, total_number_folds, t_dict_A, {}, tvt_dict_A, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) folds_A.append(one_fold) train_names_A.append(this_train_names) valid_names_A.append(this_valid_names) test_names_A.append(this_test_names) return folds_A, train_names_A, valid_names_A, test_names_A, folds_B, train_names_B, valid_names_B, test_names_B
def __build_folds(self, total_number_folds, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool): # Load files lists t_dict_A = pkl.load( open(self.database_path + '/train_only_A.pkl', 'rb')) tv_dict_A = pkl.load( open(self.database_path + '/train_and_valid_A.pkl', 'rb')) t_dict_B = pkl.load( open(self.database_path + '/train_only_B.pkl', 'rb')) tvt_dict_B = pkl.load( open(self.database_path + '/train_and_valid_B.pkl', 'rb')) if total_number_folds == -1: total_number_folds = len(tvt_dict_A.keys()) folds_step0 = [] train_names_0 = [] valid_names_0 = [] test_names_0 = [] folds_step1 = [] train_names_1 = [] valid_names_1 = [] test_names_1 = [] # Build the list of split_matrices for current_fold_number in range(total_number_folds): # Pretraining is done one time on one single fold (8/1/1 split -> number of k-folds = 10) one_fold_B, this_train_names_B, this_valid_names_B, this_test_names_B = build_one_fold( current_fold_number, total_number_folds, t_dict_B, {}, tvt_dict_B, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) one_fold_A, this_train_names_A, this_valid_names_A, this_test_names_A = build_one_fold( current_fold_number, total_number_folds, t_dict_A, tv_dict_A, {}, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) folds_step0.append(one_fold_B) train_names_0.append(this_train_names_B) valid_names_0.append(this_valid_names_B) test_names_0.append(this_test_names_B) one_fold_step1 = { 'train': one_fold_A['train'] + one_fold_A['test'], # we can use test data in this case 'valid': one_fold_A['valid'], 'test': one_fold_B['test'] } folds_step1.append(one_fold_step1) train_names_1.append(this_train_names_A + this_test_names_A) valid_names_1.append(this_valid_names_A) test_names_1.append(this_test_names_B) return folds_step0, train_names_0, valid_names_0, test_names_0, folds_step1, train_names_1, valid_names_1, test_names_1
def __build_folds(self, total_number_folds, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool): # Load files lists t_dict_A = pkl.load(open(self.database_path + '/train_only_A.pkl', 'rb')) tvt_dict_A = pkl.load(open(self.database_path + '/train_and_valid_A.pkl', 'rb')) t_dict_B = pkl.load(open(self.database_path + '/train_only_B.pkl', 'rb')) tvt_dict_B = pkl.load(open(self.database_path + '/train_and_valid_B.pkl', 'rb')) t_dict = {} # Init to avoid copy by reference t_dict.update(t_dict_A) t_dict.update(t_dict_B) tv_dict = {} tv_dict.update(tvt_dict_B) # Don't use B for testing tvt_dict = {} tvt_dict.update(tvt_dict_A) if total_number_folds == -1: total_number_folds = len(tvt_dict.keys()) folds = [] train_names = [] valid_names = [] test_names = [] # Build the list of split_matrices for current_fold_number in range(total_number_folds): one_fold, this_train_names, this_valid_names, this_test_names = build_one_fold(current_fold_number, total_number_folds, t_dict, tv_dict, tvt_dict, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) folds.append(one_fold) train_names.append(this_train_names) valid_names.append(this_valid_names) test_names.append(this_test_names) return folds, train_names, valid_names, test_names
def __build_folds(self, total_number_folds, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, dataset_id): # Load files lists t_dict = pkl.load(open(self.database_path + '/train_only_' + dataset_id + '.pkl', 'rb')) tv_dict = {} tvt_dict = pkl.load(open(self.database_path + '/train_and_valid_' + dataset_id + '.pkl', 'rb')) # Lists of files t_names_shuffle = self.sort_and_shuffle_dict(t_dict) tv_names_shuffle = self.sort_and_shuffle_dict(tv_dict) tvt_names_shuffle = self.sort_and_shuffle_dict(tvt_dict) if total_number_folds == -1: # Number of files total_number_folds = len(tvt_dict.keys()) folds = [] train_names = [] valid_names = [] test_names = [] # Build the list of split_matrices for current_fold_number in range(total_number_folds): one_fold, this_train_names, this_valid_names, this_test_names = build_one_fold(current_fold_number, total_number_folds, t_dict, tv_dict, tvt_dict, t_names_shuffle, tv_names_shuffle, tvt_names_shuffle, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, embedded_piano_bool, mask_orch_bool, self.random_inst) folds.append(one_fold) train_names.append(this_train_names) valid_names.append(this_valid_names) test_names.append(this_test_names) return folds, train_names, valid_names, test_names
def __build_folds(self, total_number_folds, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks): t_dict = pkl.load(open(self.database_path + '/train_only_A.pkl', 'rb')) tv_dict = {} tvt_dict = pkl.load( open(self.database_path + '/train_and_valid_A.pkl', 'rb')) if total_number_folds == -1: total_number_folds = len(tvt_dict.keys()) folds = [] train_names = [] valid_names = [] test_names = [] # Build the list of split_matrices for current_fold_number in range(total_number_folds): one_fold, this_train_names, this_valid_names, this_test_names = build_one_fold( current_fold_number, total_number_folds, t_dict, tv_dict, tvt_dict, temporal_order, train_batch_size, long_range_pred, num_max_contiguous_blocks, self.random_inst) folds.append(one_fold) train_names.append(this_train_names) valid_names.append(this_valid_names) test_names.append(this_test_names) self.K_folds = folds self.train_names = train_names self.valid_names = valid_names self.test_names = test_names return