def dict2array(self, directory_files_feature_dict: dict, label_list: list, normalize: bool): """ :param directory_files_feature_dict: :param label_list: :param normalize: :return: expert_feature_2d_array: Extracted feature with expert system (Numpy 2D array) :return mel_spectrogram_array: Extracted mel-spectrogram (Numpy 3D array) :return list_array: Labels in numpy array """ # Convert extracted feature and label to numpy array expert_feature_array, mel_spectrogram_array = self.AFE.dict2array(directory_files_feature_dict) label_array = np.array(label_list) # Normalize expert feature if normalize is True: # Remove NaNs from array expert_feature_array, mel_spectrogram_array, label_array = DataProcess.remove_nan_from_array(expert_feature_array, mel_spectrogram_array, label_array) # Take stats from expert feature DataProcess.take_dataset_stats(expert_feature_array, 'backend/expert_feature_mean_list.txt') expert_feature_array = DataProcess.min_max_normalize(expert_feature_array) # Convert dimension of mel-spectrogram array mel_spectrogram_array = mel_spectrogram_array.reshape(mel_spectrogram_array.shape[0], mel_spectrogram_array.shape[1], mel_spectrogram_array.shape[2], 1) return expert_feature_array, mel_spectrogram_array, label_array
def main(): # Case of loading pre-extracted features and/or pre-trained feature pre_trained_model_file = "" # Set Conditions run_feature_extraction = False run_training = True # Instantiate mgc main class MGC = MusicGenreClassification(AudioDatasetMaker, AudioFeatureExtraction, Classifier, music_dataset_path="processed_data", setting_file="config/master_config.ini") # Make label from genre names in processed_music_data MGC.make_label() # Feature extraction/ Load pre-extracted feature if run_feature_extraction is True: # Apply feature extraction directory_files_feature_dict, label_list = MGC.feature_extraction() print("Cleaning extracted feature...") # Apply data processing to extracted feature expert_feature_array, mel_spectrogram_array, label_array = MGC.dict2array(directory_files_feature_dict, label_list, MGC.cfg.normalize) # Save extracted data expert_feature_array, mel_spectrogram_array, label_array = MGC.save_data(expert_feature_array, mel_spectrogram_array, label_array) print("Saved feature!") # Load pre-extracted feature. Train/Test separation if MGC.CLF.selected_classifier == 'cnn' or MGC.CLF.selected_classifier == 'resnet' or MGC.CLF.selected_classifier == 'rnn': data_array, label_array = DataProcess.read_data_from_array("backend/feature/mel_spectrogram") train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array) else: data_array, label_array = DataProcess.read_data_from_array("backend/feature/expert") train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array) # Run training or load pre-trained model print("Training model...") model = MGC.training(run_training, train_data, train_label, pre_trained_model_file, output_model_directory_path="backend/model") print("Trained model!") # Write metadata json file metadata = {"words": [label for label in os.listdir('processed_data')]} FileUtil.dict2json(metadata, 'backend/model/metadata.json') # Test model performance loss, accuracy = MGC.test(model, test_data, test_label)
def extract_dataset(self, dataset_path: str, stats_type: str): """ Feature extraction to dataset Extract time series feature as 2D pandas dataframe and 3D numpy array, as well as label vector as list :param dataset_path: path to dataset :param stats_type: type of statistics for 2D feature :return all_2d_dataframe: 2D feature pandas dataframe across all frames :return all_3d_array: 3D feature numpy array across all frames :return label_list: list of numerical label vector """ # Get folder names under data set path directory_names = FileUtil.get_folder_names(dataset_path, sort=True) # Get file names and store them into a dictionary directory_files_dict = {} for directory in directory_names: directory_files_dict[directory] = FileUtil.get_file_names(os.path.join(dataset_path, directory)) # Extract all features and store them into list all_2d_dataframe = pd.DataFrame() dir_num = 0 label_list = [] for directory, audio_files in directory_files_dict.items(): # Apply feature extraction to a directory file_feature_stat_dict, class_3d_feature = self.extract_directory(os.path.join(dataset_path, directory), stats_type) # Convert dictionary to data frame class_2d_dataframe = DataProcess.dict2dataframe(file_feature_stat_dict, segment_feature=True) # Add label to 2D feature data frame class_2d_dataframe_with_label = DataProcess.add_label(class_2d_dataframe, directory) # Combine 2D feature data frame all_2d_dataframe = all_2d_dataframe.append(class_2d_dataframe_with_label) # Append 3D arrays if dir_num == 0: all_3d_array = class_3d_feature else: all_3d_array = np.dstack((all_3d_array, class_3d_feature)) # Make label as list a = [dir_num] * len(audio_files) label_list.extend(a) dir_num += 1 # Transpose 3D array all_3d_array = all_3d_array.T return all_2d_dataframe, all_3d_array, label_list
def data_process(self, dataframe): """ Apply data process to features :param dataframe: extracted feature in data frame :param label_name: name of label column in data frame :return processed_dataframe: extracted feature in pandas data frame """ # Make a copy of dataframe processed_dataframe = dataframe.copy() # Apply normalization to data frame processed_dataframe = DataProcess.normalize_dataframe( processed_dataframe, self.cfg.label_name) # Factorize label processed_dataframe = DataProcess.factorize_label( processed_dataframe, self.cfg.label_name) return processed_dataframe
def prediction_process(input_audio_file_path: str): """ Process one audio file, make prediction to it. :param: input_audio_file_path: Path to input audio file """ # Feature extraction feature_dict, file_short_feature_list = AFE.extract_file(input_audio_file_path) # Stats across frames stats_dict = AFE.get_feature_stats(feature_dict, "mean") # Segment dictionary data segmented_dict = DataProcess.segment_feature_dict(stats_dict) # Data shape formatting dataframe = pd.DataFrame.from_dict(segmented_dict, orient='index') #mean_data = genfromtxt(mean_list, delimiter=',') #std_data = genfromtxt(std_list, delimiter=',') #data_numpy = (np.array(dataframe[0]) - mean_data) #reshaped_data = data_numpy.reshape(1, -1) # 3D feature feature_2d = np.array(file_short_feature_list) feature_3d = feature_2d[newaxis, :, :] print(feature_3d.shape) # Make prediction result = CLF.predict(model, feature_2d) return list(result[0])
def make_dataset_loader(self, train_data_with_label, test_data_with_label, validation_rate): """ Dataset loader for Torch :return train loader, test loader """ # Make Dataset loader train_loader, validation_loader, test_loader = DataProcess.torch_data_loader( train_data_with_label, test_data_with_label, validation_rate) return train_loader, validation_loader, test_loader
def main(): # Case of loading pre-extracted features and/or pre-trained feature pre_trained_model_file = "" # Set Conditions run_feature_extraction = True run_training = True # Instantiate mgc main class MGC = MusicGenreClassification(AudioDatasetMaker, AudioFeatureExtraction, Classifier, music_dataset_path="processed_music_data_small", setting_file="config/master_config.ini") # Make label from genre names in processed_music_data MGC.make_label() # Feature extraction/ Load pre-extracted feature if run_feature_extraction is True: # Apply feature extraction directory_files_feature_dict, label_list = MGC.feature_extraction() # Apply data processing to extracted feature expert_feature_array, mel_spectrogram_array, label_array = MGC.dict2array(directory_files_feature_dict, label_list, MGC.cfg.normalize) # Save extracted data expert_feature_array, mel_spectrogram_array, label_array = MGC.save_data(expert_feature_array, mel_spectrogram_array, label_array) # Load pre-extracted feature. Train/Test separation if MGC.CLF.selected_classifier == 'cnn' or MGC.CLF.selected_classifier == 'resnet' or MGC.CLF.selected_classifier == 'rnn': data_array, label_array = DataProcess.read_data_from_array("backend/feature/mel_spectrogram") train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array) else: data_array, label_array = DataProcess.read_data_from_array("backend/feature/expert") train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array) # Run training or load pre-trained model model = MGC.training(run_training, train_data, train_label, pre_trained_model_file, output_model_directory_path="backend/model") # Test model performance accuracy = MGC.test(model, test_data, test_label) print("Test accuracy is {0}% \n".format(accuracy))
def save_data(self, expert_feature_array, mel_spectrogram_array, label_array): """ Save extracted feature into directories. :param expert_feature_array: Extracted feature with expert system (Numpy 2D array) :param mel_spectrogram_array: Extracted mel-spectrogram (Numpy 3D array) :param label_array: :return: """ # Remove NaNs from array expert_feature_array = DataProcess.remove_nan_from_array(expert_feature_array) # Take stats from expert feature DataProcess.take_dataset_stats(expert_feature_array, 'backend/normalized_expert_feature_mean_list.txt') # Save data np.save(os.path.join('backend/feature/expert', "data"), expert_feature_array) np.save(os.path.join('backend/feature/expert', "label"), label_array) np.save(os.path.join('backend/feature/mel_spectrogram', "data"), expert_feature_array) np.save(os.path.join('backend/feature/mel_spectrogram', "label"), label_array) return expert_feature_array, mel_spectrogram_array, label_array
def read_3D_dataset(self, input_data_directory_with_date): """ Read 3D data set :param input_data_directory_with_date: name of the directory where train and test data exist :return train_data: train data :return train_label: train label :return test_data: test data :return test_label: test label """ # Read data set train_data, test_data, train_label, test_label = DataProcess.read_3D_dataset( input_data_directory_with_date, self.cfg.label_name) return train_data, test_data, train_label, test_label
def data_process(self, dataframe): """ Apply data process to features :param dataframe: extracted feature in data frame :param label_name: name of label column in data frame :return processed_dataframe: extracted feature in pandas data frame """ # Make a copy of dataframe processed_dataframe = dataframe.copy() # Centerize data and apply standardization to data frame #processed_dataframe = DataProcess.normalize_dataframe(processed_dataframe, self.cfg.label_name) centerized_dataframe, mean_list = DataProcess.centerize_dataframe(processed_dataframe, self.cfg.label_name) cleaned_dataframe, std_list = DataProcess.standardize_dataframe(centerized_dataframe, self.cfg.label_name) # Factorize label cleaned_dataframe = DataProcess.factorize_label(cleaned_dataframe, self.cfg.label_name) # Write out mean values FileUtil.list2csv(mean_list, "../mean_list.csv") FileUtil.list2csv(std_list, "../std_list.csv") return cleaned_dataframe
def make_dataset_from_array(self, feature_array, label_array): """ Take all data and label as input. Split into train/test dataset. :param feature_array: extracted feature in 2D numpy array or 3D numpy array :param label_array: labels in numpy array :return train_data: train data :return train_label: train label :return test_data: test data :return test_label: test label """ train_data, test_data, train_label, test_label = DataProcess.make_dataset_from_array(feature_array, label_array, self.cfg.test_rate, self.cfg.shuffle) return train_data, test_data, train_label, test_label
def extract_file(self, input_audio_file: str): """ Feature extraction to one audio file :param input_audio_file: name of the audio file :return dictionary of extracted features from audio file {key: name of feature, value: list of array(number of frames)} :return file_short_feature_list: short-term features across whole audio file stored into list """ # Pre-processing to audio file processed_audio = self.pre_processing(input_audio_file) # Apply feature extraction to all frames and store into dictionary feature_dict = {} frame_number = 0 long_frame_audio = [] long_frame_power_spectrum = [] # Store whole short-term features in list file_short_feature_list = [] for short_frame_audio in processed_audio: # Extract short-term features short_frame_power_spectrum, short_feature_dict = self.extract_short_frame(short_frame_audio) # Create a feature vector and append file_short_feature_list.append(DataProcess.flatten_list(list(short_feature_dict.values()))) # Store short-term features in dictionary for short_feature_type in self.short_feature_list: feature_dict.setdefault(short_feature_type, []).append(short_feature_dict[short_feature_type]) # Extract long-term features if frame_number == self.cfg.long_frame_length: long_feature_dict = self.extract_long_frame(long_frame_audio, long_frame_power_spectrum) # Store long-term features in dictionary for long_feature in self.long_feature_list: feature_dict.setdefault(long_feature, []).append(long_feature_dict[long_feature]) # Reset for long-term feature frame_number = 0 long_frame_audio = [] long_frame_power_spectrum = [] # Update short frame stack frame_number += 1 long_frame_audio.append(short_frame_audio) long_frame_power_spectrum.append(short_frame_power_spectrum) return feature_dict, file_short_feature_list
def test(self, model, test_data, test_label) -> float: """ Make predictions and output the result from a given model to test data set :param model: trained model :param test_data: test data :param test_label: test data :return Over all test score (accuracy) """ print("Test Started") if self.selected_classifier == "cnn" or self.selected_classifier == "resnet" or self.selected_classifier == "rnn": # Make Torch dataset loader for test test_loader = DataProcess.torch_test_data_loader( test_data, test_label) # Test model performance return self.classifier.test(model, test_loader) else: # Test model performance return self.classifier.test(model, test_data, test_label)
def training(self, train_data, train_label, visualize=None): """ Training with train data set :param train_data: training data :param train_label: training data :param visualize: True/False to visualize training history :return model: trained model """ print("Train Started") if self.selected_classifier == "cnn" or self.selected_classifier == "resnet" or self.selected_classifier == "rnn": # Make Torch dataset loader for train train_loader, validation_loader = DataProcess.torch_train_data_loader( train_data, train_label, self.validation_rate) # Train model return self.classifier.training(train_loader, validation_loader, visualize=visualize) else: # Train model return self.classifier.training(train_data, train_label, visualize)
def make_3D_dataset(self, feature_3D_array, label_list: list, output_directory: str): """ Make data set :param feature_3D_array: extracted feature in data frame :param output_directory: output directory to write out the train and test data :return train_data: train data :return train_label: train label :return test_data: test data :return test_label: test label """ # Make directory if it does not exist if not os.path.isdir(output_directory): os.mkdir(output_directory) # Get time and make a new directory name directory_name_with_time = os.path.join(output_directory, FileUtil.get_time().replace(":", "_")) train_data, test_data, train_label, test_label = DataProcess.make_3D_dataset(feature_3D_array, label_list, self.cfg.test_rate, self.cfg.shuffle, directory_name_with_time) return train_data, test_data, train_label, test_label