def dict2array(self, directory_files_feature_dict: dict, label_list: list, normalize: bool):
"""
:param directory_files_feature_dict:
:param label_list:
:param normalize:
:return: expert_feature_2d_array: Extracted feature with expert system (Numpy 2D array)
:return mel_spectrogram_array: Extracted mel-spectrogram (Numpy 3D array)
:return list_array: Labels in numpy array
"""
# Convert extracted feature and label to numpy array
expert_feature_array, mel_spectrogram_array = self.AFE.dict2array(directory_files_feature_dict)
label_array = np.array(label_list)
# Normalize expert feature
if normalize is True:
# Remove NaNs from array
expert_feature_array, mel_spectrogram_array, label_array = DataProcess.remove_nan_from_array(expert_feature_array,
mel_spectrogram_array,
label_array)
# Take stats from expert feature
DataProcess.take_dataset_stats(expert_feature_array, 'backend/expert_feature_mean_list.txt')
expert_feature_array = DataProcess.min_max_normalize(expert_feature_array)
# Convert dimension of mel-spectrogram array
mel_spectrogram_array = mel_spectrogram_array.reshape(mel_spectrogram_array.shape[0],
mel_spectrogram_array.shape[1],
mel_spectrogram_array.shape[2],
1)
return expert_feature_array, mel_spectrogram_array, label_array
def main():
# Case of loading pre-extracted features and/or pre-trained feature
pre_trained_model_file = ""
# Set Conditions
run_feature_extraction = False
run_training = True
# Instantiate mgc main class
MGC = MusicGenreClassification(AudioDatasetMaker,
AudioFeatureExtraction,
Classifier,
music_dataset_path="processed_data",
setting_file="config/master_config.ini")
# Make label from genre names in processed_music_data
MGC.make_label()
# Feature extraction/ Load pre-extracted feature
if run_feature_extraction is True:
# Apply feature extraction
directory_files_feature_dict, label_list = MGC.feature_extraction()
print("Cleaning extracted feature...")
# Apply data processing to extracted feature
expert_feature_array, mel_spectrogram_array, label_array = MGC.dict2array(directory_files_feature_dict,
label_list,
MGC.cfg.normalize)
# Save extracted data
expert_feature_array, mel_spectrogram_array, label_array = MGC.save_data(expert_feature_array,
mel_spectrogram_array,
label_array)
print("Saved feature!")
# Load pre-extracted feature. Train/Test separation
if MGC.CLF.selected_classifier == 'cnn' or MGC.CLF.selected_classifier == 'resnet' or MGC.CLF.selected_classifier == 'rnn':
data_array, label_array = DataProcess.read_data_from_array("backend/feature/mel_spectrogram")
train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array)
else:
data_array, label_array = DataProcess.read_data_from_array("backend/feature/expert")
train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array)
# Run training or load pre-trained model
print("Training model...")
model = MGC.training(run_training,
train_data,
train_label,
pre_trained_model_file,
output_model_directory_path="backend/model")
print("Trained model!")
# Write metadata json file
metadata = {"words": [label for label in os.listdir('processed_data')]}
FileUtil.dict2json(metadata, 'backend/model/metadata.json')
# Test model performance
loss, accuracy = MGC.test(model, test_data, test_label)
def extract_dataset(self, dataset_path: str, stats_type: str):
"""
Feature extraction to dataset
Extract time series feature as 2D pandas dataframe and 3D numpy array, as well as label vector as list
:param dataset_path: path to dataset
:param stats_type: type of statistics for 2D feature
:return all_2d_dataframe: 2D feature pandas dataframe across all frames
:return all_3d_array: 3D feature numpy array across all frames
:return label_list: list of numerical label vector
"""
# Get folder names under data set path
directory_names = FileUtil.get_folder_names(dataset_path, sort=True)
# Get file names and store them into a dictionary
directory_files_dict = {}
for directory in directory_names:
directory_files_dict[directory] = FileUtil.get_file_names(os.path.join(dataset_path, directory))
# Extract all features and store them into list
all_2d_dataframe = pd.DataFrame()
dir_num = 0
label_list = []
for directory, audio_files in directory_files_dict.items():
# Apply feature extraction to a directory
file_feature_stat_dict, class_3d_feature = self.extract_directory(os.path.join(dataset_path, directory), stats_type)
# Convert dictionary to data frame
class_2d_dataframe = DataProcess.dict2dataframe(file_feature_stat_dict, segment_feature=True)
# Add label to 2D feature data frame
class_2d_dataframe_with_label = DataProcess.add_label(class_2d_dataframe, directory)
# Combine 2D feature data frame
all_2d_dataframe = all_2d_dataframe.append(class_2d_dataframe_with_label)
# Append 3D arrays
if dir_num == 0:
all_3d_array = class_3d_feature
else:
all_3d_array = np.dstack((all_3d_array, class_3d_feature))
# Make label as list
a = [dir_num] * len(audio_files)
label_list.extend(a)
dir_num += 1
# Transpose 3D array
all_3d_array = all_3d_array.T
return all_2d_dataframe, all_3d_array, label_list
def data_process(self, dataframe):
"""
Apply data process to features
:param dataframe: extracted feature in data frame
:param label_name: name of label column in data frame
:return processed_dataframe: extracted feature in pandas data frame
"""
# Make a copy of dataframe
processed_dataframe = dataframe.copy()
# Apply normalization to data frame
processed_dataframe = DataProcess.normalize_dataframe(
processed_dataframe, self.cfg.label_name)
# Factorize label
processed_dataframe = DataProcess.factorize_label(
processed_dataframe, self.cfg.label_name)
return processed_dataframe
def prediction_process(input_audio_file_path: str):
"""
Process one audio file, make prediction to it.
:param: input_audio_file_path: Path to input audio file
"""
# Feature extraction
feature_dict, file_short_feature_list = AFE.extract_file(input_audio_file_path)
# Stats across frames
stats_dict = AFE.get_feature_stats(feature_dict, "mean")
# Segment dictionary data
segmented_dict = DataProcess.segment_feature_dict(stats_dict)
# Data shape formatting
dataframe = pd.DataFrame.from_dict(segmented_dict, orient='index')
#mean_data = genfromtxt(mean_list, delimiter=',')
#std_data = genfromtxt(std_list, delimiter=',')
#data_numpy = (np.array(dataframe[0]) - mean_data)
#reshaped_data = data_numpy.reshape(1, -1)
# 3D feature
feature_2d = np.array(file_short_feature_list)
feature_3d = feature_2d[newaxis, :, :]
print(feature_3d.shape)
# Make prediction
result = CLF.predict(model, feature_2d)
return list(result[0])
def make_dataset_loader(self, train_data_with_label, test_data_with_label,
validation_rate):
"""
Dataset loader for Torch
:return train loader, test loader
"""
# Make Dataset loader
train_loader, validation_loader, test_loader = DataProcess.torch_data_loader(
train_data_with_label, test_data_with_label, validation_rate)
return train_loader, validation_loader, test_loader
def main():
# Case of loading pre-extracted features and/or pre-trained feature
pre_trained_model_file = ""
# Set Conditions
run_feature_extraction = True
run_training = True
# Instantiate mgc main class
MGC = MusicGenreClassification(AudioDatasetMaker, AudioFeatureExtraction, Classifier,
music_dataset_path="processed_music_data_small",
setting_file="config/master_config.ini")
# Make label from genre names in processed_music_data
MGC.make_label()
# Feature extraction/ Load pre-extracted feature
if run_feature_extraction is True:
# Apply feature extraction
directory_files_feature_dict, label_list = MGC.feature_extraction()
# Apply data processing to extracted feature
expert_feature_array, mel_spectrogram_array, label_array = MGC.dict2array(directory_files_feature_dict,
label_list,
MGC.cfg.normalize)
# Save extracted data
expert_feature_array, mel_spectrogram_array, label_array = MGC.save_data(expert_feature_array, mel_spectrogram_array, label_array)
# Load pre-extracted feature. Train/Test separation
if MGC.CLF.selected_classifier == 'cnn' or MGC.CLF.selected_classifier == 'resnet' or MGC.CLF.selected_classifier == 'rnn':
data_array, label_array = DataProcess.read_data_from_array("backend/feature/mel_spectrogram")
train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array)
else:
data_array, label_array = DataProcess.read_data_from_array("backend/feature/expert")
train_data, test_data, train_label, test_label = MGC.make_dataset_from_array(data_array, label_array)
# Run training or load pre-trained model
model = MGC.training(run_training, train_data, train_label, pre_trained_model_file, output_model_directory_path="backend/model")
# Test model performance
accuracy = MGC.test(model, test_data, test_label)
print("Test accuracy is {0}% \n".format(accuracy))
def save_data(self, expert_feature_array, mel_spectrogram_array, label_array):
"""
Save extracted feature into directories.
:param expert_feature_array: Extracted feature with expert system (Numpy 2D array)
:param mel_spectrogram_array: Extracted mel-spectrogram (Numpy 3D array)
:param label_array:
:return:
"""
# Remove NaNs from array
expert_feature_array = DataProcess.remove_nan_from_array(expert_feature_array)
# Take stats from expert feature
DataProcess.take_dataset_stats(expert_feature_array, 'backend/normalized_expert_feature_mean_list.txt')
# Save data
np.save(os.path.join('backend/feature/expert', "data"), expert_feature_array)
np.save(os.path.join('backend/feature/expert', "label"), label_array)
np.save(os.path.join('backend/feature/mel_spectrogram', "data"), expert_feature_array)
np.save(os.path.join('backend/feature/mel_spectrogram', "label"), label_array)
return expert_feature_array, mel_spectrogram_array, label_array
def read_3D_dataset(self, input_data_directory_with_date):
"""
Read 3D data set
:param input_data_directory_with_date: name of the directory where train and test data exist
:return train_data: train data
:return train_label: train label
:return test_data: test data
:return test_label: test label
"""
# Read data set
train_data, test_data, train_label, test_label = DataProcess.read_3D_dataset(
input_data_directory_with_date, self.cfg.label_name)
return train_data, test_data, train_label, test_label
def data_process(self, dataframe):
"""
Apply data process to features
:param dataframe: extracted feature in data frame
:param label_name: name of label column in data frame
:return processed_dataframe: extracted feature in pandas data frame
"""
# Make a copy of dataframe
processed_dataframe = dataframe.copy()
# Centerize data and apply standardization to data frame
#processed_dataframe = DataProcess.normalize_dataframe(processed_dataframe, self.cfg.label_name)
centerized_dataframe, mean_list = DataProcess.centerize_dataframe(processed_dataframe, self.cfg.label_name)
cleaned_dataframe, std_list = DataProcess.standardize_dataframe(centerized_dataframe, self.cfg.label_name)
# Factorize label
cleaned_dataframe = DataProcess.factorize_label(cleaned_dataframe, self.cfg.label_name)
# Write out mean values
FileUtil.list2csv(mean_list, "../mean_list.csv")
FileUtil.list2csv(std_list, "../std_list.csv")
return cleaned_dataframe
def make_dataset_from_array(self, feature_array, label_array):
"""
Take all data and label as input. Split into train/test dataset.
:param feature_array: extracted feature in 2D numpy array or 3D numpy array
:param label_array: labels in numpy array
:return train_data: train data
:return train_label: train label
:return test_data: test data
:return test_label: test label
"""
train_data, test_data, train_label, test_label = DataProcess.make_dataset_from_array(feature_array,
label_array,
self.cfg.test_rate,
self.cfg.shuffle)
return train_data, test_data, train_label, test_label
def extract_file(self, input_audio_file: str):
"""
Feature extraction to one audio file
:param input_audio_file: name of the audio file
:return dictionary of extracted features from audio file
{key: name of feature, value: list of array(number of frames)}
:return file_short_feature_list: short-term features across whole audio file stored into list
"""
# Pre-processing to audio file
processed_audio = self.pre_processing(input_audio_file)
# Apply feature extraction to all frames and store into dictionary
feature_dict = {}
frame_number = 0
long_frame_audio = []
long_frame_power_spectrum = []
# Store whole short-term features in list
file_short_feature_list = []
for short_frame_audio in processed_audio:
# Extract short-term features
short_frame_power_spectrum, short_feature_dict = self.extract_short_frame(short_frame_audio)
# Create a feature vector and append
file_short_feature_list.append(DataProcess.flatten_list(list(short_feature_dict.values())))
# Store short-term features in dictionary
for short_feature_type in self.short_feature_list:
feature_dict.setdefault(short_feature_type, []).append(short_feature_dict[short_feature_type])
# Extract long-term features
if frame_number == self.cfg.long_frame_length:
long_feature_dict = self.extract_long_frame(long_frame_audio, long_frame_power_spectrum)
# Store long-term features in dictionary
for long_feature in self.long_feature_list:
feature_dict.setdefault(long_feature, []).append(long_feature_dict[long_feature])
# Reset for long-term feature
frame_number = 0
long_frame_audio = []
long_frame_power_spectrum = []
# Update short frame stack
frame_number += 1
long_frame_audio.append(short_frame_audio)
long_frame_power_spectrum.append(short_frame_power_spectrum)
return feature_dict, file_short_feature_list
def test(self, model, test_data, test_label) -> float:
"""
Make predictions and output the result from a given model to test data set
:param model: trained model
:param test_data: test data
:param test_label: test data
:return Over all test score (accuracy)
"""
print("Test Started")
if self.selected_classifier == "cnn" or self.selected_classifier == "resnet" or self.selected_classifier == "rnn":
# Make Torch dataset loader for test
test_loader = DataProcess.torch_test_data_loader(
test_data, test_label)
# Test model performance
return self.classifier.test(model, test_loader)
else:
# Test model performance
return self.classifier.test(model, test_data, test_label)
def training(self, train_data, train_label, visualize=None):
"""
Training with train data set
:param train_data: training data
:param train_label: training data
:param visualize: True/False to visualize training history
:return model: trained model
"""
print("Train Started")
if self.selected_classifier == "cnn" or self.selected_classifier == "resnet" or self.selected_classifier == "rnn":
# Make Torch dataset loader for train
train_loader, validation_loader = DataProcess.torch_train_data_loader(
train_data, train_label, self.validation_rate)
# Train model
return self.classifier.training(train_loader,
validation_loader,
visualize=visualize)
else:
# Train model
return self.classifier.training(train_data, train_label, visualize)
def make_3D_dataset(self, feature_3D_array, label_list: list, output_directory: str):
"""
Make data set
:param feature_3D_array: extracted feature in data frame
:param output_directory: output directory to write out the train and test data
:return train_data: train data
:return train_label: train label
:return test_data: test data
:return test_label: test label
"""
# Make directory if it does not exist
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
# Get time and make a new directory name
directory_name_with_time = os.path.join(output_directory, FileUtil.get_time().replace(":", "_"))
train_data, test_data, train_label, test_label = DataProcess.make_3D_dataset(feature_3D_array, label_list,
self.cfg.test_rate, self.cfg.shuffle,
directory_name_with_time)
return train_data, test_data, train_label, test_label
