def __init__(self, cv_method='5-folder'): self.__classifier = Classifier() if cv_method == 'LOO': self.__cv = LeaveOneOut() elif cv_method == '10-folder': self.__cv = StratifiedKFold(10) elif cv_method == '5-folder': self.__cv = StratifiedKFold(5) else: self.__cv = None
def LoadTrainInfo(model_folder): train_info = {} ##Load normalizaiton normalizer = Normalizer() normalization_path = '' for sub_file in os.listdir(model_folder): if sub_file.rfind('_normalization_training.csv') != -1: normalization_path = os.path.join(model_folder, sub_file) if not os.path.exists(normalization_path): print('Check the normalization name : zero_center_normalization') else: normalizer.LoadInfo(normalization_path) train_info['normalizer'] = normalizer ## Load selected features selected_feature_path = os.path.join(model_folder, 'feature_select_info.csv') selected_feature_list = [] with open(selected_feature_path, 'r', newline='') as f: f_reader = csv.reader(f) for index in f_reader: if index[0] == 'selected_feature': selected_feature_list = index[1:] if selected_feature_list == []: print('No selected features') train_info['selected_features'] = selected_feature_list ## Load FAE model classifier = Classifier() classifier.Load(model_folder) train_info['classifier'] = classifier return train_info
class CrossValidation: ''' CrossValidation is the base class to explore the hpyer-parameters. Now it supported Leave-one-lout (LOO), 10-folder, and 5-folders. A classifier must be set before run CV. A training metric and validation metric will be returned. If a testing data container was also set, the test metric will be return. ''' def __init__(self, cv_method='5-folder'): self.__classifier = Classifier() if cv_method == 'LOO': self.__cv = LeaveOneOut() elif cv_method == '10-folder': self.__cv = StratifiedKFold(10) elif cv_method == '5-folder': self.__cv = StratifiedKFold(5) else: self.__cv = None def SetClassifier(self, classifier): self.__classifier = classifier def GetClassifier(self): return self.__classifier def SetCV(self, cv): if cv == 'LOO': self.__cv = LeaveOneOut() elif cv == '10-folder': self.__cv = StratifiedKFold(10) elif cv == '5-folder': self.__cv = StratifiedKFold(5) def GetCV(self): return self.__cv def SaveResult(self, info, store_path): info = dict(sorted(info.items(), key= lambda item: item[0])) write_info = [] for key in info.keys(): temp_list = [] temp_list.append(key) if isinstance(info[key], (numbers.Number, str)): temp_list.append(info[key]) else: temp_list.extend(info[key]) write_info.append(temp_list) write_info.sort() # write_info = [[key].extend(info[key]) for key in info.keys()] if os.path.isdir(store_path): store_path = os.path.join(store_path, 'result.csv') with open(store_path, 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) write_info.sort() writer.writerows(write_info) def Run(self, data_container, test_data_container=DataContainer(), store_folder=''): train_pred_list, train_label_list, val_pred_list, val_label_list = [], [], [], [] data = data_container.GetArray() label = data_container.GetLabel() val_index_store = [] for train_index, val_index in self.__cv.split(data, label): val_index_store.extend(val_index) train_data = data[train_index, :] train_label = label[train_index] val_data = data[val_index, :] val_label = label[val_index] self.__classifier.SetData(train_data, train_label) self.__classifier.Fit() train_prob = self.__classifier.Predict(train_data) val_prob = self.__classifier.Predict(val_data) train_pred_list.extend(train_prob) train_label_list.extend(train_label) val_pred_list.extend(val_prob) val_label_list.extend(val_label) total_train_label = np.asarray(train_label_list, dtype=np.uint8) total_train_pred = np.asarray(train_pred_list, dtype=np.float32) train_metric = EstimateMetirc(total_train_pred, total_train_label, 'train') total_label = np.asarray(val_label_list, dtype=np.uint8) total_pred = np.asarray(val_pred_list, dtype=np.float32) val_metric = EstimateMetirc(total_pred, total_label, 'val') self.__classifier.SetDataContainer(data_container) self.__classifier.Fit() test_metric = {} if test_data_container.GetArray().size > 0: test_data = test_data_container.GetArray() test_label = test_data_container.GetLabel() test_pred = self.__classifier.Predict(test_data) test_metric = EstimateMetirc(test_pred, test_label, 'test') if store_folder: if not os.path.exists(store_folder): os.mkdir(store_folder) info = {} info.update(train_metric) info.update(val_metric) np.save(os.path.join(store_folder, 'train_predict.npy'), total_train_pred) np.save(os.path.join(store_folder, 'val_predict.npy'), total_pred) np.save(os.path.join(store_folder, 'train_label.npy'), total_train_label) np.save(os.path.join(store_folder, 'val_label.npy'), total_label) cv_info_path = os.path.join(store_folder, 'cv_info.csv') df = pd.DataFrame(data=val_index_store) df.to_csv(cv_info_path) DrawROCList(total_train_pred, total_train_label, store_path=os.path.join(store_folder, 'train_ROC.jpg'), is_show=False) DrawROCList(total_pred, total_label, store_path=os.path.join(store_folder, 'val_ROC.jpg'), is_show=False) if test_data_container.GetArray().size > 0: info.update(test_metric) np.save(os.path.join(store_folder, 'test_predict.npy'), test_pred) np.save(os.path.join(store_folder, 'test_label.npy'), test_label) DrawROCList(test_pred, test_label, store_path=os.path.join(store_folder, 'test_ROC.jpg'), is_show=False) self.__classifier.Save(store_folder) self.SaveResult(info, store_folder) return train_metric, val_metric, test_metric
def __init__(self): self._raw_classifier = Classifier() self.__classifier = Classifier() self._hyper_parameter_manager = HyperParameterManager() self.__classifier_parameter_list = [{}]
def __init__(self): self._classifier = Classifier()