def backward_selection(self, max_features, X_train, y_train): # First select all features. selected_features = X_train.columns.tolist() ra = RegressionAlgorithms() re = RegressionEvaluation() # Select from the features that are still in the selection. for i in range(0, (len(X_train.columns) - max_features)): best_perf = sys.float_info.max worst_feature = '' for f in selected_features: temp_selected_features = copy.deepcopy(selected_features) temp_selected_features.remove(f) # Determine the score without the feature. pred_y_train, pred_y_test = ra.decision_tree(X_train[temp_selected_features], y_train, X_train[temp_selected_features]) perf = re.mean_squared_error(y_train, pred_y_train) # If we score better (i.e. a lower mse) without the feature than what we have seen so far # this is the worst feature. if perf < best_perf: best_perf = perf worst_feature = f # Remove the worst feature. selected_features.remove(worst_feature) return selected_features
def forward_selection( max_features: int, X_train: pd.DataFrame, y_train: pd.Series) -> Tuple[List[str], List[str], List[float]]: """ Select the given number of features for regression, that show the best accuracy, using forward selection. The method uses the given features and labels to train a decision tree and determine the mse of the predictions. The method returns the selected features as well as the the scores. :param max_features: Number of features to select. :param X_train: Features as DataFrame. :param y_train: True values corresponding to given features. :return: Selected features and scores. """ ordered_features = [] ordered_scores = [] # Start with no features selected_features = [] ra = RegressionAlgorithms() re = RegressionEvaluation() # Select the appropriate number of features for i in range(0, max_features): # Determine the features left to select features_left = list(set(X_train.columns) - set(selected_features)) best_perf = sys.float_info.max best_feature = '' # Iterate over all features left for f in features_left: temp_selected_features = copy.deepcopy(selected_features) temp_selected_features.append(f) # Determine the mse of a decision tree learner when adding the feature pred_y_train, pred_y_test = ra.decision_tree( X_train[temp_selected_features], y_train, X_train[temp_selected_features]) perf = re.mean_squared_error(y_train, pred_y_train) # If the performance is better than seen so far (aiming for low mse) set the current feature to the best # feature and the same for the best performance if perf < best_perf: best_perf = perf best_feature = f # Select the feature with the best performance selected_features.append(best_feature) ordered_features.append(best_feature) ordered_scores.append(best_perf) return selected_features, ordered_features, ordered_scores
def forward_selection(self, max_features, X_train, y_train): ordered_features = [] ordered_scores = [] # Start with no features. selected_features = [] ra = RegressionAlgorithms() re = RegressionEvaluation() prev_best_perf = sys.float_info.max # Select the appropriate number of features. for i in range(0, max_features): print i #Determine the features left to select. features_left = list(set(X_train.columns) - set(selected_features)) best_perf = sys.float_info.max best_feature = '' # For all features we can still select... for f in features_left: temp_selected_features = copy.deepcopy(selected_features) temp_selected_features.append(f) # Determine the mse of a decision tree learner if we were to add # the feature. pred_y_train, pred_y_test = ra.decision_tree( X_train[temp_selected_features], y_train, X_train[temp_selected_features]) perf = re.mean_squared_error(y_train, pred_y_train) # If the performance is better than what we have seen so far (we aim for low mse) # we set the current feature to the best feature and the same for the best performance. if perf < best_perf: best_perf = perf best_feature = f # We select the feature with the best performance. selected_features.append(best_feature) prev_best_perf = best_perf ordered_features.append(best_feature) ordered_scores.append(best_perf) return selected_features, ordered_features, ordered_scores
def backward_selection(max_features, X_train, y_train): """ Select the given number of features for regression, that show the best accuracy, using backward selection. The method uses the given features and labels to train a decision tree and determine the mse of the predictions. :param max_features: Number of features to select. :param X_train: Features as DataFrame. :param y_train: True values corresponding to given features. :return: Selected features. """ # First select all features selected_features = X_train.columns.tolist() ra = RegressionAlgorithms() re = RegressionEvaluation() # Select from the features that are still in the selection for i in range(0, (len(X_train.columns) - max_features)): best_perf = sys.float_info.max worst_feature = '' for f in selected_features: temp_selected_features = copy.deepcopy(selected_features) temp_selected_features.remove(f) # Determine the score without the feature pred_y_train, pred_y_test = ra.decision_tree( X_train[temp_selected_features], y_train, X_train[temp_selected_features]) perf = re.mean_squared_error(y_train, pred_y_train) # If scoring better (i.e. a lower mse) without the feature than seen so far this is the worst feature if perf < best_perf: best_perf = perf worst_feature = f # Remove the worst feature selected_features.remove(worst_feature) return selected_features
performance_te_svm_std = std_te regr_train_y, regr_test_y = learner.k_nearest_neighbor(selected_train_X, train_y, selected_test_X, gridsearch=True) mean_tr, std_tr = eval.mean_squared_error_with_std(train_y, regr_train_y) mean_te, std_te = eval.mean_squared_error_with_std(test_y, regr_test_y) performance_tr_knn = mean_tr performance_tr_knn_std = std_tr performance_te_knn = mean_te performance_te_knn_std = std_te regr_train_y, regr_test_y = learner.decision_tree( selected_train_X, train_y, selected_test_X, gridsearch=True, export_tree_path=export_tree_path) mean_tr, std_tr = eval.mean_squared_error_with_std(train_y, regr_train_y) mean_te, std_te = eval.mean_squared_error_with_std(test_y, regr_test_y) performance_tr_dt = mean_tr performance_tr_dt_std = std_tr performance_te_dt = mean_te performance_te_dt_std = std_te scores_with_sd = [ (overall_performance_tr_nn, overall_performance_tr_nn_std, overall_performance_te_nn, overall_performance_te_nn_std), (overall_performance_tr_rf, overall_performance_tr_rf_std, overall_performance_te_rf, overall_performance_te_rf_std),