def fit(self, X, y): self.feature_encoder_ = CustomOrdinalFeatureEncoder() self.class_encoder_ = CustomLabelEncoder() if isinstance(X, pd.DataFrame): self.categories_ = X.columns if self.encode_data: X = self.feature_encoder_.fit_transform(X) y = self.class_encoder_.fit_transform(y) classifier_ = NaiveBayes(encode_data=False, n_intervals=self.n_intervals, metric=self.metric) self.n_features = X.shape[1] if self.encode_data: self.unique_values = [ values.shape[0] for values in self.feature_encoder_.categories_ ] else: self.unique_values = [ np.unique(X[:, j]).shape[0] for j in range(X.shape[1]) ] random.seed(self.seed) np.random.seed(self.seed) self.size = np.ceil(np.sqrt(X.shape[1])) best_individual = self.execute_algorithm(X, y) self.best_features = best_individual self.classifier_ = NaiveBayes(encode_data=False, metric=self.metric) self.classifier_.fit( np.concatenate( [feature.transform(X) for feature in self.best_features], axis=1), y) return self
def test_incremental_validation(X=None, y=None, iterations=10, verbose=1): if not X: X, y = make_classification(n_samples=500, n_features=1000, n_informative=20, n_redundant=1, n_repeated=0, n_classes=2, n_clusters_per_class=2, weights=None, class_sep=1, hypercube=False, scale=1.0, shuffle=True, random_state=0) X //= 10 # --> To be able to evaluate categoricalNB # classifiers nb_classifier = NaiveBayes(encode_data=True) nb_classifier_no_encoding = NaiveBayes(encode_data=False) custom_encoder = CustomOrdinalFeatureEncoder() cnb = CategoricalNB() # accumulators categorical_nb = [] custom_nb_val_1 = [] custom_nb_val_2 = [] custom_nb_val_3 = [] custom_nb_val_4 = [] for i in range(iterations): if verbose: print(f"Iteration {i}") ts = time() X2 = custom_encoder.fit_transform(X) ts = time() score_2 = nb_classifier.leave_one_out_cross_val(X, y) custom_nb_val_1.append(time() - ts) ts = time() score_4 = cross_leave_one_out(nb_classifier, X, y) custom_nb_val_3.append(time() - ts) ts = time() X2 = custom_encoder.fit_transform(X) score_5 = cross_leave_one_out(nb_classifier_no_encoding, X2, y) custom_nb_val_4.append(time() - ts) if i == 0: score_1 = score_2 scores = [score_1, score_2, score_4, score_5] assert all(score == scores[0] for score in scores) print("Categorical with scikit loo: ", np.mean(categorical_nb[1:])) print("Custom with scikit loo: ", np.mean(custom_nb_val_3[1:])) print("Custom with scikit loo (pre-encoding): ", np.mean(custom_nb_val_4[1:])) print("Custom with first incremental: ", np.mean(custom_nb_val_1[1:]))
def simple_evaluate(self, individual, X, y): classifier_ = NaiveBayes(encode_data=False, metric=self.metric) return classifier_.leave_one_out_cross_val(transform_features( individual[0] + individual[1], X), y, fit=True)
def time_comparison(combinations=None, n_iterations=15, verbose=1, seed=200): column_names = [ "Classifier", "n_samples", "n_features", "Average Fit Time", "STD Fit Time", "Average Predict Time", "STD Predict Time", "Score" ] results = [] if combinations is None: columns = range(10, 40010, 5000) rows = [10, 100, 1000] combinations = list(product(rows, columns)) + list( product(columns, rows)) combinations += list(product([10, 100, 1000], [500000])) combinations += list(product([500000], [10, 100, 1000])) clf_no_encoding = NaiveBayes(encode_data=False, alpha=1) clf_encoding = NaiveBayes(encode_data=True, alpha=1, discretize=False) clf_categorical_sklearn = CategoricalNB(alpha=1) clf_gaussian_sklearn = GaussianNB() progress_bar = tqdm(total=len(combinations), bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}') X = [] y = [] for n_samples, n_features in combinations: if verbose: progress_bar.set_postfix({ "n_samples": n_samples, "n_features": n_features }) progress_bar.update(1) progress_bar.refresh() del X del y X, y = make_classification(n_samples=n_samples, n_features=n_features, flip_y=0.01, class_sep=1.0, hypercube=True, shift=0.0, scale=2.0, shuffle=True, random_state=seed) X = make_discrete(X, m=1) X_train, X_test, y_train, y_test = X, X, y, y gaussian_nb_fit_time = [] gaussian_nb_predict_time = [] gaussian_nb_score = [] gaussian_nb_errors = 0 categorical_nb_fit_time = [] categorical_nb_predict_time = [] categorical_nb_score = [] categorical_nb_errors = 0 custom_no_encoding_nb_fit_time = [] custom_no_encoding_nb_predict_time = [] custom_no_encoding_nb_score = [] custom_no_encoding_nb_errors = 0 custom_encoding_nb_fit_time = [] custom_encoding_nb_predict_time = [] custom_encoding_nb_score = [] custom_encoding_nb_errors = 0 for _ in range(n_iterations): gaussian_nb_errors += evaluate(X_train, y_train, X_test, y_test, clf_gaussian_sklearn, gaussian_nb_fit_time, gaussian_nb_predict_time, gaussian_nb_score) categorical_nb_errors += evaluate(X_train, y_train, X_test, y_test, clf_categorical_sklearn, categorical_nb_fit_time, categorical_nb_predict_time, categorical_nb_score) custom_no_encoding_nb_errors += evaluate( X_train, y_train, X_test, y_test, clf_no_encoding, custom_no_encoding_nb_fit_time, custom_no_encoding_nb_predict_time, custom_no_encoding_nb_score) custom_encoding_nb_errors += evaluate( X_train, y_train, X_test, y_test, clf_encoding, custom_encoding_nb_fit_time, custom_encoding_nb_predict_time, custom_encoding_nb_score) update_df(results, "Gaussian", n_samples, n_features, gaussian_nb_fit_time, gaussian_nb_predict_time, gaussian_nb_score, gaussian_nb_errors) update_df(results, "Categorical", n_samples, n_features, categorical_nb_fit_time, categorical_nb_predict_time, categorical_nb_score, categorical_nb_errors) update_df(results, "Custom with encoding", n_samples, n_features, custom_encoding_nb_fit_time, custom_encoding_nb_predict_time, custom_encoding_nb_score, custom_encoding_nb_errors) update_df(results, "Custom without encoding", n_samples, n_features, custom_no_encoding_nb_fit_time, custom_no_encoding_nb_predict_time, custom_no_encoding_nb_score, custom_no_encoding_nb_errors) results_df = pd.DataFrame(results, columns=column_names) results_df.drop_duplicates(["Classifier", "n_samples", "n_features"], inplace=True) results_df.to_csv("backup.csv") return results_df
def filter_features(self, X, y): '''After the rank is built this perform the greedy wrapper search''' check_is_fitted(self) self.classifier = NaiveBayes(encode_data=False, n_intervals=self.n_intervals, metric=self.metric) current_score = np.NINF first_iteration = True current_features = [] current_data = None if self.use_initials: # Original Features have already been taken into account rank_iter = filter( lambda x: not isinstance(self.all_feature_constructors[x], DummyFeatureConstructor), iter(self.rank)) # Deep copy to avoid issues when modifying the list current_features = deepcopy(self.initial_backward_features) current_data = np.concatenate( [f.transform(X) for f in current_features], axis=1) # Get initial LOO score current_score = self.evaluate_leave_one_out_cross_val( self.classifier, current_features, current_data, y, fit=True) else: # Iterator over the sorted list of indexes rank_iter = iter(self.rank) if self.verbose: progress_bar = tqdm(total=len(self.rank), bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}') iteration = 0 iterations_without_improvements = 0 # Loop for including {block size} elements at a time # Rank is an iterator, so the for loop is not sequential! for feature_constructor_index in rank_iter: iteration += 1 if self.verbose: progress_bar.set_postfix({ "n_features": len(current_features), "score": current_score }) progress_bar.update(1) progress_bar.refresh() # Add block size features new_X = [ self.all_feature_constructors[feature_constructor_index]. transform(X) ] selected_features = [ self.all_feature_constructors[feature_constructor_index] ] for _ in range(self.block_size - 1): try: index = next(rank_iter) selected_features.append( self.all_feature_constructors[index]) new_X.append( self.all_feature_constructors[index].transform(X)) if self.verbose: progress_bar.update(1) progress_bar.refresh() except: # Block size does not divide the number of elements in the rank. The search is halted break # Evaluate features new_X = np.concatenate(new_X, axis=1) if iteration == 1 and not self.use_initials: current_data = new_X current_score = self.evaluate_leave_one_out_cross_val( self.classifier, selected_features, current_data, y, fit=True) current_features = selected_features first_iteration = False if self.max_iterations <= iteration or ( len(current_features) + self.block_size) > self.max_features: break continue data = np.concatenate([current_data, new_X], axis=1) self.classifier.add_features(new_X, y) # LOO evaluation score = self.evaluate_leave_one_out_cross_val(self.classifier, current_features + selected_features, data, y, fit=False) if score > current_score: current_score = score current_data = data current_features.extend(selected_features) iterations_without_improvements = 0 else: iterations_without_improvements += 1 # Remove last added block for feature_index_to_remove in range( data.shape[1], data.shape[1] - new_X.shape[1], -1): self.classifier.remove_feature(feature_index_to_remove - 1) if self.strategy == "eager" and self.max_err < iterations_without_improvements: # Stops as soon as no impovement break if self.max_iterations <= iteration or ( len(current_features) + self.block_size) > self.max_features: break if self.verbose: progress_bar.close() print( f"\nFinal number of included features: {len(current_features)} - Final Score: {current_score}" ) self.final_feature_constructors = current_features return self
def acfs_score_comparison(datasets, seed, base_path, params, n_splits=3, n_repeats=5, n_intervals=5, metric="accuracy", send_email=False, email_data=dict(), verbose=True): # List to store results and column names for the csv result = [] columns = [ "Database", "Number of attributes", "NBScore", "NBScore STD", "ACFCS Score", "ACFCS Score STD", "Configuration", "Nodes", "Contruction Matrix", "Selection Matrix", "Selected_attributes", "Original" ] dataset_tqdm = tqdm(datasets) # Instantiate the classifier acfcs = ACFCS(verbose=0, metric=metric) nb = NaiveBayes(encode_data=False, n_intervals=n_intervals, metric=metric) # Execute algorithm on datasets for database in dataset_tqdm: name, label = database if not os.path.exists(base_path + name): print(f"{name} doesnt' exist") continue # Assume UCI REPO like data test = f"{name}.test.csv" data = f"{name}.data.csv" X, y = get_X_y_from_database(base_path, name, data, test, label) # Update progressbar dataset_tqdm.set_postfix({"DATABASE": name}) # Set up data structures to store results nb_score = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_score = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_selection_matrix = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_construction_matrix = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_nodes = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_dummy = np.zeros(shape=(len(params), n_splits * n_repeats)) acfcs_selected = np.zeros(shape=(len(params), n_splits * n_repeats)) # Create splits for the experiments rskf = RepeatedStratifiedKFold(n_splits=n_splits, n_repeats=n_repeats, random_state=seed) seed_tqdm = tqdm(rskf.split(X, y), leave=False, total=n_splits * n_repeats, bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}' ) if verbose else rskf.split(X, y) # Execute experiments for i, data in enumerate(seed_tqdm): train_index, test_index = data X_train, X_test = X.iloc[train_index], X.iloc[test_index] y_train, y_test = y.iloc[train_index], y.iloc[test_index] # Encode the data c = CustomOrdinalFeatureEncoder(n_intervals=n_intervals) X_train = c.fit_transform(X_train) X_test = c.transform(X_test) l = CustomLabelEncoder() y_train = l.fit_transform(y_train) y_test = l.transform(y_test) # Assess the classifiers reusing info to speed up evaluation nb.fit(X_train, y_train) naive_bayes_score = nb.score(X_test, y_test) acfcs.reset_cache() for conf_index, conf in enumerate(params): acfcs.set_params(**conf) acfcs.fit(X_train, y_train, init_graph=conf_index == 0) # score acfcs_score_conf = acfcs.score(X_test, y_test) if verbose: seed_tqdm.set_postfix({ "config": conf_index, "nb_score": naive_bayes_score, "ant_score": acfcs_score_conf }) # Get data n_original_features = len( list( filter( lambda x: isinstance(x, DummyFeatureConstructor), acfcs.best_features))) n_selected = len(acfcs.best_features) selection_matrix = len(acfcs.afg.pheromone_selection) construction_matrix = len(acfcs.afg.pheromone_construction) nodes = len(acfcs.afg.nodes) # Update nb_score[conf_index, i] = naive_bayes_score acfcs_score[conf_index, i] = acfcs_score_conf acfcs_selection_matrix[conf_index, i] = selection_matrix acfcs_construction_matrix[conf_index, i] = construction_matrix acfcs_nodes[conf_index, i] = nodes acfcs_dummy[conf_index, i] = n_original_features acfcs_selected[conf_index, i] = n_selected # Insert the final result - averaged metrics for this database. for conf_index, conf in enumerate(params): row = [ name, X.shape[1], np.mean(nb_score[conf_index]), np.std(nb_score[conf_index]), np.mean(acfcs_score[conf_index]), np.std(acfcs_score[conf_index]), conf, np.mean(acfcs_nodes[conf_index]), np.mean(acfcs_construction_matrix[conf_index]), np.mean(acfcs_selection_matrix[conf_index]), np.mean(acfcs_selected[conf_index]), np.mean(acfcs_dummy[conf_index]) ] result.append(row) result = pd.DataFrame(result, columns=columns) if send_email: from tfg.utils import send_results send_results("ACFCS", email_data, result) return result
def ranker_score_comparison(datasets, seed, base_path, params, n_splits=3, n_repeats=5, n_intervals=5, metric="accuracy", send_email=False, email_data=dict(), share_rank=True): result = [] columns = ["Database", "Number of attributes", "NBScore", "NBScore STD", "Ranker Score", "Ranker Score STD", "Configuration", "Combinations", "Selected_attributes", "Original"] dataset_tqdm = tqdm(datasets) # Instantiate the classifier r = RankerLogicalFeatureConstructor(n_intervals=n_intervals, metric=metric) nb = NaiveBayes(encode_data=False, n_intervals=n_intervals, metric=metric) # Execute algorithm on datasets for database in dataset_tqdm: name, label = database if not os.path.exists(base_path + name): print(f"{name} doesnt' exist") continue # Assume UCI REPO like data test = f"{name}.test.csv" data = f"{name}.data.csv" X, y = get_X_y_from_database(base_path, name, data, test, label) dataset_tqdm.set_postfix({"DATABASE": name}) # Set up data structures to store results nb_score = np.zeros(shape=(len(params), n_splits*n_repeats)) r_score = np.zeros(shape=(len(params), n_splits*n_repeats)) r_combinations = np.zeros(shape=(len(params), n_splits*n_repeats)) r_selected = np.zeros(shape=(len(params), n_splits*n_repeats)) r_dummy = np.zeros(shape=(len(params), n_splits*n_repeats)) r_total_constructed = np.zeros(shape=(len(params), n_splits*n_repeats)) r_total_selected = np.zeros(shape=(len(params), n_splits*n_repeats)) r_original_selected = np.zeros(shape=(len(params), n_splits*n_repeats)) rskf = RepeatedStratifiedKFold(n_splits=n_splits, n_repeats=n_repeats, random_state=seed) seed_tqdm = tqdm(rskf.split(X, y), leave=False, total=n_splits*n_repeats, bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}') for i, data in enumerate(seed_tqdm): train_index, test_index = data X_train, X_test = X.iloc[train_index], X.iloc[test_index] y_train, y_test = y.iloc[train_index], y.iloc[test_index] c = CustomOrdinalFeatureEncoder(n_intervals=n_intervals) X_train = c.fit_transform(X_train) X_test = c.transform(X_test) l = CustomLabelEncoder() y_train = l.fit_transform(y_train) y_test = l.transform(y_test) # Assess the classifiers nb.fit(X=X_train, y=y_train) naive_bayes_score = nb.score(X_test, y_test) for conf_index, conf in enumerate(params): seed_tqdm.set_postfix({"config": conf_index}) r.set_params(**conf) # Fit if conf_index == 0 or not share_rank: # The rank is computed from scratch r.fit(X_train, y_train) else: r.filter_features(r.feature_encoder_.transform( X_train), r.class_encoder_.transform(y_train)) # score ranker_score = r.score(X_test, y_test) # Get data n_original_features = len(list(filter(lambda x: isinstance( x, DummyFeatureConstructor), r.final_feature_constructors))) n_combinations = len(r.all_feature_constructors) n_selected = len(r.final_feature_constructors) # Update nb_score[conf_index, i] = naive_bayes_score r_score[conf_index, i] = ranker_score r_combinations[conf_index, i] = n_combinations r_selected[conf_index, i] = n_selected r_dummy[conf_index, i] = n_original_features # Insert to final result averaged metrics for this dataset for conf_index, conf in enumerate(params): row = [name, X.shape[1], np.mean(nb_score[conf_index]), np.std(nb_score[conf_index]), np.mean(r_score[conf_index]), np.std(r_score[conf_index]), conf, np.mean(r_combinations[conf_index]), np.mean(r_selected[conf_index]), np.mean(r_dummy[conf_index])] result.append(row) result = pd.DataFrame(result, columns=columns) if send_email: from tfg.utils import send_results send_results("RANKER", email_data, result) return result
def fit(self, X, y): # Parse input if isinstance(y, pd.DataFrame): y = y.to_numpy() if self.encode_data: self.feature_encoder_ = CustomOrdinalFeatureEncoder( n_intervals=self.n_intervals) self.class_encoder_ = CustomLabelEncoder() X = self.feature_encoder_.fit_transform(X) y = self.class_encoder_.fit_transform(y) if isinstance(X, pd.DataFrame): X = X.to_numpy() check_X_y(X, y) # Reset the stored results for new fit self.reset_evaluation() # Generate rank if self.use_graph: # Construct the minimum graph and create rank graph = AntFeatureGraphMI(seed=None, connections=1).compute_graph( X, y, ("AND", "OR", "XOR")) self.all_feature_constructors = graph.get_rank() elif self.prune is not None: # Construct the rank with pruning by selecting pais that maximise SU(X_iX_j,Y) feature_combinations = list( combinations(list(range(X.shape[1])), 2)) + [(i, i) for i in range(X.shape[1])] rank_pairs = [ symmetrical_uncertainty_two_variables(X[:, i], X[:, j], y) for i, j in feature_combinations ] rank_pairs_index = np.argsort(rank_pairs)[::-1] # Create the unsorted list self.all_feature_constructors = [] for index in rank_pairs_index[:self.prune]: i, j = feature_combinations[index] if i == j: from tfg.feature_construction import create_feature self.all_feature_constructors.extend([ create_feature("OR", [(i, n), (i, m)]) for n, m in combinations(np.unique(X[:, i]), 2) ]) else: self.all_feature_constructors.extend( construct_features(X[:, [i, j]], operators=self.operators, same_feature=False)) else: # Create the unsorted list of all features self.all_feature_constructors = construct_features( X, operators=self.operators) if self.verbose: print( f"Total number of constructed features: {len(self.all_feature_constructors)}" ) self.all_feature_constructors.extend( [DummyFeatureConstructor(j) for j in range(X.shape[1])]) self.symmetrical_uncertainty_rank = [] # Sort the ranking for feature_constructor in self.all_feature_constructors: feature = feature_constructor.transform(X) su = symmetrical_uncertainty(f1=feature.flatten(), f2=y) self.symmetrical_uncertainty_rank.append(su) # Store the descending order index self.rank = np.argsort(self.symmetrical_uncertainty_rank)[::-1] # If the initial variables are if self.use_initials: classifier = NaiveBayes(encode_data=False, n_intervals=self.n_intervals, metric=self.metric) classifier.fit(X, y) current_features = [ DummyFeatureConstructor(j) for j in range(X.shape[1]) ] # Store the backward result to reuse it for other executions self.initial_backward_features = backward_search( X, y, current_features, classifier) # Feature Subset Selection (FSS) from the rank self.filter_features(X, y) return self
def genetic_score_comparison(datasets, seed, base_path, params, n_splits=3, n_repeats=5, n_intervals=5, metric="accuracy", send_email=False, email_data=dict(), verbose=True, version=1): result = [] columns = [ "Database", "Number of attributes", "NBScore", "NBScore STD", "Genetic Score", "Genetic Score STD", "Configuration", "Selected_attributes", "Original" ] dataset_tqdm = tqdm(datasets) # Instantiate the classifier if version == 1: # First Version - No flexibility in the number of attributes (bad performance) # clf = GeneticProgramming(seed=seed, metric=metric) clf = GeneticProgrammingFlexibleLogic(seed=seed, metric=metric) elif version == 2: # Version with flexibility clf = GeneticProgrammingFlexibleLogic(seed=seed, metric=metric) else: # Guided mutation based on SU clf = GeneticProgrammingRankMutation(seed=seed, metric=metric) nb = NaiveBayes(encode_data=False, n_intervals=n_intervals, metric=metric) # Execute algorithm on datasets for database in dataset_tqdm: name, label = database if not os.path.exists(base_path + name): print(f"{name} doesnt' exist") continue # Assume UCI REPO like data test = f"{name}.test.csv" data = f"{name}.data.csv" X, y = get_X_y_from_database(base_path, name, data, test, label) dataset_tqdm.set_postfix({"DATABASE": name}) # Set up data structures to store results nb_score = np.zeros(shape=(len(params), n_splits * n_repeats)) clf_score = np.zeros(shape=(len(params), n_splits * n_repeats)) clf_selected = np.zeros(shape=(len(params), n_splits * n_repeats)) clf_dummy = np.zeros(shape=(len(params), n_splits * n_repeats)) # Create splits for the experiments rskf = RepeatedStratifiedKFold(n_splits=n_splits, n_repeats=n_repeats, random_state=seed) seed_tqdm = tqdm(rskf.split(X, y), leave=False, total=n_splits * n_repeats, bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}' ) if verbose else rskf.split(X, y) # Execute experiments for i, data in enumerate(seed_tqdm): train_index, test_index = data X_train, X_test = X.iloc[train_index], X.iloc[test_index] y_train, y_test = y.iloc[train_index], y.iloc[test_index] # Encode the data c = CustomOrdinalFeatureEncoder(n_intervals=n_intervals) X_train = c.fit_transform(X_train) X_test = c.transform(X_test) l = CustomLabelEncoder() y_train = l.fit_transform(y_train) y_test = l.transform(y_test) # Assess the classifiers reusing info to speed up evaluation nb.fit(X_train, y_train) naive_bayes_score = nb.score(X_test, y_test) # Reset evaluation-cache for new split clf.reset_evaluation() for conf_index, conf in enumerate(params): if verbose: seed_tqdm.set_postfix({"config": conf_index}) clf.set_params(**conf) clf.fit(X_train, y_train) # score genetic_score = clf.score(X_test, y_test) # Get data n_original_features = len( list( filter( lambda x: isinstance(x, DummyFeatureConstructor), clf.best_features))) n_selected = len(clf.best_features) # Update nb_score[conf_index, i] = naive_bayes_score clf_score[conf_index, i] = genetic_score clf_selected[conf_index, i] = n_selected clf_dummy[conf_index, i] = n_original_features # Insert to final result averaged metrics for this database for conf_index, conf in enumerate(params): row = [ name, X.shape[1], np.mean(nb_score[conf_index]), np.std(nb_score[conf_index]), np.mean(clf_score[conf_index]), np.std(clf_score[conf_index]), conf, np.mean(clf_selected[conf_index]), np.mean(clf_dummy[conf_index]) ] result.append(row) result = pd.DataFrame(result, columns=columns) if send_email: from tfg.utils import send_results send_results(f"GENETIC_{version}", email_data, result) return result
def fit(self, X, y, init_graph=True): self.feature_encoder_ = CustomOrdinalFeatureEncoder() self.class_encoder_ = CustomLabelEncoder() self.categories_ = None if isinstance(X, pd.DataFrame): self.categories_ = X.columns if self.encode_data: X = self.feature_encoder_.fit_transform(X) y = self.class_encoder_.fit_transform(y) if init_graph: if self.graph_strategy == "full": #Full graph self.afg = AntFeatureGraph(seed=self.seed).compute_graph( X, y, ("XOR", "OR", "AND")) else: #Pruned graph self.afg = AntFeatureGraphMI( seed=self.seed, connections=self.connections).compute_graph( X, y, ("XOR", "OR", "AND")) else: self.afg.reset_pheromones() if self.verbose: print(f"Number of nodes: {len(self.afg.nodes)}") random.seed(self.seed) best_score = 0 self.best_features = [] iterations_without_improvement = 0 iterator = tqdm(range(self.iterations)) if self.verbose else range( self.iterations) beta = self.beta distance_from_best = -1 for iteration in iterator: if self.verbose: iterator.set_postfix({ "best_score": best_score, "n_features": len(self.best_features), "p_matrix_c": len(self.afg.pheromone_construction), "p_matrix_s": len(self.afg.pheromone_selection), "distance_from_best": distance_from_best }) ants = [ Ant(ant_id=i, alpha=self.alpha, beta=beta, metric=self.metric, use_initials=self.use_initials, cache_loo=self.cache_loo, cache_heuristic=self.cache_heuristic, step=self.step) for i in range(self.ants) ] beta *= (1 - self.beta_evaporation_rate) results = [] for ant in ants: results.append( ant.run(X=X, y=y, graph=self.afg, random_generator=random, parallel=self.parallel, max_errors=self.max_errors)) results = np.array(results) self.afg.update_pheromone_matrix_evaporation(self.evaporation_rate) distance_from_best = np.mean(np.abs(results - best_score)) best_ant = np.argmax(results) if self.update_strategy == "best": ant = ants[best_ant] self.afg.intensify(ant.current_features, self.intensification_factor, 1, self.use_initials) else: for ant_score, ant in zip(results, ants): self.afg.intensify(ant.current_features, self.intensification_factor, ant_score, self.use_initials) if results[best_ant] >= best_score: iterations_without_improvement = 0 ant = ants[best_ant] best_score = results[best_ant] self.best_features = ant.current_features else: iterations_without_improvement += 1 if iterations_without_improvement > self.early_stopping: break self.classifier_ = NaiveBayes(encode_data=False, metric=self.metric) if self.final_selection == "BEST": pass else: #An ant traverses the graph deterministically to obtain the features final_ant = FinalAnt(ant_id=0, alpha=self.alpha, beta=beta, metric=self.metric, use_initials=self.use_initials, cache_loo=self.cache_loo, cache_heuristic=self.cache_heuristic, step=self.step) final_ant.run(X=X, y=y, graph=self.afg, random_generator=random, parallel=self.parallel) self.best_features = final_ant.current_features #Train model with final features self.classifier_.fit( np.concatenate( [feature.transform(X) for feature in self.best_features], axis=1), y) if self.save_features: #Save to features to dict translate_features(features=self.best_features, feature_encoder=self.feature_encoder_, categories=self.categories_, path=self.path, filename=self.filename) return self
def fit_fssj(self, X, y): self.evaluate = memoize(_evaluate, attribute_to_cache="columns") current_best = None current_columns = deque() available_columns = list(range(X.shape[1])) best_score = -float("inf") stop = False while not stop: update = False stop = True # self.classifier.encode_data=True if self.verbose: print("Current Best: ", current_columns, " Score: ", best_score, "Available columns: ", available_columns) for new_columns, new_available_columns, column_to_delete, column_to_add, delete in self._generate_neighbors_fssj( current_columns=current_columns, individual=current_best, original_data=X, available_columns=available_columns): if delete: action = "JOIN" # Update classifier and get validation result self.classifier.add_features(column_to_add, y) self.classifier.remove_feature(column_to_delete) neighbor = np.concatenate([current_best, column_to_add], axis=1) neighbor = np.delete(neighbor, column_to_delete, axis=1) score = self.evaluate(self.classifier, neighbor, y, columns=new_columns, fit=False) # Restore the column for the next iteration if neighbor.shape[1] == 1: self.classifier.fit(current_best, y) else: self.classifier.remove_feature(neighbor.shape[1] - 1) self.classifier.add_features( current_best[:, column_to_delete].reshape(-1, 1), y, index=[column_to_delete]) else: action = "ADD" if current_best is None: neighbor = column_to_add self.classifier.fit(neighbor, y) else: neighbor = np.concatenate( [current_best, column_to_add], axis=1) self.classifier.add_features(column_to_add, y) score = self.evaluate(self.classifier, neighbor, y, columns=new_columns, fit=False) if current_best is None: self.classifier = NaiveBayes(encode_data=True) else: self.classifier.remove_feature(neighbor.shape[1] - 1) if self.verbose == 2: print("\tNeighbour: ", new_columns, " Score: ", score, "Available columns: ", new_available_columns) if score > best_score: stop = False best_columns = new_columns best_available_columns = new_available_columns best_action = action best_score = score best_column_to_delete = column_to_delete best_column_to_add = column_to_add update = True if score == 1.0: stop = True break if update: current_columns = best_columns available_columns = best_available_columns if best_action == "JOIN": self.classifier.add_features(best_column_to_add, y) self.classifier.remove_feature(best_column_to_delete) current_best = np.concatenate( [current_best, best_column_to_add], axis=1) current_best = np.delete(current_best, best_column_to_delete, axis=1) else: if current_best is None: current_best = best_column_to_add self.classifier.fit(current_best, y) else: current_best = np.concatenate( [current_best, best_column_to_add], axis=1) self.classifier.add_features(best_column_to_add, y) if self.verbose: print("Final best: ", list(current_columns), " Score: ", best_score) self.features_ = current_columns self.feature_transformer = lambda X: join_columns( X, columns=self.features_) model = self.classifier.fit(self.feature_transformer(X), y) return self
def scoring_comparison(base_path,datasets,verbose=1,test_size=0.3,seed=None,n_iterations=30): column_names = ["dataset", "custom_training_score", "custom_test_score", "categorical_training_score", "categorical_test_score"] data =[] clf_no_encoding = NaiveBayes(encode_data=True) clf_categorical_sklearn = CategoricalNB() datasets_iter = tqdm(datasets, bar_format='{l_bar}{bar:20}{r_bar}{bar:-10b}') c = CustomOrdinalFeatureEncoder() l = CustomLabelEncoder() for dataset in datasets_iter: dataset_name, label = dataset data_filename = f"{dataset_name}.data.csv" test_filename = f"{dataset_name}.test.csv" X, y = get_X_y_from_database(base_path=base_path, name = dataset_name, data = data_filename, test = test_filename, label = label) custom_train = [] custom_test = [] sklearn_train = [] sklearn_test = [] X = c.fit_transform(X) y = l.fit_transform(y) for iteration in range(n_iterations): if verbose: datasets_iter.set_postfix({"Dataset": dataset_name, "seed":iteration}) datasets_iter.refresh() try: X_train,X_test,y_train,y_test = train_test_split(X,y, test_size=test_size, random_state=seed+iteration, shuffle=True, stratify=y) except: #Not enough values to stratify y X_train,X_test,y_train,y_test = train_test_split(X,y, test_size=test_size, random_state=seed+iteration, shuffle=True ) #Fit clf_no_encoding.fit(X_train,y_train) clf_categorical_sklearn.min_categories = [1+np.max(np.concatenate([X_train[:,j],X_test[:,j]])) for j in range(X_train.shape[1])] clf_categorical_sklearn.fit(X_train,y_train) #Predict custom_train.append(clf_no_encoding.score(X_train,y_train)) custom_test.append(clf_no_encoding.score(X_test,y_test)) sklearn_train.append(clf_categorical_sklearn.score(X_train,y_train)) sklearn_test.append(clf_categorical_sklearn.score(X_test,y_test)) data.append([dataset_name,np.mean(custom_train),np.mean(custom_test),np.mean(sklearn_train),np.mean(sklearn_test)]) return pd.DataFrame(data,columns = column_names)
def explore(self, X, y, graph, random_generator, parallel, max_errors=0): ''' Search method that follows the following steps: 1. The initial node is connected to all the others (roulette wheel selection is performed) 2. There are 2 type of nodes (corresponding to an original feature (2.1) or corresponding to a value of a feature (2.2)): 2.1. If the selected node is an original feature we add it to the selected subset and go to step 3. 2.2. If the selected node is part of a logical feature then we select another node (the CONSTRUCTION step will not return full original features) 3. Compute the score 3.1. If it improves the previous one 3.1.1 Add the feature to the current subset 3.1.2 Update the score 3.1.3 Select another node (SELECTION step) 3.1.4 Go to step 2 3.2. If not, the exploration ends Note: Threading does not speed up the calculations as they are CPU bound and in python only I/O operations will benefit from this parallelism GPU improvement would reduce the time of the exploration. ''' self.step = math.ceil(math.log2(X.shape[1])) self.current_features = [] selected_nodes = set() constructed_nodes = set() classifier = NaiveBayes(encode_data=False, metric=self.metric) current_score = np.NINF score = 0 if self.use_initials: self.current_features = [ DummyFeatureConstructor(j) for j in range(X.shape[1]) ] classifier.fit(X, y) current_transformed_features_numpy = np.concatenate( [f.transform(X) for f in self.current_features], axis=1) score = self.evaluate_loo(self.current_features, classifier, current_transformed_features_numpy, y) current_score = score selected_nodes.update(graph.get_original_ids()) if len(self.current_features) == 0: current_transformed_features_numpy = None initial, pheromones, heuristics = graph.get_initial_nodes( selected_nodes) probabilities = self.compute_probability(pheromones, heuristics) index = self.choose_next(probabilities, random_generator) node_id, selected_node = initial[index] # SU variable contains the MIFS-SU for the selected variable current_su = 0 su = heuristics[index] is_fitted = self.use_initials feature_constructor = None n_errors = 0 number_steps = 1 while True: current_score = score if selected_node[1] is None: # Original Feature feature_constructor = DummyFeatureConstructor(selected_node[0]) selected_nodes.add(node_id) else: # Need to construct next feature and compute heuristic value for the feature to replace temporal su from half-var neighbours, pheromones = graph.get_neighbours( selected_node, constructed_nodes, step="CONSTRUCTION") if len(neighbours) == 0: break if self.beta != 0: if parallel: with concurrent.futures.ThreadPoolExecutor( ) as executor: futures = [] for neighbour in neighbours: futures.append( executor.submit( self.compute_neighbour_sufs, neighbour=neighbour, transformed_features= current_transformed_features_numpy, constructors=self.current_features, selected_node=selected_node, current_su=current_su, X=X, y=y)) concurrent.futures.wait( futures, timeout=None, return_when='ALL_COMPLETED') su = [future.result() for future in futures] else: su = [ self.compute_neighbour_sufs( neighbour=neighbour, transformed_features= current_transformed_features_numpy, selected_node=selected_node, constructors=self.current_features, current_su=current_su, X=X, y=y) for neighbour in neighbours ] else: #Avoid unnecessary evaluation su = np.ones(len(neighbours)) probabilities = self.compute_probability( pheromones, np.array(su)) index = self.choose_next(probabilities, random_generator) su = su[index] feature_constructor = create_feature( neighbours[index][2], [selected_node, neighbours[index][1]]) constructed_nodes.add( frozenset( (node_id, neighbours[index][0], neighbours[index][2]))) node_id, selected_node = neighbours[index][:2] # Assess new feature transformed_feature = feature_constructor.transform(X) if is_fitted: classifier.add_features(transformed_feature, y) else: classifier.fit(transformed_feature, y) is_fitted = True if current_transformed_features_numpy is None: current_transformed_features_numpy = transformed_feature else: current_transformed_features_numpy = append_column_to_numpy( current_transformed_features_numpy, transformed_feature) if number_steps >= self.step: score = self.evaluate_loo( self.current_features + [feature_constructor], classifier, current_transformed_features_numpy, y) if score <= current_score: if n_errors >= max_errors: break else: n_errors += 1 else: n_errors = 0 number_steps = 0 else: number_steps += 1 current_su = su self.current_features.append(feature_constructor) current_score = score # Select next neighbours, pheromones = graph.get_neighbours(selected_node, selected_nodes, step="SELECTION") # Compute heuristic su = [] if len(neighbours) == 0: break if self.beta != 0: for neighbour, pheromone in zip(neighbours, pheromones): if neighbour[1][1] is None: # Original variable su.append( self.compute_sufs_cached( current_su, current_transformed_features_numpy, X[:, neighbour[1][0]], self.current_features, DummyFeatureConstructor(neighbour[1][0]), y, minimum=0)) else: # This is a temporal variable that will not be finally selected but only used to calculate the heuristic su.append( self.compute_sufs_cached( current_su, current_transformed_features_numpy, X[:, neighbour[1][0]] == neighbour[1][1], self.current_features, FeatureOperand(feature_index=neighbour[1][0], value=neighbour[1][1]), y, minimum=0)) else: su = np.ones(len(neighbours)) probabilities = self.compute_probability(pheromones, np.array(su)) index = self.choose_next(probabilities, random_generator) su = su[index] node_id, selected_node = neighbours[index][:2] if current_transformed_features_numpy.shape[1] > len( self.current_features): current_transformed_features_numpy = np.delete( current_transformed_features_numpy, -1, axis=1) self.final_score = self.evaluate_loo( self.current_features, classifier, current_transformed_features_numpy, y) return self.final_score