def test_lower_threshold(df_correlated_single): transformer = DropCorrelatedFeatures(variables=None, method="pearson", threshold=0.6) X = transformer.fit_transform(df_correlated_single) # expected result df = df_correlated_single.drop(["var_2", "var_4"], axis=1) # test init params assert transformer.method == "pearson" assert transformer.threshold == 0.6 assert transformer.variables is None # test fit attrs assert transformer.variables_ == [ "var_0", "var_1", "var_2", "var_3", "var_4", "var_5", ] assert transformer.features_to_drop_ == {"var_2", "var_4"} assert transformer.correlated_feature_sets_ == [{ "var_1", "var_2", "var_4" }] # test transform output pd.testing.assert_frame_equal(X, df)
def test_default_params(df_correlated_single): transformer = DropCorrelatedFeatures(variables=None, method="pearson", threshold=0.8) X = transformer.fit_transform(df_correlated_single) # expected result df = df_correlated_single.drop("var_2", axis=1) # test init params assert transformer.method == "pearson" assert transformer.threshold == 0.8 assert transformer.variables == [ "var_0", "var_1", "var_2", "var_3", "var_4", "var_5", ] # test fit attrs pd.testing.assert_frame_equal(transformer.correlated_matrix_, df_correlated_single.corr()) assert transformer.correlated_features_ == {"var_2"} assert transformer.correlated_feature_sets_ == [{"var_1", "var_2"}] # test transform output pd.testing.assert_frame_equal(X, df)
def clean_data(X): X.dropna(subset=['target'], inplace=True) y = X.pop('target') X.drop(columns='ID', inplace=True) X['v22'] = X['v22'].apply(az_to_int) cat_cols = X.select_dtypes(include=['object']).columns.tolist() con_cols = X.select_dtypes(include=['number']).columns.tolist() num_missing_imputer = SimpleImputer(strategy='median') cat_missing_imputer = CategoricalImputer(fill_value='__MISS__') rare_label_encoder = RareLabelEncoder(tol=0.01, n_categories=10, replace_with='__OTHER__') cat_freq_encoder = CountFrequencyEncoder(encoding_method="frequency") X[con_cols] = num_missing_imputer.fit_transform(X[con_cols]) X[cat_cols] = cat_missing_imputer.fit_transform(X[cat_cols]) X[cat_cols] = rare_label_encoder.fit_transform(X[cat_cols]) X[cat_cols] = cat_freq_encoder.fit_transform(X[cat_cols]) # more cleaning trimmer = Winsorizer(capping_method='quantiles', tail='both', fold=0.005) X = trimmer.fit_transform(X) undersampler = RandomUnderSampler(sampling_strategy=0.7, random_state=1234) X, Y = undersampler.fit_resample(X, y) quasi_constant = DropConstantFeatures(tol=0.998) X = quasi_constant.fit_transform(X) print(f"Quasi Features to drop {quasi_constant.features_to_drop_}") # Remove duplicated features¶ duplicates = DropDuplicateFeatures() X = duplicates.fit_transform(X) print(f"Duplicate feature sets {duplicates.duplicated_feature_sets_}") print(f"Dropping duplicate features {duplicates.features_to_drop_}") drop_corr = DropCorrelatedFeatures(method="pearson", threshold=0.95, missing_values="ignore") X = drop_corr.fit_transform(X) print(f"Drop correlated feature sets {drop_corr.correlated_feature_sets_}") print(f"Dropping correlared features {drop_corr.features_to_drop_}") X['target'] = Y return X
def test_callable_method(df_correlated_double, random_uniform_method): X = df_correlated_double transformer = DropCorrelatedFeatures(variables=None, method=random_uniform_method, threshold=0.6) Xt = transformer.fit_transform(X) # test no empty dataframe assert not Xt.empty # test fit attrs assert len(transformer.correlated_feature_sets_) > 0 assert len(transformer.features_to_drop_) > 0 assert len(transformer.variables_) > 0 assert transformer.n_features_in_ == len(X.columns)
def test_error_method_supplied(df_correlated_double): X = df_correlated_double method = "hola" transformer = DropCorrelatedFeatures(variables=None, method=method, threshold=0.8) with pytest.raises(ValueError) as errmsg: _ = transformer.fit_transform(X) exceptionmsg = errmsg.value.args[0] assert ( exceptionmsg == "method must be either 'pearson', 'spearman', 'kendall', or a callable," + f" '{method}' was supplied")
def test_more_than_1_correlated_group(df_correlated_double): transformer = DropCorrelatedFeatures(variables=None, method="pearson", threshold=0.6) X = transformer.fit_transform(df_correlated_double) # expected result df = df_correlated_double.drop(["var_6", "var_7", "var_8", "var_9"], axis=1) # test fit attrs assert transformer.features_to_drop_ == { "var_6", "var_7", "var_8", "var_9" } assert transformer.correlated_feature_sets_ == [ {"var_0", "var_8"}, {"var_4", "var_6", "var_7", "var_9"}, ] # test transform output pd.testing.assert_frame_equal(X, df)