def test_indicators_are_correct():
transformer = Winsorizer(tail="left",
capping_method="quantiles",
fold=0.1,
add_indicators=True)
df = pd.DataFrame({"col": np.arange(100).astype(np.float64)})
df_out = transformer.fit_transform(df)
expected_ind = np.r_[np.repeat(True, 10),
np.repeat(False, 90)].astype(np.float64)
pd.testing.assert_frame_equal(
df_out.drop("col", axis=1),
df.assign(col_left=expected_ind).drop("col", axis=1))
transformer.set_params(tail="right")
df_out = transformer.fit_transform(df)
expected_ind = np.r_[np.repeat(False, 90),
np.repeat(True, 10)].astype(np.float64)
pd.testing.assert_frame_equal(
df_out.drop("col", axis=1),
df.assign(col_right=expected_ind).drop("col", axis=1),
)
transformer.set_params(tail="both")
df_out = transformer.fit_transform(df)
expected_ind_left = np.r_[np.repeat(True, 10),
np.repeat(False, 90)].astype(np.float64)
expected_ind_right = np.r_[np.repeat(False, 90),
np.repeat(True, 10)].astype(np.float64)
pd.testing.assert_frame_equal(
df_out.drop("col", axis=1),
df.assign(col_left=expected_ind_left,
col_right=expected_ind_right).drop("col", axis=1),
)
def test_indicators_are_added(df_normal_dist):
transformer = Winsorizer(tail="both",
capping_method="quantiles",
fold=0.1,
add_indicators=True)
X = transformer.fit_transform(df_normal_dist)
# test that the number of output variables is correct
assert X.shape[1] == 3 * df_normal_dist.shape[1]
assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0)
transformer = Winsorizer(tail="left",
capping_method="quantiles",
fold=0.1,
add_indicators=True)
X = transformer.fit_transform(df_normal_dist)
assert X.shape[1] == 2 * df_normal_dist.shape[1]
assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0)
transformer = Winsorizer(tail="right",
capping_method="quantiles",
fold=0.1,
add_indicators=True)
X = transformer.fit_transform(df_normal_dist)
assert X.shape[1] == 2 * df_normal_dist.shape[1]
assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0)
def test_iqr_capping_both_tails_with_fold_1(df_normal_dist):
# test case 3: IQR, both tails, fold 1
transformer = Winsorizer(capping_method="iqr", tail="both", fold=1)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(df_transf["var"] > 0.21180113880445128,
0.21180113880445128, df_transf["var"])
df_transf["var"] = np.where(df_transf["var"] < -0.20247907173293223,
-0.20247907173293223, df_transf["var"])
# test fit params
assert np.round(transformer.right_tail_caps_["var"],
3) == np.round(0.21180113880445128, 3)
assert np.round(transformer.left_tail_caps_["var"],
3) == np.round(-0.20247907173293223, 3)
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert np.round(X["var"].max(), 3) <= np.round(0.21180113880445128, 3)
assert np.round(X["var"].min(), 3) >= np.round(-0.20247907173293223, 3)
assert np.round(df_normal_dist["var"].max(), 3) > np.round(
0.21180113880445128, 3)
assert np.round(df_normal_dist["var"].min(), 3) < np.round(
-0.20247907173293223, 3)
def test_gaussian_capping_both_tails_with_fold_2(df_normal_dist):
# test case 2: mean and std, both tails, different fold value
transformer = Winsorizer(capping_method="gaussian", tail="both", fold=2)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(df_transf["var"] > 0.20857275540714884,
0.20857275540714884, df_transf["var"])
df_transf["var"] = np.where(df_transf["var"] < -0.19661115230025186,
-0.19661115230025186, df_transf["var"])
# test fit params
assert np.round(transformer.right_tail_caps_["var"],
3) == np.round(0.20857275540714884, 3)
assert np.round(transformer.left_tail_caps_["var"],
3) == np.round(-0.19661115230025186, 3)
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert np.round(X["var"].max(), 3) <= np.round(0.20857275540714884, 3)
assert np.round(X["var"].min(), 3) >= np.round(-0.19661115230025186, 3)
assert np.round(df_normal_dist["var"].max(), 3) > np.round(
0.20857275540714884, 3)
assert np.round(df_normal_dist["var"].min(), 3) < np.round(
-0.19661115230025186, 3)
def test_transformer_ignores_na_in_df(df_na):
# test case 7: dataset contains na and transformer is asked to ignore them
transformer = Winsorizer(
capping_method="gaussian",
tail="right",
fold=1,
variables=["Age", "Marks"],
missing_values="ignore",
)
X = transformer.fit_transform(df_na)
# expected output
df_transf = df_na.copy()
df_transf["Age"] = np.where(df_transf["Age"] > 38.79255087111844,
38.79255087111844, df_transf["Age"])
df_transf["Marks"] = np.where(df_transf["Marks"] > 0.8970309389976613,
0.8970309389976613, df_transf["Marks"])
# test fit params
transformer.right_tail_caps_ = {
key: round(value, 3)
for (key, value) in transformer.right_tail_caps_.items()
}
assert transformer.right_tail_caps_ == {
"Age": 38.793,
"Marks": 0.897,
}
assert transformer.left_tail_caps_ == {}
assert transformer.n_features_in_ == 6
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert np.round(X["Age"].max(), 3) <= np.round(38.79255087111844, 3)
assert np.round(df_na["Age"].max(), 3) > np.round(38.79255087111844, 3)
def test_quantile_capping_both_tails_with_fold_10_percent(df_normal_dist):
# test case 5: quantiles, both tails, fold 10%
transformer = Winsorizer(capping_method="quantiles", tail="both", fold=0.1)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(df_transf["var"] > 0.14712481122898166,
0.14712481122898166, df_transf["var"])
df_transf["var"] = np.where(df_transf["var"] < -0.12366227743232801,
-0.12366227743232801, df_transf["var"])
# test fit params
assert np.round(transformer.right_tail_caps_["var"],
3) == np.round(0.14712481122898166, 3)
assert np.round(transformer.left_tail_caps_["var"],
3) == np.round(-0.12366227743232801, 3)
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert np.round(X["var"].max(), 3) <= np.round(0.14712481122898166, 3)
assert np.round(X["var"].min(), 3) >= np.round(-0.12366227743232801, 3)
assert np.round(df_normal_dist["var"].max(), 3) > np.round(
0.14712481122898166, 3)
assert np.round(df_normal_dist["var"].min(), 3) < np.round(
-0.12366227743232801, 3)
def test_transformer_ignores_na_in_df(df_na):
# test case 7: dataset contains na and transformer is asked to ignore them
transformer = Winsorizer(
capping_method="gaussian",
tail="right",
fold=1,
variables=["Age", "Marks"],
missing_values="ignore",
)
X = transformer.fit_transform(df_na)
# expected output
df_transf = df_na.copy()
df_transf["Age"] = np.where(df_transf["Age"] > 38.79255087111844,
38.79255087111844, df_transf["Age"])
df_transf["Marks"] = np.where(df_transf["Marks"] > 0.8970309389976613,
0.8970309389976613, df_transf["Marks"])
# test fit params
assert transformer.right_tail_caps_ == {
"Age": 38.79255087111844,
"Marks": 0.8970309389976613,
}
assert transformer.left_tail_caps_ == {}
assert transformer.input_shape_ == (8, 6)
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert X["Age"].max() <= 38.79255087111844
assert df_na["Age"].max() > 38.79255087111844
def clean_data(X):
X.dropna(subset=['target'], inplace=True)
y = X.pop('target')
X.drop(columns='ID', inplace=True)
X['v22'] = X['v22'].apply(az_to_int)
cat_cols = X.select_dtypes(include=['object']).columns.tolist()
con_cols = X.select_dtypes(include=['number']).columns.tolist()
num_missing_imputer = SimpleImputer(strategy='median')
cat_missing_imputer = CategoricalImputer(fill_value='__MISS__')
rare_label_encoder = RareLabelEncoder(tol=0.01, n_categories=10, replace_with='__OTHER__')
cat_freq_encoder = CountFrequencyEncoder(encoding_method="frequency")
X[con_cols] = num_missing_imputer.fit_transform(X[con_cols])
X[cat_cols] = cat_missing_imputer.fit_transform(X[cat_cols])
X[cat_cols] = rare_label_encoder.fit_transform(X[cat_cols])
X[cat_cols] = cat_freq_encoder.fit_transform(X[cat_cols])
# more cleaning
trimmer = Winsorizer(capping_method='quantiles', tail='both', fold=0.005)
X = trimmer.fit_transform(X)
undersampler = RandomUnderSampler(sampling_strategy=0.7, random_state=1234)
X, Y = undersampler.fit_resample(X, y)
quasi_constant = DropConstantFeatures(tol=0.998)
X = quasi_constant.fit_transform(X)
print(f"Quasi Features to drop {quasi_constant.features_to_drop_}")
# Remove duplicated features¶
duplicates = DropDuplicateFeatures()
X = duplicates.fit_transform(X)
print(f"Duplicate feature sets {duplicates.duplicated_feature_sets_}")
print(f"Dropping duplicate features {duplicates.features_to_drop_}")
drop_corr = DropCorrelatedFeatures(method="pearson", threshold=0.95, missing_values="ignore")
X = drop_corr.fit_transform(X)
print(f"Drop correlated feature sets {drop_corr.correlated_feature_sets_}")
print(f"Dropping correlared features {drop_corr.features_to_drop_}")
X['target'] = Y
return X
def test_gaussian_capping_right_tail_with_fold_1(df_normal_dist):
# test case 1: mean and std, right tail
transformer = Winsorizer(capping_method="gaussian", tail="right", fold=1)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(
df_transf["var"] > 0.10727677848029868, 0.10727677848029868, df_transf["var"]
)
# test init params
assert transformer.capping_method == "gaussian"
assert transformer.tail == "right"
assert transformer.fold == 1
# test fit attr
assert np.round(transformer.right_tail_caps_["var"], 3) == np.round(
0.10727677848029868, 3
)
assert transformer.left_tail_caps_ == {}
assert transformer.input_shape_ == (100, 1)
# test transform outputs
pd.testing.assert_frame_equal(X, df_transf)
assert np.round(X["var"].max(), 3) <= np.round(0.10727677848029868, 3)
assert np.round(df_normal_dist["var"].max(), 3) > np.round(0.10727677848029868, 3)
def test_indicators_filter_variables(df_vartypes):
transformer = Winsorizer(
variables=["Age", "Marks"],
tail="both",
capping_method="quantiles",
fold=0.1,
add_indicators=True,
)
X = transformer.fit_transform(df_vartypes)
assert X.shape[1] == df_vartypes.shape[1] + 4
transformer.set_params(tail="left")
X = transformer.fit_transform(df_vartypes)
assert X.shape[1] == df_vartypes.shape[1] + 2
transformer.set_params(tail="right")
X = transformer.fit_transform(df_vartypes)
assert X.shape[1] == df_vartypes.shape[1] + 2
def test_iqr_capping_left_tail_with_fold_2(df_normal_dist):
# test case 4: IQR, left tail, fold 2
transformer = Winsorizer(capping_method="iqr", tail="left", fold=0.8)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(df_transf["var"] < -0.17486039103044,
-0.17486039103044, df_transf["var"])
# test fit params
assert transformer.right_tail_caps_ == {}
assert transformer.left_tail_caps_ == {"var": -0.17486039103044}
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert X["var"].min() >= -0.17486039103044
assert df_normal_dist["var"].min() < -0.17486039103044
def test_quantile_capping_both_tails_with_fold_15_percent(df_normal_dist):
# test case 6: quantiles, right tail, fold 15%
transformer = Winsorizer(capping_method="quantiles",
tail="right",
fold=0.15)
X = transformer.fit_transform(df_normal_dist)
# expected output
df_transf = df_normal_dist.copy()
df_transf["var"] = np.where(df_transf["var"] > 0.11823196128033647,
0.11823196128033647, df_transf["var"])
# test fit params
assert transformer.right_tail_caps_ == {"var": 0.11823196128033647}
assert transformer.left_tail_caps_ == {}
# test transform output
pd.testing.assert_frame_equal(X, df_transf)
assert X["var"].max() <= 0.11823196128033647
assert df_normal_dist["var"].max() > 0.11823196128033647