def test_user_provides_grouping_label_name_and_variable_list(df_enc_big):
# test case 2: user provides alternative grouping value and variable list
encoder = RareLabelEncoder(tol=0.15,
n_categories=5,
variables=["var_A", "var_B"],
replace_with="Other")
X = encoder.fit_transform(df_enc_big)
# expected output
df = {
"var_A": ["A"] * 6 + ["B"] * 10 + ["Other"] * 4 + ["D"] * 10 +
["Other"] * 4 + ["G"] * 6,
"var_B": ["A"] * 10 + ["B"] * 6 + ["Other"] * 4 + ["D"] * 10 +
["Other"] * 4 + ["G"] * 6,
"var_C": ["A"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 + ["E"] * 2 +
["F"] * 2 + ["G"] * 6,
}
df = pd.DataFrame(df)
# test init params
assert encoder.tol == 0.15
assert encoder.n_categories == 5
assert encoder.replace_with == "Other"
assert encoder.variables == ["var_A", "var_B"]
# test fit attr
assert encoder.variables_ == ["var_A", "var_B"]
assert encoder.n_features_in_ == 3
# test transform output
pd.testing.assert_frame_equal(X, df)
def test_defo_params_plus_automatically_find_variables(df_enc_big):
# test case 1: defo params, automatically select variables
encoder = RareLabelEncoder(tol=0.06,
n_categories=5,
variables=None,
replace_with="Rare")
X = encoder.fit_transform(df_enc_big)
# expected output
df = {
"var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_C": ["A"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
}
df = pd.DataFrame(df)
# test init params
assert encoder.tol == 0.06
assert encoder.n_categories == 5
assert encoder.replace_with == "Rare"
assert encoder.variables is None
# test fit attr
assert encoder.variables_ == ["var_A", "var_B", "var_C"]
assert encoder.n_features_in_ == 3
# test transform output
pd.testing.assert_frame_equal(X, df)
def test_variables_cast_as_category(df_enc_big):
# test case 1: defo params, automatically select variables
encoder = RareLabelEncoder(tol=0.06,
n_categories=5,
variables=None,
replace_with="Rare")
df_enc_big = df_enc_big.copy()
df_enc_big["var_B"] = df_enc_big["var_B"].astype("category")
X = encoder.fit_transform(df_enc_big)
# expected output
df = {
"var_A": ["A"] * 6 + ["B"] * 10 + ["C"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_B": ["A"] * 10 + ["B"] * 6 + ["C"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_C": ["A"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
}
df = pd.DataFrame(df)
# test fit attr
assert encoder.variables_ == ["var_A", "var_B", "var_C"]
assert encoder.n_features_in_ == 3
# test transform output
pd.testing.assert_frame_equal(X, df)
def clean_data(X):
X.dropna(subset=['target'], inplace=True)
y = X.pop('target')
X.drop(columns='ID', inplace=True)
X['v22'] = X['v22'].apply(az_to_int)
cat_cols = X.select_dtypes(include=['object']).columns.tolist()
con_cols = X.select_dtypes(include=['number']).columns.tolist()
num_missing_imputer = SimpleImputer(strategy='median')
cat_missing_imputer = CategoricalImputer(fill_value='__MISS__')
rare_label_encoder = RareLabelEncoder(tol=0.01, n_categories=10, replace_with='__OTHER__')
cat_freq_encoder = CountFrequencyEncoder(encoding_method="frequency")
X[con_cols] = num_missing_imputer.fit_transform(X[con_cols])
X[cat_cols] = cat_missing_imputer.fit_transform(X[cat_cols])
X[cat_cols] = rare_label_encoder.fit_transform(X[cat_cols])
X[cat_cols] = cat_freq_encoder.fit_transform(X[cat_cols])
# more cleaning
trimmer = Winsorizer(capping_method='quantiles', tail='both', fold=0.005)
X = trimmer.fit_transform(X)
undersampler = RandomUnderSampler(sampling_strategy=0.7, random_state=1234)
X, Y = undersampler.fit_resample(X, y)
quasi_constant = DropConstantFeatures(tol=0.998)
X = quasi_constant.fit_transform(X)
print(f"Quasi Features to drop {quasi_constant.features_to_drop_}")
# Remove duplicated features¶
duplicates = DropDuplicateFeatures()
X = duplicates.fit_transform(X)
print(f"Duplicate feature sets {duplicates.duplicated_feature_sets_}")
print(f"Dropping duplicate features {duplicates.features_to_drop_}")
drop_corr = DropCorrelatedFeatures(method="pearson", threshold=0.95, missing_values="ignore")
X = drop_corr.fit_transform(X)
print(f"Drop correlated feature sets {drop_corr.correlated_feature_sets_}")
print(f"Dropping correlared features {drop_corr.features_to_drop_}")
X['target'] = Y
return X
def test_max_n_categories(df_enc_big):
# test case 6: user provides the maximum number of categories they want
rare_encoder = RareLabelEncoder(tol=0.10,
max_n_categories=4,
n_categories=5)
X = rare_encoder.fit_transform(df_enc_big)
df = {
"var_A": ["A"] * 6 + ["B"] * 10 + ["Rare"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_B": ["A"] * 10 + ["B"] * 6 + ["Rare"] * 4 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
"var_C": ["Rare"] * 4 + ["B"] * 6 + ["C"] * 10 + ["D"] * 10 +
["Rare"] * 4 + ["G"] * 6,
}
df = pd.DataFrame(df)
pd.testing.assert_frame_equal(X, df)
def test_max_n_categories_with_numeric_var(df_enc_numeric):
# ignore_format=True
rare_encoder = RareLabelEncoder(tol=0.10,
max_n_categories=2,
n_categories=1,
ignore_format=True)
X = rare_encoder.fit_transform(df_enc_numeric[["var_A", "var_B"]])
df = df_enc_numeric[["var_A", "var_B"]].copy()
df.replace({3: "Rare"}, inplace=True)
# massive workaround because for some reason, doing a normal pd.assert_equal
# was telling me that 2 columns that were identical, were actually not.
# I think there was a problem with the type of each number perhaps
for i in range(len(df)):
assert str(list(X["var_A"])[i]) == str(list(df["var_A"])[i])
assert str(list(X["var_B"])[i]) == str(list(df["var_B"])[i])
def transform(self, X, y=None):
pd.options.mode.chained_assignment = None # default='warn' - turn off warning about data overwrite
for category in self.categories:
x = X[category].copy() # not use copy to intentionally change value
idx_nan = x.loc[pd.isnull(x)].index # find nan values in analyzed feature column
# replace missing values
x[idx_nan] = 'MISS'
encoder = RareLabelEncoder(tol=self.tol, n_categories=self.n_categories,
max_n_categories=self.max_n_categories,
replace_with=self.replace_with)
x = x.to_frame(name=category) # convert pd.series to dataframe
x = encoder.fit_transform(x)
X[category] = x
if not self.impute_missing_label:
X[category].loc[idx_nan] = np.nan
pd.options.mode.chained_assignment = 'warn' # default='warn' - turn on warning about data overwrite
return X
#so to avoid ranking these features, we apply rare label + weight of evidence encoding for C5
#this is for the Logistic Regression model only, as ordinality isn't a problem for tree-based models
# rare label encoding:
# we set the threshold to 0.1
# categories with proporation lower than 0.1 may not have any class label 1 due to the label imbalance
# and this will impede the application of WOE encoding (log 0 is undefined)
encoder = RareLabelEncoder(tol=0.1,
n_categories=2,
variables=[
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8',
'C9', 'C10', 'C11', 'C12'
],
replace_with='Rare')
train_enc = encoder.fit_transform(X_sm_c)
#WOE encoding:
woe_encoder = WoEEncoder(variables=[
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12'
])
train_enc1 = woe_encoder.fit_transform(train_enc, X_sm['newlabel'])
train_enc1
"""# 3. Model Building
# Logistic Regression
"""
#reassemble training dataset
#for categorical features, use the datasets after applying rare label + WOE encoding