def test_log_base_e_plus_automatically_find_variables(df_vartypes):
# test case 1: log base e, automatically select variables
transformer = LogTransformer(base="e", variables=None)
X = transformer.fit_transform(df_vartypes)
# expected output
transf_df = df_vartypes.copy()
transf_df["Age"] = [2.99573, 3.04452, 2.94444, 2.89037]
transf_df["Marks"] = [-0.105361, -0.223144, -0.356675, -0.510826]
# test init params
assert transformer.base == "e"
assert transformer.variables is None
# test fit attr
assert transformer.variables_ == ["Age", "Marks"]
assert transformer.n_features_in_ == 5
# test transform output
pd.testing.assert_frame_equal(X, transf_df)
# test inverse_transform
Xit = transformer.inverse_transform(X)
# convert numbers to original format.
Xit["Age"] = Xit["Age"].round().astype("int64")
Xit["Marks"] = Xit["Marks"].round(1)
# test
pd.testing.assert_frame_equal(Xit, df_vartypes)
def test_log_base_10_plus_user_passes_var_list(df_vartypes):
# test case 2: log base 10, user passes variables
transformer = LogTransformer(base="10", variables="Age")
X = transformer.fit_transform(df_vartypes)
# expected output
transf_df = df_vartypes.copy()
transf_df["Age"] = [1.30103, 1.32222, 1.27875, 1.25527]
# test init params
assert transformer.base == "10"
assert transformer.variables == "Age"
# test fit attr
assert transformer.variables_ == ["Age"]
assert transformer.n_features_in_ == 5
# test transform output
pd.testing.assert_frame_equal(X, transf_df)
# test inverse_transform
Xit = transformer.inverse_transform(X)
# convert numbers to original format.
Xit["Age"] = Xit["Age"].round().astype("int64")
# test
pd.testing.assert_frame_equal(Xit, df_vartypes)
def test_log_base_10_plus_user_passes_var_list(df_vartypes):
# test case 2: log base 10, user passes variables
transformer = LogTransformer(base="10", variables="Age")
X = transformer.fit_transform(df_vartypes)
# expected output
transf_df = df_vartypes.copy()
transf_df["Age"] = [1.30103, 1.32222, 1.27875, 1.25527]
# test init params
assert transformer.base == "10"
assert transformer.variables == ["Age"]
# test fit attr
assert transformer.input_shape_ == (4, 5)
# test transform output
pd.testing.assert_frame_equal(X, transf_df)
def test_log_base_e_plus_automatically_find_variables(df_vartypes):
# test case 1: log base e, automatically select variables
transformer = LogTransformer(base="e", variables=None)
X = transformer.fit_transform(df_vartypes)
# expected output
transf_df = df_vartypes.copy()
transf_df["Age"] = [2.99573, 3.04452, 2.94444, 2.89037]
transf_df["Marks"] = [-0.105361, -0.223144, -0.356675, -0.510826]
# test init params
assert transformer.base == "e"
assert transformer.variables == ["Age", "Marks"]
# test fit attr
assert transformer.input_shape_ == (4, 5)
# test transform output
pd.testing.assert_frame_equal(X, transf_df)
def test_inverse_e_plus_user_passes_var_list(df_vartypes):
# test case 7: inverse log, user passes variables
transformer = LogTransformer(variables="Age")
Xt = transformer.fit_transform(df_vartypes)
X = transformer.inverse_transform(Xt)
# convert floats to int
X["Age"] = X["Age"].round().astype("int64")
# test init params
assert transformer.base == "e"
assert transformer.variables == "Age"
# test fit attr
assert transformer.variables_ == ["Age"]
assert transformer.n_features_in_ == 5
# test transform output
pd.testing.assert_frame_equal(X, df_vartypes)
print(mean_imputer.imputer_dict_)
X_train = mean_imputer.transform(X_train)
X_test = mean_imputer.transform(X_test)
# %% Varief whether there are missing value.
X_train[cat_vars_with_na].isnull().sum()
[var for var in cat_vars_with_na if X_test[var].isnull().sum() > 0]
#%% Temporal variables.
def elapsed_years(df, var):
df[var] = df['YrSold'] - df[var]
return df
for var in ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']：
X_train = elapsed_years(X_train, var)
X_test = elapsed_years(X_test, var)
# now we drop YrSold.
drop_features = DropFeatures(features_to_drop=['YrSold'])
X_train = mean_imputer.fit_transform(X_train)
X_test = mean_imputer.transform(X_test)
# %% Numerical variable -- transformation.
log_transformer = LogTransformer(
variables=["LotFrontage", "1stFlrSF", "GrLivArea"],
)
X_train = log_transformer.fit_transform(X_train)
X_test = log_transformer.transform(X_test)
# check that test set does not contain null values in the engineered variables
[var for var in ["LotFrontage", "1stFlrSF", "GrLivArea"] if X_test[var].isnull().sum() > 0]