def test_NumericalEncoder_default_and_null_values(): np.random.seed(123) df = get_sample_df(100, seed=123) df.index = np.arange(len(df)) df["cat_col_1"] = df["text_col"].apply(lambda s: s[0:3]) df.loc[0:10, "cat_col_1"] = None # All modalities are kept, __null__ category is created encoder = NumericalEncoder(encoding_type="num", min_modalities_number=2, max_cum_proba=0.8, max_na_percentage=0) res = encoder.fit_transform(df) assert "__default__" in encoder.model.variable_modality_mapping[ "cat_col_1"] assert "__null__" in encoder.model.variable_modality_mapping["cat_col_1"] df["cat_col_1"] = "zzz" # Never seen value res = encoder.transform(df) assert res["cat_col_1"].unique( )[0] == encoder.model.variable_modality_mapping["cat_col_1"]["__default__"] df["cat_col_1"] = None res = encoder.transform(df) assert res["cat_col_1"].unique( )[0] == encoder.model.variable_modality_mapping["cat_col_1"]["__null__"]
def test_NumericalEncoder_num(): ###################### ### Numerical Mode ### ###################### np.random.seed(123) df = get_sample_df(100, seed=123) ind = np.arange(len(df)) df.index = ind np.random.shuffle(ind) df["cat_col_1"] = df["text_col"].apply(lambda s: s[0:3]) df["cat_col_2"] = df["text_col"].apply(lambda s: s[3:6]) encoder = NumericalEncoder(encoding_type="num") encoder.fit(df) res = encoder.transform(df) assert res.shape == df.shape assert (res.index == df.index).all() assert encoder.get_feature_names() == encoder.model._feature_names assert encoder.get_feature_names() == list(res.columns) df2 = df.copy() df2.loc[0, "cat_col_1"] = "something-new" df2.loc[1, "cat_col_2"] = None # Something None res2 = encoder.transform(df2) assert res2.loc[0, "cat_col_1"] == -1 assert res2.loc[1, "cat_col_2"] == -1 df_with_none = df.copy() df_with_none["cat_col_3"] = df_with_none["cat_col_1"] df_with_none.loc[list(range(25)), "cat_col_3"] = None encoder2 = NumericalEncoder(encoding_type="num") res2 = encoder2.fit_transform(df_with_none) assert (df_with_none["cat_col_3"].isnull() == ( res2["cat_col_3"] == 0)).all()
def test_NumericalEncoder_dummy_output_dtype(): np.random.seed(123) df = get_sample_df(100, seed=123) ind = np.arange(len(df)) df.index = ind df["cat_col_1"] = df["text_col"].apply(lambda s: s[0:3]) df["cat_col_2"] = df["text_col"].apply(lambda s: s[3:6]) encoder = NumericalEncoder(encoding_type="dummy") encoder.fit(df) res = encoder.transform(df) assert (res.dtypes[res.columns.str.startswith("cat_col_")] == "int32" ).all() # check default encoding type = int32
def test_NumericalEncoder_num_output_dtype(): np.random.seed(123) df = get_sample_df(100, seed=123) ind = np.arange(len(df)) df.index = ind np.random.shuffle(ind) df["cat_col_1"] = df["text_col"].apply(lambda s: s[0:3]) df["cat_col_2"] = df["text_col"].apply(lambda s: s[3:6]) encoder = NumericalEncoder(encoding_type="num") encoder.fit(df) res = encoder.transform(df) assert res.dtypes["cat_col_1"] == "int32" assert res.dtypes["cat_col_2"] == "int32"
def test_NumericalEncoder_drop_used_unused_columns(drop_used_columns, drop_unused_columns, columns_to_use): # This test will verify the behavior of the encoder regarding the fact to drop or keep the use/unused columns df = pd.DataFrame({ "obj1": ["a", "b", "c", "d"] * 25, "obj2": ["AA", "BB"] * 50, "num1": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] * 10, "num2": [100, 101, 102, 103, 104] * 20, "num3": [0.01, 0.02, 0.03, 0.04, 0.05] * 20, }) df1 = df.loc[0:20, ] df2 = df.loc[20:] # for drop_used_columns, drop_unused_columns, columns_to_use in list(itertools.product((True,False),(True,False),("all","object",["num1","num2","num3"]))): resulting_columns = { col: ["%s__%s" % (col, str(v)) for v in df[col].value_counts().index] for col in df.columns } if columns_to_use == "all": cols = list(df.columns) elif columns_to_use == "object": cols = list(df.columns[df.dtypes == "object"]) else: cols = columns_to_use if drop_used_columns: columns_A = [] else: columns_A = cols columns_B = [] for c in cols: columns_B += resulting_columns[c] if drop_unused_columns: columns_C = [] else: columns_C = [c for c in df.columns if c not in cols] final_columns = columns_A + columns_C + columns_B encoder = NumericalEncoder(columns_to_use=columns_to_use, drop_used_columns=drop_used_columns, drop_unused_columns=drop_unused_columns) df1_transformed = encoder.fit_transform(df1) df2_transformed = encoder.transform(df2) assert df1_transformed.shape[0] == df1.shape[0] assert df2_transformed.shape[0] == df2.shape[0] assert type(df1_transformed) == type(df1) assert type(df2_transformed) == type(df2) assert (df1_transformed.index == df1.index).all() assert (df2_transformed.index == df2.index).all() assert df1_transformed.shape[1] == df2_transformed.shape[1] assert list(df1_transformed.columns) == list(df2_transformed.columns) assert len(df1_transformed.columns) == len(final_columns) assert set(df1_transformed) == set(final_columns) # assert list(df1_transformed.columns) == final_columns encoder = NumericalEncoder() encoder.fit(df) pickled_encoder = pickle.dumps(encoder) unpickled_encoder = pickle.loads(pickled_encoder) assert type(unpickled_encoder) == type(encoder) X1 = encoder.transform(df) X2 = unpickled_encoder.transform(df) assert X1.shape == X2.shape assert (X1 == X2).all().all()
def test_NumericalEncoder_dummy(): #################### ### One Hot Mode ### #################### np.random.seed(123) df = get_sample_df(100, seed=123) ind = np.arange(len(df)) df.index = ind df["cat_col_1"] = df["text_col"].apply(lambda s: s[0:3]) df["cat_col_2"] = df["text_col"].apply(lambda s: s[3:6]) encoder = NumericalEncoder(encoding_type="dummy") encoder.fit(df) res = encoder.transform(df) assert encoder.model._dummy_size == len(encoder.model._dummy_feature_names) assert encoder.model._dummy_size == sum( len(v) for k, v in encoder.model.variable_modality_mapping.items()) assert res.shape[0] == df.shape[0] assert res.shape[1] == len(df["cat_col_1"].value_counts()) + len( df["cat_col_2"].value_counts()) + 3 assert (res.index == df.index).all() col = ["float_col", "int_col", "text_col"] col1 = [ "cat_col_1__%s" % c for c in list(df["cat_col_1"].value_counts().index) ] col2 = [ "cat_col_2__%s" % c for c in list(df["cat_col_2"].value_counts().index) ] assert col1 == encoder.columns_mapping["cat_col_1"] assert col2 == encoder.columns_mapping["cat_col_2"] assert encoder.get_feature_names() == col + col1 + col2 assert (res.loc[:, col1 + col2]).isnull().sum().sum() == 0 assert (res.loc[:, col1 + col2]).max().max() == 1 assert (res.loc[:, col1 + col2]).min().min() == 0 assert ((df["cat_col_1"] == "aaa") == (res["cat_col_1__aaa"] == 1)).all() df2 = df.copy() df2.loc[0, "cat_col_1"] = "something-new" df2.loc[1, "cat_col_2"] = None # Something None res2 = encoder.transform(df2) assert res2.loc[0, col1].sum() == 0 # no dummy activated assert res2.loc[ 0, "cat_col_2__" + df2.loc[0, "cat_col_2"]] == 1 # activated in the right position assert res2.loc[0, col2].sum() == 1 # only one dummy activate assert res2.loc[1, col2].sum() == 0 # no dummy activated assert res2.loc[ 1, "cat_col_1__" + df2.loc[1, "cat_col_1"]] == 1 # activated in the right position assert res2.loc[1, col1].sum() == 1 df_with_none = df.copy() df_with_none["cat_col_3"] = df_with_none["cat_col_1"] df_with_none.loc[0:25, "cat_col_3"] = None encoder2 = NumericalEncoder(encoding_type="dummy") res2 = encoder2.fit_transform(df_with_none) col3b = [c for c in res2.columns if c.startswith("cat_col_3")] assert col3b[0] == "cat_col_3____null__" assert list(res2.columns) == col + col1 + col2 + col3b assert list(res2.columns) == encoder2.get_feature_names() assert (res2.loc[:, col1 + col2 + col3b]).isnull().sum().sum() == 0 assert (res2.loc[:, col1 + col2 + col3b]).max().max() == 1 assert (res2.loc[:, col1 + col2 + col3b]).min().min() == 0 assert (df_with_none["cat_col_3"].isnull() == ( res2["cat_col_3____null__"] == 1)).all() df3 = df.copy() df3["cat_col_many"] = [ "m_%d" % x for x in np.ceil(np.minimum(np.exp(np.random.rand(100) * 5), 50)).astype(np.int32) ] encoder3 = NumericalEncoder(encoding_type="dummy") res3 = encoder3.fit_transform(df3) colm = [c for c in res3.columns if c.startswith("cat_col_many")] vc = df3["cat_col_many"].value_counts() colmb = [ "cat_col_many__" + c for c in list(vc.index[vc >= encoder3.min_nb_observations]) + ["__default__"] ] assert colm == colmb