def test_easy_preprocessor_cat_cols(): # Create dataframe with 1 numerical and 3 categorical features. cat_1_unique = ['a', 'b', 'c'] cat_2_unique = ['D', 'E', 'F', 'G'] cat_3_unique = [-2.0, -1.0] data = pd.DataFrame({ 'cat_1': ['a', 'b', 'a', '', '', 'c', 'a', 'c', 'a', 'a'], 'cat_2': ['D', 'D', 'E', np.NaN, 'D', np.NaN, 'E', 'F', 'F', 'G'], 'cat_3': [-2., -1., -1., -2., -1., '', -1., -2., '', np.NaN], 'num': np.sin(range(10)), # Valid continuous feature. }) # Preprocess data, i.e. replace empty strings with NaNs, impute NaNs, and # encode categorical variables with OneHotEncoder. ep = EasyPreprocessor() data_t = ep.fit_transform(data) cat_all = [cat_1_unique, cat_2_unique, cat_3_unique] n_unique_cats = sum(len(cats) for cats in cat_all) # The number of features after preprocessing must be equal to # the number of unique categories within the dataframe + the number of # remaining valid features. assert data_t.shape[1] == n_unique_cats + 1 cat_pipe = ep.ct_.named_transformers_['categorical'] ohe = cat_pipe.named_steps['onehotencoder'] # The category sets detected by OneHotEncoder inside EasyPreprocessor # must match the ones specified in cat_all. assert len(ohe.categories_) == len(cat_all) for cat_list, ohe_cat_list in zip(cat_all, ohe.categories_): assert set(cat_list) == set(ohe_cat_list)
def test_titanic_feature_names(): path = os.path.dirname(__file__) titanic = pd.read_csv(os.path.join(path, '../datasets/titanic.csv')) ep = EasyPreprocessor() ep.fit(clean(titanic.drop('survived', axis=1))) expected_names = [ 'sibsp', 'parch', 'age_dabl_continuous', 'fare_dabl_continuous', 'body_dabl_continuous', 'pclass_1', 'pclass_2', 'pclass_3', 'sex_female', 'sex_male', 'sibsp_0', 'sibsp_1', 'sibsp_2', 'sibsp_3', 'sibsp_4', 'sibsp_5', 'sibsp_8', 'parch_0', 'parch_1', 'parch_2', 'parch_3', 'parch_4', 'parch_5', 'parch_6', 'parch_9', 'embarked_?', 'embarked_C', 'embarked_Q', 'embarked_S', 'boat_1', 'boat_10', 'boat_11', 'boat_12', 'boat_13', 'boat_13 15', 'boat_13 15 B', 'boat_14', 'boat_15', 'boat_15 16', 'boat_16', 'boat_2', 'boat_3', 'boat_4', 'boat_5', 'boat_5 7', 'boat_5 9', 'boat_6', 'boat_7', 'boat_8', 'boat_8 10', 'boat_9', 'boat_?', 'boat_A', 'boat_B', 'boat_C', 'boat_C D', 'boat_D', 'age_?_0.0', 'age_?_1.0', 'body_?_0.0', 'body_?_1.0' ] assert ep.get_feature_names() == expected_names # without clean X = ep.fit_transform(titanic.drop('survived', axis=1)) # FIXME can't do that yet # assert ep.get_feature_names() == expected_names_no_clean assert not np.isnan(X).any()
def test_simple_preprocessor(): sp = EasyPreprocessor() trans = sp.fit_transform(X_cat) assert trans.shape == (3, 6) iris = load_iris() sp = EasyPreprocessor() sp.fit(iris.data)