def test_select_dtypes_include_using_list_like(self):
df = DataFrame(
{
"a": list("abc"),
"b": list(range(1, 4)),
"c": np.arange(3, 6).astype("u1"),
"d": np.arange(4.0, 7.0, dtype="float64"),
"e": [True, False, True],
"f": pd.Categorical(list("abc")),
"g": pd.date_range("20130101", periods=3),
"h": pd.date_range("20130101", periods=3, tz="US/Eastern"),
"i": pd.date_range("20130101", periods=3, tz="CET"),
"j": pd.period_range("2013-01", periods=3, freq="M"),
"k": pd.timedelta_range("1 day", periods=3),
}
)
all = column_all(df)
assert all == ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']
num = column_number(df)
assert num == ['b', 'c', 'd', 'k']
num_exclude_time = column_number_exclude_timedelta(df)
assert num_exclude_time == ['b', 'c', 'd']
o_c_b = column_object_category_bool(df)
assert o_c_b == ['a', 'e', 'f']
o = column_object(df)
assert o == ['a']
c = column_category(df)
assert c == ['f']
b = column_bool(df)
assert b == ['e']
t = column_timedelta(df)
assert t == ['k']
d_tz = column_datetimetz(df)
assert d_tz == ['h', 'i']
d = column_datetime(df)
assert d == ['g']
all_d = column_all_datetime(df)
assert all_d == ['g', 'h', 'i']
skewed = column_skewness_kurtosis(df, 0.5)
assert skewed == ['b', 'c', 'd']
def test_in_dataframe_mapper(self):
df = dsutils.load_bank()
df.drop(['id'], axis=1, inplace=True)
X_train, X_test = train_test_split(df.head(100),
test_size=0.2,
random_state=42)
ftt = FeatureGenerationTransformer(
task='binary',
trans_primitives=['cross_categorical'],
categories_cols=column_object_category_bool(X_train))
dfm = DataFrameMapper(features=[(X_train.columns.to_list(), ftt)],
input_df=True,
df_out=True)
X_t = dfm.fit_transform(X_train)
assert X_t.shape == (80, 62)
def test_pipeline(self):
df = dsutils.load_bank()
df.drop(['id'], axis=1, inplace=True)
X_train, X_test = train_test_split(df.head(100),
test_size=0.2,
random_state=42)
ftt = FeatureGenerationTransformer(
task='binary',
trans_primitives=['cross_categorical'],
categories_cols=column_object_category_bool(X_train))
preprocessor = general_preprocessor()
pipe = Pipeline(steps=[('feature_gen', ftt), ('processor',
preprocessor)])
X_t = pipe.fit_transform(X_train)
print(X_t.columns)
assert X_t.shape == (80, 62)
def test_feature_generation_with_selection(self):
df = dsutils.load_bank().head(1000)
df.drop(['id'], axis=1, inplace=True)
y = df.pop('y')
ftt = FeatureGenerationTransformer(
task='binary',
trans_primitives=[
'add_numeric', 'divide_numeric', 'cross_categorical'
],
categories_cols=column_object_category_bool(df),
feature_selection_args={'ratio_select_cols': 0.2})
with pytest.raises(AssertionError) as err:
ftt.fit(df)
assert err.value == '`y` must be provided for feature selection.'
ftt.fit(df, y)
x_t = ftt.transform(df)
assert x_t.shape[1] == 35
def test_feature_tools_categorical_cross(self):
df = dsutils.load_bank()
df.drop(['id'], axis=1, inplace=True)
X_train, X_test = train_test_split(df.head(100),
test_size=0.2,
random_state=42)
cat_cols = column_object_category_bool(X_train)
ftt = FeatureGenerationTransformer(
task='binary',
trans_primitives=['cross_categorical'],
categories_cols=cat_cols)
ftt.fit(X_train)
x_t = ftt.transform(X_train)
columns = set(x_t.columns.to_list())
for i_left in range(len(cat_cols) - 1):
for i_right in range(i_left + 1, len(cat_cols)):
assert f'CROSS_CATEGORICAL_{cat_cols[i_left]}__{cat_cols[i_right]}' in columns \
or f'CROSS_CATEGORICAL_{cat_cols[i_right]}__{cat_cols[i_left]}' in columns
def test_feature_selection(self):
df = dsutils.load_bank().head(1000)
df.drop(['id'], axis=1, inplace=True)
y = df.pop('y')
ftt = FeatureGenerationTransformer(
task='binary',
trans_primitives=[
'add_numeric', 'divide_numeric', 'cross_categorical'
],
categories_cols=column_object_category_bool(df))
ftt.fit(df)
x_t = ftt.transform(df)
fst = FeatureSelectionTransformer('binary',
ratio_select_cols=0.2,
reserved_cols=ftt.original_cols)
fst.fit(x_t, y)
assert len(fst.scores_.items()) == 99
assert len(fst.columns_) == 35
x_t2 = fst.transform(x_t)
assert x_t2.shape[1] == 35
def get_x_data_character(X_train, get_step):
cnt_x_all = len(col_se.column_all(X_train))
cnt_x_date = len(col_se.column_all_datetime(X_train))
cnt_x_category = len(col_se.column_object_category_bool(X_train))
cnt_x_num = len(col_se.column_number(X_train))
try:
kwargs = get_step('feature_generation').transformer_kwargs
if kwargs['text_cols'] != None:
cnt_x_text = len(kwargs['text_cols'])
cnt_x_all -= len(col_se.column_text(X_train)) - len(kwargs['text_cols'])
else:
cnt_x_text = len(col_se.column_text(X_train))
if kwargs['latlong_cols'] != None:
cnt_x_latlong = len(kwargs['latlong_cols'])
cnt_x_all += len(kwargs['latlong_cols'])
else:
cnt_x_latlong = 0
except:
cnt_x_text = len(col_se.column_text(X_train))
cnt_x_latlong = 0
cnt_x_others = cnt_x_all - cnt_x_date - cnt_x_category - cnt_x_num - cnt_x_text - cnt_x_latlong
x_types = {
'experimentType': 'compete',
'featureDistribution':{
'nContinuous':cnt_x_num,
'nText':cnt_x_text,
'nDatetime':cnt_x_date,
'nCategorical':cnt_x_category,
'nLocation':cnt_x_latlong,
'nOthers':cnt_x_others
}
}
return x_types