def test_sum(self):
"""
:return:
"""
cols = ['C1', 'D', 'E', 'F']
X = self.create_dataset(n_rows=1000)
X_t = self.create_dataset(n_rows=100)
enc = encoders.SumEncoder(verbose=1, cols=cols)
enc.fit(X, None)
self.verify_numeric(enc.transform(X_t))
enc = encoders.SumEncoder(verbose=1)
enc.fit(X, None)
self.verify_numeric(enc.transform(X_t))
enc = encoders.SumEncoder(verbose=1, drop_invariant=True)
enc.fit(X, None)
self.verify_numeric(enc.transform(X_t))
enc = encoders.SumEncoder(verbose=1, return_df=False)
enc.fit(X, None)
self.assertTrue(isinstance(enc.transform(X_t), np.ndarray))
def _fit_sum(self, df, y, target, parameter):
sum_encoder = ce.SumEncoder()
sum_encoder.fit(df[target].map(to_str))
name = ['continuous_' + remove_continuous_discrete_prefix(x) + '_poly' for x in
sum_encoder.get_feature_names()]
self.trans_ls.append(('catboost', name, target, sum_encoder))
def sum_coding():
X, _, _ = get_mushroom_data()
print(X.info())
enc = ce.SumEncoder()
enc.fit(X, None)
out = enc.transform(X)
print(out.info())
del enc, _, X, out
def apply_sum_encoding(df, categorical_columns):
if not isinstance(df, pd.DataFrame):
raise DataFrameTypeError('df', df)
import category_encoders as ce
encoder = ce.SumEncoder(cols=categorical_columns).fit(df.values)
X_transformed = encoder.transform(df)
X.drop(['intercept'], inplace=True, axis=1)
return X_transformed
def test_HandleUnknown_HaveOnlyKnown_ExpectSecondColumn(self):
train = ['A', 'B']
encoder = encoders.SumEncoder(handle_unknown='indicator',
handle_missing='value')
result = encoder.fit_transform(train)
expected = [a_encoding, b_encoding]
self.assertEqual(result.values.tolist(), expected)
def test_HandleMissingIndicator_NanInTrain_ExpectAsColumn(self):
train = ['A', 'B', np.nan]
encoder = encoders.SumEncoder(handle_missing='indicator',
handle_unknown='value')
result = encoder.fit_transform(train)
expected = [a_encoding, b_encoding, c_encoding]
self.assertEqual(result.values.tolist(), expected)
def create_features(self, df_train, df_test):
encoder = ce.SumEncoder(cols=self.columns)
encoder.fit(df_train[self.columns],
df_train[self.target_column].values.tolist())
encoded_train = encoder.transform(df_train[self.columns])
encoded_test = encoder.transform(df_test[self.columns])
for column in encoded_train.columns:
self.train[column + '_SumEncoder'] = encoded_train[column]
self.test[column + '_SumEncoder'] = encoded_test[column]
def test_sum_encoder_preserve_dimension_4(self):
train = ['A', 'B', 'C']
test = ['D', 'B', 'C', None]
encoder = encoders.SumEncoder()
encoder.fit(train)
test_t = encoder.transform(test)
expected = [[1, 0, 0], b_encoding, c_encoding, [1, 0, 0]]
self.assertEqual(test_t.values.tolist(), expected)
def main(params, inputs, outputs):
columns_param = params.columns
data = inputs.data
data_new = outputs.data_new
data_0 = pd.read_pickle(data)
encoder = ce.SumEncoder(cols=[col for col in columns_param.split(",")])
data_1 = encoder.fit_transform(data_0)
data_1.to_pickle(data_new)
def test_sum_encoder_preserve_dimension_3(self):
train = ['A', 'B', 'C']
test = ['A', 'B', 'C', None]
encoder = encoders.SumEncoder(handle_unknown='value',
handle_missing='value')
encoder.fit(train)
test_t = encoder.transform(test)
expected = [a_encoding, b_encoding, c_encoding, [1, 0, 0]]
self.assertEqual(test_t.values.tolist(), expected)
def test_HandleUnknown_HaveNoUnknownInTrain_ExpectIndicatorInTest(self):
train = ['A', 'B']
test = ['A', 'B', 'C']
encoder = encoders.SumEncoder(handle_unknown='indicator',
handle_missing='value')
encoder.fit(train)
result = encoder.transform(test)
expected = [a_encoding, b_encoding, c_encoding]
self.assertEqual(result.values.tolist(), expected)
def test_sum_np(self):
"""
:return:
"""
X = self.create_array(n_rows=1000)
X_t = self.create_array(n_rows=100)
enc = encoders.SumEncoder(verbose=1)
enc.fit(X, None)
self.verify_numeric(enc.transform(X_t))
def __init__(self, encoder_type, columns_name=None):
"""
:param encoder_type:
:param columns_name: list， 特征名组成的列表名
"""
if encoder_type == "BackwardDe": # 反向差分编码
self.encoder = ce.BackwardDifferenceEncoder(cols=columns_name)
elif encoder_type == "BaseN": # BaseN编码
self.encoder = ce.BaseNEncoder(cols=columns_name)
elif encoder_type == "Binary": # 二值编码
self.encoder = ce.BinaryEncoder(cols=columns_name)
elif encoder_type == "Catboost":
self.encoder = ce.CatBoostEncoder(cols=columns_name)
elif encoder_type == "Hash":
self.encoder = ce.HashingEncoder(cols=columns_name)
elif encoder_type == "Helmert":
self.encoder = ce.HelmertEncoder(cols=columns_name)
elif encoder_type == "JamesStein":
self.encoder = ce.JamesSteinEncoder(cols=columns_name)
elif encoder_type == "LOO": # LeaveOneOutEncoder 编码
self.encoder = ce.LeaveOneOutEncoder(cols=columns_name)
elif encoder_type == "ME":
self.encoder = ce.MEstimateEncoder(cols=columns_name) # M估计编码器
elif encoder_type == "OneHot":
self.encoder = ce.OneHotEncoder(cols=columns_name)
elif encoder_type == "OridinalEncoder": # 原始编码
self.encoder = ce.OrdinalEncoder(cols=columns_name)
elif encoder_type == "Sum": # 求和编码
self.encoder = ce.SumEncoder(cols=columns_name)
elif encoder_type == "Polynomial": # 多项式编码
self.encoder = ce.PolynomialEncoder(cols=columns_name)
elif encoder_type == "Target": # 目标编码
self.encoder = ce.TargetEncoder(cols=columns_name)
elif encoder_type == "WOE": # WOE 编码器
self.encoder = ce.WOEEncoder(cols=columns_name)
else:
raise ValueError("请选择正确的编码方式")
def test_sum_encoder_2cols(self):
train = [['A', 'A'], ['B', 'B'], ['C', 'C']]
encoder = encoders.SumEncoder()
encoder.fit(train)
obtained = encoder.transform(train)
expected = [
[1, a_encoding[1], a_encoding[2], a_encoding[1], a_encoding[2]],
[1, b_encoding[1], b_encoding[2], b_encoding[1], b_encoding[2]],
[1, c_encoding[1], c_encoding[2], c_encoding[1], c_encoding[2]]
]
self.assertEqual(obtained.values.tolist(), expected)
def get_encoder_dict():
encoder_dict = {
'OneHotEncoder': ce.OneHotEncoder(),
'BinaryEncoder': ce.BinaryEncoder(),
'HashingEncoder': ce.HashingEncoder(),
'LabelEncoder': le.MultiColumnLabelEncoder(),
'FrequencyEncoder': fe.FrequencyEncoder(),
'TargetEncoder': ce.TargetEncoder(),
'HelmertEncoder': ce.HelmertEncoder(),
'JamesSteinEncoder': ce.JamesSteinEncoder(),
'BaseNEncoder': ce.BaseNEncoder(),
'SumEncoder': ce.SumEncoder(),
}
return encoder_dict
def test_sum(self):
"""
:return:
"""
cols = ['C1', 'D', 'E', 'F']
enc = encoders.SumEncoder(verbose=1, cols=cols)
X = self.create_dataset(n_rows=1000)
X_test = enc.fit_transform(X, None)
for dt in X_test.dtypes:
numeric = False
if dt == int or dt == float:
numeric = True
self.assertTrue(numeric)
def test_sum_encoder_2StringCols_ExpectCorrectOrder(self):
train = pd.DataFrame(
{
'col1': [1, 2, 3, 4],
'col2': ['A', 'B', 'C', 'D'],
'col3': [1, 2, 3, 4],
'col4': ['A', 'B', 'C', 'A']
},
columns=['col1', 'col2', 'col3', 'col4'])
expected_columns = [
'intercept', 'col1', 'col2_0', 'col2_1', 'col2_2', 'col3',
'col4_0', 'col4_1'
]
encoder = encoders.SumEncoder()
encoder.fit(train)
columns = encoder.transform(train).columns.values
self.assertItemsEqual(expected_columns, columns)
def sum_encode(self, feature, verbose, drop_invariant, return_df,
handle_unknown, handle_missing):
se = ce.SumEncoder(verbose=verbose,
drop_invariant=drop_invariant,
return_df=return_df,
handle_unknown=handle_unknown,
handle_missing=handle_missing)
if self.check_any_missing() is False:
if feature == 'all':
self.data = se.fit_transform(self.data.iloc[:, :-1])
self.data = pd.concat([self.data, self.target_data], axis=1)
else:
temp = se.fit_transform(self.data.loc[:, feature])
self.data = self.data.drop(feature, axis=1)
self.data = pd.concat([self.data, temp], axis=1)
target = self.data.pop(self.target_name)
self.data = pd.concat([self.data, target], axis=1)
self.update_data()
def fit(self, X, y=None):
if self.type == 'backdiff':
self.encoder = ce.BackwardDifferenceEncoder(
handle_unknown='ignore')
if self.type == 'binenc':
self.encoder = ce.BinaryEncoder(handle_unknown='impute')
if self.type == 'hashenc':
self.encoder = ce.HashingEncoder()
if self.type == 'helmenc':
self.encoder = ce.HelmertEncoder(handle_unknown='impute')
if self.type == 'onehot':
self.encoder = ce.OneHotEncoder(handle_unknown='ignore')
if self.type == 'ordenc':
self.encoder = ce.OrdinalEncoder(handle_unknown='impute')
if self.type == 'sumenc':
self.encoder = ce.SumEncoder(handle_unknown='ignore')
if self.type == 'polyenc':
self.encoder = ce.PolynomialEncoder(handle_unknown='impute')
self.encoder.fit(X, y)
return self
def test_sum_encoder_2StringCols_ExpectCorrectOrder(self):
train = pd.DataFrame(
{
'col1': [1, 2, 3, 4],
'col2': ['A', 'B', 'C', 'D'],
'col3': [1, 2, 3, 4],
'col4': ['A', 'B', 'C', 'A']
},
columns=['col1', 'col2', 'col3', 'col4'])
expected_columns = [
'intercept', 'col1', 'col2_0', 'col2_1', 'col2_2', 'col3',
'col4_0', 'col4_1'
]
encoder = encoders.SumEncoder(handle_unknown='value',
handle_missing='value')
encoder.fit(train)
columns = encoder.transform(train).columns.values
self.assertTrue(np.array_equal(expected_columns, columns))
def get_model(PARAMS):
"""return model for provided params
:param PARAMS: dictionary with model params
:type PARAMS: dicr
:return: model pipeline
:rtype: sklearn pipeline
"""
try:
te_dict = {
'CatBoostEncoder': ce.CatBoostEncoder(),
'HashingEncoder': ce.HashingEncoder(),
'HelmertEncoder': ce.HelmertEncoder(),
'LeaveOneOutEncoder': ce.LeaveOneOutEncoder(),
'OneHotEncoder': ce.OneHotEncoder(),
'TargetEncoder': ce.TargetEncoder(),
'WOEEncoder': ce.WOEEncoder(),
'BackwardDifferenceEncoder': ce.BackwardDifferenceEncoder(),
'BaseNEncoder': ce.BaseNEncoder(),
'BinaryEncoder': ce.BinaryEncoder(),
'CountEncoder': ce.CountEncoder(),
'JamesSteinEncoder': ce.JamesSteinEncoder(),
'MEstimateEncoder': ce.MEstimateEncoder(),
'PolynomialEncoder': ce.PolynomialEncoder(),
'SumEncoder': ce.SumEncoder()
}
pipe = make_pipeline(
helpers.PrepareData(extraxt_year=True, unicode_text=True),
ColumnTransformer([
('num', helpers.PassThroughOrReplace(), [
'flat_size', 'rooms', 'floor', 'number_of_floors',
'year_of_building', 'GC_latitude', 'GC_longitude'
]),
('te_producer', te_dict.get(PARAMS['te_producer']),
'producer_name'),
('te_road', te_dict.get(PARAMS['te_road']), 'GC_addr_road'),
('te_neighbourhood', te_dict.get(PARAMS['te_neighbourhood']),
'GC_addr_neighbourhood'),
('te_suburb', te_dict.get(PARAMS['te_suburb']),
'GC_addr_suburb'),
('te_postcode', te_dict.get(PARAMS['te_postcode']),
'GC_addr_postcode'),
('txt_name',
TfidfVectorizer(lowercase=True,
ngram_range=(1,
PARAMS['txt_name__ngram_range']),
max_features=PARAMS['txt_name__max_features'],
dtype=np.float32,
binary=PARAMS['txt_name__binary'],
use_idf=PARAMS['txt_name__use_idf']), 'name'),
('txt_dscr',
TfidfVectorizer(lowercase=True,
ngram_range=(1,
PARAMS['txt_dscr__ngram_range']),
max_features=PARAMS['txt_dscr__max_features'],
dtype=np.float32,
binary=PARAMS['txt_dscr__binary'],
use_idf=PARAMS['txt_dscr__use_idf']),
'description'),
]),
TransformedTargetRegressor(
regressor=lgb.LGBMRegressor(**PARAMS, random_state=seed),
func=np.log1p,
inverse_func=np.expm1))
return pipe
except BaseException as e:
LOG.error(e)
return None
def __init__(
self,
encoder_name,
reduction_method=None,
ngram_range=(2, 4),
categories="auto",
dtype=np.float64,
handle_unknown="ignore",
clf_type=None,
n_components=None,
):
self.ngram_range = ngram_range
self.encoder_name = encoder_name
self.categories = categories
self.dtype = dtype
self.clf_type = clf_type
self.handle_unknown = handle_unknown
self.reduction_method = reduction_method
self.n_components = n_components
self.encoders_dict = {
"OneHotEncoder":
OneHotEncoder(handle_unknown="ignore"),
"OneHotEncoder-1":
OneHotEncoderRemoveOne(handle_unknown="ignore"),
"Categorical":
None,
"OneHotEncoderDense":
OneHotEncoder(handle_unknown="ignore", sparse=False),
"OneHotEncoderDense-1":
OneHotEncoderRemoveOne(handle_unknown="ignore", sparse=False),
"SimilarityEncoder":
SimilarityEncoder(ngram_range=self.ngram_range, random_state=10),
"NgramNaiveFisherKernel":
NgramNaiveFisherKernel(ngram_range=self.ngram_range,
random_state=10),
"ngrams_hot_vectorizer": [],
"NgramsCountVectorizer":
CountVectorizer(analyzer="char", ngram_range=self.ngram_range),
"NgramsTfIdfVectorizer":
TfidfVectorizer(analyzer="char",
ngram_range=self.ngram_range,
smooth_idf=False),
"WordNgramsTfIdfVectorizer":
TfidfVectorizer(analyzer="word",
ngram_range=(1, 1),
smooth_idf=False),
"TargetEncoder":
TargetEncoder(clf_type=self.clf_type, handle_unknown="ignore"),
"MDVEncoder":
MDVEncoder(self.clf_type),
"BackwardDifferenceEncoder":
cat_enc.BackwardDifferenceEncoder(),
"BinaryEncoder":
cat_enc.BinaryEncoder(),
"HashingEncoder":
cat_enc.HashingEncoder(),
"HelmertEncoder":
cat_enc.HelmertEncoder(),
"SumEncoder":
cat_enc.SumEncoder(),
"PolynomialEncoder":
cat_enc.PolynomialEncoder(),
"BaseNEncoder":
cat_enc.BaseNEncoder(),
"LeaveOneOutEncoder":
cat_enc.LeaveOneOutEncoder(),
"NgramsLDA":
Pipeline([
(
"ngrams_count",
CountVectorizer(analyzer="char",
ngram_range=self.ngram_range),
),
(
"LDA",
LatentDirichletAllocation(n_components=self.n_components,
learning_method="batch"),
),
]),
"NMF":
Pipeline([
(
"ngrams_count",
CountVectorizer(analyzer="char",
ngram_range=self.ngram_range),
),
("NMF", NMF(n_components=self.n_components)),
]),
"WordNMF":
Pipeline([
("ngrams_count",
CountVectorizer(analyzer="word", ngram_range=(1, 1))),
("NMF", NMF(n_components=self.n_components)),
]),
"NgramsMultinomialMixture":
NgramsMultinomialMixture(n_topics=self.n_components, max_iters=10),
"AdHocNgramsMultinomialMixture":
AdHocNgramsMultinomialMixture(n_iters=0),
"AdHocIndependentPDF":
AdHocIndependentPDF(),
"OnlineGammaPoissonFactorization":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
rho=0.99,
r=None,
tol=1e-4,
random_state=18,
init="k-means++",
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=10,
),
"OnlineGammaPoissonFactorization2":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=0.3,
rho=None,
batch_size=256,
tol=1e-4,
random_state=18,
init="k-means++",
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20,
),
"OnlineGammaPoissonFactorization3":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=0.3,
rho=None,
batch_size=256,
tol=1e-4,
random_state=18,
init="k-means",
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20,
),
"OnlineGammaPoissonFactorization4":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=None,
rho=0.95,
batch_size=256,
tol=1e-4,
random_state=18,
init="k-means",
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20,
),
"WordOnlineGammaPoissonFactorization":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=0.3,
tol=1e-4,
random_state=18,
init="k-means++",
ngram_range=(1, 1),
analizer="word",
rescale_W=True,
max_iter_e_step=10,
),
"OnlineGammaPoissonFactorization_fast":
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=0.3,
ngram_range=(3, 3),
max_iter=1,
min_iter=1,
tol=1e-4,
random_state=18,
init="k-means++",
rescale_W=False,
),
"MinHashEncoder":
MinHashEncoder(n_components=self.n_components),
"PretrainedFastText":
PretrainedFastText(n_components=self.n_components),
"PretrainedFastText_fr":
PretrainedFastText(n_components=self.n_components,
language="french"),
"PretrainedFastText_hu":
PretrainedFastText(n_components=self.n_components,
language="hungarian"),
None:
FunctionTransformer(None, validate=True),
"Passthrough":
PasstroughEncoder(),
}
self.list_1D_array_methods = [
"NgramsCountVectorizer",
"NgramsTfIdfVectorizer",
"WordNgramsTfIdfVectorizer",
"ngrams_hot_vectorizer",
"NgramsLDA",
"NMF",
"WordNMF",
"NgramsMultinomialMixture",
"NgramsMultinomialMixtureKMeans2",
"AdHocNgramsMultinomialMixture",
"AdHocIndependentPDF",
"GammaPoissonFactorization",
"OnlineGammaPoissonFactorization",
"WordOnlineGammaPoissonFactorization",
"OnlineGammaPoissonFactorization2",
"OnlineGammaPoissonFactorization3",
"OnlineGammaPoissonFactorization4",
"OnlineGammaPoissonFactorization_fast",
"MinHashEncoder",
"MinMeanMinHashEncoder",
]
def _sum_encoding(self, X, y):
self.sum_encoder = ce.SumEncoder(cols=self.columns)
self.sum_encoder.fit(X, y)
'color_30', 'color_31', 'color_32', 'color_33', 'color_34', 'color_35',
'color_36', 'color_37', 'color_38', 'color_39', 'color_40', 'color_41',
'color_42', 'color_43', 'color_44', 'color_45', 'color_46', 'color_47',
'color_48', 'color_49', 'color_50', 'color_51', 'color_52', 'color_53',
'color_54', 'color_55', 'color_56', 'color_57', 'color_58', 'color_59',
'color_60', 'color_61', 'color_62', 'color_63', 'color_64', 'color_65',
'color_66', 'color_67', 'color_68', 'color_69', 'color_70', 'color_71',
'color_72', 'color_73', 'color_74', 'color_75', 'color_76', 'color_77',
'color_78', 'color_79', 'color_80', 'color_81', 'color_82', 'color_83',
'color_84', 'color_85', 'color_86', 'color_87', 'color_88', 'color_89',
'color_90', 'color_91', 'color_92', 'color_93', 'color_94', 'color_95',
'color_96', 'color_97', 'color_98'
]
]
encoder = ce.SumEncoder(cols=nominal_columns).fit(X0, y)
X = encoder.transform(X0)
# Permutaiton
for _ in range(5):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
# KNN
clf_knn = KNeighborsClassifier(n_neighbors=3)
clf_knn.fit(X_train, y_train)
pi_knn = PermutationImportance.PermutationImportance(clf_knn)
pi_knn.fit(X_test, y_test, features=features)
pi_knn.store('p100-sum-knn')
# KNN
clf_rfc = RandomForestClassifier()
clf_rfc.fit(X_train, y_train)
pi_rfc = PermutationImportance.PermutationImportance(clf_rfc)
pi_rfc.fit(X_test, y_test, features=features)
'soybean.arff', 'spectrometer.arff', 'sponge.arff', 'tic-tac-toe.arff', 'trains.arff', 'vote.arff', 'vowel.arff']
# We painstakingly initialize each encoder here because that gives us the freedom to initialize the
# encoders with any setting we want.
encoders = [category_encoders.BackwardDifferenceEncoder(),
category_encoders.BaseNEncoder(),
category_encoders.BinaryEncoder(),
category_encoders.HashingEncoder(),
category_encoders.HelmertEncoder(),
category_encoders.JamesSteinEncoder(),
category_encoders.LeaveOneOutEncoder(),
category_encoders.MEstimateEncoder(),
category_encoders.OneHotEncoder(),
category_encoders.OrdinalEncoder(),
category_encoders.PolynomialEncoder(),
category_encoders.SumEncoder(),
category_encoders.TargetEncoder(),
category_encoders.WOEEncoder()]
# Initialization
if os.path.isfile('./output/result.csv'):
os.remove('./output/result.csv')
# Loop over datasets, then over encoders, and finally, over the models
for dataset_name in datasets:
X, y, fold_count = arff_loader.load(dataset_name)
# Random permutation (needed for CatBoost encoder)
perm = np.random.permutation(len(X))
X = X.iloc[perm].reset_index(drop=True)
y = y.iloc[perm].reset_index(drop=True)
def pipeline(df, target, cat_columns, models):
n_rows, n_cols = df.shape
metrics = {
"n_rows": [],
"n_cols": [],
"cardinality": [],
"model": [],
"column": [],
"encoder": [],
"rmse": [],
"mae": [],
"fit_time": [],
"rmse_change": [],
"mae_change": [],
"fit_time_change": [],
}
columns = cat_columns
for model_name in models:
base_rmse, base_mae, base_fit_time = model(
df=df,
target=target,
encoder=np.nan,
col=np.nan,
model_name=model_name,
encoder_type="basic",
encoder_name=[],
)
_append_metric(
row_list=metrics,
n_rows=n_rows,
n_cols=n_cols,
cardinality=np.nan,
model_name=model_name,
column=np.nan,
name="basic",
rmse=base_rmse,
mae=base_mae,
fit_time=base_fit_time,
base_rmse=base_rmse,
base_mae=base_mae,
base_fit_time=base_fit_time,
)
for column in columns:
print()
print(column)
cardinality = df[column].nunique()
print("ohe")
rmse, mae, fit_time = model(
df=df,
target=target,
encoder=np.nan,
col=column,
model_name=model_name,
encoder_type="basic",
encoder_name="One Hot Encoder (pd.dummies)",
)
_append_metric(
row_list=metrics,
n_rows=n_rows,
n_cols=n_cols,
cardinality=cardinality,
model_name=model_name,
column=column,
name="One Hot Encoder (pd.dummies)",
rmse=rmse,
mae=mae,
fit_time=fit_time,
base_rmse=base_rmse,
base_mae=base_mae,
base_fit_time=base_fit_time,
)
encoders = [
("Sum Encoder(sleepmind)", SumEncoder()),
("BinaryEncoder", ce.BinaryEncoder(cols=[column])),
("HashingEncoder", ce.HashingEncoder(cols=[column])),
("OneHotEncoder", ce.OneHotEncoder(cols=[column])),
("OrdinalEncoder", ce.OrdinalEncoder(cols=[column])),
("BaseNEncoder", ce.BaseNEncoder(cols=[column])),
(
"BackwardDifferenceEncoder",
ce.BackwardDifferenceEncoder(cols=[column]),
),
("HelmertEncoder", ce.HelmertEncoder(cols=[column])),
("SumEncoder", ce.SumEncoder(cols=[column])),
("PolynomialEncoder", ce.PolynomialEncoder(cols=[column])),
("TargetEncoder", ce.TargetEncoder(cols=[column])),
("LeaveOneOutEncoder", ce.LeaveOneOutEncoder(cols=[column])),
(
"XAM_bayesian_targetEncoder",
BayesianTargetEncoder(columns=[column],
prior_weight=3,
suffix=""),
),
]
for name, encoder in encoders:
print(name)
rmse, mae, fit_time = model(
df=df,
target=target,
encoder=encoder,
col=column,
model_name=model_name,
encoder_type="sklearn_encoding",
encoder_name=name,
)
_append_metric(
row_list=metrics,
n_rows=n_rows,
n_cols=n_cols,
cardinality=cardinality,
model_name=model_name,
column=column,
name=name,
rmse=rmse,
mae=mae,
fit_time=fit_time,
base_rmse=base_rmse,
base_mae=base_mae,
base_fit_time=base_fit_time,
)
bayes_encoders = [
("hcc_BayesEncoding", BayesEncoding),
("hcc_BayesEncodingKfold", BayesEncodingKfold),
("LOOEncoding", LOOEncoding),
("LOOEncodingKfold", LOOEncodingKfold),
]
for name, bayes_encoder in bayes_encoders:
print(name)
rmse, mae, fit_time = model(
df=df,
target=target,
encoder=bayes_encoder,
col=column,
model_name=model_name,
encoder_name=name,
encoder_type="basic",
hcc_ind=1,
)
_append_metric(
row_list=metrics,
n_rows=n_rows,
n_cols=n_cols,
cardinality=cardinality,
model_name=model_name,
column=column,
name=name,
rmse=rmse,
mae=mae,
fit_time=fit_time,
base_rmse=base_rmse,
base_mae=base_mae,
base_fit_time=base_fit_time,
)
results = pd.DataFrame(metrics)
return results
def get_factors(model, df, fnum, fname, nvalues, dtype, encoder, rounding,
sentinel):
r"""Convert the original feature to a factor.
Parameters
----------
model : alphapy.Model
Model object with the feature specifications.
df : pandas.DataFrame
Dataframe containing the column ``fname``.
fnum : int
Feature number, strictly for logging purposes
fname : str
Name of the text column in the dataframe ``df``.
nvalues : int
The number of unique values.
dtype : str
The values ``'float64'``, ``'int64'``, or ``'bool'``.
encoder : alphapy.features.Encoders
Type of encoder to apply.
rounding : int
Number of places to round.
sentinel : float
The number to be imputed for NaN values.
Returns
-------
all_features : numpy array
The features that have been transformed to factors.
"""
logger.info("Feature %d: %s is a factor of type %s with %d unique values",
fnum, fname, dtype, nvalues)
logger.info("Encoding: %s", encoder)
# Extract model data
feature_map = model.feature_map
model_type = model.specs['model_type']
target_value = model.specs['target_value']
# get feature
feature = df[fname]
# convert float to factor
if dtype == 'float64':
logger.info("Rounding: %d", rounding)
feature = feature.apply(float_factor, args=[rounding])
# encoders
pd_features = pd.DataFrame()
enc = None
ef = pd.DataFrame(feature)
if encoder == Encoders.factorize:
pd_factors = pd.factorize(feature)[0]
pd_features = pd.DataFrame(pd_factors)
elif encoder == Encoders.onehot:
pd_features = pd.get_dummies(feature)
elif encoder == Encoders.ordinal:
enc = ce.OrdinalEncoder(cols=[fname])
elif encoder == Encoders.binary:
enc = ce.BinaryEncoder(cols=[fname])
elif encoder == Encoders.helmert:
enc = ce.HelmertEncoder(cols=[fname])
elif encoder == Encoders.sumcont:
enc = ce.SumEncoder(cols=[fname])
elif encoder == Encoders.polynomial:
enc = ce.PolynomialEncoder(cols=[fname])
elif encoder == Encoders.backdiff:
enc = ce.BackwardDifferenceEncoder(cols=[fname])
else:
raise ValueError("Unknown Encoder %s" % encoder)
# If encoding worked, calculate target percentages for classifiers.
pd_exists = not pd_features.empty
enc_exists = enc is not None
all_features = None
if pd_exists or enc_exists:
if pd_exists:
all_features = pd_features
elif enc_exists:
all_features = enc.fit_transform(ef, None)
# Calculate target percentages for factors
if (model_type == ModelType.classification
and fname in feature_map['crosstabs']):
# Get the crosstab for this feature
ct = feature_map['crosstabs'][fname]
# map target percentages to the new feature
ct_map = ct.to_dict()[target_value]
ct_feature = df[[fname]].applymap(ct_map.get)
# impute sentinel for any values that could not be mapped
ct_feature.fillna(value=sentinel, inplace=True)
# concatenate all generated features
all_features = np.column_stack((all_features, ct_feature))
logger.info("Applied target percentages for %s", fname)
else:
raise RuntimeError("Encoding for feature %s failed" % fname)
return all_features
def __init__(self, name: str):
super().__init__(name, '_sum_', ce.SumEncoder(cols=[name]), -1.0, 1.0)
def __init__(self,
encoder_name,
reduction_method=None,
ngram_range=(2, 4),
categories='auto',
dtype=np.float64,
handle_unknown='ignore',
clf_type=None,
n_components=None):
self.ngram_range = ngram_range
self.encoder_name = encoder_name
self.categories = categories
self.dtype = dtype
self.clf_type = clf_type
self.handle_unknown = handle_unknown
self.reduction_method = reduction_method
self.n_components = n_components
self.encoders_dict = {
'OneHotEncoder':
OneHotEncoder(handle_unknown='ignore'),
'OneHotEncoder-1':
OneHotEncoderRemoveOne(handle_unknown='ignore'),
'Categorical':
None,
'OneHotEncoderDense':
OneHotEncoder(handle_unknown='ignore', sparse=False),
'OneHotEncoderDense-1':
OneHotEncoderRemoveOne(handle_unknown='ignore', sparse=False),
'SimilarityEncoder':
SimilarityEncoder(ngram_range=self.ngram_range, random_state=10),
'NgramNaiveFisherKernel':
NgramNaiveFisherKernel(ngram_range=self.ngram_range,
random_state=10),
'ngrams_hot_vectorizer': [],
'NgramsCountVectorizer':
CountVectorizer(analyzer='char', ngram_range=self.ngram_range),
'NgramsTfIdfVectorizer':
TfidfVectorizer(analyzer='char',
ngram_range=self.ngram_range,
smooth_idf=False),
'WordNgramsTfIdfVectorizer':
TfidfVectorizer(analyzer='word',
ngram_range=(1, 1),
smooth_idf=False),
'TargetEncoder':
TargetEncoder(clf_type=self.clf_type, handle_unknown='ignore'),
'MDVEncoder':
MDVEncoder(self.clf_type),
'BackwardDifferenceEncoder':
cat_enc.BackwardDifferenceEncoder(),
'BinaryEncoder':
cat_enc.BinaryEncoder(),
'HashingEncoder':
cat_enc.HashingEncoder(),
'HelmertEncoder':
cat_enc.HelmertEncoder(),
'SumEncoder':
cat_enc.SumEncoder(),
'PolynomialEncoder':
cat_enc.PolynomialEncoder(),
'BaseNEncoder':
cat_enc.BaseNEncoder(),
'LeaveOneOutEncoder':
cat_enc.LeaveOneOutEncoder(),
'NgramsLDA':
Pipeline([
('ngrams_count',
CountVectorizer(analyzer='char',
ngram_range=self.ngram_range)),
(
'LDA',
LatentDirichletAllocation(n_components=self.n_components,
learning_method='batch'),
)
]),
'NMF':
Pipeline([('ngrams_count',
CountVectorizer(analyzer='char',
ngram_range=self.ngram_range)),
('NMF', NMF(n_components=self.n_components))]),
'WordNMF':
Pipeline([('ngrams_count',
CountVectorizer(analyzer='word', ngram_range=(1, 1))),
('NMF', NMF(n_components=self.n_components))]),
'NgramsMultinomialMixture':
NgramsMultinomialMixture(n_topics=self.n_components, max_iters=10),
'AdHocNgramsMultinomialMixture':
AdHocNgramsMultinomialMixture(n_iters=0),
'AdHocIndependentPDF':
AdHocIndependentPDF(),
'OnlineGammaPoissonFactorization':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
rho=.99,
r=None,
tol=1e-4,
random_state=18,
init='k-means++',
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=10),
'OnlineGammaPoissonFactorization2':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=.3,
rho=None,
batch_size=256,
tol=1e-4,
random_state=18,
init='k-means++',
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20),
'OnlineGammaPoissonFactorization3':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=.3,
rho=None,
batch_size=256,
tol=1e-4,
random_state=18,
init='k-means',
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20),
'OnlineGammaPoissonFactorization4':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=None,
rho=.95,
batch_size=256,
tol=1e-4,
random_state=18,
init='k-means',
ngram_range=self.ngram_range,
rescale_W=True,
max_iter_e_step=20),
'WordOnlineGammaPoissonFactorization':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=.3,
tol=1e-4,
random_state=18,
init='k-means++',
ngram_range=(1, 1),
analizer='word',
rescale_W=True,
max_iter_e_step=10),
'OnlineGammaPoissonFactorization_fast':
gamma_poisson_factorization.OnlineGammaPoissonFactorization(
n_topics=self.n_components,
r=.3,
ngram_range=(3, 3),
max_iter=1,
min_iter=1,
tol=1e-4,
random_state=18,
init='k-means++',
rescale_W=False),
'MinHashEncoder':
MinHashEncoder(n_components=self.n_components),
'PretrainedFastText':
PretrainedFastText(n_components=self.n_components),
'PretrainedFastText_fr':
PretrainedFastText(n_components=self.n_components,
language='french'),
'PretrainedFastText_hu':
PretrainedFastText(n_components=self.n_components,
language='hungarian'),
None:
FunctionTransformer(None, validate=True),
'Passthrough':
PasstroughEncoder(),
}
self.list_1D_array_methods = [
'NgramsCountVectorizer',
'NgramsTfIdfVectorizer',
'WordNgramsTfIdfVectorizer',
'ngrams_hot_vectorizer',
'NgramsLDA',
'NMF',
'WordNMF',
'NgramsMultinomialMixture',
'NgramsMultinomialMixtureKMeans2',
'AdHocNgramsMultinomialMixture',
'AdHocIndependentPDF',
'GammaPoissonFactorization',
'OnlineGammaPoissonFactorization',
'WordOnlineGammaPoissonFactorization',
'OnlineGammaPoissonFactorization2',
'OnlineGammaPoissonFactorization3',
'OnlineGammaPoissonFactorization4',
'OnlineGammaPoissonFactorization_fast',
'MinHashEncoder',
'MinMeanMinHashEncoder',
]
def encode_all(df,dfv,dfk,encoder_to_use,handle_missing='return_nan'):
encoders_used = {}
for col in encoder_to_use:
if encoder_to_use[col] == 'ColumnDropper':
df = df.drop(columns = col)
dfv = dfv.drop(columns = col)
dfk = dfk.drop(columns = col)
encoders_used[col] = 'ColumnDropper'
if encoder_to_use[col]=='BackwardDifferenceEncoder':
encoder=ce.BackwardDifferenceEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='BaseNEncoder':
encoder=ce.BaseNEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,base=3)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='BinaryEncoder':
encoder=ce.BinaryEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='CatBoostEncoder':
encoder=ce.CatBoostEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,sigma=None,a=2)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
# if encoder_to_use[col]=='HashingEncoder':
# encoder=ce.HashingEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
# encoder.fit(X=df,y=df['set_clicked'])
# df=encoder.transform(df)
# encoders_used[col]=encoder
if encoder_to_use[col]=='HelmertEncoder':
encoder=ce.HelmertEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='JamesSteinEncoder':
encoder=ce.JamesSteinEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing, model='binary')
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='LeaveOneOutEncoder':
encoder=ce.LeaveOneOutEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,sigma=None)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='MEstimateEncoder':
encoder=ce.MEstimateEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,randomized=True,sigma=None,m=2)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
encoders_used[col]=encoder
if encoder_to_use[col]=='OneHotEncoder':
encoder=ce.OneHotEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,use_cat_names=True)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='OrdinalEncoder':
encoder=ce.OrdinalEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='SumEncoder':
encoder=ce.SumEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='PolynomialEncoder':
encoder=ce.PolynomialEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='TargetEncoder':
encoder=ce.TargetEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,min_samples_leaf=10, smoothing=5)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
if encoder_to_use[col]=='WOEEncoder':
encoder=ce.WOEEncoder(cols=[col],return_df=1,drop_invariant=1,handle_missing=handle_missing,randomized=True,sigma=None)
encoder.fit(X=df,y=df['set_clicked'])
df=encoder.transform(df)
dfv=encoder.transform(dfv)
dfk=encoder.transform(dfk)
encoders_used[col]=encoder
# print("Encoding done for - ",col)
print("Completed encoder - ",datetime.datetime.now())
return df, dfv, dfk, encoders_used