def test_error_handling(self): for encoder_name in encoders.__all__: with self.subTest(encoder_name=encoder_name): # we exclude some columns X = th.create_dataset(n_rows=100) X = X.drop(['unique_str', 'none'], axis=1) X_t = th.create_dataset(n_rows=50, extras=True) X_t = X_t.drop(['unique_str', 'none'], axis=1) # illegal state, we have to first train the encoder... enc = getattr(encoders, encoder_name)() with self.assertRaises(ValueError): enc.transform(X) # wrong count of attributes enc = getattr(encoders, encoder_name)() enc.fit(X, y) with self.assertRaises(ValueError): enc.transform(X_t.iloc[:, 0:3]) # no cols enc = getattr(encoders, encoder_name)(cols=[]) enc.fit(X, y) self.assertTrue(enc.transform(X_t).equals(X_t))
def test_one_hot(self): enc = encoders.OneHotEncoder(verbose=1, return_df=False) enc.fit(X) self.assertEqual( enc.transform(X_t).shape[1], enc.transform(X).shape[1], 'We have to get the same count of columns despite the presence of a new value' ) enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='indicator') enc.fit(X) out = enc.transform(X_t) self.assertIn('extra_-1', out.columns.values) enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan') enc.fit(X) out = enc.transform(X_t) self.assertEqual( len([x for x in out.columns.values if str(x).startswith('extra_')]), 3) enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='error') # The exception is already raised in fit() because transform() is called there to get # feature_names right. enc.fit(X) with self.assertRaises(ValueError): enc.transform(X_t) enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan', use_cat_names=True) enc.fit(X) out = enc.transform(X_t) self.assertIn('extra_A', out.columns.values) enc = encoders.OneHotEncoder(verbose=1, return_df=True, use_cat_names=True, handle_unknown='indicator') enc.fit(X) out = enc.transform(X_t) self.assertIn('extra_-1', out.columns.values) # test inverse_transform X_i = th.create_dataset(n_rows=100, has_missing=False) X_i_t = th.create_dataset(n_rows=50, has_missing=False) cols = ['underscore', 'none', 'extra', 321, 'categorical'] enc = encoders.OneHotEncoder(verbose=1, use_cat_names=True, cols=cols) enc.fit(X_i) obtained = enc.inverse_transform(enc.transform(X_i_t)) th.verify_inverse_transform(X_i_t, obtained)
def test_inverse_transform(self): # we do not allow None in these data (but "none" column without any missing value is ok) X = th.create_dataset(n_rows=100, has_missing=False) X_t = th.create_dataset(n_rows=50, has_missing=False) cols = ['underscore', 'none', 321, 'categorical', 'categorical_int'] for encoder_name in ['BaseNEncoder', 'BinaryEncoder', 'OneHotEncoder', 'OrdinalEncoder']: with self.subTest(encoder_name=encoder_name): # simple run enc = getattr(encoders, encoder_name)(verbose=1, cols=cols) enc.fit(X) th.verify_inverse_transform(X_t, enc.inverse_transform(enc.transform(X_t)))
def test_handle_unknown_error(self): # BaseN has problems with None -> ignore None X = th.create_dataset(n_rows=100, has_missing=False) X_t = th.create_dataset(n_rows=50, extras=True, has_missing=False) for encoder_name in (set(encoders.__all__) - {'HashingEncoder'}): # HashingEncoder supports new values by design -> excluded with self.subTest(encoder_name=encoder_name): # new value during scoring enc = getattr(encoders, encoder_name)(handle_unknown='error') enc.fit(X, y) with self.assertRaises(ValueError): _ = enc.transform(X_t)
from unittest import TestCase import numpy as np import category_encoders as encoders import tests.helpers as th # data definitions X = th.create_dataset(n_rows=100) np_y = np.random.default_rng(42).standard_normal(100) > 0.5 class TestGLMMEncoder(TestCase): def test_continuous(self): cols = [ 'unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical', 'categorical_int' ] enc = encoders.GLMMEncoder(cols=cols, binomial_target=False) # TODO: fix this test IRL # enc.fit(X, np_y) #th.verify_numeric(enc.transform(X)) def test_binary(self): cols = [ 'unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical', 'categorical_int' ] enc = encoders.GLMMEncoder(cols=cols, binomial_target=True) # TODO: fix this test IRL #enc.fit(X, np_y) #th.verify_numeric(enc.transform(X))
import pandas as pd from unittest import TestCase # or `from unittest import ...` if on Python 3.4+ import tests.helpers as th import numpy as np import category_encoders as encoders np_X = th.create_array(n_rows=100) np_X_t = th.create_array(n_rows=50, extras=True) np_y = np.random.randn(np_X.shape[0]) > 0.5 np_y_t = np.random.randn(np_X_t.shape[0]) > 0.5 X = th.create_dataset(n_rows=100) X_t = th.create_dataset(n_rows=50, extras=True) y = pd.DataFrame(np_y) y_t = pd.DataFrame(np_y_t) class TestWeightOfEvidenceEncoder(TestCase): def test_woe(self): cols = [ 'unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical', 'categorical_int' ] # balanced label with balanced features X_balanced = pd.DataFrame(data=['1', '1', '1', '2', '2', '2'], columns=['col1']) y_balanced = [True, False, True, False, True, False] enc = encoders.WOEEncoder() enc.fit(X_balanced, y_balanced) X1 = enc.transform(X_balanced)
# sample num of data data_lines = 10000 # benchmarking result format result_cols = [ 'encoder', 'used_processes', 'X_shape', 'min_time(s)', 'average_time(s)', 'max_cpu_utilization(%)', 'average_cpu_utilization(%)' ] results = [] cpu_utilization = multiprocessing.Manager().Queue() # define data_set np_X = th.create_array(n_rows=data_lines) np_y = np.random.randn(np_X.shape[0]) > 0.5 X = th.create_dataset(n_rows=data_lines) X_t = th.create_dataset(n_rows=int(data_lines / 2), extras=True) cols = [ 'unique_str', 'underscore', 'extra', 'none', 'invariant', 321, 'categorical', 'na_categorical' ] def get_cpu_utilization(): """ new process for recording cpu utilization record cpu utilization every [cpu_sampling_rate] second & calculate its mean value the value is the cpu utilization during every encoding """ global cpu_utilization