def _discretise_features(self, X: pd.DataFrame) -> pd.DataFrame: """ Helper method to discretise input data using parameters in `discretiser_kwargs` and `discretiser_alg`. The splitting thresholds are extracted from the training data Args: X (pd.DataFrame): a dataframe to be discretised Returns: a discretised version of the input dataframe """ X = X.copy() for col in self.discretiser_alg.keys(): if self.discretiser_alg[col] == "unsupervised": if self.discretiser_kwargs[col]["method"] == "fixed": X[col] = Discretiser( **self.discretiser_kwargs[col]).transform( X[col].values) else: discretiser = Discretiser( **self.discretiser_kwargs[col]).fit( self._discretise_data[col].values) X[col] = discretiser.transform(X[col].values) else: if self.discretiser_alg[col] == "tree": discretiser = DecisionTreeSupervisedDiscretiserMethod( mode="single", tree_params=self.discretiser_kwargs[col]) elif self.discretiser_alg[col] == "mdlp": discretiser = MDLPSupervisedDiscretiserMethod( self.discretiser_kwargs[col]) discretiser.fit( dataframe=self._discretise_data, feat_names=[col], target=self._target_name, target_continuous=False, ) X[col] = discretiser.transform(X[[col]]) return X
def test_transform_splits_using_defined_split_points(self): """transforming should be done using the provided numeric split points""" d = Discretiser(method="fixed", numeric_split_points=[10, 20, 30]) transformed = d.transform(np.array([9, 10, 11, 19, 20, 21, 29, 30, 31])) assert np.array_equal(transformed, [0, 1, 1, 1, 2, 2, 2, 3, 3])
def test_transform_smaller_than_fit_range_goes_into_first_bucket(self): """If a value smaller than the input is transformed, then it should go into the minimum bucket""" arr = np.array([n + 1 for n in range(10)]) np.random.shuffle(arr) d = Discretiser(method="uniform", num_buckets=4) d.fit(arr) assert np.array_equal([0], d.transform(np.array([-101])))
def test_transform_gauss(self): """Fitting gauss data should transform to predictable buckets""" arr = np.random.normal(loc=0, scale=1, size=1000000) np.random.shuffle(arr) d = Discretiser(method="quantile", num_buckets=4) d.fit(arr) unique, counts = np.unique(d.transform(arr), return_counts=True) # check all 4 buckets are used assert np.array_equal([0, 1, 2, 3], unique) assert np.array_equal([250000 for n in range(4)], counts)
def test_transform_uniform(self): """Fitting uniform data should transform to predictable buckets""" arr = np.array(range(100001)) np.random.shuffle(arr) d = Discretiser(method="percentiles", percentile_split_points=[0.10, 0.40, 0.85]) d.fit(arr) unique, counts = np.unique(d.transform(arr), return_counts=True) # check all 4 buckets are used assert np.array_equal([0, 1, 2, 3], unique) assert np.array_equal([10000, 30000, 45000, 15001], counts)
def test_transform_outlier(self): """transforming outliers should put the expected amount of data in each bucket""" arr = np.array(range(100001)) np.random.shuffle(arr) d = Discretiser(method="outlier", outlier_percentile=0.2) d.fit(arr) unique, counts = np.unique(d.transform(arr), return_counts=True) # check all 3 buckets are used assert np.array_equal([0, 1, 2], unique) # check largest difference in outliers is 1 print(counts) assert np.abs(counts[0] - counts[2]) <= 1
def test_transform_uneven_split(self): """Data that cannot be split evenly between buckets should be transformed into near-even buckets""" arr = np.array([n + 1 for n in range(10)]) np.random.shuffle(arr) d = Discretiser(method="uniform", num_buckets=4) d.fit(arr) unique, counts = np.unique(d.transform(arr), return_counts=True) # check all 4 buckets are used assert np.array_equal([0, 1, 2, 3], unique) # check largest difference in distribution is 1 item assert (np.max(counts) - np.min(counts)) <= 1
def test_fit_transform(self): """fit transform should give the same result as calling fit and transform separately""" arr = np.array([n + 1 for n in range(10)]) np.random.shuffle(arr) d1 = Discretiser(method="uniform", num_buckets=4) d1.fit(arr) r1 = d1.transform(arr) d2 = Discretiser(method="uniform", num_buckets=4) r2 = d2.fit_transform(arr) assert np.array_equal(r1, r2)
def test_fit_transform(self): """fit transform should give the same result as calling fit and transform separately""" arr = np.array([n + 1 for n in range(10)]) np.random.shuffle(arr) d1 = Discretiser(method="outlier", outlier_percentile=0.2) d1.fit(arr) r1 = d1.transform(arr) d2 = Discretiser(method="outlier", outlier_percentile=0.2) r2 = d2.fit_transform(arr) assert np.array_equal(r1, r2)
def test_fit_transform(self): """fit transform should give the same result as calling fit and transform separately""" arr = np.array([n + 1 for n in range(10)]) np.random.shuffle(arr) d1 = Discretiser(method="percentiles", percentile_split_points=[0.10, 0.40, 0.85]) d1.fit(arr) r1 = d1.transform(arr) d2 = Discretiser(method="percentiles", percentile_split_points=[0.10, 0.40, 0.85]) r2 = d2.fit_transform(arr) assert np.array_equal(r1, r2)