def _discretise_features(self, X: pd.DataFrame) -> pd.DataFrame:
"""
Helper method to discretise input data using parameters in
`discretiser_kwargs` and `discretiser_alg`.
The splitting thresholds are extracted from the training data
Args:
X (pd.DataFrame): a dataframe to be discretised
Returns:
a discretised version of the input dataframe
"""
X = X.copy()
for col in self.discretiser_alg.keys():
if self.discretiser_alg[col] == "unsupervised":
if self.discretiser_kwargs[col]["method"] == "fixed":
X[col] = Discretiser(
**self.discretiser_kwargs[col]).transform(
X[col].values)
else:
discretiser = Discretiser(
**self.discretiser_kwargs[col]).fit(
self._discretise_data[col].values)
X[col] = discretiser.transform(X[col].values)
else:
if self.discretiser_alg[col] == "tree":
discretiser = DecisionTreeSupervisedDiscretiserMethod(
mode="single",
tree_params=self.discretiser_kwargs[col])
elif self.discretiser_alg[col] == "mdlp":
discretiser = MDLPSupervisedDiscretiserMethod(
self.discretiser_kwargs[col])
discretiser.fit(
dataframe=self._discretise_data,
feat_names=[col],
target=self._target_name,
target_continuous=False,
)
X[col] = discretiser.transform(X[[col]])
return X
def test_transform_splits_using_defined_split_points(self):
"""transforming should be done using the provided numeric split points"""
d = Discretiser(method="fixed", numeric_split_points=[10, 20, 30])
transformed = d.transform(np.array([9, 10, 11, 19, 20, 21, 29, 30,
31]))
assert np.array_equal(transformed, [0, 1, 1, 1, 2, 2, 2, 3, 3])
def test_transform_smaller_than_fit_range_goes_into_first_bucket(self):
"""If a value smaller than the input is transformed, then it
should go into the minimum bucket"""
arr = np.array([n + 1 for n in range(10)])
np.random.shuffle(arr)
d = Discretiser(method="uniform", num_buckets=4)
d.fit(arr)
assert np.array_equal([0], d.transform(np.array([-101])))
def test_transform_gauss(self):
"""Fitting gauss data should transform to predictable buckets"""
arr = np.random.normal(loc=0, scale=1, size=1000000)
np.random.shuffle(arr)
d = Discretiser(method="quantile", num_buckets=4)
d.fit(arr)
unique, counts = np.unique(d.transform(arr), return_counts=True)
# check all 4 buckets are used
assert np.array_equal([0, 1, 2, 3], unique)
assert np.array_equal([250000 for n in range(4)], counts)
def test_transform_uniform(self):
"""Fitting uniform data should transform to predictable buckets"""
arr = np.array(range(100001))
np.random.shuffle(arr)
d = Discretiser(method="percentiles",
percentile_split_points=[0.10, 0.40, 0.85])
d.fit(arr)
unique, counts = np.unique(d.transform(arr), return_counts=True)
# check all 4 buckets are used
assert np.array_equal([0, 1, 2, 3], unique)
assert np.array_equal([10000, 30000, 45000, 15001], counts)
def test_transform_outlier(self):
"""transforming outliers should put the expected amount of data in each bucket"""
arr = np.array(range(100001))
np.random.shuffle(arr)
d = Discretiser(method="outlier", outlier_percentile=0.2)
d.fit(arr)
unique, counts = np.unique(d.transform(arr), return_counts=True)
# check all 3 buckets are used
assert np.array_equal([0, 1, 2], unique)
# check largest difference in outliers is 1
print(counts)
assert np.abs(counts[0] - counts[2]) <= 1
def test_transform_uneven_split(self):
"""Data that cannot be split evenly between buckets should be transformed
into near-even buckets"""
arr = np.array([n + 1 for n in range(10)])
np.random.shuffle(arr)
d = Discretiser(method="uniform", num_buckets=4)
d.fit(arr)
unique, counts = np.unique(d.transform(arr), return_counts=True)
# check all 4 buckets are used
assert np.array_equal([0, 1, 2, 3], unique)
# check largest difference in distribution is 1 item
assert (np.max(counts) - np.min(counts)) <= 1
def test_fit_transform(self):
"""fit transform should give the same result as calling fit and
transform separately"""
arr = np.array([n + 1 for n in range(10)])
np.random.shuffle(arr)
d1 = Discretiser(method="uniform", num_buckets=4)
d1.fit(arr)
r1 = d1.transform(arr)
d2 = Discretiser(method="uniform", num_buckets=4)
r2 = d2.fit_transform(arr)
assert np.array_equal(r1, r2)
def test_fit_transform(self):
"""fit transform should give the same result as calling fit and
transform separately"""
arr = np.array([n + 1 for n in range(10)])
np.random.shuffle(arr)
d1 = Discretiser(method="outlier", outlier_percentile=0.2)
d1.fit(arr)
r1 = d1.transform(arr)
d2 = Discretiser(method="outlier", outlier_percentile=0.2)
r2 = d2.fit_transform(arr)
assert np.array_equal(r1, r2)
def test_fit_transform(self):
"""fit transform should give the same result as calling fit and
transform separately"""
arr = np.array([n + 1 for n in range(10)])
np.random.shuffle(arr)
d1 = Discretiser(method="percentiles",
percentile_split_points=[0.10, 0.40, 0.85])
d1.fit(arr)
r1 = d1.transform(arr)
d2 = Discretiser(method="percentiles",
percentile_split_points=[0.10, 0.40, 0.85])
r2 = d2.fit_transform(arr)
assert np.array_equal(r1, r2)
