def test_list_to_dataframe(openml_id):
X_pandas, y_pandas = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X_pandas, y_pandas, random_state=1)
X_list = X_train.values.tolist()
validator = FeatureValidator()
validator.fit(X_list)
transformed_X = validator.transform(X_list)
for i, col in enumerate(X_pandas.columns):
if is_numeric_dtype(X_pandas[col].dtype):
# convert dtype translates 72.0 to 72. Be robust against this!
assert is_numeric_dtype(transformed_X[i].dtype)
else:
assert X_pandas[col].dtype.name == transformed_X[i].dtype.name, col
# Also make sure that at testing time
# this work
transformed_X = validator.transform(X_test.values.tolist())
for i, col in enumerate(X_pandas.columns):
if is_numeric_dtype(X_pandas[col].dtype):
# convert dtype translates 72.0 to 72. Be robust against this!
assert is_numeric_dtype(transformed_X[i].dtype)
else:
assert X_pandas[col].dtype.name == transformed_X[i].dtype.name, col
def test_features_unsupported_calls_are_raised():
"""
Makes sure we raise a proper message to the user,
when providing not supported data input or using the validator in a way that is not
expected
"""
validator = FeatureValidator()
with pytest.raises(ValueError, match=r"Auto-sklearn does not support time"):
validator.fit(
pd.DataFrame({'datetime': [pd.Timestamp('20180310')]})
)
with pytest.raises(ValueError, match=r"Auto-sklearn only supports.*yet, the provided input"):
validator.fit({'input1': 1, 'input2': 2})
validator = FeatureValidator()
with pytest.raises(ValueError, match=r"has unsupported dtype string"):
validator.fit(pd.DataFrame([{'A': 1, 'B': 2}], dtype='string'))
with pytest.raises(ValueError, match=r"The feature dimensionality of the train and test"):
validator.fit(X_train=np.array([[1, 2, 3], [4, 5, 6]]),
X_test=np.array([[1, 2, 3, 4], [4, 5, 6, 7]]),
)
with pytest.raises(ValueError, match=r"Cannot call transform on a validator that is not fit"):
validator.transform(np.array([[1, 2, 3], [4, 5, 6]]))
validator = FeatureValidator(feat_type=['Numerical'])
with pytest.raises(ValueError, match=r"providing the option feat_type to the fit method is.*"):
validator.fit(pd.DataFrame([[1, 2, 3], [4, 5, 6]]))
with pytest.raises(ValueError, match=r"feat_type does not have same number of.*"):
validator.fit(np.array([[1, 2, 3], [4, 5, 6]]))
validator = FeatureValidator(feat_type=[1, 2, 3])
with pytest.raises(ValueError, match=r"feat_type must only contain strings.*"):
validator.fit(np.array([[1, 2, 3], [4, 5, 6]]))
validator = FeatureValidator(feat_type=['1', '2', '3'])
with pytest.raises(ValueError, match=r"Only `Categorical` and `Numerical` are.*"):
validator.fit(np.array([[1, 2, 3], [4, 5, 6]]))
def test_no_encoder_created(input_data_featuretest):
"""
Makes sure that for numerical only features, no encoder is created
"""
validator = FeatureValidator()
validator.fit(input_data_featuretest)
validator.transform(input_data_featuretest)
assert validator.encoder is None
def test_featurevalidator_new_data_after_fit(openml_id, train_data_type,
test_data_type):
# List is currently not supported as infer_objects
# cast list objects to type objects
if train_data_type == 'list' or test_data_type == 'list':
pytest.skip()
validator = FeatureValidator()
if train_data_type == 'numpy':
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True,
as_frame=False)
elif train_data_type == 'pandas':
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True,
as_frame=True)
else:
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True,
as_frame=True)
X = X.values.tolist()
y = y.values.tolist()
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X, y, random_state=1)
validator.fit(X_train)
transformed_X = validator.transform(X_test)
# Basic Checking
if sparse.issparse(input_data_featuretest):
assert sparse.issparse(transformed_X)
else:
assert isinstance(transformed_X, np.ndarray)
assert np.shape(X_test) == np.shape(transformed_X)
# And then check proper error messages
if train_data_type == 'pandas':
old_dtypes = copy.deepcopy(validator.dtypes)
validator.dtypes = ['dummy' for dtype in X_train.dtypes]
with pytest.raises(
ValueError,
match=r"hanging the dtype of the features after fit"):
transformed_X = validator.transform(X_test)
validator.dtypes = old_dtypes
if test_data_type == 'pandas':
columns = X_test.columns.tolist()
random.shuffle(columns)
X_test = X_test[columns]
with pytest.raises(
ValueError,
match=r"Changing the column order of the features"):
transformed_X = validator.transform(X_test)
def test_no_new_category_after_fit():
"""
This test makes sure that we can actually pass new categories to the estimator
without throwing an error
"""
# Then make sure we catch categorical extra categories
x = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [5, 6, 7, 8]}, dtype='category')
validator = FeatureValidator()
validator.fit(x)
x['A'] = x['A'].apply(lambda x: x * x)
validator.transform(x)
def test_encoder_created(input_data_featuretest):
"""
This test ensures an encoder is created if categorical data is provided
"""
validator = FeatureValidator()
validator.fit(input_data_featuretest)
transformed_X = validator.transform(input_data_featuretest)
assert validator.encoder is not None
# Make sure that the encoded features are actually encoded. Categorical columns are at
# the start after transformation. In our fixtures, this is also honored prior encode
enc_columns, feature_types = validator._get_columns_to_encode(
input_data_featuretest)
# At least one categorical
assert 'categorical' in validator.feat_type
# Numerical if the original data has numerical only columns
if np.any([
pd.api.types.is_numeric_dtype(input_data_featuretest[col])
for col in input_data_featuretest.columns
]):
assert 'numerical' in validator.feat_type
for i, feat_type in enumerate(feature_types):
if 'numerical' in feat_type:
np.testing.assert_array_equal(
transformed_X[:, i], input_data_featuretest[
input_data_featuretest.columns[i]].to_numpy())
elif 'categorical' in feat_type:
np.testing.assert_array_equal(
transformed_X[:, i],
# Expect always 0, 1... because we use a ordinal encoder
np.array([0, 1]))
else:
raise ValueError(feat_type)
def test_featurevalidator_supported_types(input_data_featuretest):
validator = FeatureValidator()
validator.fit(input_data_featuretest, input_data_featuretest)
transformed_X = validator.transform(input_data_featuretest)
if sparse.issparse(input_data_featuretest):
assert sparse.issparse(transformed_X)
else:
assert isinstance(transformed_X, np.ndarray)
assert np.shape(input_data_featuretest) == np.shape(transformed_X)
assert np.issubdtype(transformed_X.dtype, np.number)
assert validator._is_fitted
def test_featurevalidator_supported_types(input_data_featuretest):
validator = FeatureValidator()
validator.fit(input_data_featuretest, input_data_featuretest)
transformed_X = validator.transform(input_data_featuretest)
if sparse.issparse(input_data_featuretest):
assert sparse.issparse(transformed_X)
elif isinstance(input_data_featuretest, list):
assert isinstance(transformed_X, pd.DataFrame)
else:
assert isinstance(transformed_X, type(input_data_featuretest))
assert np.shape(input_data_featuretest) == np.shape(transformed_X)
assert validator._is_fitted
def test_unknown_encode_value():
x = pd.DataFrame([
{
'a': -41,
'b': -3,
'c': 'a',
'd': -987.2
},
{
'a': -21,
'b': -3,
'c': 'a',
'd': -9.2
},
{
'a': 0,
'b': -4,
'c': 'b',
'd': -97.2
},
{
'a': -51,
'b': -3,
'c': 'a',
'd': 987.2
},
{
'a': 500,
'b': -3,
'c': 'a',
'd': -92
},
])
x['c'] = x['c'].astype('category')
validator = FeatureValidator()
# Make sure that this value is honored
validator.fit(x)
x['c'].cat.add_categories(['NA'], inplace=True)
x.loc[0, 'c'] = 'NA' # unknown value
x_t = validator.transform(x)
# The first row should have a -1 as we added a new categorical there
expected_row = [-1, -41, -3, -987.2]
assert expected_row == x_t[0].tolist()
def test_featurevalidator_fitontypeA_transformtypeB(input_data_featuretest):
"""
Check if we can fit in a given type (numpy) yet transform
if the user changes the type (pandas then)
"""
validator = FeatureValidator()
validator.fit(input_data_featuretest, input_data_featuretest)
if isinstance(input_data_featuretest, pd.DataFrame):
complementary_type = input_data_featuretest.to_numpy()
elif isinstance(input_data_featuretest, np.ndarray):
complementary_type = pd.DataFrame(input_data_featuretest)
elif isinstance(input_data_featuretest, list):
complementary_type = pd.DataFrame(input_data_featuretest)
elif sparse.issparse(input_data_featuretest):
complementary_type = sparse.csr_matrix(input_data_featuretest.todense())
else:
raise ValueError(type(input_data_featuretest))
transformed_X = validator.transform(complementary_type)
assert np.shape(input_data_featuretest) == np.shape(transformed_X)
assert validator._is_fitted
def test_featurevalidator_new_data_after_fit(openml_id,
train_data_type, test_data_type):
# List is currently not supported as infer_objects
# cast list objects to type objects
if train_data_type == 'list' or test_data_type == 'list':
pytest.skip()
validator = FeatureValidator()
if train_data_type == 'numpy':
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True, as_frame=False)
elif train_data_type == 'pandas':
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True, as_frame=True)
else:
X, y = sklearn.datasets.fetch_openml(data_id=openml_id,
return_X_y=True, as_frame=True)
X = X.values.tolist()
y = y.values.tolist()
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X, y, random_state=1)
validator.fit(X_train)
transformed_X = validator.transform(X_test)
# Basic Checking
if sparse.issparse(input_data_featuretest):
assert sparse.issparse(transformed_X)
elif isinstance(input_data_featuretest, list):
assert isinstance(transformed_X, pd.DataFrame)
else:
assert isinstance(transformed_X, type(X_train))
assert np.shape(X_test) == np.shape(transformed_X)
class InputValidator(BaseEstimator):
"""
Makes sure the input data complies with Auto-sklearn requirements.
Categorical inputs are encoded via a Label Encoder, if the input
is a dataframe.
This class also perform checks for data integrity and flags the user
via informative errors.
Attributes
----------
feat_type: typing.Optional[typing.List[str]]
In case the dataset is not a pandas DataFrame:
+ If provided, this list indicates which columns should be treated as categorical
it is internally transformed into a dictionary that indicates a mapping from
column index to categorical/numerical
+ If not provided, by default all columns are treated as numerical
If the input dataset is of type pandas dataframe, this argument
must be none, as the column type will be inferred from the pandas dtypes.
is_classification: bool
For classification task, this flag indicates that the target data
should be encoded
feature_validator: FeatureValidator
A FeatureValidator instance used to validate and encode feature columns to match
sklearn expectations on the data
target_validator: TargetValidator
A TargetValidator instance used to validate and encode (in case of classification)
the target values
"""
def __init__(
self,
feat_type: typing.Optional[typing.List[str]] = None,
is_classification: bool = False,
logger_port: typing.Optional[int] = None,
) -> None:
self.feat_type = feat_type
self.is_classification = is_classification
self.logger_port = logger_port
if self.logger_port is not None:
self.logger = get_named_client_logger(
name='Validation',
port=self.logger_port,
)
else:
self.logger = logging.getLogger('Validation')
self.feature_validator = FeatureValidator(feat_type=self.feat_type,
logger=self.logger)
self.target_validator = TargetValidator(
is_classification=self.is_classification, logger=self.logger)
self._is_fitted = False
def fit(
self,
X_train: SUPPORTED_FEAT_TYPES,
y_train: SUPPORTED_TARGET_TYPES,
X_test: typing.Optional[SUPPORTED_FEAT_TYPES] = None,
y_test: typing.Optional[SUPPORTED_TARGET_TYPES] = None,
) -> BaseEstimator:
"""
Validates and fit a categorical encoder (if needed) to the features, and
a encoder for targets in the case of classification. Specifically:
For features:
+ Valid data types are enforced (List, np.ndarray, pd.DataFrame, pd.Series, scipy
sparse) as well as dimensionality checks
+ If the provided data is a pandas DataFrame with categorical/boolean/int columns,
such columns will be encoded using an Ordinal Encoder
For targets:
+ Checks for dimensionality as well as missing values are performed.
+ If performing a classification task, the data is going to be encoded
Parameters
----------
X_train: SUPPORTED_FEAT_TYPES
A set of features that are going to be validated (type and dimensionality
checks). If this data contains categorical columns, an encoder is going to
be instantiated and trained with this data.
y_train: SUPPORTED_TARGET_TYPES
A set of targets that are going to be encoded if the task is for classification
X_test: typing.Optional[SUPPORTED_FEAT_TYPES]
A hold out set of features used for checking
y_test: SUPPORTED_TARGET_TYPES
A hold out set of targets used for checking. Additionally, if the current task
is a classification task, this y_test categories are also going to be used to
fit a pre-processing encoding (to prevent errors on unseen classes).
Returns
-------
self
"""
# Check that the data is valid
if np.shape(X_train)[0] != np.shape(y_train)[0]:
raise ValueError(
"Inconsistent number of train datapoints for features and targets,"
" {} for features and {} for targets".format(
np.shape(X_train)[0],
np.shape(y_train)[0],
))
if X_test is not None and np.shape(X_test)[0] != np.shape(y_test)[0]:
raise ValueError(
"Inconsistent number of test datapoints for features and targets,"
" {} for features and {} for targets".format(
np.shape(X_test)[0],
np.shape(y_test)[0],
))
self.feature_validator.fit(X_train, X_test)
self.target_validator.fit(y_train, y_test)
self._is_fitted = True
return self
def transform(
self,
X: SUPPORTED_FEAT_TYPES,
y: typing.Optional[SUPPORTED_TARGET_TYPES] = None,
) -> typing.Tuple[np.ndarray, typing.Optional[np.ndarray]]:
"""
Transform the given target or features to a numpy array
Parameters
----------
X: SUPPORTED_FEAT_TYPES
A set of features to transform
y: typing.Optional[SUPPORTED_TARGET_TYPES]
A set of targets to transform
Return
------
np.ndarray:
The transformed features array
np.ndarray:
The transformed targets array
"""
if not self._is_fitted:
raise NotFittedError(
"Cannot call transform on a validator that is not fitted")
X_transformed = self.feature_validator.transform(X)
if y is not None:
return X_transformed, self.target_validator.transform(y)
else:
return X_transformed, y
def test_sparse_output_is_csr(input_data_featuretest):
validator = FeatureValidator()
validator.fit(input_data_featuretest, input_data_featuretest)
transformed_X = validator.transform(input_data_featuretest)
assert sparse.issparse(transformed_X)
assert isinstance(transformed_X, sparse.csr_matrix)