def test_make_pipeline_text_columns(input_type, problem_type):
X = pd.DataFrame({
"numerical": [1, 2, 3, 1, 2],
"categorical": ["a", "b", "a", "c", "c"],
"text": [
"string one", "another",
"text for a column, this should be a text column!!", "text string",
"hello world"
]
})
y = pd.Series([0, 0, 1, 1, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": None,
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": None,
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [
OneHotEncoder, StandardScaler, estimator_class
]
elif estimator_class.model_family == ModelFamily.CATBOOST:
estimator_components = [estimator_class]
else:
estimator_components = [OneHotEncoder, estimator_class]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == [Imputer, TextFeaturizer
] + estimator_components
else:
assert pipeline.component_graph == [
Imputer, TextFeaturizer
] + delayed_features + estimator_components
def test_make_pipeline_only_text_columns(input_type, problem_type):
X = pd.DataFrame({
"text": [
"string one", "the evalml team is full of wonderful people",
"text for a column, this should be a text column!!", "text string",
"hello world"
],
"another text": [
"ladidididididida", "cats are great",
"text for a column, this should be a text column!!", "text string",
"goodbye world"
]
})
y = pd.Series([0, 0, 1, 1, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": None,
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": None,
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
standard_scaler = []
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
standard_scaler = [StandardScaler]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == [
TextFeaturizer
] + standard_scaler + [estimator_class]
else:
assert pipeline.component_graph == [
TextFeaturizer
] + delayed_features + standard_scaler + [estimator_class]
def test_make_pipeline_no_nulls(input_type, problem_type):
X = pd.DataFrame({
"numerical": [1, 2, 3, 1, 2],
"categorical": ["a", "b", "a", "c", "c"],
"some dates":
pd.date_range('2000-02-03', periods=5, freq='W')
})
y = pd.Series([0, 1, 1, 0, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [
OneHotEncoder, StandardScaler, estimator_class
]
elif estimator_class.model_family == ModelFamily.CATBOOST:
estimator_components = [estimator_class]
else:
estimator_components = [OneHotEncoder, estimator_class]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == [Imputer
] + estimator_components
else:
assert pipeline.component_graph == [
Imputer, DateTimeFeaturizer
] + delayed_features + estimator_components
def test_make_pipeline_only_datetime_columns(input_type, problem_type):
X = pd.DataFrame({
"some dates":
pd.date_range('2000-02-03', periods=5, freq='W'),
"some other dates":
pd.date_range('2000-05-19', periods=5, freq='W')
})
y = pd.Series([0, 0, 1, 1, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
standard_scaler = []
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
standard_scaler = [StandardScaler]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == standard_scaler + [
estimator_class
]
else:
assert pipeline.component_graph == [
DateTimeFeaturizer
] + delayed_features + standard_scaler + [estimator_class]
def test_make_pipeline_no_column_names(input_type, problem_type):
X = pd.DataFrame([[1, "a", np.nan], [2, "b", np.nan], [5, "b", np.nan]])
y = pd.Series([0, 0, 1])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": None,
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": None,
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [
OneHotEncoder, StandardScaler, estimator_class
]
elif estimator_class.model_family == ModelFamily.CATBOOST:
estimator_components = [estimator_class]
else:
estimator_components = [OneHotEncoder, estimator_class]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == [DropNullColumns, Imputer
] + estimator_components
else:
assert pipeline.component_graph == [
DropNullColumns, Imputer
] + delayed_features + estimator_components
def test_make_pipeline_all_nan_no_categoricals(input_type, problem_type):
# testing that all_null column is not considered categorical
X = pd.DataFrame({
"all_null": [np.nan, np.nan, np.nan, np.nan, np.nan],
"num": [1, 2, 3, 4, 5]
})
y = pd.Series([0, 0, 1, 1, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": None,
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": None,
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
assert pipeline.custom_hyperparameters is None
delayed_features = []
if is_time_series(
problem_type
) and estimator_class.model_family != ModelFamily.ARIMA:
delayed_features = [DelayedFeatureTransformer]
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [StandardScaler, estimator_class]
else:
estimator_components = [estimator_class]
assert pipeline.component_graph == [
DropNullColumns, Imputer
] + delayed_features + estimator_components
def test_make_pipeline_no_datetimes(input_type, problem_type):
X = pd.DataFrame({"numerical": [1, 2, 3, 1, 2],
"categorical": ["a", "b", "a", "c", "c"],
"all_null": [np.nan, np.nan, np.nan, np.nan, np.nan]})
y = pd.Series([0, 1, 1, 0, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
pipeline = make_pipeline(X, y, estimator_class, problem_type)
assert isinstance(pipeline, type(pipeline_class))
assert pipeline.custom_hyperparameters is None
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
else:
delayed_features = []
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [OneHotEncoder, StandardScaler, estimator_class]
elif estimator_class.model_family == ModelFamily.CATBOOST:
estimator_components = [estimator_class]
else:
estimator_components = [OneHotEncoder, estimator_class]
assert pipeline.component_graph == [DropNullColumns, Imputer] + delayed_features + estimator_components
def test_make_pipeline_numpy_input(problem_type):
X = np.array([[1, 2, 0, np.nan], [2, 2, 1, np.nan], [5, 1, np.nan,
np.nan]])
y = np.array([0, 0, 1, 0])
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
pipeline = make_pipeline(X, y, estimator_class, problem_type)
assert isinstance(pipeline, type(pipeline_class))
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
else:
delayed_features = []
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [StandardScaler, estimator_class]
else:
estimator_components = [estimator_class]
assert pipeline.component_graph == [
DropNullColumns, Imputer
] + delayed_features + estimator_components
def make_data_splitter(X, y, problem_type, problem_configuration=None, n_splits=3, shuffle=True, random_state=0):
"""Given the training data and ML problem parameters, compute a data splitting method to use during AutoML search.
Arguments:
X (pd.DataFrame, ww.DataTable): The input training data of shape [n_samples, n_features].
y (pd.Series, ww.DataColumn): The target training data of length [n_samples].
problem_type (ProblemType): the type of machine learning problem.
problem_configuration (dict, None): Additional parameters needed to configure the search. For example,
in time series problems, values should be passed in for the gap and max_delay variables.
n_splits (int, None): the number of CV splits, if applicable. Default 3.
shuffle (bool): whether or not to shuffle the data before splitting, if applicable. Default True.
random_state (int, np.random.RandomState): The random seed/state. Defaults to 0.
Returns:
sklearn.model_selection.BaseCrossValidator: data splitting method.
"""
problem_type = handle_problem_types(problem_type)
data_splitter = None
if problem_type == ProblemTypes.REGRESSION:
data_splitter = KFold(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
elif problem_type in [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]:
data_splitter = StratifiedKFold(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
elif is_time_series(problem_type):
if not problem_configuration:
raise ValueError("problem_configuration is required for time series problem types")
data_splitter = TimeSeriesSplit(n_splits=n_splits, gap=problem_configuration.get('gap'),
max_delay=problem_configuration.get('max_delay'))
if X.shape[0] > _LARGE_DATA_ROW_THRESHOLD:
data_splitter = TrainingValidationSplit(test_size=_LARGE_DATA_PERCENT_VALIDATION, shuffle=True)
return data_splitter
def test_split_data(problem_type, data_type, X_y_binary, X_y_multi,
X_y_regression, make_data_type):
if is_binary(problem_type):
X, y = X_y_binary
if is_multiclass(problem_type):
X, y = X_y_multi
if is_regression(problem_type):
X, y = X_y_regression
problem_configuration = None
if is_time_series(problem_type):
problem_configuration = {'gap': 1, 'max_delay': 7}
X = make_data_type(data_type, X)
y = make_data_type(data_type, y)
test_pct = 0.25
X_train, X_test, y_train, y_test = split_data(
X,
y,
test_size=test_pct,
problem_type=problem_type,
problem_configuration=problem_configuration)
test_size = len(X) * test_pct
train_size = len(X) - test_size
assert len(X_train) == train_size
assert len(X_test) == test_size
assert len(y_train) == train_size
assert len(y_test) == test_size
assert isinstance(X_train, ww.DataTable)
assert isinstance(X_test, ww.DataTable)
assert isinstance(y_train, ww.DataColumn)
assert isinstance(y_test, ww.DataColumn)
def split_data(X, y, problem_type, problem_configuration=None, test_size=.2, random_seed=0):
"""Splits data into train and test sets.
Arguments:
X (ww.DataTable, pd.DataFrame or np.ndarray): data of shape [n_samples, n_features]
y (ww.DataColumn, pd.Series, or np.ndarray): target data of length [n_samples]
problem_type (str or ProblemTypes): type of supervised learning problem. see evalml.problem_types.problemtype.all_problem_types for a full list.
problem_configuration (dict): Additional parameters needed to configure the search. For example,
in time series problems, values should be passed in for the date_index, gap, and max_delay variables.
test_size (float): What percentage of data points should be included in the test set. Defaults to 0.2 (20%).
random_seed (int): Seed for the random number generator. Defaults to 0.
Returns:
ww.DataTable, ww.DataTable, ww.DataColumn, ww.DataColumn: Feature and target data each split into train and test sets
"""
X = infer_feature_types(X)
y = infer_feature_types(y)
data_splitter = None
if is_time_series(problem_type):
data_splitter = TrainingValidationSplit(test_size=test_size, shuffle=False, stratify=None, random_seed=random_seed)
elif is_regression(problem_type):
data_splitter = ShuffleSplit(n_splits=1, test_size=test_size, random_state=random_seed)
elif is_classification(problem_type):
data_splitter = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_seed)
train, test = next(data_splitter.split(X.to_dataframe(), y.to_series()))
X_train = X.iloc[train]
X_test = X.iloc[test]
y_train = y.iloc[train]
y_test = y.iloc[test]
return X_train, X_test, y_train, y_test
def test_make_pipeline_datetime_no_categorical(input_type, problem_type):
X = pd.DataFrame({"numerical": [1, 2, 3, 1, 2],
"some dates": pd.date_range('2000-02-03', periods=5, freq='W')})
y = pd.Series([0, 1, 1, 0, 0])
if input_type == 'ww':
X = ww.DataTable(X)
y = ww.DataColumn(y)
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
pipeline = make_pipeline(X, y, estimator_class, problem_type)
assert isinstance(pipeline, type(pipeline_class))
assert pipeline.custom_hyperparameters is None
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
else:
delayed_features = []
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [StandardScaler, estimator_class]
elif estimator_class.model_family == ModelFamily.CATBOOST:
estimator_components = [estimator_class]
else:
estimator_components = [estimator_class]
assert pipeline.component_graph == [Imputer, DateTimeFeaturizer] + delayed_features + estimator_components
def test_make_pipeline_numpy_input(problem_type):
X = np.array([[1, 2, 0, np.nan], [2, 2, 1, np.nan], [5, 1, np.nan,
np.nan]])
y = np.array([0, 0, 1, 0])
estimators = get_estimators(problem_type=problem_type)
pipeline_class = _get_pipeline_base_class(problem_type)
if problem_type == ProblemTypes.MULTICLASS:
y = pd.Series([0, 2, 1, 2])
for estimator_class in estimators:
if problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": None,
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": None,
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert isinstance(pipeline, pipeline_class)
delayed_features = []
if is_time_series(problem_type):
delayed_features = [DelayedFeatureTransformer]
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
estimator_components = [StandardScaler, estimator_class]
else:
estimator_components = [estimator_class]
if estimator_class.model_family == ModelFamily.ARIMA:
assert pipeline.component_graph == [DropNullColumns, Imputer
] + estimator_components
else:
assert pipeline.component_graph == [
DropNullColumns, Imputer
] + delayed_features + estimator_components
def make_data_splitter(X,
y,
problem_type,
problem_configuration=None,
n_splits=3,
shuffle=True,
random_state=None,
random_seed=0):
"""Given the training data and ML problem parameters, compute a data splitting method to use during AutoML search.
Arguments:
X (ww.DataTable, pd.DataFrame): The input training data of shape [n_samples, n_features].
y (ww.DataColumn, pd.Series): The target training data of length [n_samples].
problem_type (ProblemType): The type of machine learning problem.
problem_configuration (dict, None): Additional parameters needed to configure the search. For example,
in time series problems, values should be passed in for the gap and max_delay variables. Defaults to None.
n_splits (int, None): The number of CV splits, if applicable. Defaults to 3.
shuffle (bool): Whether or not to shuffle the data before splitting, if applicable. Defaults to True.
random_state (None, int): Deprecated - use random_seed instead.
random_seed (int): Seed for the random number generator. Defaults to 0.
Returns:
sklearn.model_selection.BaseCrossValidator: Data splitting method.
"""
random_seed = deprecate_arg("random_state", "random_seed", random_state,
random_seed)
problem_type = handle_problem_types(problem_type)
if is_time_series(problem_type):
if not problem_configuration:
raise ValueError(
"problem_configuration is required for time series problem types"
)
return TimeSeriesSplit(
n_splits=n_splits,
gap=problem_configuration.get('gap'),
max_delay=problem_configuration.get('max_delay'))
if X.shape[0] > _LARGE_DATA_ROW_THRESHOLD:
if problem_type == ProblemTypes.REGRESSION:
return TrainingValidationSplit(
test_size=_LARGE_DATA_PERCENT_VALIDATION, shuffle=shuffle)
elif problem_type in [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]:
return BalancedClassificationDataTVSplit(
test_size=_LARGE_DATA_PERCENT_VALIDATION,
shuffle=shuffle,
random_seed=random_seed)
if problem_type == ProblemTypes.REGRESSION:
return KFold(n_splits=n_splits,
random_state=random_seed,
shuffle=shuffle)
elif problem_type in [ProblemTypes.BINARY, ProblemTypes.MULTICLASS]:
return BalancedClassificationDataCVSplit(n_splits=n_splits,
random_seed=random_seed,
shuffle=shuffle)
def _get_preprocessing_components(X, y, problem_type, estimator_class):
"""Given input data, target data and an estimator class, construct a recommended preprocessing chain to be combined with the estimator and trained on the provided data.
Arguments:
X (ww.DataTable): The input data of shape [n_samples, n_features]
y (ww.DataColumn): The target data of length [n_samples]
problem_type (ProblemTypes or str): Problem type
estimator_class (class): A class which subclasses Estimator estimator for pipeline
Returns:
list[Transformer]: A list of applicable preprocessing components to use with the estimator
"""
X_pd = X.to_dataframe()
pp_components = []
all_null_cols = X_pd.columns[X_pd.isnull().all()]
if len(all_null_cols) > 0:
pp_components.append(DropNullColumns)
input_logical_types = set(X.logical_types.values())
types_imputer_handles = {
logical_types.Boolean, logical_types.Categorical, logical_types.Double,
logical_types.Integer
}
if len(input_logical_types.intersection(types_imputer_handles)) > 0:
pp_components.append(Imputer)
text_columns = list(X.select('natural_language').columns)
if len(text_columns) > 0:
pp_components.append(TextFeaturizer)
datetime_cols = X.select(["Datetime"])
add_datetime_featurizer = len(datetime_cols.columns) > 0
if add_datetime_featurizer:
pp_components.append(DateTimeFeaturizer)
if is_time_series(problem_type):
pp_components.append(DelayedFeatureTransformer)
categorical_cols = X.select('category')
if len(categorical_cols.columns) > 0 and estimator_class not in {
CatBoostClassifier, CatBoostRegressor
}:
pp_components.append(OneHotEncoder)
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
pp_components.append(StandardScaler)
return pp_components
def test_type_checks(problem_type):
assert is_regression(problem_type) == (problem_type in [
ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION
])
assert is_binary(problem_type) == (problem_type in [
ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY
])
assert is_multiclass(problem_type) == (problem_type in [
ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS
])
assert is_classification(problem_type) == (problem_type in [
ProblemTypes.BINARY, ProblemTypes.MULTICLASS,
ProblemTypes.TIME_SERIES_BINARY, ProblemTypes.TIME_SERIES_MULTICLASS
])
assert is_time_series(problem_type) == (problem_type in [
ProblemTypes.TIME_SERIES_BINARY, ProblemTypes.TIME_SERIES_MULTICLASS,
ProblemTypes.TIME_SERIES_REGRESSION
])
def test_make_pipeline_custom_hyperparameters(problem_type):
X = pd.DataFrame({
"all_null": [np.nan, np.nan, np.nan, np.nan, np.nan],
"categorical": ["a", "b", "a", "c", "c"],
"some dates":
pd.date_range('2000-02-03', periods=5, freq='W')
})
custom_hyperparameters = {
'Imputer': {
'numeric_impute_strategy': ['median']
}
}
y = pd.Series([0, 0, 1, 0, 0])
estimators = get_estimators(problem_type=problem_type)
for estimator_class in estimators:
for problem_type in estimator_class.supported_problem_types:
parameters = {}
if is_time_series(problem_type):
parameters = {
"pipeline": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
},
"Time Series Baseline Estimator": {
"date_index": "some dates",
"gap": 1,
"max_delay": 1
}
}
pipeline = make_pipeline(X, y, estimator_class, problem_type,
parameters, custom_hyperparameters)
assert pipeline.custom_hyperparameters == custom_hyperparameters
pipeline2 = make_pipeline(X, y, estimator_class, problem_type,
parameters)
assert not pipeline2.custom_hyperparameters
def _get_preprocessing_components(X, y, problem_type, estimator_class, sampler_name=None):
"""Given input data, target data and an estimator class, construct a recommended preprocessing chain to be combined with the estimator and trained on the provided data.
Arguments:
X (ww.DataTable): The input data of shape [n_samples, n_features]
y (ww.DataColumn): The target data of length [n_samples]
problem_type (ProblemTypes or str): Problem type
estimator_class (class): A class which subclasses Estimator estimator for pipeline,
sampler_name (str): The name of the sampler component to add to the pipeline. Defaults to None
Returns:
list[Transformer]: A list of applicable preprocessing components to use with the estimator
"""
X_pd = X.to_dataframe()
pp_components = []
all_null_cols = X_pd.columns[X_pd.isnull().all()]
if len(all_null_cols) > 0:
pp_components.append(DropNullColumns)
input_logical_types = set(X.logical_types.values())
types_imputer_handles = {logical_types.Boolean, logical_types.Categorical, logical_types.Double, logical_types.Integer}
if len(input_logical_types.intersection(types_imputer_handles)) > 0:
pp_components.append(Imputer)
text_columns = list(X.select('natural_language').columns)
if len(text_columns) > 0:
pp_components.append(TextFeaturizer)
index_columns = list(X.select('index').columns)
if len(index_columns) > 0:
pp_components.append(DropColumns)
datetime_cols = X.select(["Datetime"])
add_datetime_featurizer = len(datetime_cols.columns) > 0
if add_datetime_featurizer and estimator_class.model_family != ModelFamily.ARIMA:
pp_components.append(DateTimeFeaturizer)
if is_time_series(problem_type) and estimator_class.model_family != ModelFamily.ARIMA:
pp_components.append(DelayedFeatureTransformer)
categorical_cols = X.select('category')
if len(categorical_cols.columns) > 0 and estimator_class not in {CatBoostClassifier, CatBoostRegressor}:
pp_components.append(OneHotEncoder)
sampler_components = {
"Undersampler": Undersampler,
"SMOTE Oversampler": SMOTESampler,
"SMOTENC Oversampler": SMOTENCSampler,
"SMOTEN Oversampler": SMOTENSampler
}
if sampler_name is not None:
try:
import_or_raise("imblearn.over_sampling", error_msg="imbalanced-learn is not installed")
pp_components.append(sampler_components[sampler_name])
except ImportError:
logger.debug(f'Could not import imblearn.over_sampling, so defaulting to use Undersampler')
pp_components.append(Undersampler)
if estimator_class.model_family == ModelFamily.LINEAR_MODEL:
pp_components.append(StandardScaler)
return pp_components
def explain_predictions_best_worst(pipeline,
input_features,
y_true,
num_to_explain=5,
top_k_features=3,
include_shap_values=False,
metric=None,
output_format="text"):
"""Creates a report summarizing the top contributing features for the best and worst points in the dataset as measured by error to true labels.
XGBoost models and CatBoost multiclass classifiers are not currently supported.
Arguments:
pipeline (PipelineBase): Fitted pipeline whose predictions we want to explain with SHAP.
input_features (ww.DataTable, pd.DataFrame): Input data to evaluate the pipeline on.
y_true (ww.DataColumn, pd.Series): True labels for the input data.
num_to_explain (int): How many of the best, worst, random data points to explain.
top_k_features (int): How many of the highest/lowest contributing feature to include in the table for each
data point.
include_shap_values (bool): Whether SHAP values should be included in the table. Default is False.
metric (callable): The metric used to identify the best and worst points in the dataset. Function must accept
the true labels and predicted value or probabilities as the only arguments and lower values
must be better. By default, this will be the absolute error for regression problems and cross entropy loss
for classification problems.
output_format (str): Either "text" or "dict". Default is "text".
Returns:
str, dict, or pd.DataFrame - A report explaining the top contributing features for the best/worst predictions in the input_features.
For each of the best/worst rows of input_features, the predicted values, true labels, metric value,
feature names, prediction contribution, and SHAP Value (optional) will be listed.
Raises:
ValueError: if input_features does not have more than twice the requested features to explain.
ValueError: if y_true and input_features have mismatched lengths.
ValueError: if an output_format outside of "text", "dict" or "dataframe is provided.
"""
input_features = infer_feature_types(input_features)
input_features = _convert_woodwork_types_wrapper(
input_features.to_dataframe())
y_true = infer_feature_types(y_true)
y_true = _convert_woodwork_types_wrapper(y_true.to_series())
if not (input_features.shape[0] >= num_to_explain * 2):
raise ValueError(
f"Input features must be a dataframe with more than {num_to_explain * 2} rows! "
"Convert to a dataframe and select a smaller value for num_to_explain if you do not have "
"enough data.")
if y_true.shape[0] != input_features.shape[0]:
raise ValueError(
"Parameters y_true and input_features must have the same number of data points. Received: "
f"true labels: {y_true.shape[0]} and {input_features.shape[0]}")
if output_format not in {"text", "dict", "dataframe"}:
raise ValueError(
f"Parameter output_format must be either text, dict, or dataframe. Received {output_format}"
)
if not metric:
metric = DEFAULT_METRICS[pipeline.problem_type]
try:
if is_regression(pipeline.problem_type):
if is_time_series(pipeline.problem_type):
y_pred = pipeline.predict(input_features, y=y_true).to_series()
else:
y_pred = pipeline.predict(input_features).to_series()
y_pred_values = None
y_true_no_nan, y_pred_no_nan = drop_rows_with_nans(y_true, y_pred)
errors = metric(y_true_no_nan, y_pred_no_nan)
else:
if is_time_series(pipeline.problem_type):
y_pred = pipeline.predict_proba(input_features,
y=y_true).to_dataframe()
y_pred_values = pipeline.predict(input_features,
y=y_true).to_series()
else:
y_pred = pipeline.predict_proba(input_features).to_dataframe()
y_pred_values = pipeline.predict(input_features).to_series()
y_true_no_nan, y_pred_no_nan, y_pred_values_no_nan = drop_rows_with_nans(
y_true, y_pred, y_pred_values)
errors = metric(pipeline._encode_targets(y_true_no_nan),
y_pred_no_nan)
except Exception as e:
tb = traceback.format_tb(sys.exc_info()[2])
raise PipelineScoreError(exceptions={metric.__name__: (e, tb)},
scored_successfully={})
errors = pd.Series(errors, index=y_pred_no_nan.index)
sorted_scores = errors.sort_values()
best_indices = sorted_scores.index[:num_to_explain]
worst_indices = sorted_scores.index[-num_to_explain:]
index_list = best_indices.tolist() + worst_indices.tolist()
pipeline_features = pipeline.compute_estimator_features(
input_features, y_true).to_dataframe()
data = _ReportData(pipeline, pipeline_features, input_features, y_true,
y_pred, y_pred_values, errors, index_list, metric)
report_creator = _report_creator_factory(
data,
report_type="explain_predictions_best_worst",
output_format=output_format,
top_k_features=top_k_features,
include_shap_values=include_shap_values,
num_to_explain=num_to_explain)
return report_creator(data)
def __init__(self,
X_train=None,
y_train=None,
problem_type=None,
objective='auto',
max_iterations=None,
max_time=None,
patience=None,
tolerance=None,
data_splitter=None,
allowed_pipelines=None,
allowed_model_families=None,
start_iteration_callback=None,
add_result_callback=None,
error_callback=None,
additional_objectives=None,
random_seed=0,
n_jobs=-1,
tuner_class=None,
optimize_thresholds=True,
ensembling=False,
max_batches=None,
problem_configuration=None,
train_best_pipeline=True,
pipeline_parameters=None,
_ensembling_split_size=0.2,
_pipelines_per_batch=5,
engine=None):
"""Automated pipeline search
Arguments:
X_train (pd.DataFrame, ww.DataTable): The input training data of shape [n_samples, n_features]. Required.
y_train (pd.Series, ww.DataColumn): The target training data of length [n_samples]. Required for supervised learning tasks.
problem_type (str or ProblemTypes): type of supervised learning problem. See evalml.problem_types.ProblemType.all_problem_types for a full list.
objective (str, ObjectiveBase): The objective to optimize for. Used to propose and rank pipelines, but not for optimizing each pipeline during fit-time.
When set to 'auto', chooses:
- LogLossBinary for binary classification problems,
- LogLossMulticlass for multiclass classification problems, and
- R2 for regression problems.
max_iterations (int): Maximum number of iterations to search. If max_iterations and
max_time is not set, then max_iterations will default to max_iterations of 5.
max_time (int, str): Maximum time to search for pipelines.
This will not start a new pipeline search after the duration
has elapsed. If it is an integer, then the time will be in seconds.
For strings, time can be specified as seconds, minutes, or hours.
patience (int): Number of iterations without improvement to stop search early. Must be positive.
If None, early stopping is disabled. Defaults to None.
tolerance (float): Minimum percentage difference to qualify as score improvement for early stopping.
Only applicable if patience is not None. Defaults to None.
allowed_pipelines (list(class)): A list of PipelineBase subclasses indicating the pipelines allowed in the search.
The default of None indicates all pipelines for this problem type are allowed. Setting this field will cause
allowed_model_families to be ignored.
allowed_model_families (list(str, ModelFamily)): The model families to search. The default of None searches over all
model families. Run evalml.pipelines.components.utils.allowed_model_families("binary") to see options. Change `binary`
to `multiclass` or `regression` depending on the problem type. Note that if allowed_pipelines is provided,
this parameter will be ignored.
data_splitter (sklearn.model_selection.BaseCrossValidator): Data splitting method to use. Defaults to StratifiedKFold.
tuner_class: The tuner class to use. Defaults to SKOptTuner.
optimize_thresholds (bool): Whether or not to optimize the binary pipeline threshold. Defaults to True.
start_iteration_callback (callable): Function called before each pipeline training iteration.
Callback function takes three positional parameters: The pipeline class, the pipeline parameters, and the AutoMLSearch object.
add_result_callback (callable): Function called after each pipeline training iteration.
Callback function takes three positional parameters: A dictionary containing the training results for the new pipeline, an untrained_pipeline containing the parameters used during training, and the AutoMLSearch object.
error_callback (callable): Function called when `search()` errors and raises an Exception.
Callback function takes three positional parameters: the Exception raised, the traceback, and the AutoMLSearch object.
Must also accepts kwargs, so AutoMLSearch is able to pass along other appropriate parameters by default.
Defaults to None, which will call `log_error_callback`.
additional_objectives (list): Custom set of objectives to score on.
Will override default objectives for problem type if not empty.
random_seed (int): Seed for the random number generator. Defaults to 0.
n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
ensembling (boolean): If True, runs ensembling in a separate batch after every allowed pipeline class has been iterated over.
If the number of unique pipelines to search over per batch is one, ensembling will not run. Defaults to False.
max_batches (int): The maximum number of batches of pipelines to search. Parameters max_time, and
max_iterations have precedence over stopping the search.
problem_configuration (dict, None): Additional parameters needed to configure the search. For example,
in time series problems, values should be passed in for the gap and max_delay variables.
train_best_pipeline (boolean): Whether or not to train the best pipeline before returning it. Defaults to True.
pipeline_parameters (dict): A dict of the parameters used to initalize a pipeline with.
_ensembling_split_size (float): The amount of the training data we'll set aside for training ensemble metalearners. Only used when ensembling is True.
Must be between 0 and 1, exclusive. Defaults to 0.2
_pipelines_per_batch (int): The number of pipelines to train for every batch after the first one.
The first batch will train a baseline pipline + one of each pipeline family allowed in the search.
engine (EngineBase or None): The engine instance used to evaluate pipelines. If None, a SequentialEngine will
be used.
"""
if X_train is None:
raise ValueError('Must specify training data as a 2d array using the X_train argument')
if y_train is None:
raise ValueError('Must specify training data target values as a 1d vector using the y_train argument')
try:
self.problem_type = handle_problem_types(problem_type)
except ValueError:
raise ValueError('choose one of (binary, multiclass, regression) as problem_type')
if is_time_series(self.problem_type):
warnings.warn("Time series support in evalml is still in beta, which means we are still actively building "
"its core features. Please be mindful of that when running search().")
self.tuner_class = tuner_class or SKOptTuner
self.start_iteration_callback = start_iteration_callback
self.add_result_callback = add_result_callback
self.error_callback = error_callback or log_error_callback
self.data_splitter = data_splitter
self.optimize_thresholds = optimize_thresholds
self.ensembling = ensembling
if objective == 'auto':
objective = get_default_primary_search_objective(self.problem_type.value)
objective = get_objective(objective, return_instance=False)
self.objective = self._validate_objective(objective)
if self.data_splitter is not None and not issubclass(self.data_splitter.__class__, BaseCrossValidator):
raise ValueError("Not a valid data splitter")
if not objective.is_defined_for_problem_type(self.problem_type):
raise ValueError("Given objective {} is not compatible with a {} problem.".format(self.objective.name, self.problem_type.value))
if additional_objectives is None:
additional_objectives = get_core_objectives(self.problem_type)
# if our main objective is part of default set of objectives for problem_type, remove it
existing_main_objective = next((obj for obj in additional_objectives if obj.name == self.objective.name), None)
if existing_main_objective is not None:
additional_objectives.remove(existing_main_objective)
else:
additional_objectives = [get_objective(o) for o in additional_objectives]
additional_objectives = [self._validate_objective(obj) for obj in additional_objectives]
self.additional_objectives = additional_objectives
self.objective_name_to_class = {o.name: o for o in [self.objective] + self.additional_objectives}
if not isinstance(max_time, (int, float, str, type(None))):
raise TypeError(f"Parameter max_time must be a float, int, string or None. Received {type(max_time)} with value {str(max_time)}..")
if isinstance(max_time, (int, float)) and max_time < 0:
raise ValueError(f"Parameter max_time must be None or non-negative. Received {max_time}.")
if max_batches is not None and max_batches < 0:
raise ValueError(f"Parameter max_batches must be None or non-negative. Received {max_batches}.")
if max_iterations is not None and max_iterations < 0:
raise ValueError(f"Parameter max_iterations must be None or non-negative. Received {max_iterations}.")
self.max_time = convert_to_seconds(max_time) if isinstance(max_time, str) else max_time
self.max_iterations = max_iterations
self.max_batches = max_batches
self._pipelines_per_batch = _pipelines_per_batch
if not self.max_iterations and not self.max_time and not self.max_batches:
self.max_batches = 1
logger.info("Using default limit of max_batches=1.\n")
if patience and (not isinstance(patience, int) or patience < 0):
raise ValueError("patience value must be a positive integer. Received {} instead".format(patience))
if tolerance and (tolerance > 1.0 or tolerance < 0.0):
raise ValueError("tolerance value must be a float between 0.0 and 1.0 inclusive. Received {} instead".format(tolerance))
self.patience = patience
self.tolerance = tolerance or 0.0
self._results = {
'pipeline_results': {},
'search_order': [],
}
self.random_seed = random_seed
self.n_jobs = n_jobs
self.plot = None
try:
self.plot = PipelineSearchPlots(self)
except ImportError:
logger.warning("Unable to import plotly; skipping pipeline search plotting\n")
self.allowed_pipelines = allowed_pipelines
self.allowed_model_families = allowed_model_families
self._automl_algorithm = None
self._start = 0.0
self._baseline_cv_scores = {}
self.show_batch_output = False
self._validate_problem_type()
self.problem_configuration = self._validate_problem_configuration(problem_configuration)
self._train_best_pipeline = train_best_pipeline
self._best_pipeline = None
self._searched = False
self.X_train = infer_feature_types(X_train)
self.y_train = infer_feature_types(y_train)
self.ensembling_indices = None
default_data_splitter = make_data_splitter(self.X_train, self.y_train, self.problem_type, self.problem_configuration,
n_splits=3, shuffle=True, random_seed=self.random_seed)
self.data_splitter = self.data_splitter or default_data_splitter
self.pipeline_parameters = pipeline_parameters if pipeline_parameters is not None else {}
self.search_iteration_plot = None
self._interrupted = False
if self.allowed_pipelines is None:
logger.info("Generating pipelines to search over...")
allowed_estimators = get_estimators(self.problem_type, self.allowed_model_families)
logger.debug(f"allowed_estimators set to {[estimator.name for estimator in allowed_estimators]}")
self.allowed_pipelines = [make_pipeline(self.X_train, self.y_train, estimator, self.problem_type, custom_hyperparameters=self.pipeline_parameters) for estimator in allowed_estimators]
if self.allowed_pipelines == []:
raise ValueError("No allowed pipelines to search")
check_all_pipeline_names_unique(self.allowed_pipelines)
run_ensembling = self.ensembling
if run_ensembling and len(self.allowed_pipelines) == 1:
logger.warning("Ensembling is set to True, but the number of unique pipelines is one, so ensembling will not run.")
run_ensembling = False
if run_ensembling and self.max_iterations is not None:
# Baseline + first batch + each pipeline iteration + 1
first_ensembling_iteration = (1 + len(self.allowed_pipelines) + len(self.allowed_pipelines) * self._pipelines_per_batch + 1)
if self.max_iterations < first_ensembling_iteration:
run_ensembling = False
logger.warning(f"Ensembling is set to True, but max_iterations is too small, so ensembling will not run. Set max_iterations >= {first_ensembling_iteration} to run ensembling.")
else:
logger.info(f"Ensembling will run at the {first_ensembling_iteration} iteration and every {len(self.allowed_pipelines) * self._pipelines_per_batch} iterations after that.")
if self.max_batches and self.max_iterations is None:
self.show_batch_output = True
if run_ensembling:
ensemble_nth_batch = len(self.allowed_pipelines) + 1
num_ensemble_batches = (self.max_batches - 1) // ensemble_nth_batch
if num_ensemble_batches == 0:
run_ensembling = False
logger.warning(f"Ensembling is set to True, but max_batches is too small, so ensembling will not run. Set max_batches >= {ensemble_nth_batch + 1} to run ensembling.")
else:
logger.info(f"Ensembling will run every {ensemble_nth_batch} batches.")
self.max_iterations = (1 + len(self.allowed_pipelines) +
self._pipelines_per_batch * (self.max_batches - 1 - num_ensemble_batches) +
num_ensemble_batches)
else:
self.max_iterations = 1 + len(self.allowed_pipelines) + (self._pipelines_per_batch * (self.max_batches - 1))
if run_ensembling:
if not (0 < _ensembling_split_size < 1):
raise ValueError(f"Ensembling split size must be between 0 and 1 exclusive, received {_ensembling_split_size}")
X_shape = ww.DataTable(np.arange(self.X_train.shape[0]))
_, ensembling_indices, _, _ = split_data(X_shape, self.y_train, problem_type=self.problem_type, test_size=_ensembling_split_size, random_seed=self.random_seed)
self.ensembling_indices = ensembling_indices.to_dataframe()[0].tolist()
if not engine:
self._engine = SequentialEngine()
else:
self._engine = engine
self.automl_config = AutoMLConfig(self.ensembling_indices,
self.data_splitter, self.problem_type,
self.objective, self.additional_objectives, self.optimize_thresholds,
self.error_callback, self.random_seed)
self.allowed_model_families = list(set([p.model_family for p in (self.allowed_pipelines)]))
logger.debug(f"allowed_pipelines set to {[pipeline.name for pipeline in self.allowed_pipelines]}")
logger.debug(f"allowed_model_families set to {self.allowed_model_families}")
if len(self.problem_configuration):
pipeline_params = {**{'pipeline': self.problem_configuration}, **self.pipeline_parameters}
else:
pipeline_params = self.pipeline_parameters
self._automl_algorithm = IterativeAlgorithm(
max_iterations=self.max_iterations,
allowed_pipelines=self.allowed_pipelines,
tuner_class=self.tuner_class,
random_seed=self.random_seed,
n_jobs=self.n_jobs,
number_features=self.X_train.shape[1],
pipelines_per_batch=self._pipelines_per_batch,
ensembling=run_ensembling,
pipeline_params=pipeline_params
)
