def score(self, X, y, objectives): """Evaluate model performance on current and additional objectives. Arguments: X (ww.DataTable, pd.DataFrame or np.ndarray): Data of shape [n_samples, n_features] y (pd.Series, ww.DataColumn): True labels of length [n_samples] objectives (list): Non-empty list of objectives to score on Returns: dict: Ordered dictionary of objective scores """ # Only converting X for the call to _score_all_objectives if X is None: X = pd.DataFrame() X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) y_predicted = self.predict(X, y) y_shifted = y.shift(-self.gap) objectives = [ get_objective(o, return_instance=True) for o in objectives ] y_shifted, y_predicted = drop_rows_with_nans(y_shifted, y_predicted) return self._score_all_objectives(X, y_shifted, y_predicted, y_pred_proba=None, objectives=objectives)
def transform(self, X, y=None): """Transforms data X by creating new features using existing DateTime columns, and then dropping those DateTime columns Arguments: X (ww.DataTable, pd.DataFrame): Data to transform y (ww.DataColumn, pd.Series, optional): Ignored. Returns: ww.DataTable: Transformed X """ X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) X_t = X features_to_extract = self.parameters["features_to_extract"] if len(features_to_extract) == 0: return _convert_to_woodwork_structure(X_t) for col_name in self._date_time_col_names: for feature in features_to_extract: name = f"{col_name}_{feature}" features, categories = self._function_mappings[feature]( X_t[col_name], self.encode_as_categories) X_t[name] = features if categories: self._categories[name] = categories X_t = X_t.drop(self._date_time_col_names, axis=1) return _convert_to_woodwork_structure(X_t)
def transform(self, X, y=None): """Transforms data X by creating new features using existing text columns Arguments: X (ww.DataTable, pd.DataFrame): Data to transform y (ww.DataColumn, pd.Series, optional): Ignored. Returns: ww.DataTable: Transformed X """ X = _convert_to_woodwork_structure(X) if self._features is None or len(self._features) == 0: return X X = _convert_woodwork_types_wrapper(X.to_dataframe()) text_columns = self._get_text_columns(X) es = self._make_entity_set(X, text_columns) X_nlp_primitives = ft.calculate_feature_matrix(features=self._features, entityset=es) if X_nlp_primitives.isnull().any().any(): X_nlp_primitives.fillna(0, inplace=True) X_lsa = self._lsa.transform(X[text_columns]).to_dataframe() X_nlp_primitives.set_index(X.index, inplace=True) X_t = pd.concat( [X.drop(text_columns, axis=1), X_nlp_primitives, X_lsa], axis=1) return _convert_to_woodwork_structure(X_t)
def predict(self, X, y=None, objective=None): """Make predictions using selected features. Arguments: X (ww.DataTable, pd.DataFrame, or np.ndarray): Data of shape [n_samples, n_features] y (ww.DataColumn, pd.Series, np.ndarray, None): The target training targets of length [n_samples] objective (Object or string): The objective to use to make predictions Returns: pd.Series: Predicted values. """ if X is None: X = pd.DataFrame() X = _convert_to_woodwork_structure(X) y = _convert_to_woodwork_structure(y) X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_woodwork_types_wrapper(y.to_series()) features = self.compute_estimator_features(X, y) features_no_nan, y = drop_rows_with_nans(features, y) y_arg = None if self.estimator.predict_uses_y: y_arg = y predictions = self.estimator.predict(features_no_nan, y_arg) predictions = predictions.rename(self.input_target_name) return pad_with_nans(predictions, max(0, features.shape[0] - predictions.shape[0]))
def transform(self, X, y=None): """One-hot encode the input data. Arguments: X (ww.DataTable, pd.DataFrame): Features to one-hot encode. y (ww.DataColumn, pd.Series): Ignored. Returns: ww.DataTable: Transformed data, where each categorical feature has been encoded into numerical columns using one-hot encoding. """ X_copy = _convert_to_woodwork_structure(X) X_copy = _convert_woodwork_types_wrapper(X_copy.to_dataframe()) X_copy = self._handle_parameter_handle_missing(X_copy) X_t = pd.DataFrame() # Add the non-categorical columns, untouched for col in X_copy.columns: if col not in self.features_to_encode: X_t = pd.concat([X_t, X_copy[col]], axis=1) # The call to pd.concat above changes the type of the index so we will manually keep it the same. if not X_t.empty: X_t.index = X_copy.index # Call sklearn's transform on the categorical columns if len(self.features_to_encode) > 0: X_cat = pd.DataFrame(self._encoder.transform(X_copy[self.features_to_encode]).toarray(), index=X_copy.index) X_cat.columns = self.get_feature_names() X_t = pd.concat([X_t, X_cat], axis=1) return _convert_to_woodwork_structure(X_t)
def validate(self, X, y): """Check if the target or any of the features have no variance (1 unique value). Arguments: X (ww.DataTable, pd.DataFrame, np.ndarray): The input features. y (ww.DataColumn, pd.Series, np.ndarray): The target data. Returns: dict: dict of warnings/errors corresponding to features or target with no variance. """ messages = {"warnings": [], "errors": []} X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) unique_counts = X.nunique(dropna=self._dropnan).to_dict() any_nulls = (X.isnull().any()).to_dict() for name in unique_counts: message = self._check_for_errors(name, unique_counts[name], any_nulls[name]) if not message: continue DataCheck._add_message(message, messages) y_name = getattr(y, "name") if not y_name: y_name = "Y" target_message = self._check_for_errors( y_name, y.nunique(dropna=self._dropnan), y.isnull().any()) if target_message: DataCheck._add_message(target_message, messages) return messages
def transform(self, X, y=None): """Transforms input data by selecting features. If the component_obj does not have a transform method, will raise an MethodPropertyNotFoundError exception. Arguments: X (ww.DataTable, pd.DataFrame): Data to transform. y (ww.DataColumn, pd.Series, optional): Target data. Ignored. Returns: ww.DataTable: Transformed X """ X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) self.input_feature_names = list(X.columns.values) try: X_t = self._component_obj.transform(X) except AttributeError: raise MethodPropertyNotFoundError( "Feature selector requires a transform method or a component_obj that implements transform" ) X_dtypes = X.dtypes.to_dict() selected_col_names = self.get_names() col_types = {key: X_dtypes[key] for key in selected_col_names} features = pd.DataFrame(X_t, columns=selected_col_names, index=X.index).astype(col_types) return _convert_to_woodwork_structure(features)
def transform(self, X, y=None): """Transforms input by imputing missing values. 'None' and np.nan values are treated as the same. Arguments: X (ww.DataTable, pd.DataFrame): Data to transform y (ww.DataColumn, pd.Series, optional): Ignored. Returns: ww.DataTable: Transformed X """ X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) # Return early since bool dtype doesn't support nans and sklearn errors if all cols are bool if (X.dtypes == bool).all(): return _convert_to_woodwork_structure(X) X_null_dropped = X.copy() X_null_dropped.drop(self._all_null_cols, axis=1, errors='ignore', inplace=True) X_t = self._component_obj.transform(X) if X_null_dropped.empty: X_t = pd.DataFrame(X_t, columns=X_null_dropped.columns) return _convert_to_woodwork_structure(X_t) X_t = pd.DataFrame(X_t, columns=X_null_dropped.columns) X_t = X_t.infer_objects() X_t.index = X_null_dropped.index return _convert_to_woodwork_structure(X_t)
def fit_transform(self, X, y=None): """Fits on X and transforms X Arguments: X (pd.DataFrame): Data to fit and transform y (pd. DataFrame): Target data Returns: pd.DataFrame: Transformed X """ try: X2 = _convert_to_woodwork_structure(X) y2 = _convert_to_woodwork_structure(y) X2 = _convert_woodwork_types_wrapper(X2.to_dataframe()) y2 = _convert_woodwork_types_wrapper(y2.to_series()) X_t = self._component_obj.fit_transform(X2, y2) except AttributeError: try: self.fit(X, y) X_t = self.transform(X, y) except MethodPropertyNotFoundError as e: raise e if not isinstance(X_t, pd.DataFrame) and isinstance(X, pd.DataFrame): return pd.DataFrame(X_t, columns=X.columns, index=X.index) return pd.DataFrame(X_t)
def _manage_woodwork(self, X, y=None): """Function to convert the input and target data to Pandas data structures.""" X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) if y is not None: y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) return X, y
def fit(self, X, y=None): X = _convert_to_woodwork_structure(X) cat_cols = list(X.select('category').columns) X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) self._component_obj.fit(X, y, silent=True, cat_features=cat_cols) return self
def predict(self, X): X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) predictions = self._component_obj.predict(X) if predictions.ndim == 2 and predictions.shape[1] == 1: predictions = predictions.flatten() if self._label_encoder: predictions = self._label_encoder.inverse_transform( predictions.astype(np.int64)) return _convert_to_woodwork_structure(predictions)
def predict(self, X, y=None): if y is None: raise ValueError( "Cannot predict Time Series Baseline Estimator if y is None") y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) if self.gap == 0: y = y.shift(periods=1) return _convert_to_woodwork_structure(y)
def fit_transform(self, X, y=None): X = _convert_to_woodwork_structure(X) if not is_all_numeric(X): raise ValueError("PCA input must be all numeric") X = _convert_woodwork_types_wrapper(X.to_dataframe()) X_t = self._component_obj.fit_transform(X, y) X_t = pd.DataFrame( X_t, index=X.index, columns=[f"component_{i}" for i in range(X_t.shape[1])]) return _convert_to_woodwork_structure(X_t)
def predict_proba(self, X, y=None): if y is None: raise ValueError( "Cannot predict Time Series Baseline Estimator if y is None") y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) preds = self.predict(X, y).to_series().dropna(axis=0, how='any').astype('int') proba_arr = np.zeros((len(preds), y.max() + 1)) proba_arr[np.arange(len(preds)), preds] = 1 padded = pad_with_nans(pd.DataFrame(proba_arr), len(y) - len(preds)) return _convert_to_woodwork_structure(padded)
def transform(self, X, y=None): """Computes the delayed features for all features in X and y. For each feature in X, it will add a column to the output dataframe for each delay in the (inclusive) range [1, max_delay]. The values of each delayed feature are simply the original feature shifted forward in time by the delay amount. For example, a delay of 3 units means that the feature value at row n will be taken from the n-3rd row of that feature If y is not None, it will also compute the delayed values for the target variable. Arguments: X (ww.DataTable, pd.DataFrame or None): Data to transform. None is expected when only the target variable is being used. y (ww.DataColumn, pd.Series, or None): Target. Returns: ww.DataTable: Transformed X. """ if X is None: X = pd.DataFrame() # Normalize the data into pandas objects X = _convert_to_woodwork_structure(X) categorical_columns = self._get_categorical_columns(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) if self.delay_features and len(X) > 0: X_categorical = self._encode_X_while_preserving_index( X[categorical_columns]) for col_name in X: col = X[col_name] if col_name in categorical_columns: col = X_categorical[col_name] X = X.assign( **{ f"{col_name}_delay_{t}": col.shift(t) for t in range(1, self.max_delay + 1) }) # Handle cases where the target was passed in if self.delay_target and y is not None: y = _convert_to_woodwork_structure(y) if y.logical_type == logical_types.Categorical: y = self._encode_y_while_preserving_index(y) else: y = _convert_woodwork_types_wrapper(y.to_series()) X = X.assign( **{ f"target_delay_{t}": y.shift(t) for t in range(self.start_delay_for_target, self.max_delay + 1) }) return _convert_to_woodwork_structure(X)
def predict(self, X): X = _convert_to_woodwork_structure(X) strategy = self.parameters["strategy"] if strategy == "mode": predictions = pd.Series([self._mode] * len(X)) elif strategy == "random": predictions = get_random_state(self.random_state).choice( self._classes, len(X)) else: predictions = get_random_state(self.random_state).choice( self._classes, len(X), p=self._percentage_freq) return _convert_to_woodwork_structure(predictions)
def fit(self, X, y=None): X = _convert_to_woodwork_structure(X) cat_cols = list(X.select('category').columns) X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) # For binary classification, catboost expects numeric values, so encoding before. if y.nunique() <= 2: self._label_encoder = LabelEncoder() y = pd.Series(self._label_encoder.fit_transform(y)) self._component_obj.fit(X, y, silent=True, cat_features=cat_cols) return self
def fit(self, X, y=None): if y is None: raise ValueError("Cannot fit Baseline regressor if y is None") X = _convert_to_woodwork_structure(X) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) if self.parameters["strategy"] == "mean": self._prediction_value = y.mean() elif self.parameters["strategy"] == "median": self._prediction_value = y.median() self._num_features = X.shape[1] return self
def validate(self, X, y): """Check if any of the features are highly correlated with the target by using mutual information or Pearson correlation. If `method='mutual'`, supports all target and feature types. Otherwise, if `method='pearson'` only supports binary with numeric and boolean dtypes. Pearson correlation returns a value in [-1, 1], while mutual information returns a value in [0, 1]. Arguments: X (ww.DataTable, pd.DataFrame, np.ndarray): The input features to check y (ww.DataColumn, pd.Series, np.ndarray): The target data Returns: dict (DataCheckWarning): dict with a DataCheckWarning if target leakage is detected. Example: >>> import pandas as pd >>> X = pd.DataFrame({ ... 'leak': [10, 42, 31, 51, 61], ... 'x': [42, 54, 12, 64, 12], ... 'y': [13, 5, 13, 74, 24], ... }) >>> y = pd.Series([10, 42, 31, 51, 40]) >>> target_leakage_check = TargetLeakageDataCheck(pct_corr_threshold=0.95) >>> assert target_leakage_check.validate(X, y) == {"warnings": [{"message": "Column 'leak' is 95.0% or more correlated with the target",\ "data_check_name": "TargetLeakageDataCheck",\ "level": "warning",\ "code": "TARGET_LEAKAGE",\ "details": {"column": "leak"}}],\ "errors": []} """ messages = { "warnings": [], "errors": [] } X = _convert_to_woodwork_structure(X) y = _convert_to_woodwork_structure(y) if self.method == 'pearson': highly_corr_cols = self._calculate_pearson(X, y) else: X = _convert_woodwork_types_wrapper(X.to_dataframe()) y = _convert_woodwork_types_wrapper(y.to_series()) highly_corr_cols = self._calculate_mutual_information(X, y) warning_msg = "Column '{}' is {}% or more correlated with the target" messages["warnings"].extend([DataCheckWarning(message=warning_msg.format(col_name, self.pct_corr_threshold * 100), data_check_name=self.name, message_code=DataCheckMessageCode.TARGET_LEAKAGE, details={"column": col_name}).to_dict() for col_name in highly_corr_cols]) return messages
def make_pipeline(X, y, estimator, problem_type, custom_hyperparameters=None, text_columns=None): """Given input data, target data, an estimator class and the problem type, generates a pipeline class with a preprocessing chain which was recommended based on the inputs. The pipeline will be a subclass of the appropriate pipeline base class for the specified problem_type. Arguments: X (pd.DataFrame, ww.DataTable): The input data of shape [n_samples, n_features] y (pd.Series, ww.DataColumn): The target data of length [n_samples] estimator (Estimator): Estimator for pipeline problem_type (ProblemTypes or str): Problem type for pipeline to generate custom_hyperparameters (dictionary): Dictionary of custom hyperparameters, with component name as key and dictionary of parameters as the value text_columns (list): feature names which should be treated as text features. Defaults to None. Returns: class: PipelineBase subclass with dynamically generated preprocessing components and specified estimator """ X = _convert_to_woodwork_structure(X) y = _convert_to_woodwork_structure(y) problem_type = handle_problem_types(problem_type) if estimator not in get_estimators(problem_type): raise ValueError( f"{estimator.name} is not a valid estimator for problem type") preprocessing_components = _get_preprocessing_components( X, y, problem_type, text_columns, estimator) complete_component_graph = preprocessing_components + [estimator] if custom_hyperparameters and not isinstance(custom_hyperparameters, dict): raise ValueError( f"if custom_hyperparameters provided, must be dictionary. Received {type(custom_hyperparameters)}" ) hyperparameters = custom_hyperparameters base_class = _get_pipeline_base_class(problem_type) class GeneratedPipeline(base_class): custom_name = f"{estimator.name} w/ {' + '.join([component.name for component in preprocessing_components])}" component_graph = complete_component_graph custom_hyperparameters = hyperparameters return GeneratedPipeline
def fit(self, X, y=None): """Fits imputer to data. 'None' values are converted to np.nan before imputation and are treated as the same. Arguments: X (pd.DataFrame or np.ndarray): The input training data of shape [n_samples, n_features] y (pd.Series, optional): The target training data of length [n_samples] Returns: self """ X = _convert_to_woodwork_structure(X) cat_cols = list(X.select('category').columns) numeric_cols = list(X.select('numeric').columns) X = _convert_woodwork_types_wrapper(X.to_dataframe()) self._all_null_cols = set(X.columns) - set(X.dropna(axis=1, how='all').columns) X_copy = X.copy() X_null_dropped = X_copy.drop(self._all_null_cols, axis=1, errors='ignore') X_numerics = X_null_dropped[[col for col in numeric_cols if col not in self._all_null_cols]] if len(X_numerics.columns) > 0: self._numeric_imputer.fit(X_numerics, y) self._numeric_cols = X_numerics.columns X_categorical = X_null_dropped[[col for col in cat_cols if col not in self._all_null_cols]] if len(X_categorical.columns) > 0: self._categorical_imputer.fit(X_categorical, y) self._categorical_cols = X_categorical.columns return self
def validate(self, X, y=None): """Check if any set of features are likely to be multicollinear. Arguments: X (ww.DataTable, pd.DataFrame, np.ndarray): The input features to check Returns: dict: dict with a DataCheckWarning if there are any potentially multicollinear columns. """ messages = { "warnings": [], "errors": [] } X = _convert_to_woodwork_structure(X) mutual_info_df = X.mutual_information() if mutual_info_df.empty: return messages above_threshold = mutual_info_df.loc[mutual_info_df['mutual_info'] >= self.threshold] correlated_cols = [(col_1, col_2) for col_1, col_2 in zip(above_threshold['column_1'], above_threshold['column_2'])] if correlated_cols: warning_msg = "Columns are likely to be correlated: {}" messages["warnings"].append(DataCheckWarning(message=warning_msg.format(correlated_cols), data_check_name=self.name, message_code=DataCheckMessageCode.IS_MULTICOLLINEAR, details={"columns": correlated_cols}).to_dict()) return messages
def fit(self, X, y=None): X = _convert_to_woodwork_structure(X) cat_cols = list(X.select('category').columns) self.input_feature_names = list(X.columns) X, y = super()._manage_woodwork(X, y) self._component_obj.fit(X, y, silent=True, cat_features=cat_cols) return self
def transform(self, X, y=None): """Transforms data X by imputing missing values. 'None' values are converted to np.nan before imputation and are treated as the same. Arguments: X (pd.DataFrame): Data to transform y (pd.Series, optional): Ignored. Returns: pd.DataFrame: Transformed X """ X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) X_null_dropped = X.copy() X_null_dropped.drop(self._all_null_cols, inplace=True, axis=1, errors='ignore') if X_null_dropped.empty: return X_null_dropped if self._numeric_cols is not None and len(self._numeric_cols) > 0: X_numeric = X_null_dropped[self._numeric_cols] imputed = self._numeric_imputer.transform(X_numeric) imputed.index = X_null_dropped.index X_null_dropped[X_numeric.columns] = imputed if self._categorical_cols is not None and len(self._categorical_cols) > 0: X_categorical = X_null_dropped[self._categorical_cols] imputed = self._categorical_imputer.transform(X_categorical) imputed.index = X_null_dropped.index X_null_dropped[X_categorical.columns] = imputed return X_null_dropped
def fit(self, X, y=None): X = _convert_to_woodwork_structure(X) if not is_all_numeric(X): raise ValueError("PCA input must be all numeric") X = _convert_woodwork_types_wrapper(X.to_dataframe()) self._component_obj.fit(X) return self
def predict(self, X): X_encoded = self._encode_categories(X) predictions = super().predict(X_encoded) if not self._label_encoder: return predictions predictions = pd.Series(self._label_encoder.inverse_transform(predictions.to_series().astype(np.int64))) return _convert_to_woodwork_structure(predictions)
def fit(self, X, y=None): if y is None: raise ValueError("Cannot fit Baseline classifier if y is None") X = _convert_to_woodwork_structure(X) y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) vals, counts = np.unique(y, return_counts=True) self._classes = list(vals) self._percentage_freq = counts.astype(float) / len(y) self._num_unique = len(self._classes) self._num_features = X.shape[1] if self.parameters["strategy"] == "mode": self._mode = y.mode()[0] return self
def fit(self, X, y=None): X_encoded = self._encode_categories(X, fit=True) if y is not None: y = _convert_to_woodwork_structure(y) y = _convert_woodwork_types_wrapper(y.to_series()) self._component_obj.fit(X_encoded, y) return self
def transform(self, X, y=None): """Transforms data X by imputing missing values. 'None' values are converted to np.nan before imputation and are treated as the same. Arguments: X (ww.DataTable, pd.DataFrame): Data to transform y (ww.DataColumn, pd.Series, optional): Ignored. Returns: pd.DataFrame: Transformed X """ X = _convert_to_woodwork_structure(X) X = _convert_woodwork_types_wrapper(X.to_dataframe()) # Convert None to np.nan, since None cannot be properly handled X = X.fillna(value=np.nan) # Return early since bool dtype doesn't support nans and sklearn errors if all cols are bool if (X.dtypes == bool).all(): return X X_null_dropped = X.copy() X_null_dropped.drop(self._all_null_cols, axis=1, errors='ignore', inplace=True) category_cols = X_null_dropped.select_dtypes(include=['category']).columns X_t = self._component_obj.transform(X) if X_null_dropped.empty: return pd.DataFrame(X_t, columns=X_null_dropped.columns) X_t = pd.DataFrame(X_t, columns=X_null_dropped.columns) if len(category_cols) > 0: X_t[category_cols] = X_t[category_cols].astype('category') return X_t