def test_partial_dependence_baseline(): X = pd.DataFrame([[1, 0], [0, 1]]) y = pd.Series([0, 1]) pipeline = BinaryClassificationPipeline(component_graph=["Baseline Classifier"], parameters={}) pipeline.fit(X, y) with pytest.raises(ValueError, match="Partial dependence plots are not supported for Baseline pipelines"): partial_dependence(pipeline, X, features=0, grid_resolution=20)
def test_submit_scoring_jobs_multiple(self): """ Test that scoring multiple pipelines using the parallel engine produces the same results as the sequential engine. """ X, y = self.X_y_binary pipelines = [BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}), BinaryClassificationPipeline(component_graph=["Baseline Classifier"]), BinaryClassificationPipeline(component_graph=["SVM Classifier"])] def score_pipelines(pipelines, engine): futures = [] for pipeline in pipelines: futures.append(engine.submit_training_job(X=ww.DataTable(X), y=ww.DataColumn(y), automl_config=automl_data, pipeline=pipeline)) pipelines = [f.get_result() for f in futures] futures = [] for pipeline in pipelines: futures.append(engine.submit_scoring_job(X=ww.DataTable(X), y=ww.DataColumn(y), automl_config=automl_data, pipeline=pipeline, objectives=[automl_data.objective])) results = [f.get_result() for f in futures] return results par_eval_results = score_pipelines(pipelines, DaskEngine(client=self.client)) par_scores = [s["Log Loss Binary"] for s in par_eval_results] seq_eval_results = score_pipelines(pipelines, SequentialEngine()) seq_scores = [s["Log Loss Binary"] for s in seq_eval_results] # Check there are the proper number of pipelines and all their scores are same. assert len(par_eval_results) == len(pipelines) assert set(par_scores) == set(seq_scores)
def test_submit_training_jobs_multiple(self): """ Test that training multiple pipelines using the parallel engine produces the same results as the sequential engine. """ X, y = self.X_y_binary pipelines = [BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}), BinaryClassificationPipeline(component_graph=["Baseline Classifier"]), BinaryClassificationPipeline(component_graph=["SVM Classifier"])] def fit_pipelines(pipelines, engine): futures = [] for pipeline in pipelines: futures.append(engine.submit_training_job(X=X, y=y, automl_config=automl_data, pipeline=pipeline)) results = [f.get_result() for f in futures] return results # Verify all pipelines are trained and fitted. seq_pipelines = fit_pipelines(pipelines, SequentialEngine()) for pipeline in seq_pipelines: assert pipeline._is_fitted # Verify all pipelines are trained and fitted. par_pipelines = fit_pipelines(pipelines, DaskEngine(client=self.client)) for pipeline in par_pipelines: assert pipeline._is_fitted # Ensure sequential and parallel pipelines are equivalent assert len(par_pipelines) == len(seq_pipelines) for par_pipeline in par_pipelines: assert par_pipeline in seq_pipelines
def _method(hyperparameters=['default', 'other']): class MockEstimator(Estimator): name = "Mock Classifier" model_family = ModelFamily.RANDOM_FOREST supported_problem_types = [ ProblemTypes.BINARY, ProblemTypes.MULTICLASS ] if isinstance(hyperparameters, (list, tuple, Real, Categorical, Integer)): hyperparameter_ranges = {'dummy_parameter': hyperparameters} else: hyperparameter_ranges = {'dummy_parameter': [hyperparameters]} def __init__(self, dummy_parameter='default', n_jobs=-1, random_seed=0, **kwargs): super().__init__(parameters={ 'dummy_parameter': dummy_parameter, **kwargs, 'n_jobs': n_jobs }, component_obj=None, random_seed=random_seed) return [ BinaryClassificationPipeline([MockEstimator]), BinaryClassificationPipeline([MockEstimator]), BinaryClassificationPipeline([MockEstimator]) ]
def test_partial_dependence_respect_grid_resolution(): X, y = load_fraud(1000) pl = BinaryClassificationPipeline(component_graph=["DateTime Featurization Component", "One Hot Encoder", "Random Forest Classifier"]) pl.fit(X, y) dep = partial_dependence(pl, X, features="amount", grid_resolution=20) assert dep.shape[0] == 20 assert dep.shape[0] != max(X.select('categorical').describe().loc["nunique"]) + 1 dep = partial_dependence(pl, X, features="provider", grid_resolution=20) assert dep.shape[0] == X['provider'].to_series().nunique() assert dep.shape[0] != max(X.select('categorical').describe().loc["nunique"]) + 1
def test_graph_partial_dependence_regression_date_order(X_y_binary): pytest.importorskip('plotly.graph_objects', reason='Skipping plotting test because plotly not installed') X, y = X_y_binary pipeline = BinaryClassificationPipeline(component_graph=['Imputer', 'One Hot Encoder', 'DateTime Featurization Component', 'Standard Scaler', 'Logistic Regression Classifier']) X = pd.DataFrame(X, columns=[str(i) for i in range(X.shape[1])]) y = pd.Series(y) dt_series = pd.Series(pd.date_range('20200101', periods=X.shape[0])).sample(frac=1).reset_index(drop=True) X['dt_column'] = pd.to_datetime(dt_series, errors='coerce') pipeline.fit(X, y) fig = graph_partial_dependence(pipeline, X, features='dt_column', grid_resolution=5) plot_data = fig.to_dict()['data'][0] assert plot_data['type'] == 'scatter' assert plot_data['x'].tolist() == list(pd.date_range('20200101', periods=X.shape[0]))
def test_generate_code_pipeline_with_custom_components(): class CustomTransformer(Transformer): name = "My Custom Transformer" hyperparameter_ranges = {} def __init__(self, random_seed=0): parameters = {} super().__init__(parameters=parameters, component_obj=None, random_seed=random_seed) class CustomEstimator(Estimator): name = "My Custom Estimator" hyperparameter_ranges = {} supported_problem_types = [ ProblemTypes.BINARY, ProblemTypes.MULTICLASS ] model_family = ModelFamily.NONE def __init__(self, random_arg=False, random_seed=0): parameters = {'random_arg': random_arg} super().__init__(parameters=parameters, component_obj=None, random_seed=random_seed) mock_pipeline_with_custom_components = BinaryClassificationPipeline( [CustomTransformer, CustomEstimator]) expected_code = "from evalml.pipelines.binary_classification_pipeline import BinaryClassificationPipeline\n" \ "pipeline = BinaryClassificationPipeline(component_graph=[CustomTransformer, CustomEstimator], " \ "parameters={'My Custom Estimator':{'random_arg': False}}, random_seed=0)" pipeline = generate_pipeline_code(mock_pipeline_with_custom_components) assert pipeline == expected_code
def test_partial_dependence_xgboost_feature_names(problem_type, has_minimal_dependencies, X_y_binary, X_y_multi, X_y_regression): if has_minimal_dependencies: pytest.skip("Skipping because XGBoost not installed for minimal dependencies") if problem_type == ProblemTypes.REGRESSION: pipeline = RegressionPipeline(component_graph=['Simple Imputer', 'XGBoost Regressor'], parameters={'XGBoost Classifier': {'nthread': 1}}) X, y = X_y_regression elif problem_type == ProblemTypes.BINARY: pipeline = BinaryClassificationPipeline(component_graph=['Simple Imputer', 'XGBoost Classifier'], parameters={'XGBoost Classifier': {'nthread': 1}}) X, y = X_y_binary elif problem_type == ProblemTypes.MULTICLASS: pipeline = MulticlassClassificationPipeline(component_graph=['Simple Imputer', 'XGBoost Classifier'], parameters={'XGBoost Classifier': {'nthread': 1}}) X, y = X_y_multi X = pd.DataFrame(X) X = X.rename(columns={0: '<[0]'}) pipeline.fit(X, y) part_dep = partial_dependence(pipeline, X, features="<[0]", grid_resolution=20) check_partial_dependence_dataframe(pipeline, part_dep) assert not part_dep.isnull().all().all() part_dep = partial_dependence(pipeline, X, features=1, grid_resolution=20) check_partial_dependence_dataframe(pipeline, part_dep) assert not part_dep.isnull().all().all()
def test_stacked_different_input_pipelines_regression(): input_pipelines = [ RegressionPipeline([RandomForestRegressor]), BinaryClassificationPipeline([RandomForestClassifier]) ] with pytest.raises(ValueError, match="All pipelines must have the same problem type."): StackedEnsembleRegressor(input_pipelines=input_pipelines)
def test_evaluate_pipeline_handles_ensembling_indices( mock_fit, mock_score, dummy_binary_pipeline_class, stackable_classifiers): X = ww.DataTable(pd.DataFrame({"a": [i for i in range(100)]})) y = ww.DataColumn(pd.Series([i % 2 for i in range(100)])) automl = AutoMLSearch(X_train=X, y_train=y, problem_type='binary', max_batches=19, ensembling=True, _ensembling_split_size=0.25) training_indices, ensembling_indices, _, _ = split_data( ww.DataTable(np.arange(X.shape[0])), y, problem_type='binary', test_size=0.25, random_seed=0) training_indices, ensembling_indices = training_indices.to_dataframe( )[0].tolist(), ensembling_indices.to_dataframe()[0].tolist() pipeline1 = dummy_binary_pipeline_class({'Mock Classifier': {'a': 1}}) _ = evaluate_pipeline(pipeline1, automl, X, y, logger=MagicMock()) # check the fit length is correct, taking into account the data splits assert len(mock_fit.call_args[0][0]) == int(2 / 3 * len(training_indices)) input_pipelines = [ BinaryClassificationPipeline([classifier]) for classifier in stackable_classifiers ] ensemble = BinaryClassificationPipeline( [StackedEnsembleClassifier], parameters={ "Stacked Ensemble Classifier": { "input_pipelines": input_pipelines, "n_jobs": 1 } }) _ = evaluate_pipeline(ensemble, automl, X, y, logger=MagicMock()) assert len(mock_fit.call_args[0][0]) == int(2 / 3 * len(ensembling_indices))
def test_generate_code_pipeline_json_with_objects(): class CustomEstimator(Estimator): name = "My Custom Estimator" hyperparameter_ranges = {} supported_problem_types = [ ProblemTypes.BINARY, ProblemTypes.MULTICLASS ] model_family = ModelFamily.NONE def __init__(self, random_arg=False, numpy_arg=[], random_seed=0): parameters = {'random_arg': random_arg, 'numpy_arg': numpy_arg} super().__init__(parameters=parameters, component_obj=None, random_seed=random_seed) component_graph = ['Imputer', CustomEstimator] pipeline = BinaryClassificationPipeline( component_graph, custom_name="Mock Binary Pipeline with Transformer", parameters={'My Custom Estimator': { 'numpy_arg': np.array([0]) }}) generated_pipeline_code = generate_pipeline_code(pipeline) assert generated_pipeline_code == "from evalml.pipelines.binary_classification_pipeline import BinaryClassificationPipeline\n" \ "pipeline = BinaryClassificationPipeline(component_graph=['Imputer', CustomEstimator], " \ "parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None}, " \ "'My Custom Estimator':{'random_arg': False, 'numpy_arg': array([0])}}, custom_name='Mock Binary Pipeline with Transformer', random_seed=0)" pipeline = BinaryClassificationPipeline( component_graph, custom_name="Mock Binary Pipeline with Transformer", parameters={'My Custom Estimator': { 'random_arg': Imputer() }}) generated_pipeline_code = generate_pipeline_code(pipeline) assert generated_pipeline_code == "from evalml.pipelines.binary_classification_pipeline import BinaryClassificationPipeline\n" \ "pipeline = BinaryClassificationPipeline(component_graph=['Imputer', CustomEstimator], " \ "parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None}, " \ "'My Custom Estimator':{'random_arg': Imputer(categorical_impute_strategy='most_frequent', numeric_impute_strategy='mean', categorical_fill_value=None, numeric_fill_value=None), 'numpy_arg': []}}, " \ "custom_name='Mock Binary Pipeline with Transformer', random_seed=0)"
def test_submit_scoring_job_single(self): """ Test that scoring a single pipeline using the parallel engine produces the same results as simply running the score_pipeline function. """ X, y = self.X_y_binary pipeline = BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}) engine = DaskEngine(client=self.client) objectives = [automl_data.objective] pipeline_future = engine.submit_training_job(X=ww.DataTable(X), y=ww.DataColumn(y), automl_config=automl_data, pipeline=pipeline) pipeline = pipeline_future.get_result() pipeline_score_future = engine.submit_scoring_job(X=ww.DataTable(X), y=ww.DataColumn(y), automl_config=automl_data, pipeline=pipeline, objectives=objectives) assert isinstance(pipeline_score_future, DaskComputation) pipeline_score = pipeline_score_future.get_result() original_pipeline_score = pipeline.score(X=X, y=y, objectives=objectives) assert not np.isnan(pipeline_score["Log Loss Binary"]) assert pipeline_score == original_pipeline_score
def test_submit_evaluate_jobs_multiple(self): """ Test that evaluating multiple pipelines using the parallel engine produces the same results as the sequential engine. """ X, y = self.X_y_binary pipelines = [BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}), BinaryClassificationPipeline(component_graph=["Baseline Classifier"]), BinaryClassificationPipeline(component_graph=["SVM Classifier"])] def eval_pipelines(pipelines, engine): futures = [] for pipeline in pipelines: futures.append(engine.submit_evaluation_job(X=ww.DataTable(X), y=ww.DataColumn(y), automl_config=automl_data, pipeline=pipeline)) results = [f.get_result() for f in futures] return results par_eval_results = eval_pipelines(pipelines, DaskEngine(client=self.client)) par_dicts = [s.get("scores") for s in par_eval_results] par_scores = [s["cv_data"][0]["mean_cv_score"] for s in par_dicts] par_pipelines = [s.get("pipeline") for s in par_eval_results] seq_eval_results = eval_pipelines(pipelines, SequentialEngine()) seq_dicts = [s.get("scores") for s in seq_eval_results] seq_scores = [s["cv_data"][0]["mean_cv_score"] for s in seq_dicts] seq_pipelines = [s.get("pipeline") for s in seq_eval_results] # Ensure all pipelines are fitted. assert all([s._is_fitted for s in par_pipelines]) # Ensure the scores in parallel and sequence are same assert set(par_scores) == set(seq_scores) assert not any([np.isnan(s) for s in par_scores]) # Ensure the parallel and sequence pipelines match assert len(par_pipelines) == len(seq_pipelines) for par_pipeline in par_pipelines: assert par_pipeline in seq_pipelines
def test_compute_shap_values_catches_shap_tree_warnings(mock_tree_explainer, mock_debug, X_y_binary, caplog): X, y = X_y_binary pipeline = BinaryClassificationPipeline(["Random Forest Classifier"]) def raise_warning_from_shap(estimator, feature_perturbation): warnings.warn("Shap raised a warning!") mock = MagicMock() mock.shap_values.return_value = np.zeros(10) return mock mock_tree_explainer.side_effect = raise_warning_from_shap _ = _compute_shap_values(pipeline, pd.DataFrame(X)) mock_debug.debug.assert_called_with("_compute_shap_values TreeExplainer: Shap raised a warning!")
def test_submit_training_job_single(self): """ Test that training a single pipeline using the parallel engine produces the same results as simply running the train_pipeline function. """ X, y = self.X_y_binary engine = DaskEngine(client=self.client) pipeline = BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}) # Verify that engine fits a pipeline pipeline_future = engine.submit_training_job(X=X, y=y, automl_config=automl_data, pipeline=pipeline) dask_pipeline_fitted = pipeline_future.get_result() assert dask_pipeline_fitted._is_fitted # Verify parallelization has no effect on output of function original_pipeline_fitted = train_pipeline(pipeline, X, y, optimize_thresholds=automl_data.optimize_thresholds, objective=automl_data.objective) assert dask_pipeline_fitted == original_pipeline_fitted assert dask_pipeline_fitted.predict(X) == original_pipeline_fitted.predict(X)
def test_generate_code_pipeline(): custom_hyperparameters = { "Imputer": { "numeric_impute_strategy": 'most_frequent' } } binary_pipeline = BinaryClassificationPipeline( ['Imputer', 'Random Forest Classifier'], custom_hyperparameters=custom_hyperparameters) expected_code = "from evalml.pipelines.binary_classification_pipeline import BinaryClassificationPipeline\n" \ "pipeline = BinaryClassificationPipeline(component_graph=['Imputer', 'Random Forest Classifier'], " \ "parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None}, " \ "'Random Forest Classifier':{'n_estimators': 100, 'max_depth': 6, 'n_jobs': -1}}, custom_hyperparameters={'Imputer':{'numeric_impute_strategy': 'most_frequent'}}, random_seed=0)" pipeline = generate_pipeline_code(binary_pipeline) assert expected_code == pipeline regression_pipeline = RegressionPipeline( ['Imputer', 'Random Forest Regressor'], custom_name="Mock Regression Pipeline") expected_code = "from evalml.pipelines.regression_pipeline import RegressionPipeline\n" \ "pipeline = RegressionPipeline(component_graph=['Imputer', 'Random Forest Regressor'], parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None}, " \ "'Random Forest Regressor':{'n_estimators': 100, 'max_depth': 6, 'n_jobs': -1}}, custom_name='Mock Regression Pipeline', random_seed=0)" pipeline = generate_pipeline_code(regression_pipeline) assert pipeline == expected_code regression_pipeline_with_params = RegressionPipeline( ['Imputer', 'Random Forest Regressor'], custom_name="Mock Regression Pipeline", parameters={ "Imputer": { "numeric_impute_strategy": "most_frequent" }, "Random Forest Regressor": { "n_estimators": 50 } }) expected_code_params = "from evalml.pipelines.regression_pipeline import RegressionPipeline\n" \ "pipeline = RegressionPipeline(component_graph=['Imputer', 'Random Forest Regressor'], " \ "parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'most_frequent', 'categorical_fill_value': None, 'numeric_fill_value': None}, " \ "'Random Forest Regressor':{'n_estimators': 50, 'max_depth': 6, 'n_jobs': -1}}, custom_name='Mock Regression Pipeline', random_seed=0)" pipeline = generate_pipeline_code(regression_pipeline_with_params) assert pipeline == expected_code_params
def test_submit_evaluate_job_single(self): """ Test that evaluating a single pipeline using the parallel engine produces the same results as simply running the evaluate_pipeline function. """ X, y = self.X_y_binary X = ww.DataTable(X) y = ww.DataColumn(y) pipeline = BinaryClassificationPipeline(component_graph=["Logistic Regression Classifier"], parameters={"Logistic Regression Classifier": {"n_jobs": 1}}) engine = DaskEngine(client=self.client) # Verify that engine evaluates a pipeline pipeline_future = engine.submit_evaluation_job(X=X, y=y, automl_config=automl_data, pipeline=pipeline) assert isinstance(pipeline_future, DaskComputation) par_eval_results = pipeline_future.get_result() original_eval_results = evaluate_pipeline(pipeline, automl_config=automl_data, X=X, y=y, logger=JobLogger()) # Ensure we get back the same output as the parallelized function. assert len(par_eval_results) == 3 par_scores = par_eval_results.get("scores") original_eval_scores = original_eval_results.get("scores") # Compare cross validation information except training time. assert par_scores["cv_data"] == original_eval_scores["cv_data"] assert all(par_scores["cv_scores"] == original_eval_scores["cv_scores"]) assert par_scores["cv_score_mean"] == par_scores["cv_score_mean"] # Make sure the resulting pipelines are the same. assert isinstance(par_eval_results.get("pipeline"), PipelineBase) assert par_eval_results.get("pipeline") == original_eval_results.get("pipeline") # Make sure a properly filled logger comes back. assert isinstance(par_eval_results.get("logger"), JobLogger) assert par_eval_results.get("logger").logs == original_eval_results.get("logger").logs
def test_partial_dependence_datetime(problem_type, X_y_regression, X_y_binary, X_y_multi): if problem_type == 'binary': X, y = X_y_binary pipeline = BinaryClassificationPipeline(component_graph=['Imputer', 'One Hot Encoder', 'DateTime Featurization Component', 'Standard Scaler', 'Logistic Regression Classifier']) elif problem_type == 'multiclass': X, y = X_y_multi pipeline = MulticlassClassificationPipeline(component_graph=['Imputer', 'One Hot Encoder', 'DateTime Featurization Component', 'Standard Scaler', 'Logistic Regression Classifier']) else: X, y = X_y_regression pipeline = RegressionPipeline(component_graph=['Imputer', 'One Hot Encoder', 'DateTime Featurization Component', 'Standard Scaler', 'Linear Regressor']) X = pd.DataFrame(X, columns=[str(i) for i in range(X.shape[1])]) y = pd.Series(y) X['dt_column'] = pd.Series(pd.date_range('20200101', periods=X.shape[0])) pipeline.fit(X, y) part_dep = partial_dependence(pipeline, X, features='dt_column') if problem_type == 'multiclass': assert len(part_dep["partial_dependence"]) == 300 # 100 rows * 3 classes assert len(part_dep["feature_values"]) == 300 else: assert len(part_dep["partial_dependence"]) == 100 assert len(part_dep["feature_values"]) == 100 assert not part_dep.isnull().any(axis=None) part_dep = partial_dependence(pipeline, X, features=20) if problem_type == 'multiclass': assert len(part_dep["partial_dependence"]) == 300 # 100 rows * 3 classes assert len(part_dep["feature_values"]) == 300 else: assert len(part_dep["partial_dependence"]) == 100 assert len(part_dep["feature_values"]) == 100 assert not part_dep.isnull().any(axis=None) with pytest.raises(ValueError, match='Two-way partial dependence is not supported for datetime columns.'): part_dep = partial_dependence(pipeline, X, features=('0', 'dt_column')) with pytest.raises(ValueError, match='Two-way partial dependence is not supported for datetime columns.'): part_dep = partial_dependence(pipeline, X, features=(0, 20))
def test_iterative_algorithm_frozen_parameters(): class MockEstimator(Estimator): name = "Mock Classifier" model_family = ModelFamily.RANDOM_FOREST supported_problem_types = [ ProblemTypes.BINARY, ProblemTypes.MULTICLASS ] hyperparameter_ranges = { 'dummy_int_parameter': Integer(1, 10), 'dummy_categorical_parameter': Categorical(["random", "dummy", "test"]), 'dummy_real_parameter': Real(0, 1) } def __init__(self, dummy_int_parameter=0, dummy_categorical_parameter='dummy', dummy_real_parameter=1.0, n_jobs=-1, random_seed=0, **kwargs): super().__init__(parameters={ 'dummy_int_parameter': dummy_int_parameter, 'dummy_categorical_parameter': dummy_categorical_parameter, 'dummy_real_parameter': dummy_real_parameter, **kwargs, 'n_jobs': n_jobs }, component_obj=None, random_seed=random_seed) pipeline = BinaryClassificationPipeline([MockEstimator]) algo = IterativeAlgorithm(allowed_pipelines=[pipeline, pipeline, pipeline], pipeline_params={ 'pipeline': { 'date_index': "Date", "gap": 2, "max_delay": 10 } }, random_seed=0, _frozen_pipeline_parameters={ "Mock Classifier": { 'dummy_int_parameter': 6, 'dummy_categorical_parameter': "random", 'dummy_real_parameter': 0.1 } }) next_batch = algo.next_batch() assert all([ p.parameters['pipeline'] == { 'date_index': "Date", "gap": 2, "max_delay": 10 } for p in next_batch ]) assert all([ p.parameters['Mock Classifier'] == { 'dummy_int_parameter': 6, 'dummy_categorical_parameter': "random", 'dummy_real_parameter': 0.1, "n_jobs": -1 } for p in next_batch ]) scores = np.arange(0, len(next_batch)) for score, pipeline in zip(scores, next_batch): algo.add_result(score, pipeline, {"id": algo.pipeline_number}) # make sure that future batches remain in the hyperparam range for i in range(1, 5): next_batch = algo.next_batch() assert all([ p.parameters['Mock Classifier'] == { 'dummy_int_parameter': 6, 'dummy_categorical_parameter': "random", 'dummy_real_parameter': 0.1, "n_jobs": -1 } for p in next_batch ])
def decision_tree_classification_pipeline_class(X_y_categorical_classification): pipeline = BinaryClassificationPipeline(['Simple Imputer', 'One Hot Encoder', 'Standard Scaler', 'Decision Tree Classifier']) X, y = X_y_categorical_classification pipeline.fit(X, y) return pipeline
def test_stacked_estimator_in_pipeline( problem_type, X_y_binary, X_y_multi, X_y_regression, stackable_classifiers, stackable_regressors, logistic_regression_binary_pipeline_class, logistic_regression_multiclass_pipeline_class, linear_regression_pipeline_class): if problem_type == ProblemTypes.BINARY: X, y = X_y_binary base_pipeline_class = BinaryClassificationPipeline stacking_component_name = StackedEnsembleClassifier.name input_pipelines = [ BinaryClassificationPipeline([classifier]) for classifier in stackable_classifiers ] comparison_pipeline = logistic_regression_binary_pipeline_class( parameters={"Logistic Regression Classifier": { "n_jobs": 1 }}) objective = 'Log Loss Binary' elif problem_type == ProblemTypes.MULTICLASS: X, y = X_y_multi base_pipeline_class = MulticlassClassificationPipeline stacking_component_name = StackedEnsembleClassifier.name input_pipelines = [ MulticlassClassificationPipeline([classifier]) for classifier in stackable_classifiers ] comparison_pipeline = logistic_regression_multiclass_pipeline_class( parameters={"Logistic Regression Classifier": { "n_jobs": 1 }}) objective = 'Log Loss Multiclass' elif problem_type == ProblemTypes.REGRESSION: X, y = X_y_regression base_pipeline_class = RegressionPipeline stacking_component_name = StackedEnsembleRegressor.name input_pipelines = [ RegressionPipeline([regressor]) for regressor in stackable_regressors ] comparison_pipeline = linear_regression_pipeline_class( parameters={"Linear Regressor": { "n_jobs": 1 }}) objective = 'R2' parameters = { stacking_component_name: { "input_pipelines": input_pipelines, "n_jobs": 1 } } graph = ['Simple Imputer', stacking_component_name] pipeline = base_pipeline_class(component_graph=graph, parameters=parameters) pipeline.fit(X, y) comparison_pipeline.fit(X, y) assert not np.isnan(pipeline.predict(X).to_series()).values.any() pipeline_score = pipeline.score(X, y, [objective])[objective] comparison_pipeline_score = comparison_pipeline.score( X, y, [objective])[objective] if problem_type == ProblemTypes.BINARY or problem_type == ProblemTypes.MULTICLASS: assert not np.isnan( pipeline.predict_proba(X).to_dataframe()).values.any() assert (pipeline_score <= comparison_pipeline_score) else: assert (pipeline_score >= comparison_pipeline_score)
def test_binary_init(): clf = BinaryClassificationPipeline(component_graph=[ "Imputer", "One Hot Encoder", "Random Forest Classifier" ]) assert clf.parameters == { 'Imputer': { 'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None }, 'One Hot Encoder': { 'top_n': 10, 'features_to_encode': None, 'categories': None, 'drop': 'if_binary', 'handle_unknown': 'ignore', 'handle_missing': 'error' }, 'Random Forest Classifier': { 'n_estimators': 100, 'max_depth': 6, 'n_jobs': -1 } } assert clf.custom_hyperparameters is None assert clf.name == "Random Forest Classifier w/ Imputer + One Hot Encoder" assert clf.random_seed == 0 custom_hyperparameters = { "Imputer": { "numeric_impute_strategy": Categorical(["most_frequent", 'mean']) }, "Imputer_1": { "numeric_impute_strategy": Categorical(["median", 'mean']) }, "Random Forest Classifier": { "n_estimators": Categorical([50, 100]) } } parameters = {"One Hot Encoder": {"top_n": 20}} clf = BinaryClassificationPipeline( component_graph=[ "Imputer", "One Hot Encoder", "Random Forest Classifier" ], parameters=parameters, custom_hyperparameters=custom_hyperparameters, custom_name="Custom Pipeline", random_seed=42) assert clf.parameters == { 'Imputer': { 'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'mean', 'categorical_fill_value': None, 'numeric_fill_value': None }, 'One Hot Encoder': { 'top_n': 20, 'features_to_encode': None, 'categories': None, 'drop': 'if_binary', 'handle_unknown': 'ignore', 'handle_missing': 'error' }, 'Random Forest Classifier': { 'n_estimators': 100, 'max_depth': 6, 'n_jobs': -1 } } assert clf.custom_hyperparameters == custom_hyperparameters assert clf.name == "Custom Pipeline" assert clf.random_seed == 42
def test_stacked_ensemble_nonstackable_model_families(): with pytest.raises(ValueError, match="Pipelines with any of the following model families cannot be used as base pipelines"): StackedEnsembleClassifier(input_pipelines=[BinaryClassificationPipeline([BaselineClassifier])])