def test_validate_with_test_data_classification_target_filter_validations( self, target_filter_type): test_data_classification = get_toy_binary_classification_dataset() with pytest.raises( UserConfigValidationException, match="{0} can only be configured with " "filter {1} for classification".format( target_filter_type, CohortFilterMethods.METHOD_INCLUDES)): cohort_filter_classification = CohortFilter( method=CohortFilterMethods.METHOD_EXCLUDES, arg=['X'], column=target_filter_type) cohort_filter_classification._validate_with_test_data( test_data=test_data_classification, target_column="target", categorical_features=[], is_classification=True) with pytest.raises( UserConfigValidationException, match="Found a class in arg which is not present in " "test data"): cohort_filter_classification = CohortFilter( method=CohortFilterMethods.METHOD_INCLUDES, arg=['Z'], column=target_filter_type) cohort_filter_classification._validate_with_test_data( test_data=test_data_classification, target_column="target", categorical_features=[], is_classification=True)
def test_validate_with_test_data_with_dataset_validations( self): test_data = pd.DataFrame( data=[[23, 'new', 'A'], [25, 'new, ''B'], [25, 'old', 'B']], columns=["age", 'type', "target"]) with pytest.raises( UserConfigValidationException, match="{0} is a categorical feature and should be only " "configured with {1} cohort filter.".format( "type", CohortFilterMethods.METHOD_INCLUDES)): cohort_filter = CohortFilter( method=CohortFilterMethods.METHOD_EXCLUDES, arg=['new'], column='type') cohort_filter._validate_with_test_data( test_data=test_data, target_column="target", categorical_features=['type'], is_classification=True) with pytest.raises( UserConfigValidationException, match="Found a category {0} in arg which is not present " "in test data column {1}.".format('mid', 'type')): cohort_filter = CohortFilter( method=CohortFilterMethods.METHOD_INCLUDES, arg=['mid'], column='type') cohort_filter._validate_with_test_data( test_data=test_data, target_column="target", categorical_features=['type'], is_classification=True)
def test_validate_with_test_data_index_filter_validations(self): test_data = get_toy_binary_classification_dataset() cohort_filter_index_excludes = CohortFilter( method=CohortFilterMethods.METHOD_EXCLUDES, arg=[65], column=CohortFilter.INDEX) with pytest.raises( UserConfigValidationException, match="excludes filter is not supported with Index based " "selection."): cohort_filter_index_excludes._validate_with_test_data( test_data=test_data, target_column="target", categorical_features=[] ) cohort_filter_index_incorrect_args = CohortFilter( method=CohortFilterMethods.METHOD_GREATER, arg=[65.0], column=CohortFilter.INDEX) with pytest.raises( UserConfigValidationException, match="All entries in arg should be of type int."): cohort_filter_index_incorrect_args._validate_with_test_data( test_data=test_data, target_column="target", categorical_features=[] )
def test_validate_with_test_data_high_level_validations(self): test_data = get_toy_binary_classification_dataset() cohort_filter_not_a_feature = CohortFilter( method=CohortFilterMethods.METHOD_LESS, arg=[65], column='fake_column') with pytest.raises( UserConfigValidationException, match="Unknown column fake_column specified in cohort filter"): cohort_filter_not_a_feature._validate_with_test_data( test_data=test_data, target_column="target", categorical_features=[])
def test_validate_with_test_data_regression_error_filter_validations( self): test_data_regression = get_toy_regression_dataset() cohort_filter_regression = CohortFilter( method=CohortFilterMethods.METHOD_LESS, arg=[2.5], column=CohortFilter.REGRESSION_ERROR) with pytest.raises( UserConfigValidationException, match="Error cannot be configured for classification" " scenarios."): cohort_filter_regression._validate_with_test_data( test_data=test_data_regression, target_column="target", categorical_features=[], is_classification=True) with pytest.raises( UserConfigValidationException, match="Error cannot be configured with either includes" " or excludes."): cohort_filter_regression.method = \ CohortFilterMethods.METHOD_INCLUDES cohort_filter_regression._validate_with_test_data( test_data=test_data_regression, target_column="target", categorical_features=[], is_classification=False) with pytest.raises( UserConfigValidationException, match="Error cannot be configured with either includes" " or excludes."): cohort_filter_regression.method = \ CohortFilterMethods.METHOD_EXCLUDES cohort_filter_regression._validate_with_test_data( test_data=test_data_regression, target_column="target", categorical_features=[], is_classification=False) with pytest.raises( UserConfigValidationException, match="All entries in arg should be of type int or float" " for Error cohort."): cohort_filter_regression.method = \ CohortFilterMethods.METHOD_GREATER cohort_filter_regression.arg = ['val1', 'val2'] cohort_filter_regression._validate_with_test_data( test_data=test_data_regression, target_column="target", categorical_features=[], is_classification=False)
def test_validate_with_test_data_regression_target_filter_validations( self, target_filter_type, method): test_data_regression = get_toy_regression_dataset() with pytest.raises( UserConfigValidationException, match="{0} cannot be configured with " "filter {1} for regression.".format(target_filter_type, method)): cohort_filter_regression = CohortFilter( method=method, arg=[2.5], column=target_filter_type) cohort_filter_regression._validate_with_test_data( test_data=test_data_regression, target_column="target", categorical_features=[], is_classification=False)
def test_validate_with_test_data_classification_error_filter_validations( self): test_data_multiclass = get_toy_multiclass_classification_dataset() test_data_binary = get_toy_binary_classification_dataset() cohort_filter_classification_excludes = CohortFilter( method=CohortFilterMethods.METHOD_EXCLUDES, arg=[ClassificationOutcomes.FALSE_NEGATIVE], column=CohortFilter.CLASSIFICATION_OUTCOME) cohort_filter_classification_includes = CohortFilter( method=CohortFilterMethods.METHOD_INCLUDES, arg=["random"], column=CohortFilter.CLASSIFICATION_OUTCOME) with pytest.raises( UserConfigValidationException, match="Classification outcome cannot be " "configured for multi-class classification" " and regression scenarios."): cohort_filter_classification_excludes._validate_with_test_data( test_data=test_data_multiclass, target_column="target", categorical_features=[], is_classification=True ) with pytest.raises( UserConfigValidationException, match="Classification outcome cannot be " "configured for multi-class classification" " and regression scenarios."): cohort_filter_classification_excludes._validate_with_test_data( test_data=test_data_binary, target_column="target", categorical_features=[], is_classification=False ) with pytest.raises( UserConfigValidationException, match="Classification outcome can only be configured with " "cohort filter includes."): cohort_filter_classification_excludes._validate_with_test_data( test_data=test_data_binary, target_column="target", categorical_features=[], is_classification=True ) with pytest.raises( UserConfigValidationException, match="Classification outcome can only take argument values " "from False negative or False positive or True " "negative or True positive."): cohort_filter_classification_includes._validate_with_test_data( test_data=test_data_binary, target_column="target", categorical_features=[], is_classification=True)