def test_ovr_accuracy(self): clfs = [ # TODO: BUG 231482 , why doesnt FM work # FactorizationMachineBinaryClassifier(), LogisticRegressionBinaryClassifier(number_of_threads=1), FastForestBinaryClassifier(minimum_example_count_per_leaf=1, number_of_threads=1), GamBinaryClassifier(number_of_threads=1), AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier(minimum_example_count_per_leaf=1, number_of_threads=1), FastLinearBinaryClassifier(number_of_threads=1), SgdBinaryClassifier(number_of_threads=1), # SymSgdBinaryClassifier(number_of_threads=1), ] for clf in clfs: ovr = OneVsRestClassifier(classifier=clf, use_probabilities=True) metrics = accuracy(ovr) accu = metrics['Accuracy(micro-avg)'][0] # algos will have wide range of accuracy, so use low bar. Also # checks Pipeline + Ova + clf assert_greater( accu, 0.65, "{} accuracy is too low {}".format(clf.__class__, accu))
def test_averagedperceptron_unsupported_losses_syntax(self): df = get_dataset("infert").as_df().drop('row_num', axis=1) X = df y = df['case'] pipeline = Pipeline([ OneHotVectorizer(columns={ 'age1': 'age', 'parity1': 'parity', 'sp1': 'spontaneous' }), OneHotVectorizer(columns={'education_str': 'education_str'}), ColumnDuplicator(columns={'case2': 'case'}), AveragedPerceptronBinaryClassifier( feature=['age1', 'education_str'], label='case') ]) try: model = pipeline.fit(X, y, verbose=0) raise AssertionError("same column name in X and y") except RuntimeError as e: assert "If any step in the pipeline has defined Label" in str(e) X = X.drop('case', axis=1) pipeline = Pipeline([ OneHotVectorizer(columns={ 'age1': 'age', 'parity1': 'parity', 'sp1': 'spontaneous' }), OneHotVectorizer(columns={'education_str': 'education_str'}), # ColumnDuplicator(columns={'case2': 'case'}), # does not work AveragedPerceptronBinaryClassifier( feature=['age1', 'education_str'], label='case') ]) info = pipeline.get_fit_info(df)[0] assert info[-1]['inputs'] != ['Feature:Features', 'Label:case'] model = pipeline.fit(df) y_pred_withpipeline = model.predict(X) assert set(y_pred_withpipeline.columns) == { 'PredictedLabel', 'Probability', 'Score' } assert y_pred_withpipeline.shape == (248, 3)
def test_automl_usecase(self): # train featurization pipeline featurization_pipe = Pipeline([NGramFeaturizer(keep_diacritics=True, columns={'Features': ['SentimentText']})]) featurization_pipe.fit(train_set) # train learner pipeline learner_pipe = Pipeline([DatasetTransformer(featurization_pipe.model), OneVsRestClassifier(AveragedPerceptronBinaryClassifier(), feature=['Features'], label='Sentiment') ]) learner_pipe.fit(train_set) # Export the learner pipeline to ONNX onnx_path = get_tmp_file('.onnx') learner_pipe.export_to_onnx(onnx_path, 'com.microsoft.ml', onnx_version='Stable') # Perform the transform using the standard ML.Net backend start = time.time() result_standard = learner_pipe.predict(test_set) end = time.time() print('%ss done transform using standard backend' % round(end - start, 3)) # Perform the transform using the ORT backend df_tool = DFT(onnx_path) dataset = test_set.to_df() start = time.time() result_ort = df_tool.execute(dataset, ['PredictedLabel.output', 'Score.output']) end = time.time() print('%ss done transform using ORT backend (excludes df load time)' % round(end - start, 3)) # compare the results for col_tuple in (('PredictedLabel', 'PredictedLabel.output'), ('Score.0', 'Score.output.0'), ('Score.1', 'Score.output.1'), ): col_expected = result_standard.loc[:, col_tuple[0]] col_ort = result_ort.loc[:, col_tuple[1]] check_kwargs = { 'check_names': False, 'check_exact': False, 'check_dtype': True, 'check_less_precise': True } pd.testing.assert_series_equal(col_expected, col_ort, **check_kwargs)
def test_failing_predict_proba_called_with_use_probabilites_false(self): clfs = [ # TODO: BUG 231482 , why doesnt FM work # FactorizationMachineBinaryClassifier(), LogisticRegressionBinaryClassifier(), FastForestBinaryClassifier(minimum_example_count_per_leaf=1), GamBinaryClassifier(), AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier(minimum_example_count_per_leaf=1), LightGbmBinaryClassifier(), FastLinearBinaryClassifier(), SgdBinaryClassifier(), # SymSgdBinaryClassifier(), ] for clf in clfs: ovr = OneVsRestClassifier(classifier=clf, use_probabilities=False) check_predict_proba_when_trained_with_use_probabilites_false( self, ovr, clf)
def test_failing_decision_function_called_with_use_probabilites_true(self): clfs = [ # TODO: BUG 231482 , why doesnt FM work # FactorizationMachineBinaryClassifier(), LogisticRegressionBinaryClassifier(), FastForestBinaryClassifier(min_split=1), GamBinaryClassifier(), AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier(min_split=1), LightGbmBinaryClassifier(), FastLinearBinaryClassifier(), SgdBinaryClassifier(), # SymSgdBinaryClassifier(), ] for clf in clfs: ovr = OneVsRestClassifier(classifier=clf, use_probabilities=True) check_decision_function_when_trained_with_use_probabilites_true( self, ovr, clf)
def test_decision_function_produces_distribution_not_sum_to_1(self): clfs = [ # TODO: BUG 231482 , why doesnt FM work # FactorizationMachineBinaryClassifier(), LogisticRegressionBinaryClassifier(), FastForestBinaryClassifier(minimum_example_count_per_leaf=1), GamBinaryClassifier(), AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier(minimum_example_count_per_leaf=1), LightGbmBinaryClassifier(), FastLinearBinaryClassifier(), SgdBinaryClassifier(), # SymSgdBinaryClassifier(), ] for clf in clfs: ovr = OneVsRestClassifier(classifier=clf, use_probabilities=False) scoremean = decfun_average(ovr) assert_not_equal( scoremean, 1.0, '{} raw scores should not sum to 1.0 over 3 classes'.format( clf.__class__))
def test_predict_proba_produces_distribution_sum_to_1(self): clfs = [ # TODO: BUG 231482 , why doesnt FM work # FactorizationMachineBinaryClassifier(), LogisticRegressionBinaryClassifier(), FastForestBinaryClassifier(minimum_example_count_per_leaf=1), GamBinaryClassifier(), AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier(minimum_example_count_per_leaf=1), LightGbmBinaryClassifier(), FastLinearBinaryClassifier(), SgdBinaryClassifier(), # TODO: why symsgd does not sum to 1.0 # SymSgdBinaryClassifier(), ] for clf in clfs: ovr = OneVsRestClassifier(classifier=clf) probmean = proba_average(ovr) assert_equal( probmean, 1.0, '{} probabilites {} do not sum to 1.0 over 3 classes'.format( clf.__class__, probmean))
def test_learners_sweep(self): # grid search over 2 learners np.random.seed(0) df = pd.DataFrame( dict(education=['A', 'A', 'A', 'A', 'B', 'A', 'B'], workclass=['X', 'Y', 'X', 'X', 'X', 'Y', 'Y'], y=[1, 0, 1, 1, 0, 1, 0])) X = df.drop('y', axis=1) y = df['y'] cat = OneHotVectorizer() << ['education', 'workclass'] learner = FastTreesBinaryClassifier() pipe = Pipeline(steps=[('cat', cat), ('learner', learner)]) param_grid = dict(learner=[ AveragedPerceptronBinaryClassifier(), FastTreesBinaryClassifier() ]) grid = GridSearchCV(pipe, param_grid) grid.fit(X, y) assert grid.best_params_[ 'learner'].__class__.__name__ == 'AveragedPerceptronBinaryClassifier'
# AveragedPerceptronBinaryClassifier import numpy as np from nimbusml.datasets import get_dataset from nimbusml.feature_extraction.categorical import OneHotVectorizer from nimbusml.linear_model import AveragedPerceptronBinaryClassifier from sklearn.model_selection import train_test_split # use the built-in data set 'infert' to create test and train data # Unnamed: 0 education age parity induced case spontaneous stratum \ # 0 1 0.0 26.0 6.0 1.0 1.0 2.0 1.0 # 1 2 0.0 42.0 1.0 1.0 1.0 0.0 2.0 # pooled.stratum education_str # 0 3.0 0-5yrs # 1 1.0 0-5yrs np.random.seed(0) df = get_dataset("infert").as_df() # remove : and ' ' from column names, and encode categorical column df.columns = [i.replace(': ', '') for i in df.columns] df = (OneHotVectorizer() << 'education_str').fit_transform(df) X_train, X_test, y_train, y_test = \ train_test_split(df.loc[:, df.columns != 'case'], df['case']) lr = AveragedPerceptronBinaryClassifier().fit(X_train, y_train) scores = lr.predict(X_test) # Evaluate the model print('Accuracy:', np.mean(y_test == [i for i in scores]))
############################################################################### # Hinge Loss from nimbusml.linear_model import AveragedPerceptronBinaryClassifier from nimbusml.loss import Hinge # specify the loss function as a string keyword trainer1 = AveragedPerceptronBinaryClassifier(loss='hinge') # can also use the loss class instead of string trainer1 = AveragedPerceptronBinaryClassifier( loss=Hinge()) # equivalent to loss='hinge' trainer2 = AveragedPerceptronBinaryClassifier(loss=Hinge(margin=2.0))
def test_averagedperceptron(self): accuracy = get_accuracy(self, AveragedPerceptronBinaryClassifier()) # Accuracy depends on column Unnamed0 (index). assert_greater(accuracy, 0.93, "accuracy should be greater than %s" % 0.93)
'ToKeyImputer', 'ToString', 'EnsembleClassifier', 'EnsembleRegressor', 'CharTokenizer', 'WordTokenizer', 'MutualInformationSelector', 'NaiveBayesClassifier', 'CountSelector', 'KMeansPlusPlus', 'ToKey', 'ColumnSelector' } INSTANCES = { 'AveragedPerceptronBinaryClassifier': AveragedPerceptronBinaryClassifier( feature=['education_str.0-5yrs', 'education_str.6-11yrs', 'education_str.12+ yrs']), 'Binner': Binner(num_bins=3), 'CharTokenizer': CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}), 'ColumnConcatenator': ColumnConcatenator(columns={'Features': [ 'Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width', 'Setosa']}), 'ColumnSelector': ColumnSelector(columns=['Sepal_Width', 'Sepal_Length']), 'ColumnDuplicator': ColumnDuplicator(columns={'dup': 'Sepal_Width'}), 'CountSelector': CountSelector(count=5, columns=['Sepal_Width']), 'DateTimeSplitter': DateTimeSplitter(prefix='dt'), 'FastForestBinaryClassifier': FastForestBinaryClassifier(feature=['Sepal_Width', 'Sepal_Length'], label='Setosa'), 'FastLinearBinaryClassifier': FastLinearBinaryClassifier(feature=['Sepal_Width', 'Sepal_Length'],
'ethnicity', 'sex', 'native-country-region' ] file_schema = 'sep=, col=label:R4:0 col=Features:R4:9-14 col=workclass:TX:1 ' \ 'col=education:TX:2 col=marital-status:TX:3 ' \ 'col=occupation:TX:4 col=relationship:TX:5 col=ethnicity:TX:6 ' \ 'col=sex:TX:7 col=native-country-region:TX:8 header+' label_column = 'label' learners = [ FastForestBinaryClassifier(), FastForestRegressor(), FastTreesBinaryClassifier(), FastTreesRegressor(), FastTreesTweedieRegressor(), LightGbmRegressor(), LightGbmBinaryClassifier(), AveragedPerceptronBinaryClassifier(), FastLinearBinaryClassifier(), FastLinearClassifier(), FastLinearRegressor(), LogisticRegressionBinaryClassifier(), LogisticRegressionClassifier(), OnlineGradientDescentRegressor(), SgdBinaryClassifier(), # SymSgdBinaryClassifier(), OrdinaryLeastSquaresRegressor(), PoissonRegressionRegressor() ] learners_not_supported = [ NaiveBayesClassifier(), # fix in nimbusml, needs to implement ICanGetSummaryAsIDataView
def test_syntax_slots_wo_pipeline(self): # data df = get_dataset("infert").as_df() df = df.drop(['row_num', ], axis=1) X = df.drop('case', axis=1) y = df['case'] # transform xf1 = OneHotVectorizer(columns=['age', 'parity', 'education_str']) X_xf1 = xf1.fit_transform(X, verbose=0) assert "age.21" in list(X_xf1.columns) # learner # (1 .a.) model = AveragedPerceptronBinaryClassifier() # (1. b) try: model = AveragedPerceptronBinaryClassifier(feature=['age']) model.fit(X_xf1, y, verbose=0) cont = True assert False except Exception as e: # does not work cont = False print(e) if cont: y_pred = model.predict(X_xf1) assert y_pred.shape == (248, 3) pipeline = Pipeline([ OneHotVectorizer(columns=['age', 'parity', 'education_str']), AveragedPerceptronBinaryClassifier(feature='age') ]) pipeline.fit(X, y, verbose=0) y_pred_withpipeline = pipeline.predict(X) print(y_pred_withpipeline.head()) assert y_pred_withpipeline.shape == (248, 3) metrics, scores = pipeline.test(X, y, output_scores=True) print(metrics) assert scores.shape == (248, 3) assert metrics.shape == (1, 11) # back to X_xf1 print(list(X_xf1.columns)) l1 = list(sorted(set(_.split('.')[-1] for _ in X_xf1.columns))) levels = [['age', 'education', 'education_str', 'parity', 'pooled', 'spontaneous', 'stratum', 'induced'], [''] + l1] names = ['columns', 'slots'] labels = [[], []] ages = [] for _ in X_xf1.columns: spl = _.split('.') l1 = levels[0].index(spl[0]) try: l2 = levels[1].index(spl[1]) except IndexError: l2 = levels[1].index('') labels[0].append(l1) labels[1].append(l2) if spl[0] == 'age': ages.append(l2) X_xf1.columns = pandas.MultiIndex( levels=levels, labels=labels, names=names) print(X_xf1.head(n=2).T) col_ages = [('age', a) for a in ages] print(col_ages) try: model = AveragedPerceptronBinaryClassifier(feature=col_ages) model.fit(X_xf1, y, verbose=0) y_pred = model.predict(X_xf1) assert y_pred.shape == (248, 3) except Exception as e: # Does not work, probably confusion between list and tuple in nimbusml print(e) try: model = AveragedPerceptronBinaryClassifier(feature=['age']) model.fit(X_xf1, y, verbose=0) y_pred = model.predict(X_xf1) assert y_pred.shape == (248, 3) except Exception as e: # Does not work. print(e)
from nimbusml.linear_model import AveragedPerceptronBinaryClassifier # data input (as a FileDataStream) path = get_dataset('infert').as_filepath() data = FileDataStream.read_csv(path) print(data.head()) # age case education induced parity ... row_num spontaneous ... # 0 26 1 0-5yrs 1 6 ... 1 2 ... # 1 42 1 0-5yrs 1 1 ... 2 0 ... # 2 39 1 0-5yrs 2 6 ... 3 0 ... # 3 34 1 0-5yrs 2 4 ... 4 0 ... # 4 35 1 6-11yrs 1 3 ... 5 1 ... # define the training pipeline pipeline = Pipeline([ AveragedPerceptronBinaryClassifier( feature=['age', 'parity', 'spontaneous'], label='case') ]) # train, predict, and evaluate # TODO: Replace with CV metrics, predictions = pipeline.fit(data).test(data, output_scores=True) # print predictions print(predictions.head()) # PredictedLabel Score # 0 0 -0.285667 # 1 0 -1.304729 # 2 0 -2.651955 # 3 0 -2.111450 # 4 0 -0.660658 # print evaluation metrics