def test_ovr_accuracy(self):
clfs = [
# TODO: BUG 231482 , why doesnt FM work
# FactorizationMachineBinaryClassifier(),
LogisticRegressionBinaryClassifier(number_of_threads=1),
FastForestBinaryClassifier(minimum_example_count_per_leaf=1,
number_of_threads=1),
GamBinaryClassifier(number_of_threads=1),
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier(minimum_example_count_per_leaf=1,
number_of_threads=1),
FastLinearBinaryClassifier(number_of_threads=1),
SgdBinaryClassifier(number_of_threads=1),
# SymSgdBinaryClassifier(number_of_threads=1),
]
for clf in clfs:
ovr = OneVsRestClassifier(classifier=clf, use_probabilities=True)
metrics = accuracy(ovr)
accu = metrics['Accuracy(micro-avg)'][0]
# algos will have wide range of accuracy, so use low bar. Also
# checks Pipeline + Ova + clf
assert_greater(
accu, 0.65,
"{} accuracy is too low {}".format(clf.__class__, accu))
def test_averagedperceptron_unsupported_losses_syntax(self):
df = get_dataset("infert").as_df().drop('row_num', axis=1)
X = df
y = df['case']
pipeline = Pipeline([
OneHotVectorizer(columns={
'age1': 'age',
'parity1': 'parity',
'sp1': 'spontaneous'
}),
OneHotVectorizer(columns={'education_str': 'education_str'}),
ColumnDuplicator(columns={'case2': 'case'}),
AveragedPerceptronBinaryClassifier(
feature=['age1', 'education_str'], label='case')
])
try:
model = pipeline.fit(X, y, verbose=0)
raise AssertionError("same column name in X and y")
except RuntimeError as e:
assert "If any step in the pipeline has defined Label" in str(e)
X = X.drop('case', axis=1)
pipeline = Pipeline([
OneHotVectorizer(columns={
'age1': 'age',
'parity1': 'parity',
'sp1': 'spontaneous'
}),
OneHotVectorizer(columns={'education_str': 'education_str'}),
# ColumnDuplicator(columns={'case2': 'case'}), # does not work
AveragedPerceptronBinaryClassifier(
feature=['age1', 'education_str'], label='case')
])
info = pipeline.get_fit_info(df)[0]
assert info[-1]['inputs'] != ['Feature:Features', 'Label:case']
model = pipeline.fit(df)
y_pred_withpipeline = model.predict(X)
assert set(y_pred_withpipeline.columns) == {
'PredictedLabel', 'Probability', 'Score'
}
assert y_pred_withpipeline.shape == (248, 3)
def test_automl_usecase(self):
# train featurization pipeline
featurization_pipe = Pipeline([NGramFeaturizer(keep_diacritics=True, columns={'Features': ['SentimentText']})])
featurization_pipe.fit(train_set)
# train learner pipeline
learner_pipe = Pipeline([DatasetTransformer(featurization_pipe.model),
OneVsRestClassifier(AveragedPerceptronBinaryClassifier(),
feature=['Features'], label='Sentiment')
])
learner_pipe.fit(train_set)
# Export the learner pipeline to ONNX
onnx_path = get_tmp_file('.onnx')
learner_pipe.export_to_onnx(onnx_path, 'com.microsoft.ml', onnx_version='Stable')
# Perform the transform using the standard ML.Net backend
start = time.time()
result_standard = learner_pipe.predict(test_set)
end = time.time()
print('%ss done transform using standard backend' % round(end - start, 3))
# Perform the transform using the ORT backend
df_tool = DFT(onnx_path)
dataset = test_set.to_df()
start = time.time()
result_ort = df_tool.execute(dataset, ['PredictedLabel.output', 'Score.output'])
end = time.time()
print('%ss done transform using ORT backend (excludes df load time)' % round(end - start, 3))
# compare the results
for col_tuple in (('PredictedLabel', 'PredictedLabel.output'),
('Score.0', 'Score.output.0'),
('Score.1', 'Score.output.1'),
):
col_expected = result_standard.loc[:, col_tuple[0]]
col_ort = result_ort.loc[:, col_tuple[1]]
check_kwargs = {
'check_names': False,
'check_exact': False,
'check_dtype': True,
'check_less_precise': True
}
pd.testing.assert_series_equal(col_expected, col_ort, **check_kwargs)
def test_failing_predict_proba_called_with_use_probabilites_false(self):
clfs = [
# TODO: BUG 231482 , why doesnt FM work
# FactorizationMachineBinaryClassifier(),
LogisticRegressionBinaryClassifier(),
FastForestBinaryClassifier(minimum_example_count_per_leaf=1),
GamBinaryClassifier(),
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier(minimum_example_count_per_leaf=1),
LightGbmBinaryClassifier(),
FastLinearBinaryClassifier(),
SgdBinaryClassifier(),
# SymSgdBinaryClassifier(),
]
for clf in clfs:
ovr = OneVsRestClassifier(classifier=clf, use_probabilities=False)
check_predict_proba_when_trained_with_use_probabilites_false(
self, ovr, clf)
def test_failing_decision_function_called_with_use_probabilites_true(self):
clfs = [
# TODO: BUG 231482 , why doesnt FM work
# FactorizationMachineBinaryClassifier(),
LogisticRegressionBinaryClassifier(),
FastForestBinaryClassifier(min_split=1),
GamBinaryClassifier(),
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier(min_split=1),
LightGbmBinaryClassifier(),
FastLinearBinaryClassifier(),
SgdBinaryClassifier(),
# SymSgdBinaryClassifier(),
]
for clf in clfs:
ovr = OneVsRestClassifier(classifier=clf, use_probabilities=True)
check_decision_function_when_trained_with_use_probabilites_true(
self, ovr, clf)
def test_decision_function_produces_distribution_not_sum_to_1(self):
clfs = [
# TODO: BUG 231482 , why doesnt FM work
# FactorizationMachineBinaryClassifier(),
LogisticRegressionBinaryClassifier(),
FastForestBinaryClassifier(minimum_example_count_per_leaf=1),
GamBinaryClassifier(),
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier(minimum_example_count_per_leaf=1),
LightGbmBinaryClassifier(),
FastLinearBinaryClassifier(),
SgdBinaryClassifier(),
# SymSgdBinaryClassifier(),
]
for clf in clfs:
ovr = OneVsRestClassifier(classifier=clf, use_probabilities=False)
scoremean = decfun_average(ovr)
assert_not_equal(
scoremean, 1.0,
'{} raw scores should not sum to 1.0 over 3 classes'.format(
clf.__class__))
def test_predict_proba_produces_distribution_sum_to_1(self):
clfs = [
# TODO: BUG 231482 , why doesnt FM work
# FactorizationMachineBinaryClassifier(),
LogisticRegressionBinaryClassifier(),
FastForestBinaryClassifier(minimum_example_count_per_leaf=1),
GamBinaryClassifier(),
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier(minimum_example_count_per_leaf=1),
LightGbmBinaryClassifier(),
FastLinearBinaryClassifier(),
SgdBinaryClassifier(),
# TODO: why symsgd does not sum to 1.0
# SymSgdBinaryClassifier(),
]
for clf in clfs:
ovr = OneVsRestClassifier(classifier=clf)
probmean = proba_average(ovr)
assert_equal(
probmean, 1.0,
'{} probabilites {} do not sum to 1.0 over 3 classes'.format(
clf.__class__, probmean))
def test_learners_sweep(self):
# grid search over 2 learners
np.random.seed(0)
df = pd.DataFrame(
dict(education=['A', 'A', 'A', 'A', 'B', 'A', 'B'],
workclass=['X', 'Y', 'X', 'X', 'X', 'Y', 'Y'],
y=[1, 0, 1, 1, 0, 1, 0]))
X = df.drop('y', axis=1)
y = df['y']
cat = OneHotVectorizer() << ['education', 'workclass']
learner = FastTreesBinaryClassifier()
pipe = Pipeline(steps=[('cat', cat), ('learner', learner)])
param_grid = dict(learner=[
AveragedPerceptronBinaryClassifier(),
FastTreesBinaryClassifier()
])
grid = GridSearchCV(pipe, param_grid)
grid.fit(X, y)
assert grid.best_params_[
'learner'].__class__.__name__ == 'AveragedPerceptronBinaryClassifier'
# AveragedPerceptronBinaryClassifier
import numpy as np
from nimbusml.datasets import get_dataset
from nimbusml.feature_extraction.categorical import OneHotVectorizer
from nimbusml.linear_model import AveragedPerceptronBinaryClassifier
from sklearn.model_selection import train_test_split
# use the built-in data set 'infert' to create test and train data
# Unnamed: 0 education age parity induced case spontaneous stratum \
# 0 1 0.0 26.0 6.0 1.0 1.0 2.0 1.0
# 1 2 0.0 42.0 1.0 1.0 1.0 0.0 2.0
# pooled.stratum education_str
# 0 3.0 0-5yrs
# 1 1.0 0-5yrs
np.random.seed(0)
df = get_dataset("infert").as_df()
# remove : and ' ' from column names, and encode categorical column
df.columns = [i.replace(': ', '') for i in df.columns]
df = (OneHotVectorizer() << 'education_str').fit_transform(df)
X_train, X_test, y_train, y_test = \
train_test_split(df.loc[:, df.columns != 'case'], df['case'])
lr = AveragedPerceptronBinaryClassifier().fit(X_train, y_train)
scores = lr.predict(X_test)
# Evaluate the model
print('Accuracy:', np.mean(y_test == [i for i in scores]))
###############################################################################
# Hinge Loss
from nimbusml.linear_model import AveragedPerceptronBinaryClassifier
from nimbusml.loss import Hinge
# specify the loss function as a string keyword
trainer1 = AveragedPerceptronBinaryClassifier(loss='hinge')
# can also use the loss class instead of string
trainer1 = AveragedPerceptronBinaryClassifier(
loss=Hinge()) # equivalent to loss='hinge'
trainer2 = AveragedPerceptronBinaryClassifier(loss=Hinge(margin=2.0))
def test_averagedperceptron(self):
accuracy = get_accuracy(self, AveragedPerceptronBinaryClassifier())
# Accuracy depends on column Unnamed0 (index).
assert_greater(accuracy, 0.93,
"accuracy should be greater than %s" % 0.93)
'ToKeyImputer',
'ToString',
'EnsembleClassifier',
'EnsembleRegressor',
'CharTokenizer',
'WordTokenizer',
'MutualInformationSelector',
'NaiveBayesClassifier',
'CountSelector',
'KMeansPlusPlus',
'ToKey',
'ColumnSelector'
}
INSTANCES = {
'AveragedPerceptronBinaryClassifier': AveragedPerceptronBinaryClassifier(
feature=['education_str.0-5yrs', 'education_str.6-11yrs', 'education_str.12+ yrs']),
'Binner': Binner(num_bins=3),
'CharTokenizer': CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}),
'ColumnConcatenator': ColumnConcatenator(columns={'Features': [
'Sepal_Length',
'Sepal_Width',
'Petal_Length',
'Petal_Width',
'Setosa']}),
'ColumnSelector': ColumnSelector(columns=['Sepal_Width', 'Sepal_Length']),
'ColumnDuplicator': ColumnDuplicator(columns={'dup': 'Sepal_Width'}),
'CountSelector': CountSelector(count=5, columns=['Sepal_Width']),
'DateTimeSplitter': DateTimeSplitter(prefix='dt'),
'FastForestBinaryClassifier': FastForestBinaryClassifier(feature=['Sepal_Width', 'Sepal_Length'],
label='Setosa'),
'FastLinearBinaryClassifier': FastLinearBinaryClassifier(feature=['Sepal_Width', 'Sepal_Length'],
'ethnicity', 'sex', 'native-country-region'
]
file_schema = 'sep=, col=label:R4:0 col=Features:R4:9-14 col=workclass:TX:1 ' \
'col=education:TX:2 col=marital-status:TX:3 ' \
'col=occupation:TX:4 col=relationship:TX:5 col=ethnicity:TX:6 ' \
'col=sex:TX:7 col=native-country-region:TX:8 header+'
label_column = 'label'
learners = [
FastForestBinaryClassifier(),
FastForestRegressor(),
FastTreesBinaryClassifier(),
FastTreesRegressor(),
FastTreesTweedieRegressor(),
LightGbmRegressor(),
LightGbmBinaryClassifier(),
AveragedPerceptronBinaryClassifier(),
FastLinearBinaryClassifier(),
FastLinearClassifier(),
FastLinearRegressor(),
LogisticRegressionBinaryClassifier(),
LogisticRegressionClassifier(),
OnlineGradientDescentRegressor(),
SgdBinaryClassifier(),
# SymSgdBinaryClassifier(),
OrdinaryLeastSquaresRegressor(),
PoissonRegressionRegressor()
]
learners_not_supported = [
NaiveBayesClassifier(),
# fix in nimbusml, needs to implement ICanGetSummaryAsIDataView
def test_syntax_slots_wo_pipeline(self):
# data
df = get_dataset("infert").as_df()
df = df.drop(['row_num', ], axis=1)
X = df.drop('case', axis=1)
y = df['case']
# transform
xf1 = OneHotVectorizer(columns=['age', 'parity', 'education_str'])
X_xf1 = xf1.fit_transform(X, verbose=0)
assert "age.21" in list(X_xf1.columns)
# learner
# (1 .a.)
model = AveragedPerceptronBinaryClassifier()
# (1. b)
try:
model = AveragedPerceptronBinaryClassifier(feature=['age'])
model.fit(X_xf1, y, verbose=0)
cont = True
assert False
except Exception as e:
# does not work
cont = False
print(e)
if cont:
y_pred = model.predict(X_xf1)
assert y_pred.shape == (248, 3)
pipeline = Pipeline([
OneHotVectorizer(columns=['age', 'parity', 'education_str']),
AveragedPerceptronBinaryClassifier(feature='age')
])
pipeline.fit(X, y, verbose=0)
y_pred_withpipeline = pipeline.predict(X)
print(y_pred_withpipeline.head())
assert y_pred_withpipeline.shape == (248, 3)
metrics, scores = pipeline.test(X, y, output_scores=True)
print(metrics)
assert scores.shape == (248, 3)
assert metrics.shape == (1, 11)
# back to X_xf1
print(list(X_xf1.columns))
l1 = list(sorted(set(_.split('.')[-1] for _ in X_xf1.columns)))
levels = [['age', 'education', 'education_str', 'parity',
'pooled', 'spontaneous', 'stratum', 'induced'], [''] + l1]
names = ['columns', 'slots']
labels = [[], []]
ages = []
for _ in X_xf1.columns:
spl = _.split('.')
l1 = levels[0].index(spl[0])
try:
l2 = levels[1].index(spl[1])
except IndexError:
l2 = levels[1].index('')
labels[0].append(l1)
labels[1].append(l2)
if spl[0] == 'age':
ages.append(l2)
X_xf1.columns = pandas.MultiIndex(
levels=levels, labels=labels, names=names)
print(X_xf1.head(n=2).T)
col_ages = [('age', a) for a in ages]
print(col_ages)
try:
model = AveragedPerceptronBinaryClassifier(feature=col_ages)
model.fit(X_xf1, y, verbose=0)
y_pred = model.predict(X_xf1)
assert y_pred.shape == (248, 3)
except Exception as e:
# Does not work, probably confusion between list and tuple in nimbusml
print(e)
try:
model = AveragedPerceptronBinaryClassifier(feature=['age'])
model.fit(X_xf1, y, verbose=0)
y_pred = model.predict(X_xf1)
assert y_pred.shape == (248, 3)
except Exception as e:
# Does not work.
print(e)
from nimbusml.linear_model import AveragedPerceptronBinaryClassifier
# data input (as a FileDataStream)
path = get_dataset('infert').as_filepath()
data = FileDataStream.read_csv(path)
print(data.head())
# age case education induced parity ... row_num spontaneous ...
# 0 26 1 0-5yrs 1 6 ... 1 2 ...
# 1 42 1 0-5yrs 1 1 ... 2 0 ...
# 2 39 1 0-5yrs 2 6 ... 3 0 ...
# 3 34 1 0-5yrs 2 4 ... 4 0 ...
# 4 35 1 6-11yrs 1 3 ... 5 1 ...
# define the training pipeline
pipeline = Pipeline([
AveragedPerceptronBinaryClassifier(
feature=['age', 'parity', 'spontaneous'], label='case')
])
# train, predict, and evaluate
# TODO: Replace with CV
metrics, predictions = pipeline.fit(data).test(data, output_scores=True)
# print predictions
print(predictions.head())
# PredictedLabel Score
# 0 0 -0.285667
# 1 0 -1.304729
# 2 0 -2.651955
# 3 0 -2.111450
# 4 0 -0.660658
# print evaluation metrics