def test_weighting_effect(self):
data = sklearn.datasets.make_classification(
n_samples=1000, n_features=20, n_redundant=5, n_informative=5,
n_repeated=2, n_clusters_per_class=2, weights=[0.8, 0.2],
random_state=1)
for name, clf, acc_no_weighting, acc_weighting in \
[('adaboost', AdaboostClassifier, 0.709, 0.658),
('decision_tree', DecisionTree, 0.683, 0.701),
('extra_trees', ExtraTreesClassifier, 0.812, 0.8),
('gradient_boosting', GradientBoostingClassifier,
0.800, 0.760),
('random_forest', RandomForest, 0.846, 0.792),
('libsvm_svc', LibSVM_SVC, 0.571, 0.658),
('liblinear_svc', LibLinear_SVC, 0.685, 0.699),
('sgd', SGD, 0.65384615384615385, 0.38795986622073581)]:
for strategy, acc in [('none', acc_no_weighting),
('weighting', acc_weighting)]:
# Fit
data_ = copy.copy(data)
X_train = data_[0][:700]
Y_train = data_[1][:700]
X_test = data_[0][700:]
Y_test = data_[1][700:]
cs = SimpleClassificationPipeline.\
get_hyperparameter_search_space(
include={'classifier': [name]})
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(default, random_state=1)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(acc,
sklearn.metrics.f1_score(predictions, Y_test),
places=3)
# pre_transform and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:700]
Y_train = data_[1][:700]
X_test = data_[0][700:]
Y_test = data_[1][700:]
cs = SimpleClassificationPipeline.get_hyperparameter_search_space(
include={'classifier': [name]})
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(default, random_state=1)
Xt, fit_params = classifier.pre_transform(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(acc,
sklearn.metrics.f1_score(
predictions, Y_test),
places=3)
for name, pre, acc_no_weighting, acc_weighting in \
[('extra_trees_preproc_for_classification',
ExtraTreesPreprocessorClassification, 0.7142857142857143,
0.72180451127819545),
('liblinear_svc_preprocessor', LibLinear_Preprocessor,
0.5934065934065933, 0.71111111111111114)]:
for strategy, acc in [('none', acc_no_weighting),
('weighting', acc_weighting)]:
data_ = copy.copy(data)
X_train = data_[0][:700]
Y_train = data_[1][:700]
X_test = data_[0][700:]
Y_test = data_[1][700:]
cs = SimpleClassificationPipeline.get_hyperparameter_search_space(
include={'classifier': ['sgd'], 'preprocessor': [name]})
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(default, random_state=1)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(acc,
sklearn.metrics.f1_score(
predictions, Y_test),
places=3)
# pre_transform and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:700]
Y_train = data_[1][:700]
X_test = data_[0][700:]
Y_test = data_[1][700:]
cs = SimpleClassificationPipeline.get_hyperparameter_search_space(
include={'classifier': ['sgd'], 'preprocessor': [name]})
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(default, random_state=1)
Xt, fit_params = classifier.pre_transform(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(acc,
sklearn.metrics.f1_score(
predictions, Y_test),
places=3)
def test_weighting_effect(self):
data = sklearn.datasets.make_classification(
n_samples=200, n_features=10, n_redundant=2, n_informative=2,
n_repeated=2, n_clusters_per_class=2, weights=[0.8, 0.2],
random_state=1)
for name, clf, acc_no_weighting, acc_weighting in \
[('adaboost', AdaboostClassifier, 0.810, 0.735),
('decision_tree', DecisionTree, 0.780, 0.643),
('extra_trees', ExtraTreesClassifier, 0.75, 0.800),
('gradient_boosting', GradientBoostingClassifier,
0.789, 0.762),
('random_forest', RandomForest, 0.75, 0.821),
('libsvm_svc', LibSVM_SVC, 0.769, 0.72),
('liblinear_svc', LibLinear_SVC, 0.762, 0.735),
('sgd', SGD, 0.704, 0.667)
]:
for strategy, acc in [('none', acc_no_weighting),
('weighting', acc_weighting)]:
# Fit
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
include = {'classifier': [name],
'preprocessor': ['no_preprocessing']}
classifier = SimpleClassificationPipeline(
random_state=1, include=include)
cs = classifier.get_hyperparameter_search_space()
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3, msg=(name, strategy))
# fit_transformer and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
classifier.set_hyperparameters(configuration=default)
Xt, fit_params = classifier.fit_transformer(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3)
for name, pre, acc_no_weighting, acc_weighting in \
[('extra_trees_preproc_for_classification',
ExtraTreesPreprocessorClassification, 0.691, 0.692),
('liblinear_svc_preprocessor', LibLinear_Preprocessor,
0.692, 0.590)]:
for strategy, acc in [('none', acc_no_weighting),
('weighting', acc_weighting)]:
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
include = {'classifier': ['sgd'], 'preprocessor': [name]}
classifier = SimpleClassificationPipeline(
random_state=1, include=include)
cs = classifier.get_hyperparameter_search_space()
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier.set_hyperparameters(default)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3, msg=(name, strategy))
# fit_transformer and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
Xt, fit_params = classifier.fit_transformer(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3)
def test_weighting_effect(self):
data = sklearn.datasets.make_classification(
n_samples=200, n_features=10, n_redundant=2, n_informative=2,
n_repeated=2, n_clusters_per_class=2, weights=[0.8, 0.2],
random_state=1)
for name, clf, acc_no_weighting, acc_weighting, places in \
[('adaboost', AdaboostClassifier, 0.810, 0.735, 3),
('decision_tree', DecisionTree, 0.780, 0.643, 3),
('extra_trees', ExtraTreesClassifier, 0.780, 0.8, 3),
('gradient_boosting', GradientBoostingClassifier,
0.737, 0.684, 3),
('random_forest', RandomForest, 0.780, 0.789, 3),
('libsvm_svc', LibSVM_SVC, 0.769, 0.72, 3),
('liblinear_svc', LibLinear_SVC, 0.762, 0.735, 3),
('passive_aggressive', PassiveAggressive, 0.642, 0.449, 3),
('sgd', SGD, 0.818, 0.575, 2)
]:
for strategy, acc in [
('none', acc_no_weighting),
('weighting', acc_weighting)
]:
# Fit
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
include = {'classifier': [name],
'preprocessor': ['no_preprocessing']}
classifier = SimpleClassificationPipeline(
random_state=1, include=include)
cs = classifier.get_hyperparameter_search_space()
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=places, msg=(name, strategy))
# fit_transformer and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
classifier.set_hyperparameters(configuration=default)
Xt, fit_params = classifier.fit_transformer(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=places)
for name, pre, acc_no_weighting, acc_weighting in \
[('extra_trees_preproc_for_classification',
ExtraTreesPreprocessorClassification, 0.810, 0.563),
('liblinear_svc_preprocessor', LibLinear_Preprocessor,
0.837, 0.567)]:
for strategy, acc in [('none', acc_no_weighting),
('weighting', acc_weighting)]:
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
include = {'classifier': ['sgd'], 'preprocessor': [name]}
classifier = SimpleClassificationPipeline(
random_state=1, include=include)
cs = classifier.get_hyperparameter_search_space()
default = cs.get_default_configuration()
default._values['balancing:strategy'] = strategy
classifier.set_hyperparameters(default)
predictor = classifier.fit(X_train, Y_train)
predictions = predictor.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3, msg=(name, strategy))
# fit_transformer and fit_estimator
data_ = copy.copy(data)
X_train = data_[0][:100]
Y_train = data_[1][:100]
X_test = data_[0][100:]
Y_test = data_[1][100:]
default._values['balancing:strategy'] = strategy
classifier = SimpleClassificationPipeline(
default, random_state=1, include=include)
Xt, fit_params = classifier.fit_transformer(X_train, Y_train)
classifier.fit_estimator(Xt, Y_train, **fit_params)
predictions = classifier.predict(X_test)
self.assertAlmostEqual(
sklearn.metrics.f1_score(predictions, Y_test), acc,
places=3)
