def test_meta(knn_methods, voting):
pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers()
meta_des = METADES(pool_classifiers,
knn_classifier=knn_methods,
voting=voting)
meta_des.fit(X_dsel, y_dsel)
assert np.isclose(meta_des.score(X_test, y_test), 0.9787234042553191)
def test_meta_no_pool_of_classifiers():
rng = np.random.RandomState(123456)
data = load_breast_cancer()
X = data.data
y = data.target
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=rng)
scalar = StandardScaler()
X_train = scalar.fit_transform(X_train)
X_test = scalar.transform(X_test)
meta_des = METADES(random_state=rng, DSEL_perc=0.5)
meta_des.fit(X_train, y_train)
assert np.isclose(meta_des.score(X_test, y_test), 0.8936170212765957)
def initialize_ds(pool_classifiers, X, y, k=5):
knorau = KNORAU(pool_classifiers, k=k)
kne = KNORAE(pool_classifiers, k=k)
desknn = DESKNN(pool_classifiers, k=k)
ola = OLA(pool_classifiers, k=k)
lca = LCA(pool_classifiers, k=k)
mla = MLA(pool_classifiers, k=k)
mcb = MCB(pool_classifiers, k=k)
rank = Rank(pool_classifiers, k=k)
knop = KNOP(pool_classifiers, k=k)
meta = METADES(pool_classifiers, k=k)
list_ds = [knorau, kne, ola, lca, mla, desknn, mcb, rank, knop, meta]
names = [
'KNORA-U', 'KNORA-E', 'OLA', 'LCA', 'MLA', 'DESKNN', 'MCB', 'RANK',
'KNOP', 'META-DES'
]
# fit the ds techniques
for ds in list_ds:
ds.fit(X, y)
return list_ds, names
pool_classifiers = [
model_perceptron, model_svc, model_bayes, model_tree, model_knn
]
voting_classifiers = [("perceptron", model_perceptron), ("svc", model_svc),
("bayes", model_bayes), ("tree", model_tree),
("knn", model_knn)]
model_voting = VotingClassifier(estimators=voting_classifiers).fit(
X_train, y_train)
# Initializing the techniques
knorau = KNORAU(pool_classifiers)
kne = KNORAE(pool_classifiers)
desp = DESP(pool_classifiers)
metades = METADES(pool_classifiers, mode='hybrid')
# DCS techniques
ola = OLA(pool_classifiers)
mcb = MCB(pool_classifiers)
##############################################################################
# Adding stacked classifier as baseline comparison. Stacked classifier can
# be found in the static module. In this experiment we consider two types
# of stacking: one using logistic regression as meta-classifier
# (default configuration) and the other using a Decision Tree.
stacked_lr = StackedClassifier(pool_classifiers, random_state=rng)
stacked_dt = StackedClassifier(pool_classifiers,
random_state=rng,
meta_classifier=DecisionTreeClassifier())
# Fitting the DS techniques
knorau.fit(X_dsel, y_dsel)
# DRBM
logistic = linear_model.LogisticRegression()
rbm = BernoulliRBM(n_components=5, random_state=rng)
model_drbm = Pipeline(steps=[('rbm',
rbm), ('logistic', logistic)])
model_drbm.fit(X_train, Y_train)
# DRBM
pool_classifiers = [
model_lda, model_nb, model_knn, model_nmc, model_dt,
model_l2svm, model_drbm
]
print('Num Classifiers = {}'.format(len(pool_classifiers)))
metades = METADES(pool_classifiers)
metades.fit(X_dev, Y_dev)
accuracy = metades.score(X_test, Y_test)
Y_pred = metades.predict(X_test)
support_macro = precision_recall_fscore_support(Y_test,
Y_pred,
average='macro')
support_micro = precision_recall_fscore_support(Y_test,
Y_pred,
average='micro')
print('Accuracy at loop {}: {} '.format(current_loop + 1,
accuracy))
pool_classifiers.fit(X_train, y_train)
# Setting up static methods.
stacked = StackedClassifier(pool_classifiers)
static_selection = StaticSelection(pool_classifiers)
single_best = SingleBest(pool_classifiers)
# Initialize a DS technique. Here we specify the size of
# the region of competence (5 neighbors)
knorau = KNORAU(pool_classifiers, random_state=rng)
kne = KNORAE(pool_classifiers, random_state=rng)
desp = DESP(pool_classifiers, random_state=rng)
ola = OLA(pool_classifiers, random_state=rng)
mcb = MCB(pool_classifiers, random_state=rng)
knop = KNOP(pool_classifiers, random_state=rng)
meta = METADES(pool_classifiers, random_state=rng)
names = [
'Single Best', 'Static Selection', 'Stacked', 'KNORA-U', 'KNORA-E',
'DES-P', 'OLA', 'MCB', 'KNOP', 'META-DES'
]
methods = [
single_best, static_selection, stacked, knorau, kne, desp, ola, mcb, knop,
meta
]
# Fit the DS techniques
scores = []
for method, name in zip(methods, names):
method.fit(X_dsel, y_dsel)
random_state=rng)
# Considering a pool composed of 10 base classifiers
pool_classifiers = RandomForestClassifier(n_estimators=10,
random_state=rng,
max_depth=10)
pool_classifiers.fit(X_train, y_train)
ds_names = ['A Priori', 'A Posteriori', 'OLA', 'LCA', 'DES-P', 'META-DES']
# DS techniques without DFP
apriori = APriori(pool_classifiers, random_state=rng)
aposteriori = APosteriori(pool_classifiers, random_state=rng)
ola = OLA(pool_classifiers)
lca = LCA(pool_classifiers)
desp = DESP(pool_classifiers)
meta = METADES(pool_classifiers)
# FIRE-DS techniques (with DFP)
fire_apriori = APriori(pool_classifiers, DFP=True, random_state=rng)
fire_aposteriori = APosteriori(pool_classifiers, DFP=True, random_state=rng)
fire_ola = OLA(pool_classifiers, DFP=True)
fire_lca = LCA(pool_classifiers, DFP=True)
fire_desp = DESP(pool_classifiers, DFP=True)
fire_meta = METADES(pool_classifiers, DFP=True)
list_ds = [apriori, aposteriori, ola, lca, desp, meta]
list_fire_ds = [
fire_apriori, fire_aposteriori, fire_ola, fire_lca, fire_desp, fire_meta
]
scores_ds = []
# Setting up the random state to have consistent results
rng = np.random.RandomState(42)
# Generate a classification dataset
X, y = make_classification(n_samples=1000, random_state=rng)
# split the data into training and test data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33,
random_state=rng)
# Split the data into training and DSEL for DS techniques
X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train,
test_size=0.5,
random_state=rng)
# Initialize the DS techniques. DS methods can be initialized without
# specifying a single input parameter. In this example, we just pass the random
# state in order to always have the same result.
kne = KNORAE(random_state=rng)
meta = METADES(random_state=rng)
# Fitting the des techniques
kne.fit(X_dsel, y_dsel)
meta.fit(X_dsel, y_dsel)
# Calculate classification accuracy of each technique
print('Evaluating DS techniques:')
print('Classification accuracy KNORA-Eliminate: ',
kne.score(X_test, y_test))
print('Classification accuracy META-DES: ', meta.score(X_test, y_test))