def test_label_encoder_integration_sklearn_ensembles():
pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers(
encode_labels=['no', 'yes'])
knorau = KNORAU(pool_classifiers)
knorau.fit(X_dsel, y_dsel)
assert np.isclose(knorau.score(X_test, y_test), 0.97340425531914898)
def test_weights_zero():
knorau_test = KNORAU(create_pool_classifiers())
knorau_test.fit(X_dsel_ex1, y_dsel_ex1)
competences = np.zeros((1, 3))
result = knorau_test.select(competences)
assert np.all(result)
def test_select():
knorau_test = KNORAU(create_pool_classifiers())
competences = np.ones(3)
competences[0] = 0
expected = np.atleast_2d([False, True, True])
selected = knorau_test.select(competences)
assert np.array_equal(expected, selected)
def test_weights_zero():
query = np.atleast_2d([1, 1])
knora_u_test = KNORAU(create_pool_classifiers())
knora_u_test.estimate_competence = MagicMock(return_value=np.zeros(3))
result = knora_u_test.select(query)
assert np.array_equal(result, np.array([0, 1, 0]))
def test_estimate_competence_batch():
query = np.ones((3, 2))
expected = np.array([[4.0, 3.0, 4.0], [5.0, 2.0, 5.0], [2.0, 5.0, 2.0]])
knora_u_test = KNORAU(create_pool_classifiers())
knora_u_test.fit(X_dsel_ex1, y_dsel_ex1)
neighbors = neighbors_ex1
competences = knora_u_test.estimate_competence(query, neighbors)
assert np.allclose(competences, expected, atol=0.01)
def test_estimate_competence_batch(example_estimate_competence,
create_pool_classifiers):
X, y, neighbors = example_estimate_competence[0:3]
expected = np.array([[4.0, 3.0, 4.0], [5.0, 2.0, 5.0], [2.0, 5.0, 2.0]])
knora_u_test = KNORAU(create_pool_classifiers)
knora_u_test.fit(X, y)
competences = knora_u_test.estimate_competence(neighbors)
assert np.allclose(competences, expected, atol=0.01)
def test_label_encoder_integration_list_classifiers():
rng = np.random.RandomState(123456)
X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset(encode_labels=['no', 'yes'], rng=rng)
pool_classifiers = [LogisticRegression(), SVC(probability=True)]
[clf.fit(X_train, y_train) for clf in pool_classifiers]
knorau = KNORAU(pool_classifiers)
knorau.fit(X_dsel, y_dsel)
this_score = knorau.score(X_test, y_test)
assert np.isclose(this_score, 0.9574468085106383)
def test_label_encoder_integration_sklearn_ensembles_not_encoding():
rng = np.random.RandomState(123456)
X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset(
['yes', 'no'], rng)
# Train a pool of using adaboost which has label encoding problems.
pool_classifiers = AdaBoostClassifier(n_estimators=10, random_state=rng)
pool_classifiers.fit(X_train, y_train)
knorau = KNORAU(pool_classifiers)
knorau.fit(X_dsel, y_dsel)
assert np.isclose(knorau.score(X_test, y_test), 0.9521276595744681)
def DES(self, x_train, y_train,X_test, Y_test, dsel):
pool_classifiers = BaggingClassifier(linear_model.Perceptron(max_iter=5), self.pool_size)
pool_classifiers.fit(x_train, y_train)
# Initialize the DES model
knorae = KNORAE(pool_classifiers)
knorau = KNORAU(pool_classifiers)
# Preprocess the Dynamic Selection dataset (DSEL)
score1 = knorae.fit(x_train[dsel], y_train[dsel])
score2 = knorau.fit(x_train[dsel], y_train[dsel])
# Predict new examples:
# print (knorae.score(X_test, Y_test), knorau.score(X_test, Y_test))
return (score1, score2, ) + self.calc_metrics(X_test, Y_test)
def escolher_modelo(nome, x_sel, y_sel, P, k):
'''
metodo para chamar o tipo de DS
:param: x_sel: dados de treinamento da janela de validacao
:param: y_sel: rotulos da janela de validacao
:param: P: pool de classificadores
:param: k: vizinhanca
'''
# escolhendo a tecnica de selecao de classificadores
if(nome=='OLA'):
DS = OLA(P, k)
number_model = 0
elif(nome=='LCA'):
DS = LCA(P, k)
number_model = 1
elif(nome=='KNORAE'):
DS = KNORAE(P, k)
number_model = 2
elif(nome=='KNORAU'):
DS = KNORAU(P, k)
number_model = 3
# encontrando os classificadores competentes do DS escolhido
DS.fit(x_sel, y_sel)
# retornando a tecnica de DS
return DS, number_model
def fit(self, x_sel, y_sel, P, k):
'''
metodo para chamar o tipo de DS
:param: x_sel: dados de treinamento da janela de validacao
:param: y_sel: rotulos da janela de validacao
:param: P: pool de classificadores
:param: k: vizinhanca
'''
# escolhendo a tecnica de selecao de classificadores
if (self.TYPE == 'knorae'):
DS = KNORAE(P, k)
elif (self.TYPE == 'knorau'):
DS = KNORAU(P, k)
elif (self.TYPE == 'ola'):
DS = OLA(P, k)
elif (self.TYPE == 'lca'):
DS = LCA(P, k)
elif (self.TYPE == 'posteriori'):
DS = APosteriori(P, k)
elif (self.TYPE == 'priori'):
DS = APriori(P, k)
# encontrando os classificadores competentes do DS escolhido
self.DS = copy.deepcopy(DS)
self.DS.fit(x_sel, y_sel)
def __init__(
self,
name: str,
model_params: Dict[str, Any],
classifier_paths: Iterable[Tuple[str, str]],
) -> None:
super().__init__(name, model_params, classifier_paths)
self._selector = KNORAU(self.classifiers, **model_params)
def test_estimate_competence(index, expected):
query = np.atleast_2d([1, 1])
knora_u_test = KNORAU(create_pool_classifiers())
knora_u_test.fit(X_dsel_ex1, y_dsel_ex1)
knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers)
knora_u_test.neighbors = neighbors_ex1[index, :]
knora_u_test.distances = distances_ex1[index, :]
competences = knora_u_test.estimate_competence(query)
assert np.isclose(competences, expected, atol=0.01).all()
def test_classify(index, expected):
query = np.atleast_2d([1, 1])
knora_u_test = KNORAU(create_pool_classifiers())
knora_u_test.fit(X_dsel_ex1, y_dsel_ex1)
knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers)
knora_u_test.neighbors = neighbors_ex1[index, :]
knora_u_test.distances = distances_ex1[index, :]
prediction = knora_u_test.classify_instance(query)
assert prediction == expected
def fit(self, x, y):
'''
metodo para treinar a arquitetura de dois niveis
:x: dados para treinamento
:y: rotulo dos dados
:dsel_x: padroes da janela de validacao
:dsel_y: rotulos da janela de validacao
'''
# salvando os dados de trainamento
self.x_train = x
self.y_train = y
# salvando as dificuldades das instancias
self.H = self.kDN(x, y)
# treinando o nivel 1 #########################################
self.levelone = KNeighborsClassifier(self.n_vizinhos)
self.levelone.fit(x, y)
# realizando a previsao para o conjunto de treinamento
y_pred = self.levelone.predict(x)
# salvando os indices das instancias que foram classificadas erradas
indices = [i for i in range(len(y)) if y_pred[i] != y[i]]
# obtendo o limiar de dificuldade do problema
self.limiar = self.defineThreshold(indices)
###############################################################
# treinando o nivel 2 #########################################
# obtendo as instancias dificeis
x_dificeis, y_dificeis = self.hardInstances(x, y, self.limiar)
# criando o ensemble
self.ensemble = BaggingClassifier(base_estimator=Perceptron(),
max_samples=0.9,
max_features=1.0,
n_estimators=100)
self.ensemble.fit(x_dificeis, y_dificeis)
# treinando o modelo 2
self.leveltwo = KNORAU(self.ensemble.estimators_, self.n_vizinhos)
self.leveltwo.fit(x_dificeis, y_dificeis)
def train(train_index, test_index):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = Y[train_index], Y[test_index]
#train_perc = 0.7
#split_point = int(train_perc*len(train_index))
# valid_index = train_index[split_point:]
# train_index = train_index[:split_point]
# X_train, X_valid, X_test = X[train_index], X[valid_index], X[test_index]
# y_train, y_valid, y_test = Y[train_index], Y[valid_index], Y[test_index]
#print("TRAIN:", train_index, "VALID:", valid_index, "TEST:", test_index)
X_train, X_valid, y_train, y_valid = train_test_split(
X_train, y_train, test_size=0.3, random_state=seed)
pool_classifiers.fit(X_train, y_train)
validation_data, validation_labels = get_validation_data(
X_valid, y_valid, 0.5, hardness=hardness)
dynamic_selection_algorithm = None
try:
if args.dynamic_selection == True and args.dynamic_algorithm is None:
raise ValueError(
'Dynamic selection requires you provide an algorithm.')
elif args.dynamic_selection == True and args.dynamic_algorithm is not None:
if args.dynamic_algorithm == 'ola':
dynamic_selection_algorithm = OLA(pool_classifiers,
random_state=seed)
elif args.dynamic_algorithm == 'lca':
dynamic_selection_algorithm = LCA(pool_classifiers,
random_state=seed)
elif args.dynamic_algorithm == 'mcb':
dynamic_selection_algorithm = MCB(pool_classifiers,
random_state=seed)
elif args.dynamic_algorithm == 'knorau':
dynamic_selection_algorithm = KNORAU(pool_classifiers,
random_state=seed)
elif args.dynamic_algorithm == 'kne':
dynamic_selection_algorithm = KNORAE(pool_classifiers,
random_state=seed)
dynamic_selection_algorithm.fit(validation_data,
validation_labels)
preds = dynamic_selection_algorithm.predict(X_test)
else:
# Static combination by voting
preds = voting(X_test, pool_classifiers)
except Exception as error:
raise error
acc = get_accuracy_score(y_test, preds)
g1 = get_g1_score(y_test, preds, average='macro')
f1 = get_f1_score(y_test, preds)
roc = roc_auc_score(y_test, preds, average='macro')
return dict(f1=f1, g1=g1, acc=acc, roc=roc)
def gustavo_method(self, x_train, y_train, X_test, Y_test, dsel):
neigh = KNeighborsClassifier(n_neighbors=self.kNeighbors)
neigh.fit(self.x[dsel], self.y[dsel])
pool_classifiers = BaggingClassifier(linear_model.Perceptron(max_iter=5), self.pool_size)
pool_classifiers.fit(x_train, y_train)
knorau = KNORAU(pool_classifiers)
knorau.fit(x_train[dsel], y_train[dsel])
knn = KNeighborsClassifier(n_neighbors=5)
knn.fit(x_train, y_train)
score = []
for i in range(len(Y_test)):
kdn = sum(self.y[neigh.kneighbors([self.x[dsel]], return_distance=False)[0]]!=self.y[i])/self.kNeighbors
if (kdn > self.threshold):
score.append(knorau.predict(self.x[i]))
else:
score.append(knn.predict(self.x[i]))
return (score, ) + self.calc_metrics(X_test, Y_test) # dependendo da base formato nao suportado
def test_classify(index, expected):
query = np.atleast_2d([1, 1])
knora_u_test = KNORAU(create_pool_classifiers())
knora_u_test.fit(X_dsel_ex1, y_dsel_ex1)
knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers)
knora_u_test.neighbors = neighbors_ex1[index, :]
knora_u_test.distances = distances_ex1[index, :]
predictions = []
for clf in knora_u_test.pool_classifiers:
predictions.append(clf.predict(query)[0])
prediction = knora_u_test.classify_instance(query, np.array(predictions))
assert prediction == expected
def ensemble_model(self, ensemble=None):
if ensemble is not None:
self.ensemble = ensemble
if self.moo_ is None:
self.moo_ = monise(weightedScalar=self.scalarization,
singleScalar=self.scalarization,
nodeTimeLimit=2, targetSize=150,
targetGap=0, nodeGap=0.01, norm=False)
self.moo_.optimize()
self.solutions_ = []
for solution in self.moo_.solutionsList:
self.solutions_.append(solution.x)
if self.solutions_ is None:
self.solutions_ = []
for solution in self.moo_.solutionsList:
self.solutions_.append(solution.x)
if self.ensemble in ['voting', 'voting hard']:
models_t = [
("Model " + str(i), self.solutions_[i])
for i in range(len(self.solutions_))
]
ensemble_model = SimpleVoting(estimators=models_t)
if self.ensemble == 'voting soft':
models_t = [
("Model " + str(i), self.solutions_[i])
for i in range(len(self.solutions_))
]
ensemble_model = SimpleVoting(estimators=models_t, voting='soft')
if self.ensemble == 'knorau':
ensemble_model = KNORAU(self.solutions_)
ensemble_model.fit(self.X_val, self.y_val)
if self.ensemble == 'knorae':
ensemble_model = KNORAE(self.solutions_)
ensemble_model.fit(self.X_val, self.y_val)
return ensemble_model
def test_knorau():
pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers()
knorau = KNORAU(pool_classifiers, DFP=True, with_IH=True, IH_rate=0.1)
knorau.fit(X_dsel, y_dsel)
assert np.isclose(knorau.score(X_test, y_test), 0.9090909090909091)
#------------------- KONARAE TRAINING --------------------------------------
train_time_kne_start = time.time()
kne = KNORAE(pool_classifiers)
kne.fit(X_dsel, y_dsel)
train_time_kne_end = time.time()
#------------------- META DES TRAINING -------------------------------------
train_time_mdes_start = time.time()
meta = METADES(pool_classifiers)
meta.fit(X_dsel, y_dsel)
train_time_mdes_end = time.time()
#------------------- KONARAU TRAINING --------------------------------------
train_time_knu_start = time.time()
knu = KNORAU(pool_classifiers)
knu.fit(X_dsel, y_dsel)
train_time_knu_end = time.time()
#---------------------- Test KNE Pharse --------------------------
test_time_kne_start = time.time()
y_kne_pred = kne.predict(X_test)
accuracy_kne = accuracy_score(Y_test, y_kne_pred)
# print('accuracy = ', accuracy_kne)
micro_f1_kne = f1_score(Y_test - 1, y_kne_pred - 1, average='micro')
# print('micro_f1 =', micro_f1_kne)
macro_f1_kne = f1_score(Y_test - 1, y_kne_pred - 1, average='macro')
# print('macro_f1 =', macro_f1_kne)
class Arquitetura:
def __init__(self, n_vizinhos):
'''
:n_vizinhos: quantidade de vizinhos mais proximos que serao utilizados para regiao de competencia
'''
self.n_vizinhos = n_vizinhos
def kDN(self, x, y):
'''
Metodo para computar o grau de dificuldade de cada observacao em um conjunto de dados
:param: x: padroes dos dados
:param: y: respectivos rotulos
:return: dificuldades: vetor com a probabilidade de cada instancia
'''
# instanciando os vizinhos mais proximos
nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1,
algorithm='ball_tree').fit(x)
# variavel para salvar as probabilidades
dificuldades = []
# for para cada instancia do dataset
for i in range(len(x)):
# computando os vizinhos mais proximos para cada instancia
_, indices = nbrs.kneighbors([x[i]])
# verificando o rotulo dos vizinhos
cont = 0
for j in indices[0]:
if (j != i and y[j] != y[i]):
cont += 1
# computando a porcentagem
dificuldades.append(cont / (self.n_vizinhos + 1))
return dificuldades
def neighbors(self, dsel, x_query):
'''
metodo para retornar apenas os indices dos vizinhos
'''
# instanciando os vizinhos mais proximos
nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1,
algorithm='ball_tree').fit(dsel)
# computando os vizinhos mais proximos para cada instancia
_, indices = nbrs.kneighbors([x_query])
return indices
def hardInstances(self, x, y, limiar):
'''
Metodo para retornar um subconjunto de validacao apenas com as instacias faceis
:param: x: padroes dos dados
:param: y: respectivos rotulos
:return: x_new, y_new:
'''
# computando as dificulades para cada instancia
dificuldades = self.kDN(x, y)
# variaveis para salvar as novas instancias
x_new = []
y_new = []
# salvando apenas as instancias faceis
for i in range(len(dificuldades)):
if (dificuldades[i] > limiar):
x_new.append(x[i])
y_new.append(y[i])
return np.asarray(x_new), np.asarray(y_new)
def neighborhoodDifficulty(self, dsel, x_query, H):
'''
metodo para calcular o grau de dificuldade da vizinhanca
:dsel: dataset para pesquisar os vizinhos
:x_query: instancia a ser pesquisada
:H: dificuldade do dataset dsel
'''
# obtendo a vizinhanca do exemplo
indices = self.neighbors(dsel, x_query)[0]
# dificuldade da regiao
dificuldades = [H[i] for i in indices]
# media da dificuldadde da regiao
return np.min(dificuldades)
def defineThreshold(self, indices):
'''
Metodo para definir o threshold
:indices: os indices das instancias que foram classificadas incorretamente
'''
# obtendo a vizinhanca do exemplo
lista = []
for i in indices:
lista.append(
self.neighborhoodDifficulty(self.x_train, self.x_train[i],
self.H))
return np.mean(lista)
def fit(self, x, y):
'''
metodo para treinar a arquitetura de dois niveis
:x: dados para treinamento
:y: rotulo dos dados
:dsel_x: padroes da janela de validacao
:dsel_y: rotulos da janela de validacao
'''
# salvando os dados de trainamento
self.x_train = x
self.y_train = y
# salvando as dificuldades das instancias
self.H = self.kDN(x, y)
# treinando o nivel 1 #########################################
self.levelone = KNeighborsClassifier(self.n_vizinhos)
self.levelone.fit(x, y)
# realizando a previsao para o conjunto de treinamento
y_pred = self.levelone.predict(x)
# salvando os indices das instancias que foram classificadas erradas
indices = [i for i in range(len(y)) if y_pred[i] != y[i]]
# obtendo o limiar de dificuldade do problema
self.limiar = self.defineThreshold(indices)
###############################################################
# treinando o nivel 2 #########################################
# obtendo as instancias dificeis
x_dificeis, y_dificeis = self.hardInstances(x, y, self.limiar)
# criando o ensemble
self.ensemble = BaggingClassifier(base_estimator=Perceptron(),
max_samples=0.9,
max_features=1.0,
n_estimators=100)
self.ensemble.fit(x_dificeis, y_dificeis)
# treinando o modelo 2
self.leveltwo = KNORAU(self.ensemble.estimators_, self.n_vizinhos)
self.leveltwo.fit(x_dificeis, y_dificeis)
# verificando se o ola acerta os exemplos errados pelo svm
###############################################################
def predict_svm(self, x):
# to predict multiple examples
if (len(x.shape) > 1):
# returning all labels
return [
self.levelone.predict(np.array([pattern]))[0] for pattern in x
]
# to predict only one example
else:
return self.levelone.predict(np.array([x]))[0]
def predict_ola(self, x):
# to predict multiple examples
if (len(x.shape) > 1):
# returning all labels
return [
self.leveltwo.predict(np.array([pattern]))[0] for pattern in x
]
# to predict only one example
else:
return self.leveltwo.predict(np.array([x]))[0]
def predict_one(self, x):
'''
metodo para computar a previsao de um exemplo
:x: padrao a ser predito
'''
# media da dificuldadde da regiao
media = self.neighborhoodDifficulty(self.x_train, x, self.H)
# verificando a dificuldade da instancia
if (media >= self.limiar):
return self.leveltwo.predict(np.array([x]))[0]
else:
return self.levelone.predict(np.array([x]))[0]
def predict(self, x):
'''
metodo para computar a previsao de um exemplo
:x: padrao a ser predito
'''
# to predict multiple examples
if (len(x.shape) > 1):
# returning all labels
return [self.predict_one(pattern) for pattern in x]
# to predict only one example
else:
return self.predict_one(x)
def test_predict_proba():
X = X_dsel_ex1
y = y_dsel_ex1
clf1 = Perceptron()
clf1.fit(X, y)
KNORAU([clf1, clf1])
def test_predict_proba(create_X_y):
X, y = create_X_y
clf1 = Perceptron()
clf1.fit(X, y)
KNORAU([clf1, clf1]).fit(X, y)
def test_weights_zero():
knorau_test = KNORAU()
competences = np.zeros((1, 3))
result = knorau_test.select(competences)
assert np.all(result)
def test_knorau(knn_methods):
pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers()
knorau = KNORAU(pool_classifiers, knn_classifier=knn_methods)
knorau.fit(X_dsel, y_dsel)
assert np.isclose(knorau.score(X_test, y_test), 0.97340425531914898)
# Split the data into training and DSEL for DS techniques
X_train, X_dsel, y_train, y_dsel = train_test_split(X_train,
y_train,
test_size=0.5)
# Considering a pool composed of 10 base classifiers
# Calibrating Perceptrons to estimate probabilities
model = CalibratedClassifierCV(Perceptron(max_iter=100))
# Train a pool of 10 classifiers
pool_classifiers = BaggingClassifier(model, n_estimators=100)
pool_classifiers.fit(X_train, y_train)
# Initialize the DS techniques
knorau = KNORAU(pool_classifiers)
kne = KNORAE(pool_classifiers)
desp = DESP(pool_classifiers)
ola = OLA(pool_classifiers)
mcb = MCB(pool_classifiers)
apriori = APriori(pool_classifiers)
meta = METADES(pool_classifiers)
# Fit the des techniques
knorau.fit(X_dsel, y_dsel)
kne.fit(X_dsel, y_dsel)
desp.fit(X_dsel, y_dsel)
# Fit the dcs techniques
ola.fit(X_dsel, y_dsel)
mcb.fit(X_dsel, y_dsel)
X = data.data
y = data.target
# split the data into training and test data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
RF = RandomForestClassifier()
RF.fit(X_train, y_train)
X_train, X_dsel, y_train, y_dsel = train_test_split(X, y, test_size=0.50)
# Training a random forest to be used as the pool of classifiers. We set the maximum depth of the tree so that it
# can estimate probabilities
pool_classifiers = RandomForestClassifier(n_estimators=10, max_depth=5)
pool_classifiers.fit(X_train, y_train)
# Initialize a DS technique. Here we specify the size of the region of competence (5 neighbors)
knorau = KNORAU(pool_classifiers)
kne = KNORAE(pool_classifiers, k=5)
desp = DESP(pool_classifiers, k=5)
ola = OLA(pool_classifiers, k=5)
mcb = MCB(pool_classifiers, k=5)
meta = METADES(pool_classifiers, k=5)
# Fit the DS techniques
knorau.fit(X_dsel, y_dsel)
kne.fit(X_dsel, y_dsel)
desp.fit(X_dsel, y_dsel)
meta.fit(X_dsel, y_dsel)
ola.fit(X_dsel, y_dsel)
mcb.fit(X_dsel, y_dsel)
# Calculate classification accuracy of each technique
def test_knorau():
pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers()
knorau = KNORAU(pool_classifiers, DFP=True)
knorau.fit(X_dsel, y_dsel)
assert np.isclose(knorau.score(X_test, y_test), 0.90606060606060601)
test_size=0.50,
random_state=rng)
# Training a random forest to be used as the pool of classifiers.
# We set the maximum depth of the tree so that it
# can estimate probabilities
pool_classifiers = RandomForestClassifier(n_estimators=100, max_depth=5,
random_state=rng)
pool_classifiers.fit(X_train, y_train)
stacked = StackedClassifier(pool_classifiers, LogisticRegression())
stacked.fit(X_dsel, y_dsel)
# Initialize a DS technique. Here we specify the size of
# the region of competence (5 neighbors)
knorau = KNORAU(pool_classifiers, random_state=rng)
kne = KNORAE(pool_classifiers, k=5, random_state=rng)
desp = DESP(pool_classifiers, k=5, random_state=rng)
ola = OLA(pool_classifiers, k=5, random_state=rng)
mcb = MCB(pool_classifiers, k=5, random_state=rng)
meta = METADES(pool_classifiers, k=5, random_state=rng)
# Fit the DS techniques
knorau.fit(X_dsel, y_dsel)
kne.fit(X_dsel, y_dsel)
desp.fit(X_dsel, y_dsel)
meta.fit(X_dsel, y_dsel)
ola.fit(X_dsel, y_dsel)
mcb.fit(X_dsel, y_dsel)
###############################################################################