def test_label_encoder_integration_sklearn_ensembles(): pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers( encode_labels=['no', 'yes']) knorau = KNORAU(pool_classifiers) knorau.fit(X_dsel, y_dsel) assert np.isclose(knorau.score(X_test, y_test), 0.97340425531914898)
def test_weights_zero(): knorau_test = KNORAU(create_pool_classifiers()) knorau_test.fit(X_dsel_ex1, y_dsel_ex1) competences = np.zeros((1, 3)) result = knorau_test.select(competences) assert np.all(result)
def test_select(): knorau_test = KNORAU(create_pool_classifiers()) competences = np.ones(3) competences[0] = 0 expected = np.atleast_2d([False, True, True]) selected = knorau_test.select(competences) assert np.array_equal(expected, selected)
def test_weights_zero(): query = np.atleast_2d([1, 1]) knora_u_test = KNORAU(create_pool_classifiers()) knora_u_test.estimate_competence = MagicMock(return_value=np.zeros(3)) result = knora_u_test.select(query) assert np.array_equal(result, np.array([0, 1, 0]))
def test_estimate_competence_batch(): query = np.ones((3, 2)) expected = np.array([[4.0, 3.0, 4.0], [5.0, 2.0, 5.0], [2.0, 5.0, 2.0]]) knora_u_test = KNORAU(create_pool_classifiers()) knora_u_test.fit(X_dsel_ex1, y_dsel_ex1) neighbors = neighbors_ex1 competences = knora_u_test.estimate_competence(query, neighbors) assert np.allclose(competences, expected, atol=0.01)
def test_estimate_competence_batch(example_estimate_competence, create_pool_classifiers): X, y, neighbors = example_estimate_competence[0:3] expected = np.array([[4.0, 3.0, 4.0], [5.0, 2.0, 5.0], [2.0, 5.0, 2.0]]) knora_u_test = KNORAU(create_pool_classifiers) knora_u_test.fit(X, y) competences = knora_u_test.estimate_competence(neighbors) assert np.allclose(competences, expected, atol=0.01)
def test_label_encoder_integration_list_classifiers(): rng = np.random.RandomState(123456) X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset(encode_labels=['no', 'yes'], rng=rng) pool_classifiers = [LogisticRegression(), SVC(probability=True)] [clf.fit(X_train, y_train) for clf in pool_classifiers] knorau = KNORAU(pool_classifiers) knorau.fit(X_dsel, y_dsel) this_score = knorau.score(X_test, y_test) assert np.isclose(this_score, 0.9574468085106383)
def test_label_encoder_integration_sklearn_ensembles_not_encoding(): rng = np.random.RandomState(123456) X_dsel, X_test, X_train, y_dsel, y_test, y_train = load_dataset( ['yes', 'no'], rng) # Train a pool of using adaboost which has label encoding problems. pool_classifiers = AdaBoostClassifier(n_estimators=10, random_state=rng) pool_classifiers.fit(X_train, y_train) knorau = KNORAU(pool_classifiers) knorau.fit(X_dsel, y_dsel) assert np.isclose(knorau.score(X_test, y_test), 0.9521276595744681)
def DES(self, x_train, y_train,X_test, Y_test, dsel): pool_classifiers = BaggingClassifier(linear_model.Perceptron(max_iter=5), self.pool_size) pool_classifiers.fit(x_train, y_train) # Initialize the DES model knorae = KNORAE(pool_classifiers) knorau = KNORAU(pool_classifiers) # Preprocess the Dynamic Selection dataset (DSEL) score1 = knorae.fit(x_train[dsel], y_train[dsel]) score2 = knorau.fit(x_train[dsel], y_train[dsel]) # Predict new examples: # print (knorae.score(X_test, Y_test), knorau.score(X_test, Y_test)) return (score1, score2, ) + self.calc_metrics(X_test, Y_test)
def escolher_modelo(nome, x_sel, y_sel, P, k): ''' metodo para chamar o tipo de DS :param: x_sel: dados de treinamento da janela de validacao :param: y_sel: rotulos da janela de validacao :param: P: pool de classificadores :param: k: vizinhanca ''' # escolhendo a tecnica de selecao de classificadores if(nome=='OLA'): DS = OLA(P, k) number_model = 0 elif(nome=='LCA'): DS = LCA(P, k) number_model = 1 elif(nome=='KNORAE'): DS = KNORAE(P, k) number_model = 2 elif(nome=='KNORAU'): DS = KNORAU(P, k) number_model = 3 # encontrando os classificadores competentes do DS escolhido DS.fit(x_sel, y_sel) # retornando a tecnica de DS return DS, number_model
def fit(self, x_sel, y_sel, P, k): ''' metodo para chamar o tipo de DS :param: x_sel: dados de treinamento da janela de validacao :param: y_sel: rotulos da janela de validacao :param: P: pool de classificadores :param: k: vizinhanca ''' # escolhendo a tecnica de selecao de classificadores if (self.TYPE == 'knorae'): DS = KNORAE(P, k) elif (self.TYPE == 'knorau'): DS = KNORAU(P, k) elif (self.TYPE == 'ola'): DS = OLA(P, k) elif (self.TYPE == 'lca'): DS = LCA(P, k) elif (self.TYPE == 'posteriori'): DS = APosteriori(P, k) elif (self.TYPE == 'priori'): DS = APriori(P, k) # encontrando os classificadores competentes do DS escolhido self.DS = copy.deepcopy(DS) self.DS.fit(x_sel, y_sel)
def __init__( self, name: str, model_params: Dict[str, Any], classifier_paths: Iterable[Tuple[str, str]], ) -> None: super().__init__(name, model_params, classifier_paths) self._selector = KNORAU(self.classifiers, **model_params)
def test_estimate_competence(index, expected): query = np.atleast_2d([1, 1]) knora_u_test = KNORAU(create_pool_classifiers()) knora_u_test.fit(X_dsel_ex1, y_dsel_ex1) knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers) knora_u_test.neighbors = neighbors_ex1[index, :] knora_u_test.distances = distances_ex1[index, :] competences = knora_u_test.estimate_competence(query) assert np.isclose(competences, expected, atol=0.01).all()
def test_classify(index, expected): query = np.atleast_2d([1, 1]) knora_u_test = KNORAU(create_pool_classifiers()) knora_u_test.fit(X_dsel_ex1, y_dsel_ex1) knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers) knora_u_test.neighbors = neighbors_ex1[index, :] knora_u_test.distances = distances_ex1[index, :] prediction = knora_u_test.classify_instance(query) assert prediction == expected
def fit(self, x, y): ''' metodo para treinar a arquitetura de dois niveis :x: dados para treinamento :y: rotulo dos dados :dsel_x: padroes da janela de validacao :dsel_y: rotulos da janela de validacao ''' # salvando os dados de trainamento self.x_train = x self.y_train = y # salvando as dificuldades das instancias self.H = self.kDN(x, y) # treinando o nivel 1 ######################################### self.levelone = KNeighborsClassifier(self.n_vizinhos) self.levelone.fit(x, y) # realizando a previsao para o conjunto de treinamento y_pred = self.levelone.predict(x) # salvando os indices das instancias que foram classificadas erradas indices = [i for i in range(len(y)) if y_pred[i] != y[i]] # obtendo o limiar de dificuldade do problema self.limiar = self.defineThreshold(indices) ############################################################### # treinando o nivel 2 ######################################### # obtendo as instancias dificeis x_dificeis, y_dificeis = self.hardInstances(x, y, self.limiar) # criando o ensemble self.ensemble = BaggingClassifier(base_estimator=Perceptron(), max_samples=0.9, max_features=1.0, n_estimators=100) self.ensemble.fit(x_dificeis, y_dificeis) # treinando o modelo 2 self.leveltwo = KNORAU(self.ensemble.estimators_, self.n_vizinhos) self.leveltwo.fit(x_dificeis, y_dificeis)
def train(train_index, test_index): X_train, X_test = X[train_index], X[test_index] y_train, y_test = Y[train_index], Y[test_index] #train_perc = 0.7 #split_point = int(train_perc*len(train_index)) # valid_index = train_index[split_point:] # train_index = train_index[:split_point] # X_train, X_valid, X_test = X[train_index], X[valid_index], X[test_index] # y_train, y_valid, y_test = Y[train_index], Y[valid_index], Y[test_index] #print("TRAIN:", train_index, "VALID:", valid_index, "TEST:", test_index) X_train, X_valid, y_train, y_valid = train_test_split( X_train, y_train, test_size=0.3, random_state=seed) pool_classifiers.fit(X_train, y_train) validation_data, validation_labels = get_validation_data( X_valid, y_valid, 0.5, hardness=hardness) dynamic_selection_algorithm = None try: if args.dynamic_selection == True and args.dynamic_algorithm is None: raise ValueError( 'Dynamic selection requires you provide an algorithm.') elif args.dynamic_selection == True and args.dynamic_algorithm is not None: if args.dynamic_algorithm == 'ola': dynamic_selection_algorithm = OLA(pool_classifiers, random_state=seed) elif args.dynamic_algorithm == 'lca': dynamic_selection_algorithm = LCA(pool_classifiers, random_state=seed) elif args.dynamic_algorithm == 'mcb': dynamic_selection_algorithm = MCB(pool_classifiers, random_state=seed) elif args.dynamic_algorithm == 'knorau': dynamic_selection_algorithm = KNORAU(pool_classifiers, random_state=seed) elif args.dynamic_algorithm == 'kne': dynamic_selection_algorithm = KNORAE(pool_classifiers, random_state=seed) dynamic_selection_algorithm.fit(validation_data, validation_labels) preds = dynamic_selection_algorithm.predict(X_test) else: # Static combination by voting preds = voting(X_test, pool_classifiers) except Exception as error: raise error acc = get_accuracy_score(y_test, preds) g1 = get_g1_score(y_test, preds, average='macro') f1 = get_f1_score(y_test, preds) roc = roc_auc_score(y_test, preds, average='macro') return dict(f1=f1, g1=g1, acc=acc, roc=roc)
def gustavo_method(self, x_train, y_train, X_test, Y_test, dsel): neigh = KNeighborsClassifier(n_neighbors=self.kNeighbors) neigh.fit(self.x[dsel], self.y[dsel]) pool_classifiers = BaggingClassifier(linear_model.Perceptron(max_iter=5), self.pool_size) pool_classifiers.fit(x_train, y_train) knorau = KNORAU(pool_classifiers) knorau.fit(x_train[dsel], y_train[dsel]) knn = KNeighborsClassifier(n_neighbors=5) knn.fit(x_train, y_train) score = [] for i in range(len(Y_test)): kdn = sum(self.y[neigh.kneighbors([self.x[dsel]], return_distance=False)[0]]!=self.y[i])/self.kNeighbors if (kdn > self.threshold): score.append(knorau.predict(self.x[i])) else: score.append(knn.predict(self.x[i])) return (score, ) + self.calc_metrics(X_test, Y_test) # dependendo da base formato nao suportado
def test_classify(index, expected): query = np.atleast_2d([1, 1]) knora_u_test = KNORAU(create_pool_classifiers()) knora_u_test.fit(X_dsel_ex1, y_dsel_ex1) knora_u_test.DFP_mask = np.ones(knora_u_test.n_classifiers) knora_u_test.neighbors = neighbors_ex1[index, :] knora_u_test.distances = distances_ex1[index, :] predictions = [] for clf in knora_u_test.pool_classifiers: predictions.append(clf.predict(query)[0]) prediction = knora_u_test.classify_instance(query, np.array(predictions)) assert prediction == expected
def ensemble_model(self, ensemble=None): if ensemble is not None: self.ensemble = ensemble if self.moo_ is None: self.moo_ = monise(weightedScalar=self.scalarization, singleScalar=self.scalarization, nodeTimeLimit=2, targetSize=150, targetGap=0, nodeGap=0.01, norm=False) self.moo_.optimize() self.solutions_ = [] for solution in self.moo_.solutionsList: self.solutions_.append(solution.x) if self.solutions_ is None: self.solutions_ = [] for solution in self.moo_.solutionsList: self.solutions_.append(solution.x) if self.ensemble in ['voting', 'voting hard']: models_t = [ ("Model " + str(i), self.solutions_[i]) for i in range(len(self.solutions_)) ] ensemble_model = SimpleVoting(estimators=models_t) if self.ensemble == 'voting soft': models_t = [ ("Model " + str(i), self.solutions_[i]) for i in range(len(self.solutions_)) ] ensemble_model = SimpleVoting(estimators=models_t, voting='soft') if self.ensemble == 'knorau': ensemble_model = KNORAU(self.solutions_) ensemble_model.fit(self.X_val, self.y_val) if self.ensemble == 'knorae': ensemble_model = KNORAE(self.solutions_) ensemble_model.fit(self.X_val, self.y_val) return ensemble_model
def test_knorau(): pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() knorau = KNORAU(pool_classifiers, DFP=True, with_IH=True, IH_rate=0.1) knorau.fit(X_dsel, y_dsel) assert np.isclose(knorau.score(X_test, y_test), 0.9090909090909091)
#------------------- KONARAE TRAINING -------------------------------------- train_time_kne_start = time.time() kne = KNORAE(pool_classifiers) kne.fit(X_dsel, y_dsel) train_time_kne_end = time.time() #------------------- META DES TRAINING ------------------------------------- train_time_mdes_start = time.time() meta = METADES(pool_classifiers) meta.fit(X_dsel, y_dsel) train_time_mdes_end = time.time() #------------------- KONARAU TRAINING -------------------------------------- train_time_knu_start = time.time() knu = KNORAU(pool_classifiers) knu.fit(X_dsel, y_dsel) train_time_knu_end = time.time() #---------------------- Test KNE Pharse -------------------------- test_time_kne_start = time.time() y_kne_pred = kne.predict(X_test) accuracy_kne = accuracy_score(Y_test, y_kne_pred) # print('accuracy = ', accuracy_kne) micro_f1_kne = f1_score(Y_test - 1, y_kne_pred - 1, average='micro') # print('micro_f1 =', micro_f1_kne) macro_f1_kne = f1_score(Y_test - 1, y_kne_pred - 1, average='macro') # print('macro_f1 =', macro_f1_kne)
class Arquitetura: def __init__(self, n_vizinhos): ''' :n_vizinhos: quantidade de vizinhos mais proximos que serao utilizados para regiao de competencia ''' self.n_vizinhos = n_vizinhos def kDN(self, x, y): ''' Metodo para computar o grau de dificuldade de cada observacao em um conjunto de dados :param: x: padroes dos dados :param: y: respectivos rotulos :return: dificuldades: vetor com a probabilidade de cada instancia ''' # instanciando os vizinhos mais proximos nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1, algorithm='ball_tree').fit(x) # variavel para salvar as probabilidades dificuldades = [] # for para cada instancia do dataset for i in range(len(x)): # computando os vizinhos mais proximos para cada instancia _, indices = nbrs.kneighbors([x[i]]) # verificando o rotulo dos vizinhos cont = 0 for j in indices[0]: if (j != i and y[j] != y[i]): cont += 1 # computando a porcentagem dificuldades.append(cont / (self.n_vizinhos + 1)) return dificuldades def neighbors(self, dsel, x_query): ''' metodo para retornar apenas os indices dos vizinhos ''' # instanciando os vizinhos mais proximos nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1, algorithm='ball_tree').fit(dsel) # computando os vizinhos mais proximos para cada instancia _, indices = nbrs.kneighbors([x_query]) return indices def hardInstances(self, x, y, limiar): ''' Metodo para retornar um subconjunto de validacao apenas com as instacias faceis :param: x: padroes dos dados :param: y: respectivos rotulos :return: x_new, y_new: ''' # computando as dificulades para cada instancia dificuldades = self.kDN(x, y) # variaveis para salvar as novas instancias x_new = [] y_new = [] # salvando apenas as instancias faceis for i in range(len(dificuldades)): if (dificuldades[i] > limiar): x_new.append(x[i]) y_new.append(y[i]) return np.asarray(x_new), np.asarray(y_new) def neighborhoodDifficulty(self, dsel, x_query, H): ''' metodo para calcular o grau de dificuldade da vizinhanca :dsel: dataset para pesquisar os vizinhos :x_query: instancia a ser pesquisada :H: dificuldade do dataset dsel ''' # obtendo a vizinhanca do exemplo indices = self.neighbors(dsel, x_query)[0] # dificuldade da regiao dificuldades = [H[i] for i in indices] # media da dificuldadde da regiao return np.min(dificuldades) def defineThreshold(self, indices): ''' Metodo para definir o threshold :indices: os indices das instancias que foram classificadas incorretamente ''' # obtendo a vizinhanca do exemplo lista = [] for i in indices: lista.append( self.neighborhoodDifficulty(self.x_train, self.x_train[i], self.H)) return np.mean(lista) def fit(self, x, y): ''' metodo para treinar a arquitetura de dois niveis :x: dados para treinamento :y: rotulo dos dados :dsel_x: padroes da janela de validacao :dsel_y: rotulos da janela de validacao ''' # salvando os dados de trainamento self.x_train = x self.y_train = y # salvando as dificuldades das instancias self.H = self.kDN(x, y) # treinando o nivel 1 ######################################### self.levelone = KNeighborsClassifier(self.n_vizinhos) self.levelone.fit(x, y) # realizando a previsao para o conjunto de treinamento y_pred = self.levelone.predict(x) # salvando os indices das instancias que foram classificadas erradas indices = [i for i in range(len(y)) if y_pred[i] != y[i]] # obtendo o limiar de dificuldade do problema self.limiar = self.defineThreshold(indices) ############################################################### # treinando o nivel 2 ######################################### # obtendo as instancias dificeis x_dificeis, y_dificeis = self.hardInstances(x, y, self.limiar) # criando o ensemble self.ensemble = BaggingClassifier(base_estimator=Perceptron(), max_samples=0.9, max_features=1.0, n_estimators=100) self.ensemble.fit(x_dificeis, y_dificeis) # treinando o modelo 2 self.leveltwo = KNORAU(self.ensemble.estimators_, self.n_vizinhos) self.leveltwo.fit(x_dificeis, y_dificeis) # verificando se o ola acerta os exemplos errados pelo svm ############################################################### def predict_svm(self, x): # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [ self.levelone.predict(np.array([pattern]))[0] for pattern in x ] # to predict only one example else: return self.levelone.predict(np.array([x]))[0] def predict_ola(self, x): # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [ self.leveltwo.predict(np.array([pattern]))[0] for pattern in x ] # to predict only one example else: return self.leveltwo.predict(np.array([x]))[0] def predict_one(self, x): ''' metodo para computar a previsao de um exemplo :x: padrao a ser predito ''' # media da dificuldadde da regiao media = self.neighborhoodDifficulty(self.x_train, x, self.H) # verificando a dificuldade da instancia if (media >= self.limiar): return self.leveltwo.predict(np.array([x]))[0] else: return self.levelone.predict(np.array([x]))[0] def predict(self, x): ''' metodo para computar a previsao de um exemplo :x: padrao a ser predito ''' # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [self.predict_one(pattern) for pattern in x] # to predict only one example else: return self.predict_one(x)
def test_predict_proba(): X = X_dsel_ex1 y = y_dsel_ex1 clf1 = Perceptron() clf1.fit(X, y) KNORAU([clf1, clf1])
def test_predict_proba(create_X_y): X, y = create_X_y clf1 = Perceptron() clf1.fit(X, y) KNORAU([clf1, clf1]).fit(X, y)
def test_weights_zero(): knorau_test = KNORAU() competences = np.zeros((1, 3)) result = knorau_test.select(competences) assert np.all(result)
def test_knorau(knn_methods): pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() knorau = KNORAU(pool_classifiers, knn_classifier=knn_methods) knorau.fit(X_dsel, y_dsel) assert np.isclose(knorau.score(X_test, y_test), 0.97340425531914898)
# Split the data into training and DSEL for DS techniques X_train, X_dsel, y_train, y_dsel = train_test_split(X_train, y_train, test_size=0.5) # Considering a pool composed of 10 base classifiers # Calibrating Perceptrons to estimate probabilities model = CalibratedClassifierCV(Perceptron(max_iter=100)) # Train a pool of 10 classifiers pool_classifiers = BaggingClassifier(model, n_estimators=100) pool_classifiers.fit(X_train, y_train) # Initialize the DS techniques knorau = KNORAU(pool_classifiers) kne = KNORAE(pool_classifiers) desp = DESP(pool_classifiers) ola = OLA(pool_classifiers) mcb = MCB(pool_classifiers) apriori = APriori(pool_classifiers) meta = METADES(pool_classifiers) # Fit the des techniques knorau.fit(X_dsel, y_dsel) kne.fit(X_dsel, y_dsel) desp.fit(X_dsel, y_dsel) # Fit the dcs techniques ola.fit(X_dsel, y_dsel) mcb.fit(X_dsel, y_dsel)
X = data.data y = data.target # split the data into training and test data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25) RF = RandomForestClassifier() RF.fit(X_train, y_train) X_train, X_dsel, y_train, y_dsel = train_test_split(X, y, test_size=0.50) # Training a random forest to be used as the pool of classifiers. We set the maximum depth of the tree so that it # can estimate probabilities pool_classifiers = RandomForestClassifier(n_estimators=10, max_depth=5) pool_classifiers.fit(X_train, y_train) # Initialize a DS technique. Here we specify the size of the region of competence (5 neighbors) knorau = KNORAU(pool_classifiers) kne = KNORAE(pool_classifiers, k=5) desp = DESP(pool_classifiers, k=5) ola = OLA(pool_classifiers, k=5) mcb = MCB(pool_classifiers, k=5) meta = METADES(pool_classifiers, k=5) # Fit the DS techniques knorau.fit(X_dsel, y_dsel) kne.fit(X_dsel, y_dsel) desp.fit(X_dsel, y_dsel) meta.fit(X_dsel, y_dsel) ola.fit(X_dsel, y_dsel) mcb.fit(X_dsel, y_dsel) # Calculate classification accuracy of each technique
def test_knorau(): pool_classifiers, X_dsel, y_dsel, X_test, y_test = setup_classifiers() knorau = KNORAU(pool_classifiers, DFP=True) knorau.fit(X_dsel, y_dsel) assert np.isclose(knorau.score(X_test, y_test), 0.90606060606060601)
test_size=0.50, random_state=rng) # Training a random forest to be used as the pool of classifiers. # We set the maximum depth of the tree so that it # can estimate probabilities pool_classifiers = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=rng) pool_classifiers.fit(X_train, y_train) stacked = StackedClassifier(pool_classifiers, LogisticRegression()) stacked.fit(X_dsel, y_dsel) # Initialize a DS technique. Here we specify the size of # the region of competence (5 neighbors) knorau = KNORAU(pool_classifiers, random_state=rng) kne = KNORAE(pool_classifiers, k=5, random_state=rng) desp = DESP(pool_classifiers, k=5, random_state=rng) ola = OLA(pool_classifiers, k=5, random_state=rng) mcb = MCB(pool_classifiers, k=5, random_state=rng) meta = METADES(pool_classifiers, k=5, random_state=rng) # Fit the DS techniques knorau.fit(X_dsel, y_dsel) kne.fit(X_dsel, y_dsel) desp.fit(X_dsel, y_dsel) meta.fit(X_dsel, y_dsel) ola.fit(X_dsel, y_dsel) mcb.fit(X_dsel, y_dsel) ###############################################################################