def decision_tree_bagging(Xtrain, Xtest, ytrain, ytest, ensemble_size=60): # bagging accuracies = [] ensemble_sizes = [] for i in range(1, ensemble_size): bagging = BaggingClassifier( base_estimator=tree.DecisionTreeClassifier(), n_estimators=i, bootstrap=True, max_samples=1.0, max_features=1.0) bagging.fit(Xtrain, ytrain) ypred = bagging.predict(Xtest) accuracy = np.mean(ypred == ytest) ensemble_sizes.append(i) accuracies.append(accuracy) plt.plot(ensemble_sizes, accuracies) plt.xlabel('number of estimators') plt.ylabel('accuracy') plt.grid(True) plt.title('Decision tree (bagging)') plt.show() print('Highest accuracy of bagging = %f' % np.max(accuracies))
class BaggingClassifierImpl(): def __init__(self, base_estimator=None, n_estimators=10, max_samples=1.0, max_features=1.0, bootstrap=True, bootstrap_features=False, oob_score=False, warm_start=False, n_jobs=None, random_state=None, verbose=0): self._hyperparams = { 'base_estimator': make_sklearn_compat(base_estimator), 'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features, 'bootstrap': bootstrap, 'bootstrap_features': bootstrap_features, 'oob_score': oob_score, 'warm_start': warm_start, 'n_jobs': n_jobs, 'random_state': random_state, 'verbose': verbose} self._wrapped_model = SKLModel(**self._hyperparams) def fit(self, X, y=None): if (y is not None): self._wrapped_model.fit(X, y) else: self._wrapped_model.fit(X) return self def predict(self, X): return self._wrapped_model.predict(X) def predict_proba(self, X): return self._wrapped_model.predict_proba(X) def decision_function(self, X): return self._wrapped_model.decision_function(X)
# 3.1. End ################################################################################################### # 3.2 Instanciando os classificadores ######################################################################### # 3.2.1. Bagging com DecisionTree ############################################################ # numero do modelo na tabela num_model = 0 # modelo bg = BaggingClassifier(base_estimator=DecisionTreeClassifier(), max_samples=pct_trainamento[i], max_features=1.0, n_estimators=qtd_modelos) # treinando o modelo bg.fit(x_train, y_train) # computando a previsao pred = bg.predict(x_test) # printando os resultados acuracia, auc, f1measure, gmean = printar_resultados( y_test, pred, nome_datasets[h] + '-pct-' + str(pct_trainamento[i]) + '- Bagging com DecisionTree [' + str(j) + ']') # escrevendo os resultados obtidos tabela.Adicionar_Sheet_Linha(num_model, j, [acuracia, auc, f1measure, gmean]) # 3.2.1. End ###################################################################################
# 3.3. End ################################################################################################ # 3.4. Instanciando os classificadores ########################################################## ########## instanciando o modelo Bagging+REP ########################################### # definindo o numero do modelo na tabela num_model = 0 # intanciando o classificador ensemble = BaggingClassifier(base_estimator=Perceptron(), max_samples=qtd_amostras, max_features=1.0, n_estimators = qtd_modelos) # treinando o modelo ensemble.fit(x_train, y_train) # realizando a poda ensemble = REP(x_val, y_val, ensemble) # computando a previsao pred = ensemble.predict(x_test) # computando a diversidade do ensemble q_statistic = MedidasDiversidade('q', x_val, y_val, ensemble) double_fault = MedidasDiversidade('disagreement', x_val, y_val, ensemble) # printando os resultados qtd_modelos, acuracia, auc, f1measure, gmean = printar_resultados(y_test, pred, ensemble, nome_datasets[h]+'-Bagging-REP-'+validacao[k]+'['+str(j)+']') # escrevendo os resultados obtidos
class Arquitetura: def __init__(self, n_vizinhos): ''' :n_vizinhos: quantidade de vizinhos mais proximos que serao utilizados para regiao de competencia ''' self.n_vizinhos = n_vizinhos def kDN(self, x, y): ''' Metodo para computar o grau de dificuldade de cada observacao em um conjunto de dados :param: x: padroes dos dados :param: y: respectivos rotulos :return: dificuldades: vetor com a probabilidade de cada instancia ''' # instanciando os vizinhos mais proximos nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1, algorithm='ball_tree').fit(x) # variavel para salvar as probabilidades dificuldades = [] # for para cada instancia do dataset for i in range(len(x)): # computando os vizinhos mais proximos para cada instancia _, indices = nbrs.kneighbors([x[i]]) # verificando o rotulo dos vizinhos cont = 0 for j in indices[0]: if (j != i and y[j] != y[i]): cont += 1 # computando a porcentagem dificuldades.append(cont / (self.n_vizinhos + 1)) return dificuldades def neighbors(self, dsel, x_query): ''' metodo para retornar apenas os indices dos vizinhos ''' # instanciando os vizinhos mais proximos nbrs = NearestNeighbors(n_neighbors=self.n_vizinhos + 1, algorithm='ball_tree').fit(dsel) # computando os vizinhos mais proximos para cada instancia _, indices = nbrs.kneighbors([x_query]) return indices def hardInstances(self, x, y, limiar): ''' Metodo para retornar um subconjunto de validacao apenas com as instacias faceis :param: x: padroes dos dados :param: y: respectivos rotulos :return: x_new, y_new: ''' # computando as dificulades para cada instancia dificuldades = self.kDN(x, y) # variaveis para salvar as novas instancias x_new = [] y_new = [] # salvando apenas as instancias faceis for i in range(len(dificuldades)): if (dificuldades[i] > limiar): x_new.append(x[i]) y_new.append(y[i]) return np.asarray(x_new), np.asarray(y_new) def neighborhoodDifficulty(self, dsel, x_query, H): ''' metodo para calcular o grau de dificuldade da vizinhanca :dsel: dataset para pesquisar os vizinhos :x_query: instancia a ser pesquisada :H: dificuldade do dataset dsel ''' # obtendo a vizinhanca do exemplo indices = self.neighbors(dsel, x_query)[0] # dificuldade da regiao dificuldades = [H[i] for i in indices] # media da dificuldadde da regiao return np.min(dificuldades) def defineThreshold(self, indices): ''' Metodo para definir o threshold :indices: os indices das instancias que foram classificadas incorretamente ''' # obtendo a vizinhanca do exemplo lista = [] for i in indices: lista.append( self.neighborhoodDifficulty(self.x_train, self.x_train[i], self.H)) return np.mean(lista) def fit(self, x, y): ''' metodo para treinar a arquitetura de dois niveis :x: dados para treinamento :y: rotulo dos dados :dsel_x: padroes da janela de validacao :dsel_y: rotulos da janela de validacao ''' # salvando os dados de trainamento self.x_train = x self.y_train = y # salvando as dificuldades das instancias self.H = self.kDN(x, y) # treinando o nivel 1 ######################################### self.levelone = KNeighborsClassifier(self.n_vizinhos) self.levelone.fit(x, y) # realizando a previsao para o conjunto de treinamento y_pred = self.levelone.predict(x) # salvando os indices das instancias que foram classificadas erradas indices = [i for i in range(len(y)) if y_pred[i] != y[i]] # obtendo o limiar de dificuldade do problema self.limiar = self.defineThreshold(indices) ############################################################### # treinando o nivel 2 ######################################### # obtendo as instancias dificeis x_dificeis, y_dificeis = self.hardInstances(x, y, self.limiar) # criando o ensemble self.ensemble = BaggingClassifier(base_estimator=Perceptron(), max_samples=0.9, max_features=1.0, n_estimators=100) self.ensemble.fit(x_dificeis, y_dificeis) # treinando o modelo 2 self.leveltwo = KNORAU(self.ensemble.estimators_, self.n_vizinhos) self.leveltwo.fit(x_dificeis, y_dificeis) # verificando se o ola acerta os exemplos errados pelo svm ############################################################### def predict_svm(self, x): # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [ self.levelone.predict(np.array([pattern]))[0] for pattern in x ] # to predict only one example else: return self.levelone.predict(np.array([x]))[0] def predict_ola(self, x): # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [ self.leveltwo.predict(np.array([pattern]))[0] for pattern in x ] # to predict only one example else: return self.leveltwo.predict(np.array([x]))[0] def predict_one(self, x): ''' metodo para computar a previsao de um exemplo :x: padrao a ser predito ''' # media da dificuldadde da regiao media = self.neighborhoodDifficulty(self.x_train, x, self.H) # verificando a dificuldade da instancia if (media >= self.limiar): return self.leveltwo.predict(np.array([x]))[0] else: return self.levelone.predict(np.array([x]))[0] def predict(self, x): ''' metodo para computar a previsao de um exemplo :x: padrao a ser predito ''' # to predict multiple examples if (len(x.shape) > 1): # returning all labels return [self.predict_one(pattern) for pattern in x] # to predict only one example else: return self.predict_one(x)
# from sklearn.ensemble import AdaBoostClassifier as Boost from sklearn.ensemble.bagging import BaggingClassifier as Boost from sklearn.naive_bayes import GaussianNB from csxdata import CData from SciProjects.grapes import path, indepsn if __name__ == '__main__': data = CData(path, indepsn, feature="evjarat", headers=1, cross_val=0.2, lower=True) data.transformation = "std" model = Boost(GaussianNB(), n_estimators=100) model.fit(data.learning, data.lindeps) preds = model.predict(data.testing) eq = [left == right for left, right in zip(preds, data.tindeps)] print("Acc:", sum(eq) / len(eq))
def main(): # 1. Definindo variaveis para o experimento ######################################################################### qtd_modelos = 100 qtd_execucoes = 30 qtd_amostras = 0.9 qtd_folds = 10 n_vizinhos = 7 nome_datasets = ['kc1', 'kc2'] # 1. End ############################################################################################################ # for para variar entre os datasets for h in range(len(nome_datasets)): # 2. Lendo os datasets ############################################################################################ # lendo o dataset data = pd.read_csv('dataset/'+nome_datasets[h]+'.csv') # obtendo os padroes e seus respectivos rotulos df_x = np.asarray(data.iloc[:,0:-1]) df_y = np.asarray(data.iloc[:,-1]) # 2.1. Criando a tabela para salvar os dados ################################################# # criando a tabela que vai acomodar o modelo tabela = Tabela_excel() tabela.Criar_tabela(nome_tabela='arquivos_lista03/'+nome_datasets[h], folhas=['OLA', 'LCA', 'KNORA-E', 'KNORA-U', 'Arquitetura'], cabecalho=['acuracy', 'auc', 'fmeasure', 'gmean'], largura_col=5000) # 2.1. End ##################################################################################### # 2. End ############################################################################################################ # executando os algoritmos x vezes for j in range(qtd_execucoes): # 3. Dividindo os dados para treinamento e teste ################################################################ # quebrando o dataset sem sobreposicao em 90% para treinamento e 10% para teste skf = StratifiedKFold(df_y, n_folds=qtd_folds) # tomando os indices para treinamento e teste train_index, test_index = next(iter(skf)) # obtendo os conjuntos de dados para treinamento e teste x_train = df_x[train_index] y_train = df_y[train_index] x_test = df_x[test_index] y_test = df_y[test_index] # 3. End ######################################################################################################### # 4. Gerando o pool de classificadores ########################################################################## # intanciando o classificador ensemble = BaggingClassifier(base_estimator=Perceptron(), max_samples=qtd_amostras, max_features=1.0, n_estimators = qtd_modelos) # treinando o modelo ensemble.fit(x_train, y_train) # 4. End ######################################################################################################## # 5. Instanciando os classificadores ########################################################## ################################### OLA ######################################################## executar_modelo('OLA', x_train, y_train, x_test, y_test, ensemble.estimators_, n_vizinhos, nome_datasets, h, j, tabela) ################################################################################################ ################################### LCA ######################################################## executar_modelo('LCA', x_train, y_train, x_test, y_test, ensemble.estimators_, n_vizinhos, nome_datasets, h, j, tabela) ################################################################################################ ################################### KNORAE ##################################################### executar_modelo('KNORAE', x_train, y_train, x_test, y_test, ensemble.estimators_, n_vizinhos, nome_datasets, h, j, tabela) ################################################################################################ ################################### KNORAU ##################################################### executar_modelo('KNORAU', x_train, y_train, x_test, y_test, ensemble.estimators_, n_vizinhos, nome_datasets, h, j, tabela) ################################################################################################ ################################### Arquitetura ################################################ # importando o metodo arq = Arquitetura(n_vizinhos) # treinando o metodo arq.fit(x_train, y_train) # realizando a previsao pred = arq.predict(x_test) # printando os resultados nome = 'Arquitetura' acuracia, auc, f1measure, gmean = printar_resultados(y_test, pred, nome_datasets[h]+'-'+nome+'-['+str(j)+']') # escrevendo os resultados obtidos tabela.Adicionar_Sheet_Linha(4, j, [acuracia, auc, f1measure, gmean])
from sklearn.ensemble.bagging import BaggingClassifier train = pd.read_csv("train.csv") train.drop(['Cabin'], 1, inplace=True) train = train.dropna() y = train['Survived'] train.drop(['Survived', 'PassengerId', 'Name', 'Ticket'], 1, inplace=True) train.fillna({'Age': 30}) X = pd.get_dummies(train) bag_clf = BaggingClassifier( tree.DecisionTreeClassifier(), n_estimators=500, max_samples=200, bootstrap=True, # True => bagging, False => pasting n_jobs=-1 # use all cores ) bag_clf.fit(X, y) test = pd.read_csv('test.csv') ids = test[['PassengerId']] test.drop(['PassengerId', 'Name', 'Ticket', 'Cabin'], 1, inplace=True) test.fillna(2, inplace=True) test = pd.get_dummies(test) predictions = bag_clf.predict(test) results = ids.assign(Survived=predictions) results.to_csv('titanic_result_bagging.csv', index=False)
plt.xlabel('Relative Importance') plt.title('Variable Importance based on bagging method') plt.show() except: print('不展示特征重要性') if 0: print(' '.join(['*' * 25, 'RandomForestClassifier', '*' * 25, '\n'])) from sklearn.ensemble.bagging import BaggingClassifier clf_svm0=SVC(C=10,kernel='rbf',gamma=0.1,probability=True,\ decision_function_shape='ovr',random_state=seed,class_weight='balanced') pipe_svm0 = Pipeline([('scaler', Scaler()), ('clf', clf_svm0)]) clf_bg = BaggingClassifier(base_estimator=pipe_svm0, n_estimators=10, max_samples=1.0, \ max_features=1.0, random_state=seed) start = time.time() clf_bg = clf_bg.fit(X_train, Y_train) print('Total running time is {}s'.format(time.time() - start)) judge = cross_val_score(clf_bg, X, Y, groups=None, scoring=Evaluate(score_func), cv=5) #print('Cross-validation score is {}'.format(judge)) print('Mean cross-validation score is {}'.format(judge.mean())) # Boosting method:GradBoost if 1: print(' '.join( ['*' * 25, 'GradientBoostingClassifier', '*' * 25, '\n'])) from sklearn.ensemble import GradientBoostingClassifier