def bagging_metaestimator(X, Y, vrbl_names, n_estimators, p_smpl, p_feat, n_jobs, base_estim): from sklearn.model_selection import KFold, RepeatedKFold from sklearn.ensemble import BaggingRegressor cv = KFold(n_splits=5, shuffle=True) try: X = X.values except: pass try: Y = Y.values except: pass max_feats = np.max([int(X.shape[1]*p_feat), 1]) max_n_estim = n_estimators*5 fitted_ensemble = BaggingRegressor( base_estimator=base_estim, n_estimators=max_n_estim, # Number of fittings max_samples=p_smpl, # Select e.g. 50% of training data per random sample max_features=max_feats, # Select e.g. N/3 variables randomly bootstrap=False, # bootstrap_features=False, oob_score=False, n_jobs=n_jobs, #8, random_state=70, verbose=1).fit(X, Y) final_ensemble_idx = np.zeros(max_n_estim,dtype=bool) for i, estimator in enumerate(fitted_ensemble.estimators_): true_indices = np.abs(estimator.coef_)>0 # Definition of success in fitting: at least one predictor # needs to be found if(true_indices.sum() > 0): final_ensemble_idx[i] = True fitted_ensemble.estimators_features_ = list(np.array(fitted_ensemble.estimators_features_)[final_ensemble_idx]) fitted_ensemble.estimators_features_ = fitted_ensemble.estimators_features_[0:n_estimators] fitted_ensemble.estimators_ = list(np.array(fitted_ensemble.estimators_)[final_ensemble_idx]) fitted_ensemble.estimators_ = fitted_ensemble.estimators_[0:n_estimators] return fitted_ensemble
def bagging_test(self, dataset, target): bagging_workflow_ranks = [] if self._validateDataset(dataset, target): for f in dataset.columns: if dataset[f].dtype == 'object': dataset = dataset.drop(columns=f, axis=1) meta_features_estematic = self._metafeatures( dataset, target, self.meta_functions, self.post_processing_steps) simpleImputer = SimpleImputer() X = simpleImputer.fit_transform(dataset.drop(target, axis=1)) y = dataset[target] for params in self.bagging_grid: # Combinações de Parametros for DS in self.DStechique: for pruning in self.pruning: for base_estimator in self.base_estimators: # Combinação dos algoritmos base # Skip Useless Combinations if (self._skipCombination(pruning)): continue t = time.time() # Cross Validation 4 Folds Ranks = [] kf = KFold(n_splits=4) for train_index, test_index in kf.split(X): # Slit the dataset of the current Fold X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] y_train = y_train.reset_index(drop=True) y_test = y_test.reset_index(drop=True) # Create BaggingClassifier Model bagging_workflow = BaggingRegressor( base_estimator=self. base_estimators[base_estimator], random_state=0, n_jobs=-1, **params) # Apply Learning Algorithm bagging_workflow.fit(X_train, y_train) predictions = [] # PRUNING METHODS if pruning['pruning_method'] == 1 and pruning[ 'pruning_cp'] != 0: # RE Method # Create predicts for each base-estimator for estimator, features in zip( bagging_workflow.estimators_, bagging_workflow. estimators_features_): predictions.append( estimator.predict( X_train[:, features])) # Calculate the index of the base-estimators that will remain in the bagging re_index = self._re( y_train, predictions, X_train, pruning['pruning_cp']) # The actual prunning of the bagging estimators = [] for i in re_index.values(): estimators.append( bagging_workflow.estimators_[i]) bagging_workflow.estimators_ = estimators else: # No pruning method pruning['pruning_cp'] = 0 # Dynamic Select if DS['ds'] == -1: # KNORAE bagging_workflow = KNORAE(bagging_workflow, random_state=0) bagging_workflow.fit(X_train, y_train) else: if DS['ds'] == 1: # OLA bagging_workflow = OLA( bagging_workflow, random_state=0) bagging_workflow.fit(X_train, y_train) Rank_fold = mean_squared_error( bagging_workflow.predict(X_test), y_test) # Save the current bagging rank Ranks.append(Rank_fold) # y_meta is the Meta Target Array that contains the score of the bagging workflow sys.stdout.write('\r' + "Elapsed: %.2f seconds\n" % (time.time() - t)) bagging_workflow_ranks.append(float(mean(Ranks))) return bagging_workflow_ranks
def fit( self, datasets, # Lista com datasets target_names): # Nome dos targets de todas os datasets # Por cada file abrir o csv e tirar para um array de DataFrames x_meta = [ ] # Vai conter todas as Meta-features, uma linha um exemplo de um algoritmo com um certo tipo de parametros y_meta = [ ] # Vai conter o Meta-Target, em cada linha têm a avaliação de 1-n de cada algoritmo # + parametros do bagging workflow ndataset = 0 t = time.time() for dataset, target in zip( datasets, target_names ): # Percorre todos os datasets para treino do meta-model if self._validateDataset(dataset, target): ndataset = ndataset + 1 if not self.silence: print( "________________________________________________________________________" ) print("Dataset nº ", ndataset) print("Shape: {}(examples, features)".format( np.shape(dataset))) # Number of Bagging Workflows indexBagging = 1 indexMaxBagging = 0 for params in self.bagging_grid: # Combinações de Parametros for DS in self.DStechique: # Combinações do Dynamic Selection for pruning in self.pruning: # Combinações dos Pruning Methods for base_estimator in self.base_estimators: # Combinação dos algoritmos base # Skip Useless Combinations if (self._skipCombination(pruning)): continue indexMaxBagging = indexMaxBagging + 1 # Time t = time.time() # Drop Categorial features, DecisionTree do sklearn não aceitam for f in dataset.columns: if dataset[f].dtype == 'object': if type(dataset[f]) != pd.core.series.Series: dataset = dataset.drop(columns=f, axis=1) else: dataset[f] = pd.to_numeric(dataset[f], errors='coerce') # MetaFeatures meta_features_estematic = self._metafeatures( dataset, target, self.meta_functions, self.post_processing_steps) # Convert +/- Inf to NaN dataset.replace(np.inf, np.nan) dataset.replace(-np.inf, np.nan) # Drop Columns with all NaN values dataset = dataset.dropna(axis=1, how='all') # Drop examples with some Nan Values dataset = dataset.dropna(axis=0, how='any') dataset = dataset.reset_index(drop=True) # Dividir o dataset em exemplos e os targets simpleImputer = SimpleImputer() X = simpleImputer.fit_transform(dataset.drop(target, axis=1)) y = dataset[target] # Criar base-models for params in self.bagging_grid: # Combinações de Parametros for DS in self.DStechique: for pruning in self.pruning: for base_estimator in self.base_estimators: # Combinação dos algoritmos base # Skip Useless Combinations if (self._skipCombination(pruning)): continue sys.stdout.write( '\r' + "Creating Baggings Workflows... [{}/{}]". format(indexBagging, indexMaxBagging)) meta_features = meta_features_estematic.copy( ) # Meta-features do dataset só é criado uma vez # Cross Validation 4 Folds Ranks = [] kf = KFold(n_splits=4) for train_index, test_index in kf.split(X): # Separar set do Fold atual X_train, X_test = X[train_index], X[ test_index] y_train, y_test = y[train_index], y[ test_index] y_train = y_train.reset_index(drop=True) y_test = y_test.reset_index(drop=True) # Criar modelo bagging_workflow = BaggingRegressor( base_estimator=self. base_estimators[base_estimator], random_state=0, n_jobs=-1, **params) # Treinar o modelo bagging_workflow.fit(X_train, y_train) predictions = [] # PRUNING METHODS if pruning[ 'pruning_method'] == 1 and pruning[ 'pruning_cp'] != 0: # Criar predicts para todos os base-model for estimator, features in zip( bagging_workflow.estimators_, bagging_workflow. estimators_features_): predictions.append( estimator.predict( X_train[:, features])) re_index = self._re( y_train, predictions, X_train, pruning['pruning_cp']) # Pruning the bagging_workflow estimators = [] for i in re_index.values(): estimators.append(bagging_workflow. estimators_[i]) bagging_workflow.estimators_ = estimators else: pruning['pruning_cp'] = 0 # Dynamic Select if DS['ds'] == -1: bagging_workflow = KNORAE( bagging_workflow, random_state=0) bagging_workflow.fit(X_train, y_train) Rank_fold = mean_squared_error( bagging_workflow.predict(X_test), y_test) else: if DS['ds'] == 1: bagging_workflow = OLA( bagging_workflow, random_state=0) bagging_workflow.fit( X_train, y_train) Rank_fold = mean_squared_error( bagging_workflow.predict( X_test), y_test) else: # Criar landmark do baggingworkflow atual Rank_fold = mean_squared_error( bagging_workflow.predict( X_test), y_test) Ranks.append(Rank_fold) # Adicionar ao array de metafeatures, as caracteriticas dos baggings workflows meta_features['n_estimators'] = params[ 'n_estimators'] meta_features['pruning_method'] = pruning[ 'pruning_method'] meta_features['pruning_cp'] = pruning[ 'pruning_cp'] meta_features['ds'] = DS['ds'] meta_features[ 'Algorithm'] = self.estimators_switcher[ base_estimator] # Este array é o meta target do score do algoritmo y_meta.append(float(mean(Ranks))) # Este array contem as várias metafeatures do dataset e o scores do algoritmo base/parametros a testar x_meta.append(meta_features) indexBagging = indexBagging + 1 sys.stdout.write( '\r' + " ") sys.stdout.write('\r' + "Elapsed: %.2f seconds\n" % (time.time() - t)) # Backup Data pd.DataFrame(x_meta).to_csv( "./metadata/Meta_Data_Regressor_backup.csv") pd.DataFrame(y_meta).to_csv( "./metadata/Meta_Target_Regressor_backup.csv") if not self.silence: print( "________________________________________________________________________" ) # Meta Data é a junção de todas as metafeatures com os scores dos respeticos algoritmos base self.meta_data = pd.DataFrame(x_meta) self.meta_target = np.array(y_meta) # Guardar Meta Data num ficheiro .CSV self.meta_data.to_csv('./metadata/Meta_Data_Regressor.csv') pd.DataFrame( self.meta_target).to_csv('./metadata/Meta_Target_Regressor.csv') if not self.silence: print("Meta-Data Created and Saved.") # Tratar dos dados para entrar no XGBOOST for f in self.meta_data.columns: if self.meta_data[f].dtype == 'object': lbl = LabelEncoder() lbl.fit(list(self.meta_data[f].values)) self.meta_data[f] = lbl.transform( list(self.meta_data[f].values)) self.meta_data.fillna((-999), inplace=True) self.meta_data = np.array(self.meta_data) self.meta_data = self.meta_data.astype(float) if not self.silence: print("Constructing Meta-Model:") # Criar o Meta Model XGBOOST self.meta_model = xgb.XGBRegressor(objective="reg:squarederror", colsample_bytree=0.3, learning_rate=0.1, max_depth=6, alpha=1, n_estimators=100, n_jobs=-1) # Aplicar Learning algorithm self.meta_model.fit(self.meta_data, self.meta_target) self.is_fitted = True return self
def predict(self, dataset, target): if self._validateDataset(dataset, target): for f in dataset.columns: if dataset[f].dtype == 'object': dataset = dataset.drop(columns=f, axis=1) meta_features_estematic = self._metafeatures( dataset, target, self.meta_functions, self.post_processing_steps) simpleImputer = SimpleImputer() X = simpleImputer.fit_transform(dataset.drop(target, axis=1)) y = dataset[target] BestScore = -1 score = 0 RecommendedBagging = {} bagging_combination = [] for params in self.bagging_grid: # Combinações de Parametros for DS in self.DStechique: for pruning in self.pruning: for base_estimator in self.base_estimators: # Combinação dos algoritmos base meta_features = meta_features_estematic.copy() meta_features['n_estimators'] = params[ 'n_estimators'] meta_features['pruning_method'] = pruning[ 'pruning_method'] meta_features['pruning_cp'] = pruning['pruning_cp'] meta_features['ds'] = DS['ds'] meta_features[ 'Algorithm'] = self.estimators_switcher[ base_estimator] meta_features_dic = meta_features features = [] features.append(meta_features) meta_features = pd.DataFrame(features) meta_features = meta_features[[ 'Features.Entropy.Mean', 'Features.Entropy.StandardDeviation', 'Features.Entropy.Skew', 'Features.Entropy.Kurtosis', 'Features.MutualInformation.Mean', 'Features.MutualInformation.StandardDeviation', 'Features.MutualInformation.Skew', 'Features.MutualInformation.Kurtosis', 'Features.SpearmanCorrelation.Mean', 'Features.SpearmanCorrelation.StandardDeviation', 'Features.SpearmanCorrelation.Skew', 'Features.SpearmanCorrelation.Kurtosis', 'FeaturesLabels.SpearmanCorrelation.Mean', 'FeaturesLabels.SpearmanCorrelation.StandardDeviation', 'FeaturesLabels.SpearmanCorrelation.Skew', 'FeaturesLabels.SpearmanCorrelation.Kurtosis', 'Features.Mean.Mean', 'Features.Mean.StandardDeviation', 'Features.Mean.Skew', 'Features.Mean.Kurtosis', 'Features.StandardDeviation.Mean', 'Features.StandardDeviation.StandardDeviation', 'Features.StandardDeviation.Skew', 'Features.StandardDeviation.Kurtosis', 'Features.Skew.Mean', 'Features.Skew.StandardDeviation', 'Features.Skew.Skew', 'Features.Skew.Kurtosis', 'Features.Kurtosis.Mean', 'Features.Kurtosis.StandardDeviation', 'Features.Kurtosis.Skew', 'Features.Kurtosis.Kurtosis', 'FeaturesLabels.MutualInformation.Mean', 'FeaturesLabels.MutualInformation.StandardDeviation', 'FeaturesLabels.MutualInformation.Skew', 'FeaturesLabels.MutualInformation.Kurtosis', 'Number of Examples', 'Number of Features', 'n_estimators', 'pruning_method', 'pruning_cp', 'ds', 'Algorithm' ]] meta_features.replace([np.inf, -np.inf], np.nan, inplace=True) meta_features.fillna((-999), inplace=True) bagging_combination.append( np.array(meta_features.copy())) bagging_combination = np.squeeze(np.array(bagging_combination)) scores = self.meta_model.predict(bagging_combination) BestScore = (np.abs(scores)).argmin() RecommendedBagging = bagging_combination[BestScore] # Prints e construção do Bagging previsto npSize = RecommendedBagging.size n_estimators = int(RecommendedBagging[npSize - 5]) pruning_method = int(RecommendedBagging[npSize - 4]) pruning_cp = float(RecommendedBagging[npSize - 3] / 100) ds = int(RecommendedBagging[npSize - 2]) base_estimator = int(RecommendedBagging[npSize - 1]) # String para visualização if pruning_method == 1: pruning_method_str = 'RE' else: pruning_method_str = 'None' if ds > 0.5: ds_str = 'KNORAE' else: if ds < -0.5: ds_str = 'OLA' else: ds_str = 'None' if not self.silence: print("Recommended Bagging workflow: ") print("\tNumber of models: ", n_estimators) if pruning_method != 0: print("\tPruning Method: ", pruning_method_str) print("\tPruning CutPoint: ", pruning_cp * 100) else: print("\tPruning: ", pruning_method_str) print("\tDynamic Selection: ", ds_str) print("\tAlgorithm: ", base_estimator) # BaggingWorkflow bagging_workflow = BaggingRegressor( base_estimator=self.base_estimators[ self.estimators_switcher2[base_estimator]], n_estimators=n_estimators, random_state=0, n_jobs=-1) # Dividir o dataset em exemplos e os targets X = SimpleImputer().fit_transform(dataset.drop(target, axis=1)) y = dataset[target] X_train = X y_train = y # Treinar o modelo bagging_workflow.fit(X_train, y_train) predictions = [] if pruning_method == 1 and pruning_cp != 0: if not self.silence: print("Waiting for RE") for estimator, features in zip( bagging_workflow.estimators_, bagging_workflow.estimators_features_): predictions.append(estimator.predict(X_train[:, features])) re_index = self._re(y_train, predictions, X_train, pruning_cp) # Pruning the bagging_workflow estimators = [] for i in re_index.values(): estimators.append(bagging_workflow.estimators_[i]) bagging_workflow.estimators_ = estimators # Dynamic Select if ds == -1: bagging_workflow = KNORAE(bagging_workflow, random_state=0) bagging_workflow.fit(X_train, y_train) else: if ds == 1: bagging_workflow = OLA(bagging_workflow, random_state=0) bagging_workflow.fit(X_train, y_train) return bagging_workflow else: print("Erro, não é um problema de Regressão")