def CF_QualVal(X, Y, estimator, conformalSignificance): """ Qualitative conformal predictor validation""" print("Starting qualitative conformal prediction validation") icp = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(estimator), MarginErrFunc())), BootstrapSampler()) Y = np.asarray(Y).reshape(-1, 1) loo = LeaveOneOut() predictions = [] for train, test in loo.split(X): Xn = [X[i] for i in train] Yn = [Y[i] for i in train] Xn, mux = center(Xn) Xn, wgx = scale(Xn, True) Yn = np.asarray(Yn) Xout = X[test] Yout = Y[test[0]] icp.fit(Xn, Yn) predictions.append(icp.predict(Xout, significance=0.15)) predictions = [(x[0]).tolist() for x in predictions] predictions = np.asarray(predictions) table = np.hstack((predictions, Y)) print('Error rate: {}'.format(class_mean_errors(predictions, Y, 0.15))) print('Class one: ', class_one_c(predictions, Y, 0.15)) return icp
def build(self): '''Build a new qualitative GNB model with the X and Y numpy matrices''' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # Build estimator LOG.info('Building GaussianNB model') self.estimator = GaussianNB(**self.estimator_parameters) results.append(('model', 'model type', 'GNB qualitative')) self.estimator.fit(X, Y) if not self.param.getVal('conformal'): return True, results # If conformal, then create aggregated conformal classifier self.estimator_temp = copy(self.estimator) self.estimator = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit estimator to the data self.estimator.fit(X, Y) results.append(('model', 'model type', 'conformal GNB qualitative')) return True, results
def CF_quantitative_validation(self): ''' Performs internal validation for conformal quantitative models ''' # Make a copy of original matrices. X = self.X.copy() Y = self.Y.copy() # Number of external validations for the aggregated conformal estimator. seeds = [5, 7, 35] # Interval means for each aggregated conformal estimator (out of 3) interval_means = [] # Accuracies for each aggregated conformal estimator (out of 3) accuracies = [] results = [] try: for i in range(len(seeds)): # Generate training a test sets X_train, X_test, Y_train, Y_test = train_test_split( X, Y, test_size=0.25, random_state=i, shuffle=False) # Create the aggregated conformal regressor. conformal_pred = AggregatedCp( IcpRegressor(RegressorNc(RegressorAdapter( self.estimator))), BootstrapSampler()) # Fit conformal regressor to the data conformal_pred.fit(X_train, Y_train) # Perform prediction on test set prediction = conformal_pred.predict(X_test, self.conformalSignificance) # Add the n validation interval means interval_means.append( np.mean( np.abs(prediction[:, 0]) - np.abs(prediction[:, 1]))) Y_test = Y_test.reshape(-1, 1) # Get boolean mask of instances within the applicability domain. inside_interval = ((prediction[:, 0].reshape(-1, 1) < Y_test) & (prediction[:, 1].reshape(-1, 1) > Y_test)) # Compute the accuracy (number of instances within the AD). accuracy = np.sum(inside_interval) / len(Y_test) # Add validation result to the list of accuracies. accuracies.append(accuracy) except Exception as e: LOG.error(f'Quantitative conformal validation' f' failed with exception: {e}') raise e # Compute mean interval_means and accuracy. interval_means = np.mean(interval_means) accuracies = np.mean(accuracies) # Cut into two decimals. self.conformal_accuracy = float("{0:.2f}".format(accuracies)) self.conformal_mean_interval = float("{0:.2f}".format(interval_means)) #Add quality metrics to results. results.append(('Conformal_mean_interval', 'Conformal mean interval', self.conformal_mean_interval)) results.append(('Conformal_accuracy', 'Conformal accuracy', self.conformal_accuracy)) return True, (results, )
def build(self): if not self.quantitative: print("PLSR only applies to quantitative data") return False, "PLSR only applies to quantitative data" if self.failed: return False, "Error initiating model" X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.cv: self.cv = getCrossVal(self.cv, 46, self.n, self.p) if self.tune: if self.optimiz == 'auto': super(PLSR, self).optimize(X, Y, PLS_r( **self.estimator_parameters), self.tune_parameters) elif self.optimiz == 'manual': self.optimize(X, Y, PLS_r( **self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'PLSR quantitative (optimized)')) else: print("Building Quantitative PLSR") self.estimator = PLS_r(**self.estimator_parameters) results.append(('model', 'model type', 'PLSR quantitative')) if self.conformal: underlying_model = RegressorAdapter(self.estimator) normalizing_model = RegressorAdapter( KNeighborsRegressor(n_neighbors=1)) normalizing_model = RegressorAdapter(self.estimator) normalizer = RegressorNormalizer( underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) self.conformal_pred = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) # self.conformal_pred = AggregatedCp(IcpRegressor(RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.conformal_pred.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal PLSR quantitative')) self.estimator.fit(X, Y) return True, results
def CF_QuanCal(X, Y, estimator): # X_train, X_test, y_train, y_test = train_test_split( X, Y, test_size=0.20, random_state=42) acp = AggregatedCp( IcpRegressor( RegressorNc( RegressorAdapter(estimator), AbsErrorErrFunc(), RegressorNormalizer(estimator, copy.copy(estimator), AbsErrorErrFunc())), RandomSubSampler()), ) acp.fit(X, Y) # icp.calibrate(X_test, y_test) return acp
def test_acp_regression_tree(self): # ----------------------------------------------------------------------------- # Experiment setup # ----------------------------------------------------------------------------- data = load_diabetes() idx = np.random.permutation(data.target.size) train = idx[:int(2 * idx.size / 3)] test = idx[int(2 * idx.size / 3):] truth = data.target[test] columns = ["min", "max", "truth"] significance = 0.1 # ----------------------------------------------------------------------------- # Define models # ----------------------------------------------------------------------------- models = { "ACP-RandomSubSampler": AggregatedCp( IcpRegressor( RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), RandomSubSampler(), ), "ACP-CrossSampler": AggregatedCp( IcpRegressor( RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), CrossSampler(), ), "ACP-BootstrapSampler": AggregatedCp( IcpRegressor( RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), BootstrapSampler(), ), } # ----------------------------------------------------------------------------- # Train, predict and evaluate # ----------------------------------------------------------------------------- for name, model in models.items(): model.fit(data.data[train, :], data.target[train]) prediction = model.predict(data.data[test, :]) prediction_sign = model.predict(data.data[test, :], significance=significance) table = np.vstack((prediction_sign.T, truth)).T df = pd.DataFrame(table, columns=columns) print("\n{}".format(name)) print("Error rate: {}".format( reg_mean_errors(prediction, truth, significance))) print(df)
class GNB(BaseEstimator): def __init__(self, X, Y, parameters): super(GNB, self).__init__(X, Y, parameters) self.estimator_parameters = parameters['GNB_parameters'] if self.quantitative: return else: self.name = "GNB-Classifier" def build(self): '''Build a new qualitative GNB model with the X and Y numpy matrices''' if self.failed: return False, "Error initiating model" X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.cv: self.cv = getCrossVal(self.cv, 46, self.n, self.p) if self.quantitative: print("GNB only applies to qualitative data") return False, "GNB only applies to qualitative data" else: print("Building GaussianNB model") print(self.estimator_parameters) self.estimator = GaussianNB(**self.estimator_parameters) results.append(('model', 'model type', 'GNB qualitative')) if self.conformal: self.conformal_pred = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator), MarginErrFunc())), BootstrapSampler()) self.conformal_pred.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal GNB qualitative')) self.estimator.fit(X, Y) return True, results
def CF_QualCal(X, Y, estimator): """Qualitative conformal predictor calibration""" acp = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(estimator), MarginErrFunc())), BootstrapSampler()) acp.fit(X, Y) # X_train, X_test, y_train, y_test = train_test_split( X, Y, test_size=0.30, random_state=42) # icp = IcpClassifier(ClassifierNc(ClassifierAdapter(estimator), # MarginErrFunc())) # icp.fit(X_train, y_train) # icp.calibrate(X_test, y_test) return acp
def CF_QuanVal(X, Y, estimator, conformalSignificance): print("Starting quantitative conformal prediction validation") icp = AggregatedCp(IcpRegressor(RegressorNc(RegressorAdapter(estimator))), BootstrapSampler()) # icp = AggregatedCp(IcpRegressor(RegressorNc(RegressorAdapter(estimator), # AbsErrorErrFunc(), RegressorNormalizer(estimator, # RegressorAdapter(copy.copy(estimator)), AbsErrorErrFunc())))) # icp_cv = RegIcpCvHelper(icp) # scores = conformal_cross_val_score(icp_cv, # X, # Y, # iterations=5, # folds=5, # scoring_funcs=[reg_mean_errors, reg_median_size, reg_mean_size], # significance_levels=[0.05, 0.1, 0.2, conformalSignificance]) icp.fit(X[:30], Y[:30]) prediction = icp.predict(X[30:]) prediction_sign = icp.predict(X[30:], significance=0.25) interval = prediction_sign[:, 0] - prediction_sign[:, 1] print(np.mean(interval)) print(interval) print("\n") print(prediction) print(prediction_sign) return (icp)
def SelectLabeled(self, labeled_data_x, labeled_data_y, unlabeled_data_x): # just append train data to labeled data labeled_x = np.concatenate( (self.init_labeled_data_x, labeled_data_x )) if len(labeled_data_x) > 0 else self.init_labeled_data_x labeled_y = np.concatenate( (self.init_labeled_data_y, labeled_data_y )) if len(labeled_data_x) > 0 else self.init_labeled_data_y # # create model to predict with confidence and credibility model = ClassifierAdapter( DecisionTreeClassifier(random_state=config.random_state, min_samples_leaf=config.min_samples_leaf)) model_acp = AggregatedCp( IcpClassifier(ClassifierNc(model), smoothing=True), RandomSubSampler()) model_acp.fit(labeled_x, labeled_y) s = model_acp.predict(unlabeled_data_x) # print(s) # # selection method labeled_ind = [ i for i, a in enumerate(s) if 1 - a.min() > config.confidence and a.max() > config.credibility ] unlabeled_ind = [ i for i, a in enumerate(s) if 1 - a.min() < config.confidence or a.max() < config.credibility ] labeled_unlabeled_x, labeled_unlabeled_y, unlabeled_data_x = \ np.take(unlabeled_data_x, labeled_ind, axis=0), np.take(s.argmax(axis=1), labeled_ind), np.take( unlabeled_data_x, unlabeled_ind, axis=0) # return labeled_unlabeled_x, labeled_unlabeled_y, unlabeled_data_x
def build(self): # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.param.getVal('tune'): # Optimize estimator using sklearn-gridsearch if self.estimator_parameters['optimize'] == 'auto': try: LOG.info('Optimizing PLSR using SK-LearnGridSearch') # Remove optimize key from parameter dictionary # to avoid sklearn estimator error (unexpected keyword) self.estimator_parameters.pop("optimize") super(PLSR, self).optimize(X, Y, PLS_r( **self.estimator_parameters), self.param.getDict('PLSR_optimize')) except Exception as e: LOG.error(f'Error performing SK-LearnGridSearch' f' on PLSR estimator with exception {e}') return False, f'Error performing SK-LearnGridSearch on PLSR estimator with exception {e}' # Optimize using flame implementation (recommended) elif self.estimator_parameters['optimize'] == 'manual': LOG.info('Optimizing PLSR using manual method') # Remove optimize key from parameter dictionary # to avoid sklearn estimator error (unexpected keyword) self.estimator_parameters.pop("optimize") success, message = self.optimize(X, Y, PLS_r( **self.estimator_parameters), self.param.getDict('PLSR_optimize')) if not success: return False, message else: LOG.error('Type of tune not recognized, check the input') return False, 'Type of tune not recognized, check the input' results.append(('model', 'model type', 'PLSR quantitative (optimized)')) else: LOG.info('Building Quantitative PLSR with no optimization') try: # Remove optimize key from parameters to avoid error self.estimator_parameters.pop("optimize") # as the sklearn estimator does not have this key self.estimator = PLS_r(**self.estimator_parameters) except Exception as e: LOG.error(f'Error at PLS_r instantiation with ' f'exception {e}') return False, f'Error at PLS_da instantiation with exception {e}' results.append(('model', 'model type', 'PLSR quantitative')) # Fit estimator to the data self.estimator.fit(X, Y) if not self.param.getVal('conformal'): return True, results self.estimator_temp = copy(self.estimator) try: LOG.info('Building PLSR aggregated conformal predictor') underlying_model = RegressorAdapter(self.estimator_temp) # normalizing_model = RegressorAdapter( # KNeighborsRegressor(n_neighbors=1)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) except Exception as e: LOG.error(f'Error building aggregated PLSR conformal' f' regressor with exception: {e}') return False, f'Error building aggregated PLSR conformal regressor with exception: {e}' # self.conformal_pred = AggregatedCp(IcpRegressor( # RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) # Fit conformal estimator to the data self.estimator.fit(X, Y) # overrides non-conformal results.append(('model', 'model type', 'conformal PLSR quantitative')) return True, results
idx = np.random.permutation(data.target.size) train = idx[:int(2 * idx.size / 3)] test = idx[int(2 * idx.size / 3):] truth = data.target[test].reshape(-1, 1) columns = ['C-{}'.format(i) for i in np.unique(data.target)] + ['truth'] significance = 0.1 # ----------------------------------------------------------------------------- # Define models # ----------------------------------------------------------------------------- models = { 'ACP-RandomSubSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter( DecisionTreeClassifier()))), RandomSubSampler()), 'ACP-CrossSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter( DecisionTreeClassifier()))), CrossSampler()), 'ACP-BootstrapSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter( DecisionTreeClassifier()))), BootstrapSampler()), 'CCP': CrossConformalClassifier( IcpClassifier(ClassifierNc(ClassifierAdapter( DecisionTreeClassifier())))), 'BCP': BootstrapConformalClassifier(
def build(self): '''Build a new XGBOOST model with the X and Y numpy matrices ''' try: from xgboost.sklearn import XGBClassifier from xgboost.sklearn import XGBRegressor except Exception as e: return False, 'XGboost not found, please revise your environment' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): LOG.info("Optimizing XGBOOST estimator") try: # Check type of model if self.param.getVal('quantitative'): self.estimator = XGBRegressor( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model','model type','XGBOOST quantitative (optimized)')) else: self.estimator = XGBClassifier( **self.estimator_parameters) params = self.estimator.get_params() params['num_class'] = 2 self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model','model type','XGBOOST qualitative (optimized)')) except Exception as e: return False, f'Exception optimizing XGBOOST estimator with exception {e}' else: try: if self.param.getVal('quantitative'): LOG.info("Building Quantitative XGBOOST model") # params = { # 'objective': 'reg:squarederror', # 'missing': -99.99999, # # 'max_depth': 20, # # 'learning_rate': 1.0, # # 'silent': 1, # # 'n_estimators': 25 # } # self.estimator = XGBRegressor(**params) self.estimator = XGBRegressor(**self.estimator_parameters) results.append(('model', 'model type', 'XGBOOST quantitative')) else: LOG.info("Building Qualitative XGBOOST model") # params = { # 'objective': 'binary:logistic', # 'max_depth': 3, # #'learning_rate': 0.7, # #'silent': 1, # 'n_estimators': 100 # } self.estimator = XGBClassifier(**self.estimator_parameters) results.append(('model', 'model type', 'XGBOOST qualitative')) self.estimator.fit(X, Y) print(self.estimator) except Exception as e: raise e return False, f'Exception building XGBOOST estimator with exception {e}' self.estimator_temp = copy(self.estimator) if not self.param.getVal('conformal'): return True, results # Create the conformal estimator try: # Conformal regressor if self.param.getVal('quantitative'): LOG.info("Building conformal Quantitative XGBOOST model") underlying_model = RegressorAdapter(self.estimator_temp) #normalizing_model = RegressorAdapter( #KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer( underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) results.append(('model', 'model type', 'conformal XGBOOST quantitative')) # Conformal classifier else: LOG.info("Building conformal Qualitative XGBOOST model") self.estimator = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(self.estimator_temp), MarginErrFunc() ) ), BootstrapSampler()) # Fit estimator to the data self.estimator.fit(X, Y) results.append(('model', 'model type', 'conformal XGBOOST qualitative')) except Exception as e: raise e return False, f'Exception building conformal XGBOOST estimator with exception {e}' return True, results ## Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
class XGBOOST(BaseEstimator): """ This class inherits from BaseEstimator and wraps SKLEARN RandomForestClassifier or RandomForestRegressor estimator ... Attributes ---------- estimator_parameters : dict parameter values name : string name of the estimator tune_parameters: dict Hyperparameter optimization settings Methods ------- build(X) Instance the estimator optimizing it if tune=true. """ def __init__(self, X, Y, parameters, conveyor): # Initialize parent class try: BaseEstimator.__init__(self, X, Y, parameters, conveyor) LOG.debug('Initialize BaseEstimator parent class') except Exception as e: self.conveyor.setError(f'Error initializing BaseEstimator parent class with exception: {e}') LOG.error(f'Error initializing BaseEstimator parent class with exception: {e}') return # Load estimator parameters self.estimator_parameters = self.param.getDict('XGBOOST_parameters') # Load tune parameters self.tune_parameters = self.param.getDict('XGBOOST_optimize') if self.param.getVal('quantitative'): self.estimator_parameters['objective'] = 'reg:squarederror' self.name = "XGB-Regressor" else: self.estimator_parameters['objective'] = 'binary:logistic' self.name = "XGB-Classifier" # Missing value must be defined. Otherwyse it returns 'nan' which cannot be # converted to JSON and produces trouble in different points self.estimator_parameters['missing'] = -99.99999 def build(self): '''Build a new XGBOOST model with the X and Y numpy matrices ''' try: from xgboost.sklearn import XGBClassifier from xgboost.sklearn import XGBRegressor except Exception as e: return False, 'XGboost not found, please revise your environment' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): LOG.info("Optimizing XGBOOST estimator") try: # Check type of model if self.param.getVal('quantitative'): self.estimator = XGBRegressor( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model','model type','XGBOOST quantitative (optimized)')) else: self.estimator = XGBClassifier( **self.estimator_parameters) params = self.estimator.get_params() params['num_class'] = 2 self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model','model type','XGBOOST qualitative (optimized)')) except Exception as e: return False, f'Exception optimizing XGBOOST estimator with exception {e}' else: try: if self.param.getVal('quantitative'): LOG.info("Building Quantitative XGBOOST model") # params = { # 'objective': 'reg:squarederror', # 'missing': -99.99999, # # 'max_depth': 20, # # 'learning_rate': 1.0, # # 'silent': 1, # # 'n_estimators': 25 # } # self.estimator = XGBRegressor(**params) self.estimator = XGBRegressor(**self.estimator_parameters) results.append(('model', 'model type', 'XGBOOST quantitative')) else: LOG.info("Building Qualitative XGBOOST model") # params = { # 'objective': 'binary:logistic', # 'max_depth': 3, # #'learning_rate': 0.7, # #'silent': 1, # 'n_estimators': 100 # } self.estimator = XGBClassifier(**self.estimator_parameters) results.append(('model', 'model type', 'XGBOOST qualitative')) self.estimator.fit(X, Y) print(self.estimator) except Exception as e: raise e return False, f'Exception building XGBOOST estimator with exception {e}' self.estimator_temp = copy(self.estimator) if not self.param.getVal('conformal'): return True, results # Create the conformal estimator try: # Conformal regressor if self.param.getVal('quantitative'): LOG.info("Building conformal Quantitative XGBOOST model") underlying_model = RegressorAdapter(self.estimator_temp) #normalizing_model = RegressorAdapter( #KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer( underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) results.append(('model', 'model type', 'conformal XGBOOST quantitative')) # Conformal classifier else: LOG.info("Building conformal Qualitative XGBOOST model") self.estimator = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(self.estimator_temp), MarginErrFunc() ) ), BootstrapSampler()) # Fit estimator to the data self.estimator.fit(X, Y) results.append(('model', 'model type', 'conformal XGBOOST qualitative')) except Exception as e: raise e return False, f'Exception building conformal XGBOOST estimator with exception {e}' return True, results ## Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
def build(self): '''Build a new RF model with the X and Y numpy matrices ''' if self.failed: return False X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.cv: self.cv = getCrossVal(self.cv, self.estimator_parameters["random_state"], self.n, self.p) if self.tune: if self.quantitative: self.optimize(X, Y, RandomForestRegressor(), self.tune_parameters) results.append( ('model', 'model type', 'RF quantitative (optimized)')) else: self.optimize(X, Y, RandomForestClassifier(), self.tune_parameters) results.append( ('model', 'model type', 'RF qualitative (optimized)')) else: if self.quantitative: log.info("Building Quantitative RF model") self.estimator_parameters.pop('class_weight', None) self.estimator = RandomForestRegressor( **self.estimator_parameters) results.append(('model', 'model type', 'RF quantitative')) else: log.info("Building Qualitative RF model") self.estimator = RandomForestClassifier( **self.estimator_parameters) results.append(('model', 'model type', 'RF qualitative')) if self.conformal: if self.quantitative: underlying_model = RegressorAdapter(self.estimator) normalizing_model = RegressorAdapter( KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor(RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.conformal_pred = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.conformal_pred.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal RF quantitative')) else: self.conformal_pred = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator), MarginErrFunc())), BootstrapSampler()) self.conformal_pred.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal RF qualitative')) self.estimator.fit(X, Y) return True, results #### Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
class PLSR(BaseEstimator): def __init__(self, X, Y, parameters): super(PLSR, self).__init__(X, Y, parameters) self.estimator_parameters = parameters['PLSR_parameters'] self.tune = parameters['tune'] self.tune_parameters = parameters['PLSR_optimize'] self.name = "PLSR" self.optimiz = self.estimator_parameters["optimize"] self.estimator_parameters.pop("optimize") def build(self): if not self.quantitative: print("PLSR only applies to quantitative data") return False, "PLSR only applies to quantitative data" if self.failed: return False, "Error initiating model" X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.cv: self.cv = getCrossVal(self.cv, 46, self.n, self.p) if self.tune: if self.optimiz == 'auto': super(PLSR, self).optimize(X, Y, PLS_r( **self.estimator_parameters), self.tune_parameters) elif self.optimiz == 'manual': self.optimize(X, Y, PLS_r( **self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'PLSR quantitative (optimized)')) else: print("Building Quantitative PLSR") self.estimator = PLS_r(**self.estimator_parameters) results.append(('model', 'model type', 'PLSR quantitative')) if self.conformal: underlying_model = RegressorAdapter(self.estimator) normalizing_model = RegressorAdapter( KNeighborsRegressor(n_neighbors=1)) normalizing_model = RegressorAdapter(self.estimator) normalizer = RegressorNormalizer( underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) self.conformal_pred = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) # self.conformal_pred = AggregatedCp(IcpRegressor(RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.conformal_pred.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal PLSR quantitative')) self.estimator.fit(X, Y) return True, results def optimize(self, X, Y, estimator, tune_parameters): ''' optimizes a model using a grid search over a range of values for diverse parameters''' print("Optimizing PLSR algorithm") latent_variables = tune_parameters["n_components"] r2 = 0 estimator0 = "" list_latent = [] for n_comp in latent_variables: r2_0 = 0 estimator.set_params(**{"n_components": n_comp}) y_pred = cross_val_predict(estimator, X, Y, cv=self.cv, n_jobs=1) r2_0 = r2_score(Y, y_pred) if r2_0 >= r2: r2 = r2_0 estimator0 = copy.copy(estimator) list_latent.append([n_comp, r2_0]) print("r2 per latent variable") for el in list_latent: print("Number of latent variables: %s \nr2: %s\n" % (el[0], el[1])) self.estimator = estimator0 self.estimator.fit(X, Y) print(self.estimator.get_params()) #### Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
def CF_quantitative_validation(self): ''' Performs internal validation for conformal quantitative models ''' # Make a copy of original matrices. X = self.X.copy() Y = self.Y.copy() info = [] kf = KFold(n_splits=self.param.getVal('ModelValidationN'), shuffle=True, random_state=46) # Copy Y vector to use it as template to assign predictions Y_pred = copy.copy(Y).tolist() try: for train_index, test_index in kf.split(X): # Generate training and test sets X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] # Generate training a test sets # Create the aggregated conformal regressor. conformal_pred = AggregatedCp( IcpRegressor( RegressorNc(RegressorAdapter(self.estimator_temp))), BootstrapSampler()) # Fit conformal regressor to the data conformal_pred.fit(X_train, Y_train) # Perform prediction on test set prediction = conformal_pred.predict( X_test, self.param.getVal('conformalSignificance')) # Assign the prediction its original index for index, el in enumerate(test_index): Y_pred[el] = prediction[index] except Exception as e: LOG.error(f'Quantitative conformal validation' f' failed with exception: {e}') raise e Y_pred = np.asarray(Y_pred) # Add the n validation interval means interval_mean = np.mean(np.abs((Y_pred[:, 0]) - (Y_pred[:, 1]))) # Get boolean mask of instances # within the applicability domain. inside_interval = ((Y_pred[:, 0].reshape(-1, 1) < Y) & (Y_pred[:, 1].reshape(-1, 1) > Y)) # Compute the accuracy (number of instances within the AD). accuracy = np.sum(inside_interval) / len(Y) # Cut into two decimals. self.conformal_interval_medians = (np.mean(Y_pred, axis=1)) self.conformal_accuracy = float("{0:.2f}".format(accuracy)) self.conformal_mean_interval = float("{0:.2f}".format(interval_mean)) #Add quality metrics to results. info.append(('Conformal_mean_interval', 'Conformal mean interval', self.conformal_mean_interval)) info.append(('Conformal_accuracy', 'Conformal accuracy', self.conformal_accuracy)) info.append( ('Conformal_interval_medians', 'Conformal interval medians', self.conformal_interval_medians)) info.append(('Conformal_prediction_ranges', 'Conformal prediction ranges', Y_pred)) results = {} results['quality'] = info return True, results
class PLSR(BaseEstimator): """ This class inherits from BaseEstimator and wraps SKLEARN PLSR estimator ... Attributes ---------- estimator_parameters : dict parameter values name : string name of the estimator Methods ------- build(X) Instance the estimator optimizing it if tune=true. optimize( X, Y, estimator, tune_parameters) Gridsearch specially designed for PLSR. Optimizes number of variables """ def __init__(self, X, Y, parameters, conveyor): # Initialize parent class try: BaseEstimator.__init__(self,X, Y, parameters, conveyor) LOG.debug('Initialize BaseEstimator parent class') except Exception as e: LOG.error(f'Error initializing BaseEstimator parent class with exception: {e}') self.conveyor.setError(f'Error initializing BaseEstimator parent class with exception: {e}') return # Load estimator parameters self.estimator_parameters = self.param.getDict('PLSR_parameters') # Scale is hard-coded to False for making use of external scalers self.estimator_parameters['scale'] = False self.name = "PLSR" # Check if the model is quantitative if not self.param.getVal('quantitative'): LOG.error('PLSR only applies to quantitative data') self.conveyor.setError('PLSR only applies to quantitative data') return def build(self): # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) if self.param.getVal('tune'): # Optimize estimator using sklearn-gridsearch if self.estimator_parameters['optimize'] == 'auto': try: LOG.info('Optimizing PLSR using SK-LearnGridSearch') # Remove optimize key from parameter dictionary # to avoid sklearn estimator error (unexpected keyword) self.estimator_parameters.pop("optimize") super(PLSR, self).optimize(X, Y, PLS_r( **self.estimator_parameters), self.param.getDict('PLSR_optimize')) except Exception as e: LOG.error(f'Error performing SK-LearnGridSearch' f' on PLSR estimator with exception {e}') return False, f'Error performing SK-LearnGridSearch on PLSR estimator with exception {e}' # Optimize using flame implementation (recommended) elif self.estimator_parameters['optimize'] == 'manual': LOG.info('Optimizing PLSR using manual method') # Remove optimize key from parameter dictionary # to avoid sklearn estimator error (unexpected keyword) self.estimator_parameters.pop("optimize") success, message = self.optimize(X, Y, PLS_r( **self.estimator_parameters), self.param.getDict('PLSR_optimize')) if not success: return False, message else: LOG.error('Type of tune not recognized, check the input') return False, 'Type of tune not recognized, check the input' results.append(('model', 'model type', 'PLSR quantitative (optimized)')) else: LOG.info('Building Quantitative PLSR with no optimization') try: # Remove optimize key from parameters to avoid error self.estimator_parameters.pop("optimize") # as the sklearn estimator does not have this key self.estimator = PLS_r(**self.estimator_parameters) except Exception as e: LOG.error(f'Error at PLS_r instantiation with ' f'exception {e}') return False, f'Error at PLS_da instantiation with exception {e}' results.append(('model', 'model type', 'PLSR quantitative')) # Fit estimator to the data self.estimator.fit(X, Y) if not self.param.getVal('conformal'): return True, results self.estimator_temp = copy(self.estimator) try: LOG.info('Building PLSR aggregated conformal predictor') underlying_model = RegressorAdapter(self.estimator_temp) # normalizing_model = RegressorAdapter( # KNeighborsRegressor(n_neighbors=1)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) except Exception as e: LOG.error(f'Error building aggregated PLSR conformal' f' regressor with exception: {e}') return False, f'Error building aggregated PLSR conformal regressor with exception: {e}' # self.conformal_pred = AggregatedCp(IcpRegressor( # RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) # Fit conformal estimator to the data self.estimator.fit(X, Y) # overrides non-conformal results.append(('model', 'model type', 'conformal PLSR quantitative')) return True, results def optimize(self, X, Y, estimator, tune_parameters): ''' optimizes a model using a grid search over a range of values for diverse parameters''' # Max number of latent variables latent_variables = tune_parameters['n_components'] # Best r2 r2 = 0 estimator0 = None # List to add r2 for each number of latent variables list_latent = [] try: for n_comp in latent_variables: r2_0 = 0 estimator.set_params(**{"n_components": n_comp}) y_pred = cross_val_predict(estimator, X, Y, cv=self.cv, n_jobs=1) r2_0 = r2_score(Y, y_pred) # print (n_comp, r2_0) # Update estimator0 to best current estimator if r2_0 >= r2 or estimator0 == None: r2 = r2_0 estimator0 = copy(estimator) list_latent.append([n_comp, r2_0]) except Exception as e: return False, f'Error optimizing PLSR with exception {e}' self.estimator = estimator0 # LOG.debug('Number of latent variables, r2') # for lv in list_latent: # LOG.debug(f'Number of latent variables: {lv[0]} \nr2: {lv[1]}\n') self.estimator.fit(X, Y) LOG.info(f'Estimator best parameters: {self.estimator.get_params()}') return True, 'OK' #### Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
class RF(BaseEstimator): """ This class inherits from BaseEstimator and wraps SKLEARN RandomForestClassifier or RandomForestRegressor estimator ... Attributes ---------- estimator_parameters : dict parameter values name : string name of the estimator tune_parameters: dict Hyperparameter optimization settings Methods ------- build(X) Instance the estimator optimizing it if tune=true. """ def __init__(self, X, Y, parameters): # Initialize parent class try: super(RF, self).__init__(X, Y, parameters) LOG.debug('Initialize BaseEstimator parent class') except Exception as e: LOG.error(f'Error initializing BaseEstimator parent' f'class with exception: {e}') raise e # Load estimator parameters # alternative approach based in the use of a Parameters class for storing # estimator and optimizer parameters # self.estimator_parametes = Parameters() # self.estimator_parametes.loadDict(self.param.getVal('RF_parameters')) self.estimator_parameters = self.param.getDict('RF_parameters') # Load tune parameters self.tune_parameters = self.param.getDict('RF_optimize') if self.param.getVal('quantitative'): self.name = "RF-R" self.tune_parameters.pop("class_weight") else: self.name = "RF-C" def build(self): '''Build a new RF model with the X and Y numpy matrices ''' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): try: # Check type of model if self.param.getVal('quantitative'): self.optimize( X, Y, RandomForestRegressor(**self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'RF quantitative (optimized)')) else: self.optimize( X, Y, RandomForestClassifier(**self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'RF qualitative (optimized)')) LOG.debug('RF estimator optimized') except Exception as e: LOG.error(f'Exception optimizing RF' f'estimator with exception {e}') else: try: LOG.info("Building RF model") if self.param.getVal('quantitative'): self.estimator_parameters.pop('class_weight', None) self.estimator = RandomForestRegressor( **self.estimator_parameters) results.append(('model', 'model type', 'RF quantitative')) else: LOG.info("Building Qualitative RF model") self.estimator = RandomForestClassifier( **self.estimator_parameters) results.append(('model', 'model type', 'RF qualitative')) except Exception as e: LOG.error(f'Exception building RF' f'estimator with exception {e}') self.estimator.fit(X, Y) self.estimator_temp = copy(self.estimator) # Create the conformal estimator if self.param.getVal('conformal'): try: LOG.info("Building aggregated conformal RF model") if self.param.getVal('quantitative'): underlying_model = RegressorAdapter(self.estimator_temp) #normalizing_model = RegressorAdapter( #KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal RF quantitative')) # Conformal classifier else: self.estimator = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit estimator to the data self.estimator.fit(X, Y) results.append( ('model', 'model type', 'conformal RF qualitative')) except Exception as e: LOG.error(f'Exception building aggregated conformal Random' f'Forest estimator with exception {e}') return True, results ## Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
def build(self): '''Build a new RF model with the X and Y numpy matrices ''' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) results.append(('model', 'model type', 'RF')) conformal = self.param.getVal('conformal') # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): LOG.info("Optimizing RF estimator") try: # Check type of model if self.param.getVal('quantitative'): self.estimator = RandomForestRegressor( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) # results.append(('model','model type','RF quantitative (optimized)')) else: self.estimator = RandomForestClassifier( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) # results.append(('model','model type','RF qualitative (optimized)')) except Exception as e: return False, f'Exception optimizing RF estimator with exception {e}' else: try: if self.param.getVal('quantitative'): self.estimator = RandomForestRegressor( **self.estimator_parameters) if not conformal: LOG.info("Building Quantitative RF model") # results.append(('model', 'model type', 'RF quantitative')) else: self.estimator = RandomForestClassifier( **self.estimator_parameters) if not conformal: LOG.info("Building Qualitative RF model") # results.append(('model', 'model type', 'RF qualitative')) self.estimator.fit(X, Y) except Exception as e: return False, f'Exception building RF estimator with exception {e}' if not conformal: return True, results self.estimator_temp = copy(self.estimator) # Create the conformal estimator try: # Conformal regressor if self.param.getVal('quantitative'): conformal_settings = self.param.getDict('conformal_settings') LOG.info("Building conformal Quantitative RF model") underlying_model = RegressorAdapter(self.estimator_temp) self.normalizing_model = RegressorAdapter( KNeighborsRegressor( n_neighbors=conformal_settings['KNN_NN'])) # normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, copy(self.normalizing_model), AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) # results.append(('model', 'model type', 'conformal RF quantitative')) # Conformal classifier else: LOG.info("Building conformal Qualitative RF model") self.estimator = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit estimator to the data self.estimator.fit(X, Y) # results.append(('model', 'model type', 'conformal RF qualitative')) except Exception as e: return False, f'Exception building conformal RF estimator with exception {e}' return True, results ## Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' # def optimize(self, X, Y, estimator, tune_parameters): # ''' optimizes a model using a grid search over a range of values for diverse parameters''' # def regularProject(self, Xb, results): # ''' projects a collection of query objects in a regular model, for obtaining predictions ''' # def conformalProject(self, Xb, results): # ''' projects a collection of query objects in a conformal model, for obtaining predictions ''' # def project(self, Xb, results): # ''' Uses the X matrix provided as argument to predict Y'''
def test_acp_classification_tree(self): # ----------------------------------------------------------------------------- # Experiment setup # ----------------------------------------------------------------------------- data = load_iris() idx = np.random.permutation(data.target.size) train = idx[:int(2 * idx.size / 3)] test = idx[int(2 * idx.size / 3):] truth = data.target[test].reshape(-1, 1) columns = ["C-{}".format(i) for i in np.unique(data.target)] + ["truth"] significance = 0.1 # ----------------------------------------------------------------------------- # Define models # ----------------------------------------------------------------------------- models = { "ACP-RandomSubSampler": AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(DecisionTreeClassifier()))), RandomSubSampler(), ), "ACP-CrossSampler": AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(DecisionTreeClassifier()))), CrossSampler(), ), "ACP-BootstrapSampler": AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(DecisionTreeClassifier()))), BootstrapSampler(), ), "CCP": CrossConformalClassifier( IcpClassifier( ClassifierNc(ClassifierAdapter( DecisionTreeClassifier())))), "BCP": BootstrapConformalClassifier( IcpClassifier( ClassifierNc(ClassifierAdapter( DecisionTreeClassifier())))), } # ----------------------------------------------------------------------------- # Train, predict and evaluate # ----------------------------------------------------------------------------- for name, model in models.items(): model.fit(data.data[train, :], data.target[train]) prediction = model.predict(data.data[test, :], significance=significance) table = np.hstack((prediction, truth)) df = pd.DataFrame(table, columns=columns) print("\n{}".format(name)) print("Error rate: {}".format( class_mean_errors(prediction, truth, significance))) print(df) self.assertTrue(True)
# ----------------------------------------------------------------------------- # Setup training, calibration and test indices # ----------------------------------------------------------------------------- idx = np.random.permutation(data.target.size) train = idx[:int(2 * idx.size / 3)] test = idx[int(2 * idx.size / 3):] # ----------------------------------------------------------------------------- # Train and calibrate # ----------------------------------------------------------------------------- nc_class_params = {'model_class': DecisionTreeRegressor, 'err_func': abs_error, 'inverse_err_func': abs_error_inv} rscp = AggregatedCp(IcpRegressor, RegressorNc, sampler=RandomSubSampler(), nc_class_params=nc_class_params) rscp.fit(data.data[train, :], data.target[train]) ccp = AggregatedCp(IcpRegressor, RegressorNc, sampler=CrossSampler(), nc_class_params=nc_class_params) ccp.fit(data.data[train, :], data.target[train]) bcp = AggregatedCp(IcpRegressor, RegressorNc, sampler=BootstrapSampler(), nc_class_params=nc_class_params) bcp.fit(data.data[train, :], data.target[train])
idx = np.random.permutation(data.target.size) train = idx[:int(2 * idx.size / 3)] test = idx[int(2 * idx.size / 3):] truth = data.target[test] columns = ['min', 'max', 'truth'] significance = 0.1 # ----------------------------------------------------------------------------- # Define models # ----------------------------------------------------------------------------- models = { 'ACP-RandomSubSampler': AggregatedCp( IcpRegressor(RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), RandomSubSampler()), 'ACP-CrossSampler': AggregatedCp( IcpRegressor(RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), CrossSampler()), 'ACP-BootstrapSampler': AggregatedCp( IcpRegressor(RegressorNc(RegressorAdapter(DecisionTreeRegressor()))), BootstrapSampler()) } # ----------------------------------------------------------------------------- # Train, predict and evaluate # ----------------------------------------------------------------------------- for name, model in models.iteritems():
def CF_qualitative_validation(self): ''' performs validation for conformal qualitative models ''' # Make a copy of original matrices. X = self.X.copy() Y = self.Y.copy() # Total number of class 0 correct predictions. c0_correct_all = 0 # Total number of class 0 incorrect predictions. c0_incorrect_all = 0 # Total number of class 1 correct predictions. c1_correct_all = 0 # Total number of class 1 incorrect predictions c1_incorrect_all = 0 # Total number of instances out of the applicability domain. not_predicted_all = 0 info = [] kf = KFold(n_splits=5, shuffle=True, random_state=46) # Copy Y vector to use it as template to assign predictions Y_pred = copy.copy(Y).tolist() try: for train_index, test_index in kf.split(X): # Generate training and test sets X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] # Create the aggregated conformal classifier. conformal_pred = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit the conformal classifier to the data conformal_pred.fit(X_train, Y_train) # Perform prediction on test set prediction = conformal_pred.predict( X_test, self.param.getVal('conformalSignificance')) # Assign the prediction the correct index. for index, el in enumerate(test_index): Y_pred[el] = prediction[index] # Iterate over the prediction and check the result for i in range(len(Y_pred)): real = float(Y[i]) predicted = Y_pred[i] if predicted[0] != predicted[1]: if real == 0 and predicted[0] == True: c0_correct_all += 1 if real == 0 and predicted[1] == True: c0_incorrect_all += 1 if real == 1 and predicted[1] == True: c1_correct_all += 1 if real == 1 and predicted[0] == True: c1_incorrect_all += 1 else: not_predicted_all += 1 except Exception as e: LOG.error(f'Qualitative conformal validation' f' failed with exception: {e}') raise e # Get the mean confusion matrix. self.TN = c0_correct_all self.FP = c0_incorrect_all self.TP = c1_correct_all self.FN = c1_incorrect_all not_predicted_all = not_predicted_all info.append(('TP', 'True positives in cross-validation', self.TP)) info.append(('TN', 'True negatives in cross-validation', self.TN)) info.append(('FP', 'False positives in cross-validation', self.FP)) info.append(('FN', 'False negatives in cross-validation', self.FN)) # Compute sensitivity, specificity and MCC try: self.sensitivity = (self.TP / (self.TP + self.FN)) except Exception as e: LOG.error(f'Failed to compute sensibility with' f'exception {e}') self.sensitivity = '-' try: self.specificity = (self.TN / (self.TN + self.FP)) except Exception as e: LOG.error(f'Failed to compute specificity with' f'exception {e}') self.specificity = '-' try: # Compute Matthews Correlation Coefficient self.mcc = (((self.TP * self.TN) - (self.FP * self.FN)) / np.sqrt( (self.TP + self.FP) * (self.TP + self.FN) * (self.TN + self.FP) * (self.TN + self.FN))) except Exception as e: LOG.error(f'Failed to compute Mathews Correlation Coefficient' f'exception {e}') self.mcc = '-' info.append(('Sensitivity', 'Sensitivity in cross-validation', self.sensitivity)) info.append(('Specificity', 'Specificity in cross-validation', self.specificity)) info.append( ('MCC', 'Matthews Correlation Coefficient in cross-validation', self.mcc)) try: # Compute coverage (% of compounds inside the applicability domain) self.conformal_coverage = ( self.TN + self.FP + self.TP + self.FN) / ( (self.TN + self.FP + self.TP + self.FN) + not_predicted_all) except Exception as e: LOG.error(f'Failed to compute conformal coverage with' f'exception {e}') self.conformal_coverage = '-' try: # Compute accuracy (% of correct predictions) self.conformal_accuracy = ( float(self.TN + self.TP) / float(self.FP + self.FN + self.TN + self.TP)) except Exception as e: LOG.error(f'Failed to compute conformal accuracy with' f'exception {e}') self.conformal_accuracy = '-' info.append(('Conformal_coverage', 'Conformal coverage', self.conformal_coverage)) info.append(('Conformal_accuracy', 'Conformal accuracy', self.conformal_accuracy)) results = {} results['quality'] = info #results ['classes'] = prediction return True, results
class SVM(BaseEstimator): """ This class inherits from BaseEstimator and wraps SKLEARN SVClassifier or SVRegressor estimator ... Attributes ---------- estimator_parameters : dict parameter values name : string name of the estimator tune_parameters: dict Hyperparameter optimization settings Methods ------- build(X) Instance the estimator optimizing it if tune=true. """ def __init__(self, X, Y, parameters): # Initialize parent class try: super(SVM, self).__init__(X, Y, parameters) LOG.debug('Initialize BaseEstimator parent class') except Exception as e: LOG.error(f'Error initializing BaseEstimator parent' f'class with exception: {e}') raise e # Load estimator parameters self.estimator_parameters = self.param.getDict('SVM_parameters') # Load tune parameters self.tune_parameters = self.param.getDict('SVM_optimize') if self.param.getVal('quantitative'): self.name = "SVM-R" # Remove parameters of SVC class and set the name self.estimator_parameters.pop("class_weight", None) self.estimator_parameters.pop("probability", None) self.estimator_parameters.pop("decision_function_shape", None) self.estimator_parameters.pop("random_state", None) self.tune_parameters.pop("class_weight", None) self.tune_parameters.pop("random_state", None) self.tune_parameters.pop("probability", None) else: # Remove parameters of SVR class and set the name self.estimator_parameters.pop("epsilon", None) self.name = "SVM-C" def build(self): '''Build a new SVM model with the X and Y numpy matrices''' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): try: # Check type of model if self.param.getVal('quantitative'): self.optimize(X, Y, svm.SVR(**self.estimator_parameters), self.tune_parameters) results.append(('model', 'model type', 'SVM quantitative (optimized)')) else: self.optimize(X, Y, svm.SVC(**self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'SVM qualitative (optimized)')) LOG.debug('SVM estimator optimized') except Exception as e: LOG.error(f'Exception optimizing SVM' f'estimator with exception {e}') else: try: LOG.info("Building SVM model") if self.param.getVal('quantitative'): LOG.info("Building Quantitative SVM-R model") self.estimator = svm.SVR(**self.estimator_parameters) results.append(('model', 'model type', 'SVM quantitative')) else: self.estimator = svm.SVC(**self.estimator_parameters) results.append(('model', 'model type', 'SVM qualitative')) except Exception as e: LOG.error(f'Exception building SVM' f'estimator with exception {e}') self.estimator.fit(X, Y) self.estimator_temp = copy(self.estimator) if self.param.getVal('conformal'): try: LOG.info("Building aggregated conformal SVM model") if self.param.getVal('quantitative'): underlying_model = RegressorAdapter(self.estimator_temp) # normalizing_model = RegressorAdapter( # KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor( # RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal SVM quantitative')) else: self.estimator = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) self.estimator.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal SVM qualitative')) except Exception as e: LOG.error(f'Exception building aggregated conformal SVM ' f'estimator with exception {e}') # Fit estimator to the data return True, results
# 'ACP-CrossSampler' : AggregatedCp( # IcpClassifier( # ClassifierNc( # ClassifierAdapter(gbm))), # CrossSampler()) # # 'ACP-BootstrapSampler' : AggregatedCp( # # IcpClassifier( # # ClassifierNc( # # ClassifierAdapter(DecisionTreeClassifier()))), # # BootstrapSampler()), # # 'CCP' : CrossConformalClassifier( # # IcpClassifier( # # ClassifierNc( # # ClassifierAdapter(DecisionTreeClassifier())))), # # 'BCP' : BootstrapConformalClassifier( # # IcpClassifier( # # ClassifierNc( # # ClassifierAdapter(DecisionTreeClassifier())))) # } model = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(gbm))), CrossSampler()) model.fit(x_train, y_train) print('predicting') prediction = model.predict(x_test, significance=None) np.savetxt(os.getcwd()+"/prediction/prediction_acp_cross_1.txt", prediction, delimiter=',')
classification_method = DecisionTreeClassifier() file_name = 'decision_tree.xls' ACP_Random = [] ACP_Cross = [] ACP_Boot = [] CCP = [] BCP = [] # ----------------------------------------------------------------------------- # Define models # ----------------------------------------------------------------------------- models = { 'ACP-RandomSubSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter(classification_method))), RandomSubSampler()), 'ACP-CrossSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter(classification_method))), CrossSampler()), 'ACP-BootstrapSampler': AggregatedCp( IcpClassifier(ClassifierNc(ClassifierAdapter(classification_method))), BootstrapSampler()), 'CCP': CrossConformalClassifier( IcpClassifier(ClassifierNc(ClassifierAdapter(classification_method)))), 'BCP': BootstrapConformalClassifier( IcpClassifier(ClassifierNc(ClassifierAdapter(classification_method))))
# ----------------------------------------------------------------------------- # Train and calibrate # ----------------------------------------------------------------------------- icp = IcpClassifier( ProbEstClassifierNc(DecisionTreeClassifier(), inverse_probability)) icp.fit(X[train, :], y[train]) icp.calibrate(X[calibrate, :], y[calibrate]) ccp = CrossConformalClassifier( IcpClassifier( ProbEstClassifierNc(DecisionTreeClassifier(), inverse_probability))) ccp.fit(X[train, :], y[train]) acp = AggregatedCp( IcpClassifier( ProbEstClassifierNc(DecisionTreeClassifier(), inverse_probability)), CrossSampler()) acp.fit(X[train, :], y[train]) # ----------------------------------------------------------------------------- # Predict # ----------------------------------------------------------------------------- print('# Inductive') prediction = icp.predict(X[test, :], significance=0.1) for pred, actual in zip(prediction[:5], y[test]): print(pred, actual) print('\n# Cross') prediction = ccp.predict(X[test, :], significance=0.1) for pred, actual in zip(prediction[:5], y[test]): print(pred, actual)
class Keras_nn(BaseEstimator): """ This class inherits from BaseEstimator and wraps SKLEARN RandomForestClassifier or RandomForestRegressor estimator ... Attributes ---------- estimator_parameters : dict parameter values name : string name of the estimator tune_parameters: dict Hyperparameter optimization settings Methods ------- build(X) Instance the estimator optimizing it if tune=true. """ def __init__(self, X, Y, parameters, conveyor): # Initialize parent class try: BaseEstimator.__init__(self, X, Y, parameters, conveyor) LOG.debug('Initialize BaseEstimator parent class') except Exception as e: self.conveyor.setError( f'Error initializing BaseEstimator parent class with exception: {e}' ) LOG.error( f'Error initializing BaseEstimator parent class with exception: {e}' ) return # Load estimator parameters self.estimator_parameters = self.param.getDict('Keras_parameters') # Load tune parameters self.tune_parameters = self.param.getDict('Keras_optimize') if self.param.getVal('quantitative'): self.name = "Keras-Regressor" else: self.name = "Keras-Classifier" def build(self): '''Build a new DL model with the X and Y numpy matrices ''' try: from keras.wrappers.scikit_learn import KerasClassifier from keras.wrappers.scikit_learn import KerasRegressor except Exception as e: return False, 'Keras not found, please revise your environment' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): LOG.info("Optimizing Keras estimator") try: # Check type of model if self.param.getVal('quantitative'): self.estimator = KerasRegressor( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model', 'model type', 'KERAS quantitative (optimized)')) else: self.estimator = KerasClassifier( **self.estimator_parameters) #params = self.estimator.get_params() #params['num_class'] = 2 self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model', 'model type', 'KERAS qualitative (optimized)')) except Exception as e: return False, f'Exception optimizing KERAS estimator with exception {e}' else: try: if self.param.getVal('quantitative'): LOG.info("Building Quantitative KERAS mode") self.estimator = KerasRegressor( build_fn=self.create_model, **self.estimator_parameters, verbose=0) results.append( ('model', 'model type', 'Keras quantitative')) else: LOG.info("Building Qualitative Keras model") self.estimator = KerasClassifier( build_fn=self.create_model, dim=self.X.shape[1], **self.estimator_parameters, verbose=0) results.append( ('model', 'model type', 'Keras qualitative')) self.estimator.fit(X, Y) print(self.estimator) except Exception as e: raise e return False, f'Exception building Keras estimator with exception {e}' self.estimator_temp = clone(self.estimator) if not self.param.getVal('conformal'): return True, results # Create the conformal estimator try: # Conformal regressor if self.param.getVal('quantitative'): LOG.info("Building conformal Quantitative Keras model") underlying_model = RegressorAdapter(self.estimator_temp) normalizing_model = RegressorAdapter( KNeighborsRegressor(n_neighbors=15)) # normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) results.append( ('model', 'model type', 'conformal Keras quantitative')) # Conformal classifier else: LOG.info("Building conformal Qualitative Keras model") self.estimator = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit estimator to the data print('build finished') self.estimator.fit(X, Y) results.append( ('model', 'model type', 'conformal Keras qualitative')) except Exception as e: raise e return False, f'Exception building conformal Keras estimator with exception {e}' return True, [] # Function to create model, required for KerasClassifier def create_model(self, dim=20): # create model model = Sequential() model.add(Dense(10, input_dim=dim, activation='relu')) model.add(Dense(20, activation='sigmoid')) model.add(Dense(1, activation='sigmoid')) # Compile model #if self.param.getVal('quantitative'): #loss = 'mean_squared_error' #else: loss = 'binary_crossentropy' model.compile(loss=loss, optimizer='adam', metrics=['accuracy']) return model # Overrides regular project to single class prediction def regularProject(self, Xb): ''' projects a collection of query objects in a regular model, for obtaining predictions ''' Yp = self.estimator.predict(Xb) Yp = np.asarray([x[0] for x in Yp]) # if conveyor contains experimental values for any of the objects replace the # predictions with the experimental results exp = self.conveyor.getVal('experim') if exp is not None: if len(exp) == len(Yp): for i in range(len(Yp)): if not np.isnan(exp[i]): # print (exp[i], Yp[i]) Yp[i] = exp[i] else: # if exp is nan, substitute it with a number which can be recognized # to facilitate handling and do not replace Yp exp[i] = float('-99999') self.conveyor.addVal(Yp, 'values', 'Prediction', 'result', 'objs', 'Results of the prediction', 'main') # Overrides base_model validation # def validate(self,): # results ['quality'] = [] # return True, None ## Overriding of parent methods # def CF_quantitative_validation(self): # ''' performs validation for conformal quantitative models ''' # def CF_qualitative_validation(self): # ''' performs validation for conformal qualitative models ''' # def quantitativeValidation(self): # ''' performs validation for quantitative models ''' # def qualitativeValidation(self): # ''' performs validation for qualitative models ''' # def validate(self): # ''' Validates the model and computes suitable model quality scoring values''' def optimize(self, X, Y, estimator, tune_parameters): # TODO Perhaps not suitable ''' optimizes a model using a grid search over a range of values for diverse parameters''' LOG.info( 'Keras model parameter optimization not implemented. skipping....')
def build(self): '''Build a new SVM model with the X and Y numpy matrices''' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): try: # Check type of model if self.param.getVal('quantitative'): self.optimize(X, Y, svm.SVR(**self.estimator_parameters), self.tune_parameters) results.append(('model', 'model type', 'SVM quantitative (optimized)')) else: self.optimize(X, Y, svm.SVC(**self.estimator_parameters), self.tune_parameters) results.append( ('model', 'model type', 'SVM qualitative (optimized)')) LOG.debug('SVM estimator optimized') except Exception as e: LOG.error(f'Exception optimizing SVM' f'estimator with exception {e}') else: try: LOG.info("Building SVM model") if self.param.getVal('quantitative'): LOG.info("Building Quantitative SVM-R model") self.estimator = svm.SVR(**self.estimator_parameters) results.append(('model', 'model type', 'SVM quantitative')) else: self.estimator = svm.SVC(**self.estimator_parameters) results.append(('model', 'model type', 'SVM qualitative')) except Exception as e: LOG.error(f'Exception building SVM' f'estimator with exception {e}') self.estimator.fit(X, Y) self.estimator_temp = copy(self.estimator) if self.param.getVal('conformal'): try: LOG.info("Building aggregated conformal SVM model") if self.param.getVal('quantitative'): underlying_model = RegressorAdapter(self.estimator_temp) # normalizing_model = RegressorAdapter( # KNeighborsRegressor(n_neighbors=5)) normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor( # RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal SVM quantitative')) else: self.estimator = AggregatedCp( IcpClassifier( ClassifierNc( ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) self.estimator.fit(X, Y) # overrides non-conformal results.append( ('model', 'model type', 'conformal SVM qualitative')) except Exception as e: LOG.error(f'Exception building aggregated conformal SVM ' f'estimator with exception {e}') # Fit estimator to the data return True, results
def build(self): '''Build a new DL model with the X and Y numpy matrices ''' try: from keras.wrappers.scikit_learn import KerasClassifier from keras.wrappers.scikit_learn import KerasRegressor except Exception as e: return False, 'Keras not found, please revise your environment' # Make a copy of data matrices X = self.X.copy() Y = self.Y.copy() results = [] results.append(('nobj', 'number of objects', self.nobj)) results.append(('nvarx', 'number of predictor variables', self.nvarx)) # If tune then call gridsearch to optimize the estimator if self.param.getVal('tune'): LOG.info("Optimizing Keras estimator") try: # Check type of model if self.param.getVal('quantitative'): self.estimator = KerasRegressor( **self.estimator_parameters) self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model', 'model type', 'KERAS quantitative (optimized)')) else: self.estimator = KerasClassifier( **self.estimator_parameters) #params = self.estimator.get_params() #params['num_class'] = 2 self.optimize(X, Y, self.estimator, self.tune_parameters) results.append(('model', 'model type', 'KERAS qualitative (optimized)')) except Exception as e: return False, f'Exception optimizing KERAS estimator with exception {e}' else: try: if self.param.getVal('quantitative'): LOG.info("Building Quantitative KERAS mode") self.estimator = KerasRegressor( build_fn=self.create_model, **self.estimator_parameters, verbose=0) results.append( ('model', 'model type', 'Keras quantitative')) else: LOG.info("Building Qualitative Keras model") self.estimator = KerasClassifier( build_fn=self.create_model, dim=self.X.shape[1], **self.estimator_parameters, verbose=0) results.append( ('model', 'model type', 'Keras qualitative')) self.estimator.fit(X, Y) print(self.estimator) except Exception as e: raise e return False, f'Exception building Keras estimator with exception {e}' self.estimator_temp = clone(self.estimator) if not self.param.getVal('conformal'): return True, results # Create the conformal estimator try: # Conformal regressor if self.param.getVal('quantitative'): LOG.info("Building conformal Quantitative Keras model") underlying_model = RegressorAdapter(self.estimator_temp) normalizing_model = RegressorAdapter( KNeighborsRegressor(n_neighbors=15)) # normalizing_model = RegressorAdapter(self.estimator_temp) normalizer = RegressorNormalizer(underlying_model, normalizing_model, AbsErrorErrFunc()) nc = RegressorNc(underlying_model, AbsErrorErrFunc(), normalizer) # self.conformal_pred = AggregatedCp(IcpRegressor # (RegressorNc(RegressorAdapter(self.estimator))), # BootstrapSampler()) self.estimator = AggregatedCp(IcpRegressor(nc), BootstrapSampler()) self.estimator.fit(X, Y) results.append( ('model', 'model type', 'conformal Keras quantitative')) # Conformal classifier else: LOG.info("Building conformal Qualitative Keras model") self.estimator = AggregatedCp( IcpClassifier( ClassifierNc(ClassifierAdapter(self.estimator_temp), MarginErrFunc())), BootstrapSampler()) # Fit estimator to the data print('build finished') self.estimator.fit(X, Y) results.append( ('model', 'model type', 'conformal Keras qualitative')) except Exception as e: raise e return False, f'Exception building conformal Keras estimator with exception {e}' return True, []