def train(X_train, y_train, method_name, base_classifier, T): if method_name == 'adaboost': clf = AdaBoostClassifier(base_estimator=base_classifier, n_estimators=T) elif method_name == 'RUSBoost': clf = RUSBoostClassifier(base_estimator=base_classifier, n_estimators=T, sampling_strategy='majority') elif method_name == 'SMOTEBoost': clf = OversampleBoost(oversampling_algorithm='SMOTE', base_estimator=base_classifier, n_estimators=T) elif method_name == 'SMOTETomekBoost': clf = OversampleBoost(oversampling_algorithm='SMOTE-TOMEK', base_estimator=base_classifier, n_estimators=T) elif method_name == 'SMOTEENNBoost': clf = OversampleBoost(oversampling_algorithm='SMOTE-ENN', base_estimator=base_classifier, n_estimators=T) elif method_name == 'DERSBoost': clf = DERSBoost(base_estimator=base_classifier, n_estimators=T, NGEN=50) start_time = time() clf.fit(X_train, y_train) elapsed_time = time() - start_time return clf, elapsed_time
def get_best_parameters(self, features, labels, base_estimator=None, n_iter=300, cv=3, verbose=1, random_state=1, n_jobs=-1): clf_random =\ GridSearchCV( estimator=RUSBoostClassifier(), param_grid=self.random_grid, cv=cv, verbose=verbose, n_jobs=n_jobs, iid=False, error_score=0 ) _features = features if 1 == len(features.values.shape): # imbalanced learn RUSBoostClassifier # doesn't like shapes of (N=1,) ? _features = features.values.reshape(-1, 1) clf_random.fit(_features, labels) return clf_random.best_params_
def train(classifier, df,y, user_id): ''' The main training function that runs on a seperate process''' X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.33, random_state=0) base_estimator = AdaBoostClassifier(n_estimators=10) rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format(balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) joblib.dump(rusboost, user_id+'.pkl') classifier.classifierStatus = "trained" print("Done training") return classifier
def test_rusboost_sample_weight(imbalanced_dataset, algorithm): X, y = imbalanced_dataset sample_weight = np.ones_like(y) rusboost = RUSBoostClassifier(algorithm=algorithm, random_state=0) # Predictions should be the same when sample_weight are all ones y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) y_pred_no_sample_weight = rusboost.fit(X, y).predict(X) assert_array_equal(y_pred_sample_weight, y_pred_no_sample_weight) rng = np.random.RandomState(42) sample_weight = rng.rand(y.shape[0]) y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) with pytest.raises(AssertionError): assert_array_equal(y_pred_no_sample_weight, y_pred_sample_weight)
def test_rusboost(imbalanced_dataset, algorithm): X, y = imbalanced_dataset X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=1) classes = np.unique(y) n_estimators = 500 rusboost = RUSBoostClassifier(n_estimators=n_estimators, algorithm=algorithm, random_state=0) rusboost.fit(X_train, y_train) assert_array_equal(classes, rusboost.classes_) # check that we have an ensemble of samplers and estimators with a # consistent size assert len(rusboost.estimators_) > 1 assert len(rusboost.estimators_) == len(rusboost.samplers_) assert len(rusboost.pipelines_) == len(rusboost.samplers_) # each sampler in the ensemble should have different random state assert (len({sampler.random_state for sampler in rusboost.samplers_ }) == len(rusboost.samplers_)) # each estimator in the ensemble should have different random state assert (len({est.random_state for est in rusboost.estimators_ }) == len(rusboost.estimators_)) # check the consistency of the feature importances assert len(rusboost.feature_importances_) == imbalanced_dataset[0].shape[1] # check the consistency of the prediction outpus y_pred = rusboost.predict_proba(X_test) assert y_pred.shape[1] == len(classes) assert rusboost.decision_function(X_test).shape[1] == len(classes) score = rusboost.score(X_test, y_test) assert score > 0.7, "Failed with algorithm {} and score {}".format( algorithm, score) y_pred = rusboost.predict(X_test) assert y_pred.shape == y_test.shape
def fit(self, X, Y, sample_weight=None): import sklearn.tree self.n_estimators = int(self.n_estimators) self.learning_rate = float(self.learning_rate) self.max_depth = int(self.max_depth) base_estimator = sklearn.tree.DecisionTreeClassifier( max_depth=self.max_depth) from imblearn.ensemble import RUSBoostClassifier estimator = RUSBoostClassifier(base_estimator=base_estimator, n_estimators=self.n_estimators, learning_rate=self.learning_rate, algorithm=self.algorithm, random_state=self.random_state) estimator.fit(X, Y, sample_weight=sample_weight) self.estimator = estimator return self
def get_models(self): base_lr = LogisticRegression(class_weight='balanced') ovr_lr = OneVsRestClassifier(base_lr) base_eec = EasyEnsembleClassifier(n_estimators=10) ovr_eec = OneVsRestClassifier(base_eec) base_rus = RUSBoostClassifier(n_estimators=50) ovr_rus = OneVsRestClassifier(base_rus) base_bbc = BalancedBaggingClassifier(n_estimators=10) ovr_bbc = OneVsRestClassifier(base_bbc) base_brf = BalancedRandomForestClassifier(n_estimators=100) ovr_brf = OneVsRestClassifier(base_brf) estimators = [('lr', ovr_lr), ('eec', ovr_eec), ('rus', ovr_rus), ('bbc', ovr_bbc), ('brf', ovr_brf)] return estimators
def __init__(self): from imblearn.over_sampling import SMOTE, ADASYN, SVMSMOTE, BorderlineSMOTE, RandomOverSampler from imblearn.under_sampling import ClusterCentroids, RandomUnderSampler, InstanceHardnessThreshold, NearMiss, \ TomekLinks, EditedNearestNeighbours, RepeatedEditedNearestNeighbours, AllKNN, OneSidedSelection, \ CondensedNearestNeighbour, NeighbourhoodCleaningRule from imblearn.ensemble import EasyEnsemble, EasyEnsembleClassifier, BalancedBaggingClassifier, \ BalancedRandomForestClassifier, BalanceCascade, RUSBoostClassifier self.oversamplers = { 'ADASYN': ADASYN(), 'RandomOverSampler': RandomOverSampler(), 'SMOTE': SMOTE(), 'BorderlineSMOTE': BorderlineSMOTE(), 'SVMSMOTE': SVMSMOTE() } self.undersamplers = { 'ClusterCentroids': ClusterCentroids(), 'RandomUnderSampler': RandomUnderSampler(), 'InstanceHardnessThreshold': InstanceHardnessThreshold(), 'NearMiss': NearMiss(), 'TomekLinks': TomekLinks(), 'EditedNearestNeighbours': EditedNearestNeighbours(), 'RepeatedEditedNearestNeighbours': RepeatedEditedNearestNeighbours(), 'AllKNN': AllKNN(), 'OneSidedSelection': OneSidedSelection(), 'CondensedNearestNeighbour': CondensedNearestNeighbour(), 'NeighbourhoodCleaningRule': NeighbourhoodCleaningRule() } self.ensemblesamplers = { 'EasyEnsemble': EasyEnsemble(), 'EasyEnsembleClassifier': EasyEnsembleClassifier(), 'BalancedBaggingClassifier': BalancedBaggingClassifier(), 'BalanceCascade': BalanceCascade(), 'BalancedRandomForestClassifier': BalancedRandomForestClassifier, 'RUSBoostClassifier': RUSBoostClassifier() }
def get_models(): models, names = list(), list() # LR models.append( LogisticRegression(solver='liblinear', class_weight='balanced', penalty='l2')) names.append('Logistic Regression') # Ada Boost names.append('Ada Boost') models.append(AdaBoostClassifier()) # Gradient Boosting names.append('Gradient Boosting') models.append(GradientBoostingClassifier()) # RUSBoostClassifier names.append('RUSBoost Classifier') models.append(RUSBoostClassifier()) # BalancedRandomForestClassifier names.append('RandomForestClassifier') models.append(RandomForestClassifier(class_weight='balanced')) # BalancedRandomForestClassifier names.append('EasyEnsembleClassifier') models.append(EasyEnsembleClassifier()) return models, names
def test_rusboost_error(imbalanced_dataset, boosting_params, err_msg): rusboost = RUSBoostClassifier(**boosting_params) with pytest.raises(ValueError, match=err_msg): rusboost.fit(*imbalanced_dataset)
def test_balanced_random_forest_error(imbalanced_dataset, boosting_params, err_msg): rusboost = RUSBoostClassifier(**boosting_params) with pytest.raises(ValueError, message=err_msg): rusboost.fit(*imbalanced_dataset)
def _init_classifier(self, opt): if "classifier_opt" in opt: opt = opt['classifier_opt'] if "base_estimator" in opt: b_est = self._init_classifier(opt["base_estimator"]) else: b_est = None if "n_estimators" in opt: n_estimators = opt["n_estimators"] else: n_estimators = 200 if "max_iter" in opt: max_iter = opt["max_iter"] else: max_iter = 100000 if "num_parallel_tree" in opt: num_parallel_tree = opt["num_parallel_tree"] else: num_parallel_tree = 5 if "layer_structure" in opt: layer_structure = opt["layer_structure"] else: layer_structure = (100,) if opt["type"] in ["random_forrest", "rf"]: return RandomForestClassifier(n_estimators=n_estimators, class_weight="balanced", n_jobs=-1) elif opt["type"] == "ada_boost": return AdaBoostClassifier(base_estimator=b_est, n_estimators=n_estimators) elif opt["type"] in ["logistic_regression", "lr"]: return LogisticRegression(class_weight='balanced', max_iter=max_iter) elif opt["type"] == "sgd": return SGDClassifier(class_weight='balanced', max_iter=max_iter) elif opt["type"] in ["gaussian_bayes", "bayes", "gaussian_nb"]: return GaussianNB() elif opt["type"] in ["support_vector_machine", "svm"]: return SVC(kernel='rbf', class_weight='balanced', gamma="scale") elif opt["type"] in ["multilayer_perceptron", "mlp"]: return MLPClassifier(hidden_layer_sizes=layer_structure, max_iter=max_iter) elif opt["type"] in ["decision_tree", "dt", "tree"]: return DecisionTreeClassifier() elif opt["type"] in ["b_decision_tree", "b_dt", "b_tree"]: return DecisionTreeClassifier(class_weight="balanced") elif opt["type"] in ["neighbours", "knn"]: return KNeighborsClassifier(n_neighbors=opt["n_neighbours"]) elif opt["type"] == "extra_tree": return ExtraTreesClassifier(n_estimators=n_estimators, class_weight="balanced", n_jobs=-1) elif opt["type"] == "xgboost": return XGBClassifier(objective='binary:logistic', n_estimators=n_estimators, num_parallel_tree=num_parallel_tree, tree_method="hist", booster="gbtree", n_jobs=-1) elif opt["type"] in ["b_random_forrest", "b_rf"]: return BalancedRandomForestClassifier(n_estimators=n_estimators, n_jobs=-1) elif opt["type"] == "b_bagging": return BalancedBaggingClassifier(base_estimator=b_est, n_estimators=n_estimators) elif opt["type"] == "b_boosting": return RUSBoostClassifier(base_estimator=b_est, n_estimators=n_estimators) else: raise ValueError("type: {} not recognised".format(opt["type"]))
def nosampling_pipeline(data=[], verbose=False, clean=False, plot=False): results_table = [] results = [] rand_state = 42 if clean: X = data.drop('Class', axis=1) y = data['Class'] X_vals = X.values y_vals = y.values X_inliners, y_inliners = reject_sampler.fit_resample(X_vals, y_vals) X = X_inliners y = y_inliners else: X = data.drop('Class', axis=1) y = data['Class'] X = X.values y = y.values pass sss = StratifiedKFold(n_splits=10, random_state=rand_state, shuffle=False) print("StratKFold:", sss) #List of models to be used models = [ DecisionTreeClassifier(random_state=rand_state), RUSBoostClassifier(random_state=rand_state), LogisticRegression(random_state=rand_state), BalancedBaggingClassifier(random_state=rand_state), RandomForestClassifier(random_state=rand_state), EasyEnsembleClassifier( base_estimator=RandomForestClassifier(random_state=rand_state), random_state=rand_state), BalancedRandomForestClassifier(random_state=rand_state) ] results_table = pd.DataFrame(columns=['models', 'fpr', 'tpr', 'auc']) #Create training and testing data sets depending on wheather or not they have been generated previously. #Instantiate lists to store each of the models results strategy = [] classifier = [] strategy = [] samp_technique = [] accuracy = [] f1 = [] auc = [] recall = [] precision = [] g_mean = [] start = time.time() #Run thorugh each of the models to get their performance metrics sampling_strat = 'no_sampling' for train_index, test_index in sss.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] # X_train=X_train.values # X_test=X_test.values # y_train=y_train.values # y_test=y_test.values for model in models: print( "Using lentgh of X for training: {}; Using Length of Y for training: {}" .format(len(X_train), len(y_train))) print( "Using lentgh of X for testing: {}; Using Length of Y for test: {}" .format(len(X_test), len(y_test))) print("Currently training model - {} using sampling strategy - {}". format(model.__class__.__name__, sampling_strat)) print("--" * 20) clf = model pipe = make_pipeline(clf) # LOG_REG_MODEL WITH BOTHER pipe.fit(X_train, y_train) test_preds = pipe.predict(X_test) #yproba = pipe.predict_proba(X_test)[::,1] classifier.append(model.__class__.__name__) samp_technique.append(sampling_strat) strategy.append(" %s+%s " % (str(model.__class__.__name__), sampling_strat)) f1.append(f1_score(y_test, test_preds)) accuracy.append(accuracy_score(y_test, test_preds)) auc.append(roc_auc_score(y_test, test_preds)) recall.append(recall_score(y_test, test_preds)) precision.append(precision_score(y_test, test_preds)) g_mean.append( geometric_mean_score(y_test, test_preds, average='binary')) fpr, tpr, _ = roc_curve(y_test, test_preds) auc_score = roc_auc_score(y_test, test_preds) results_table = results_table.append( { 'classifiers': model.__class__.__name__, 'fpr': fpr, 'tpr': tpr, 'auc_score': auc_score }, ignore_index=True) #Print the model and its report if verbose: print('Classification Model: ', model.__class__.__name__, '\n') print('Sampling Strategy Model: ', sampling_strat, '\n') print(confusion_matrix(y_test, test_preds), '\n') print(classification_report_imbalanced(y_test, test_preds), '\n') #round the results for convenience f1 = [float(round(n, 4)) for n in f1] auc = [float(round(n, 4)) for n in auc] g_mean = [float(round(n, 4)) for n in g_mean] accuracy = [float(round(n, 4)) for n in accuracy] precision = [float(round(n, 4)) for n in precision] recall = [float(round(n, 4)) for n in recall] #store results in dataframe results = pd.DataFrame( [ classifier, strategy, samp_technique, f1, auc, g_mean, accuracy, precision, recall ], index=[ 'classifier', 'strategy', 'samp_technique', 'f1', 'roc_auc', 'g_mean', 'accuracy', 'precision', 'recall' ], columns=[ 'DecisionTreeClassifier', 'RUSBoostClaassifier', 'LogisiticRegression', 'BalancedBaggingClassifier', 'RandomForestClassifier', 'EasyEnsembleClassifier', 'BalancedRandomForestClassifier' ]) if plot: results_table.set_index('classifiers', inplace=True) fig = plt.figure(figsize=(8, 6)) results_table.sort_values(by=['auc_score'], ascending=False) for i in results_table.index: plt.plot(results_table.loc[i]['fpr'], results_table.loc[i]['tpr'], label="{}, AUC={:.4f}".format( i, results_table.loc[i]['auc_score'])) plt.plot([0, 1], [0, 1], color='orange', linestyle='--') plt.xticks(np.arange(0.0, 1.1, step=0.1)) plt.xlabel("Flase Positive Rate", fontsize=15) plt.yticks(np.arange(0.0, 1.1, step=0.1)) plt.ylabel("True Positive Rate", fontsize=15) plt.title( 'ROC Curve for classifiers using Full data split using sampling technique: {}' .format(sampling_strat), fontweight='bold', fontsize=15) plt.legend(prop={'size': 13}, loc='lower right') plt.show() #Change orientation of the dataframe end = time.time() print("Time elapsed:", start - end) return results.transpose()
DecisionTreeClassifier(random_state=r), KNeighborsClassifier(), GaussianNB(), MultinomialNB(), LogisticRegression(random_state=r), SVC(random_state=r, kernel='sigmoid'), MLPClassifier(random_state=r), BaggingClassifier(random_state=r), RandomForestClassifier(random_state=r), GradientBoostingClassifier(random_state=r), LGBMClassifier(), XGBClassifier(random_state=r), CatBoostClassifier(random_state=r, verbose=False), BalancedBaggingClassifier(random_state=r), BalancedRandomForestClassifier(random_state=r), RUSBoostClassifier(random_state=r) ] names = [ "DecisionTree", "KNeighbors", "GaussianNB", "MultinomialNB", "LogisticRegression", "SVC", "MLPClassifier", "Ensemble-Bagging", "Ensemble-RandomForest", "Ensemble-GradientBoosting", "LightGradientBoosting", "XGBoost", "CatBoost", "BalancedBagging", "BalancedRandomForest", "RUSBoost" ] outputs = {} for name, model in zip(names, models): model.fit(x_train, y_train) output = model.predict(test_dataframe) outputs[name] = output
# initialize cv5 skf = StratifiedKFold(n_splits=5) cv5_ids = list(skf.split(full_data, labels)) # print(cv5_ids) # initialize model # lin_clf = svm.SVC(decision_function_shape='ovo', probability=True) # lin_clf = svm.LinearSVC() # lin_clf = LogisticRegression() # lin_clf = svm.SVC(kernel='sigmoid') # lin_clf = MLPClassifier((256,256), activation='relu', max_iter=1000) # lin_clf = RandomForestClassifier(n_estimators=5000, max_depth=2, random_state=0) single_clf = tree.DecisionTreeClassifier(max_depth=1) # single_clf = LogisticRegression() lin_clf = RUSBoostClassifier(base_estimator=single_clf, n_estimators=5000) # initialize booster sm = SMOTE(random_state=42) # perform cv5 precision_avg = [] recall_avg = [] fscore_avg = [] acc_avg = 0. for sp in cv5_ids: train_data, train_labels = full_data[sp[0]], labels[sp[0]] # train_data, train_labels = sm.fit_sample(train_data, train_labels) test_data, test_labels = full_data[sp[1]], labels[sp[1]] lin_clf.fit(train_data, train_labels)
def learning_model(year, class_weight): iters = 300 gap = 2 year_test = year data_test = reader.ordinary_data_reader('uscecchini28.csv', year_test, year_test) x_test = data_test.features y_test = data_test.labels test = np.c_[data_test.years, data_test.firms] ''' an if-else is used to judge whether the class_weight is None to prevent Exception from string concatenation a try-except for RusBoost with DecisionTreeClassifier using custom class_weight if we can find the right model trained last time on disk, we can directly use that model to predict the result without training twice otherwise, we have to train that model and save it on disk ''' # if class_weight is not None: # we use current_model_name to find/save the trained model with custom class_weight # current_model_name = class_weight + "_" + str(year_test) + ".m" # else: # current_model_name = str(year_test) + ".m" current_model_name = class_weight + "_" + str(year_test) + ".m" try: rusboost_model = joblib.load(current_model_name) except Exception as e: print('Running RUSBoost (training period: 1991-' + str(year_test - gap) + ', testing period: ' + str( year_test) + ', with ' + str(gap) + '-year gap)...') data_train = reader.ordinary_data_reader('uscecchini28.csv', 1991, year_test - gap) x_train = data_train.features y_train = data_train.labels newpaaer_train = data_train.newpaaers # formatter labels and newpaaers for the step: data_test.newpaaers(data_test.labels~=0) data_test.newpaaers = np.array(data_test.newpaaers) data_test.labels = np.array(data_test.labels) # replace the nan that should be remained in the array with 0 for i in range(len(data_test.newpaaers)): if np.isnan(data_test.newpaaers[i]): if data_test.labels[i] != 0: data_test.newpaaers[i] = 0 # replace all the nans remain in the array data_test.newpaaers = np.array([x for x in data_test.newpaaers if str(x) != 'nan']) # replace all the 0 back to nan for i in range(len(data_test.newpaaers)): if int(data_test.newpaaers[i]) == 0.0: data_test.newpaaers[i] = np.NaN # do the unique to get final result for newpaaer_test newpaaer_test = np.unique(data_test.newpaaers) ''' Caution: here we change the type of variable called y_train for matching the array index of formatted array newpaaer_train in the following loop ''' y_train = np.array(y_train) num_frauds = sum(y_train == 1) print(num_frauds) ''' here we use the function in1d to replace the function ismember used in matlab and a temp array for the other operation to handle serial frauds finish the step: y_train[ismember(newpaaer_train, newpaaer_test)] = 0 ''' temp_array = np.array(np.in1d(newpaaer_train, newpaaer_test)).astype(int) for i in range(len(temp_array)): if temp_array[i] == 1: y_train[i] = 0 # delete the temp array del temp_array num_frauds = num_frauds - sum(y_train == 1) print('Recode', num_frauds, 'overlapped frauds (i.e., change fraud label from 1 to 0).') start_time = time.perf_counter() rusboost_model = RUSBoostClassifier(DecisionTreeClassifier(min_samples_leaf=5, class_weight=class_weight), learning_rate=0.1, n_estimators=iters) rusboost_model.fit(x_train, y_train) end_time = time.perf_counter() t_train = end_time - start_time joblib.dump(rusboost_model, current_model_name) print(end_time - start_time) print('Training time: %.3f seconds' % t_train) start_time = time.perf_counter() predit = rusboost_model.predict(x_test) prob = rusboost_model.predict_proba(x_test) end_time = time.perf_counter() t_test = end_time - start_time print('Testing time %.3f seconds' % t_test) # test figures print("AUC: %.4f" % metrics.roc_auc_score(y_test, predit)) # np.set_printoptions(precision=4, threshold=8, edgeitems=4, linewidth=75, suppress=True, nanstr='nan', infstr='inf') print("precision: %.2f%%" % np.multiply(metrics.precision_score(y_test, predit, zero_division=0), 100)) print("recall: %.2f%%" % np.multiply(metrics.recall_score(y_test, predit), 100)) # dump part of the results(fraud probability) prob = np.around(np.delete(prob, 0, axis=1) * 100, decimals=5) data = np.c_[predit, prob] data = np.c_[test, data] file_data = pd.DataFrame(data) csv_file_name = 'data.csv' file_data.to_csv(csv_file_name, header=False, index=False)
def pipe_main(pipe=None): '''pipeline construction using sklearn estimators, final step support only classifiers currently .. note:: data flows through a pipeline consisting of steps as below: raw data --> clean --> encoding --> scaling --> feature construction --> feature selection --> resampling --> final estimator see scikit-learn preprocess & estimators parameter ---- pipe - str - in the format of 'xx_xx' of which 'xx' means steps in pipeline, default None return ---- 1) pipeline instance of chosen steps 2) if pipe is None, a dict indicating possible choice of 'steps' ''' clean = { 'clean': Split_cls(dtype_filter='not_datetime', na1='null', na2=-999), 'cleanNA': Split_cls(dtype_filter='not_datetime', na1=None, na2=None), 'cleanMean': Split_cls(dtype_filter='not_datetime', na1='most_frequent', na2='mean'), } # encode = { 'woe': Woe_encoder(max_leaf_nodes=5), 'oht': Oht_encoder(), 'ordi': Ordi_encoder(), } resample = { # over_sampling 'rover': RandomOverSampler(), 'smote': SMOTE(), 'bsmote': BorderlineSMOTE(), 'adasyn': ADASYN(), # under sampling controlled methods 'runder': RandomUnderSampler(), 'nearmiss': NearMiss(version=3), 'pcart': InstanceHardnessThreshold(), # under sampling cleaning methods 'tlinks': TomekLinks(n_jobs=-1), 'oside': OneSidedSelection(n_jobs=-1), 'cleanNN': NeighbourhoodCleaningRule(n_jobs=-1), 'enn': EditedNearestNeighbours(n_jobs=-1), 'ann': AllKNN(n_jobs=-1), 'cnn': CondensedNearestNeighbour(n_jobs=-1), # clean outliers 'inlierForest': FunctionSampler(outlier_rejection, kw_args={'method': 'IsolationForest'}), 'inlierLocal': FunctionSampler(outlier_rejection, kw_args={'method': 'LocalOutlierFactor'}), 'inlierEllip': FunctionSampler(outlier_rejection, kw_args={'method': 'EllipticEnvelope'}), 'inlierOsvm': FunctionSampler(outlier_rejection, kw_args={'method': 'OneClassSVM'}), # combine 'smoteenn': SMOTEENN(), 'smotelink': SMOTETomek(), } scale = { 'stdscale': StandardScaler(), 'maxscale': MinMaxScaler(), 'rscale': RobustScaler(quantile_range=(10, 90)), 'qauntile': QuantileTransformer(), # uniform distribution 'power': PowerTransformer(), # Gaussian distribution 'norm': Normalizer(), # default L2 norm # scale sparse data 'maxabs': MaxAbsScaler(), 'stdscalesp': StandardScaler(with_mean=False), } # feature construction feature_c = { 'pca': PCA(whiten=True), 'spca': SparsePCA(normalize_components=True, n_jobs=-1), 'ipca': IncrementalPCA(whiten=True), 'kpca': KernelPCA(kernel='rbf', n_jobs=-1), 'poly': PolynomialFeatures(degree=2), 'rtembedding': RandomTreesEmbedding(n_estimators=10), 'LDA': LinearDiscriminantAnalysis(), 'QDA': QuadraticDiscriminantAnalysis(), } # select from model feature_m = { 'fwoe': SelectFromModel(Woe_encoder(max_leaf_nodes=5)), 'flog': SelectFromModel( LogisticRegressionCV(penalty='l1', solver='saga', scoring='roc_auc')), 'fsgd': SelectFromModel(SGDClassifier(penalty="l1")), 'fsvm': SelectFromModel(LinearSVC('l1', dual=False, C=1e-2)), 'fxgb': SelectFromModel(XGBClassifier(n_jobs=-1)), 'frf': SelectFromModel(ExtraTreesClassifier(n_estimators=100, max_depth=5)), 'fRFExgb': RFE(XGBClassifier(n_jobs=-1), step=0.1, n_features_to_select=20), 'fRFErf': RFE(ExtraTreesClassifier(n_estimators=100, max_depth=5), step=0.3, n_features_to_select=20), 'fRFElog': RFE(LogisticRegressionCV(penalty='l1', solver='saga', scoring='roc_auc'), step=0.3, n_features_to_select=20) } # Univariate feature selection feature_u = { 'fchi2': GenericUnivariateSelect(chi2, 'percentile', 25), 'fMutualclf': GenericUnivariateSelect(mutual_info_classif, 'percentile', 25), 'fFclf': GenericUnivariateSelect(f_classif, 'percentile', 25), } # sklearn estimator t = all_estimators(type_filter=['classifier']) estimator = {} for i in t: try: estimator.update({i[0]: i[1]()}) except Exception: continue estimator.update( dummy=DummyClassifier(), XGBClassifier=XGBClassifier(n_jobs=-1), LogisticRegressionCV=LogisticRegressionCV(scoring='roc_auc'), EasyEnsembleClassifier=EasyEnsembleClassifier(), BalancedRandomForestClassifier=BalancedRandomForestClassifier(), RUSBoostClassifier=RUSBoostClassifier(), SVC=SVC(C=0.01, gamma='auto')) if pipe is None: feature_s = {} feature_s.update(**feature_m, **feature_u) return { 'clean': clean.keys(), 'encoding': encode.keys(), 'resample': resample.keys(), 'scale': scale.keys(), 'feature_c': feature_c.keys(), 'feature_s': feature_s.keys(), 'classifier': estimator.keys() } elif isinstance(pipe, str): l = pipe.split('_') all_keys_dict = {} all_keys_dict.update(**clean, **encode, **scale, **feature_c, **feature_m, **feature_u, **estimator, **resample) steps = [] for i in l: if all_keys_dict.get(i) is not None: steps.append((i, all_keys_dict.get(i))) else: raise KeyError( "'{}' invalid key for sklearn estimators".format(i)) return Pipeline(steps) else: raise ValueError("input pipe must be a string in format 'xx[_xx]'")
def classiferSet(pre_cost_weight=20): #xgt = xgb.XGBClassifier(learning_rate=0.1, scale_pos_weight=10, n_estimators=100, random_state=1) #80.77% xgt = xgb.XGBClassifier( learning_rate=0.1, #subsample=0.99, max_depth=3, scale_pos_weight=pre_cost_weight, n_estimators=80, #cv=5, #subsample=.99, random_state=27, nthread=2 #use more threads only for large dataset ) #84.62% ada = AdaBoostClassifier(n_estimators=100, learning_rate=.1, random_state=1234) #(0,130): .815 #gbt = GradientBoostingClassifier(n_estimators=100, subsample=1.0, learning_rate=1, random_state=1234) #(0,130): .830 gbt = GradientBoostingClassifier( n_estimators=100, subsample=0.99, learning_rate=.1, random_state=1234) #(0,130): .861 # rf = RandomForestClassifier( n_estimators=100, #max_depth=10, oob_score=True, class_weight={ 0: 1, 1: pre_cost_weight }, #class_weight='balanced', random_state=1234) #.846 brf = BalancedRandomForestClassifier(n_estimators=100, oob_score=True, class_weight={ 0: 1, 1: pre_cost_weight }, random_state=1234) rus = RUSBoostClassifier(n_estimators=100, random_state=1234) #https://www.kaggle.com/c/home-credit-default-risk/discussion/60921 #https://sites.google.com/view/lauraepp/parameters lgbm = lightgbm.LGBMClassifier( boosting_type='dart', #'gbdt', 'goss', 'dart' num_leaves=31, max_depth=-1, learning_rate=0.1, class_weight= None, #{0:1,1:pre_cost_weight}, using this is inferior to default random_state=1234) ourmodels = dict({ 'AdaBoost': ada, 'GradientBoost': gbt, 'RandomForest': rf, 'BalancedRandomForest': brf, 'RUSBoost': rus, 'XGBoost': xgt, 'LightGBM': lgbm }) return ourmodels
df = pd.read_csv('data/poker-8-9_vs_5.csv') X, y, z = prepare_data(df) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0, test_size=0.7) kf = StratifiedKFold(n_splits=10) kf.get_n_splits(X, y) bbc = BalancedBaggingClassifier( base_estimator=DecisionTreeClassifier(random_state=0), random_state=42) brfc = BalancedRandomForestClassifier(max_depth=2, random_state=0) eec = EasyEnsembleClassifier( base_estimator=DecisionTreeClassifier(random_state=0), random_state=42) rbc = RUSBoostClassifier(base_estimator=DecisionTreeClassifier(random_state=0), random_state=0) bbc_score = [] brfc_score = [] eec_score = [] rbc_score = [] for train_index, test_index in kf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] bbc.fit(X_train, y_train) brfc.fit(X_train, y_train) eec.fit(X_train, y_train) rbc.fit(X_train, y_train) y_pred_bbc = bbc.predict(X_test) y_pred_brfc = brfc.predict(X_test)
y_test = y[test_index] #classifier = CUSBoostClassifier(**a) #classifier = AdaboostClassifier(**a) #classifier = RusBoost(depth=depth, n_estimators=estimators) #classifier = AdaboostNC_Classifier(**a) #classifier = CUSBoostNC_Classifier(**a) #classifier = RusBoost(**a) classifier = RUSBoostClassifier(DecisionTreeClassifier(max_depth=8), n_estimators=64) #classifier.fit(X_train, y_train, number_of_clusters, 0.5) #CUSBoost classifier #classifier.fit(X_train, y_train) #Adaboost classifier #classifier.fit(X_train, y_train, 0.5) #AdaboostNC classifier #classifier.fit(X_train, y_train, 6, 0.5) #classifier.fit(X_train, y_train, 6, fraction/100, 8) classifier.fit(X_train, y_train) predictions = classifier.predict_proba(X_test) prediction_ = classifier.predict(X_test) auc = roc_auc_score(y_test, predictions[:, 1]) f1 = f1_score(y_test, prediction_)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, shuffle=True) model = GradientBoostingClassifier(n_estimators=100, random_state=42) imbl_methods = { 'eec': EasyEnsembleClassifier(random_state=42, sampling_strategy=1., n_jobs=-1, base_estimator=model), 'rub': RUSBoostClassifier(random_state=42, sampling_strategy=1., base_estimator=model) } for method in imbl_methods.keys(): imbl = imbl_methods[method] imbl.fit(X_train, y_train) y_hat_test = imbl.predict(X_test) y_hat_train = imbl.predict(X_train) print(f"Reults of {method}") print(imbl.score(X_test, y_test)) print("Train data") print(classification_report(y_train, y_hat_train)) print("Test data") print(classification_report(y_test, y_hat_test))
def Gridsearchcv(X_train, X_test, y_train, y_test): ############ # Scale numeric values num_transformer = Pipeline(steps=[ ('scaler', MinMaxScaler())]) preprocessor = ColumnTransformer( remainder='passthrough', transformers=[ ('num', num_transformer, make_column_selector(pattern='EDAD')) ]) ############ pipe = Pipeline([ ('preprocessor', preprocessor), ('clf', PipelineHelper([ ('svc', SVC()), ('gb', GradientBoostingClassifier()), ('xgb', XGBClassifier(use_label_encoder=False)), ('eec', EasyEnsembleClassifier()), ('rbc', RUSBoostClassifier()), ('bbc', BalancedBaggingClassifier()), ('brf', BalancedRandomForestClassifier()), ])), ]) params = { 'clf__selected_model': pipe.named_steps['clf'].generate({ # # #EasyEnsembleClassifier 'eec__n_estimators' : [10, 25, 50, 100], 'eec__warm_start' : [False, True], 'eec__replacement' : [False, True], # # #RUSBoostClassifier 'rbc__algorithm' : ['SAMME','SAMME.R'], 'rbc__n_estimators' : [10, 50, 100, 200, 500], 'rbc__learning_rate' : [1e-3, 1e-2, 1e-1, 0.5, 1.], # # #BalancedBaggingClassifier 'bbc__base_estimator': [HistGradientBoostingClassifier(), None], 'bbc__n_estimators' : [10, 50, 100, 200, 500,750,1000], 'bbc__max_samples':[0.5,0.6,0.7,0.8,0.9,1.0], 'bbc__max_features':[0.5,0.6,0.7,0.8,0.9,1.0], # #BalancedRandomForestClassifier 'brf__criterion': ['gini', 'entropy'], 'brf__n_estimators' : [int(x) for x in np.linspace(start = 20, stop = 200, num = 5)], 'brf__max_depth' : [int(x) for x in np.linspace(1, 45, num = 3)], 'brf__min_samples_split' : range(2,10), 'brf__min_samples_leaf': [1,3,5,10], 'brf__max_features' : ['auto', 'sqrt', 'log2'], # # #svm 'svc__C': [0.1, 0.5, 1, 10, 30, 40, 50, 75, 100, 500, 1000], 'svc__gamma' : [0.0001, 0.001, 0.005, 0.01, 0.05, 0.07, 0.1, 0.5, 1, 5, 10, 50], 'svc__kernel': ['rbf'], # # #gb 3780 "gb__learning_rate": [0.0001, 0.001, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2], "gb__max_depth":[3,7,8,9,10,50], "gb__max_features":["log2","sqrt"], "gb__subsample":[0.5, 0.618, 0.8, 0.85, 0.9, 0.95, 1.0], "gb__n_estimators":[10, 50, 100, 200, 300], # #xgboost 'xgb__learning_rate' : [1e-3, 1e-2, 1e-1, 0.5, 1.], 'xgb__min_child_weight': np.arange(1, 21, 5), 'xgb__subsample': np.arange(0.05, 1.01, 0.05), 'xgb__verbosity': [0], # 'xgb__booster': ['gbtree', 'gblinear' ,'dart'], # 'xgb__learning_rate' : [1e-3, 1e-2, 1e-1, 0.5, 1.], # 'xgb__min_child_weight': range(1, 21, 5), # 'xgb__subsample': np.arange(0.05, 1.01, 0.05), # 'xgb__max_depth': [15,20,25], # 'xgb__verbosity': [0], # 'xgb__n_estimators': [100], # 'xgb__max_depth': range(1, 11), # 'xgb__learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], # 'xgb__subsample': np.arange(0.05, 1.01, 0.05), # 'xgb__min_child_weight': range(1, 21), # 'xgb__verbosity': [0], # add this line to slient warning # 'xgb__n_estimators': [400, 700, 1000], # 'xgb__colsample_bytree': [0.7, 0.8], # 'xgb__max_depth': [15,20,25], # 'xgb__reg_alpha': [1.1, 1.2, 1.3], # 'xgb__reg_lambda': [1.1, 1.2, 1.3], # 'xgb__subsample': [0.7, 0.8, 0.9], # 'xgb__eval_metric' : ['mlogloss'] }), } scoring = {'ba': 'balanced_accuracy','ap': 'average_precision', 'F1' : 'f1', 'ra': 'roc_auc', 'rc': 'recall'} cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3) #cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=5) #https://towardsdatascience.com/hyper-parameter-tuning-with-randomised-grid-search-54f865d27926 #n_iter: 30,60, 100 grid = RandomizedSearchCV( pipe, params, refit = 'ba', cv = cv, verbose = 3, n_jobs=-1, n_iter = 60, scoring= scoring, return_train_score = True ) grid.fit(X_train, y_train) df_grid=pd.DataFrame(grid.cv_results_) df_grid = df_grid.sort_values(by=['mean_test_ba'],ascending=False) df_grid = df_grid[[ 'param_clf__selected_model', 'params', 'mean_fit_time', 'std_fit_time', 'mean_test_ba', 'std_test_ba', 'rank_test_ba', 'mean_test_ap', 'std_test_ap', 'rank_test_ap', 'mean_test_ra', 'std_test_ra', 'rank_test_ra', 'mean_test_F1', 'std_test_F1', 'rank_test_F1' ]] print("Best-Fit Parameters From Training Data:\n",grid.best_params_) grid_predictions = grid.best_estimator_.predict(X_test) report = classification_report(y_test, grid_predictions, output_dict=True) report = pd.DataFrame(report).transpose() print(report) print(confusion_matrix(y_test, grid_predictions)) return grid, df_grid, report
clf_results = pd.DataFrame() # define models models = { 'ExtraTrees': ExtraTreesClassifier(), 'RandomForest': RandomForestClassifier(), 'AdaBoost': AdaBoostClassifier(), 'GradientBoosting': GradientBoostingClassifier(), 'SVC': SVC(), 'LogitBoost': LogitBoost(), 'XGBClassifier': XGBClassifier(), 'ComplementNB': ComplementNB(), 'BalancedBagging': BalancedBaggingClassifier(), 'BalancedRandomForest': BalancedRandomForestClassifier(), 'RUSBoost': RUSBoostClassifier(), 'EasyEnsemble': EasyEnsembleClassifier() } # define model parameters for parameter search param_extra_trees = { 'n_estimators': [5, 10, 50, 100, 200], 'min_samples_split': [2, 4], 'max_depth': [2, 3, None], 'max_features': ['sqrt', None], 'class_weight': ['balanced'] } param_random_forest = { 'n_estimators': [5, 10, 50, 100, 200],
# eec = EasyEnsembleClassifier(n_estimators=10, # base_estimator=base_estimator, # n_jobs=-1) # eec.fit(X_train_seek, y_train_seek) # y_pred_eec = eec.predict(X_test_seek) # print('Easy ensemble classifier performance:') # print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}' # .format(balanced_accuracy_score(y_test_seek, y_pred_eec), # geometric_mean_score(y_test_seek, y_pred_eec))) # cm_eec = confusion_matrix(y_test_seek, y_pred_eec) # fig, ax = plt.subplots(ncols=2) # plot_confusion_matrix(cm_eec, classes=np.unique(dataset.target), ax=ax[0], # title='Easy ensemble classifier') base_estimator = AdaBoostClassifier(n_estimators=10) rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('RUSBoost classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format( balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) fig, ax = plt.subplots(ncols=2) plot_confusion_matrix(cm_rusboost, classes=np.unique(dataset.target), ax=ax[1], title='RUSBoost classifier') rusboost.fit(X_train_seek, y_train_seek)
exLpred.append(float(lineE[j])) #cellTypesTrue.append(lineE[int(len(lineE))-1]) exMpred.append(exLpred) #s.append("\n") exLpred = [] cellID.append(lineE[0]) #cellTypesTrue = np.array(cellTypesTrue) exMpred = np.array(exMpred) cellID = np.array(cellID) ################################### ##### Everything is ready for cell type prediction ##### rusboost = RUSBoostClassifier(random_state=0) rusboost.fit(exMtrain, cellTypesTrain) ##### Cell types prediction ##### cellTypesPred = rusboost.predict(exMpred) #accuracy_score = balanced_accuracy_score(cellTypesTrue, cellTypesPred) #print accuracy_score #classification_report(cellTypesTrue, cellTypesPred) ##### Checking performance ##### #confusionMatrix = confusion_matrix(cellTypesTrue, cellTypesPred) cellTypesProbs = rusboost.predict_proba(exMpred) #print confusionMatrix ##### Merging the cell types and probability score #####