def train(classifier, df,y, user_id): ''' The main training function that runs on a seperate process''' X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.33, random_state=0) base_estimator = AdaBoostClassifier(n_estimators=10) rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format(balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) joblib.dump(rusboost, user_id+'.pkl') classifier.classifierStatus = "trained" print("Done training") return classifier
def test_rusboost_sample_weight(imbalanced_dataset, algorithm): X, y = imbalanced_dataset sample_weight = np.ones_like(y) rusboost = RUSBoostClassifier(algorithm=algorithm, random_state=0) # Predictions should be the same when sample_weight are all ones y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) y_pred_no_sample_weight = rusboost.fit(X, y).predict(X) assert_array_equal(y_pred_sample_weight, y_pred_no_sample_weight) rng = np.random.RandomState(42) sample_weight = rng.rand(y.shape[0]) y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) with pytest.raises(AssertionError): assert_array_equal(y_pred_no_sample_weight, y_pred_sample_weight)
def test_rusboost_sample_weight(imbalanced_dataset, algorithm): X, y = imbalanced_dataset sample_weight = np.ones_like(y) rusboost = RUSBoostClassifier(algorithm=algorithm, random_state=0) # Predictions should be the same when sample_weight are all ones y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) y_pred_no_sample_weight = rusboost.fit(X, y).predict(X) assert_array_equal(y_pred_sample_weight, y_pred_no_sample_weight) rng = np.random.RandomState(42) sample_weight = rng.rand(y.shape[0]) y_pred_sample_weight = rusboost.fit(X, y, sample_weight).predict(X) with pytest.raises(AssertionError): assert_array_equal(y_pred_no_sample_weight, y_pred_sample_weight)
def test_rusboost(imbalanced_dataset, algorithm): X, y = imbalanced_dataset X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=1) classes = np.unique(y) n_estimators = 500 rusboost = RUSBoostClassifier(n_estimators=n_estimators, algorithm=algorithm, random_state=0) rusboost.fit(X_train, y_train) assert_array_equal(classes, rusboost.classes_) # check that we have an ensemble of samplers and estimators with a # consistent size assert len(rusboost.estimators_) > 1 assert len(rusboost.estimators_) == len(rusboost.samplers_) assert len(rusboost.pipelines_) == len(rusboost.samplers_) # each sampler in the ensemble should have different random state assert (len({sampler.random_state for sampler in rusboost.samplers_ }) == len(rusboost.samplers_)) # each estimator in the ensemble should have different random state assert (len({est.random_state for est in rusboost.estimators_ }) == len(rusboost.estimators_)) # check the consistency of the feature importances assert len(rusboost.feature_importances_) == imbalanced_dataset[0].shape[1] # check the consistency of the prediction outpus y_pred = rusboost.predict_proba(X_test) assert y_pred.shape[1] == len(classes) assert rusboost.decision_function(X_test).shape[1] == len(classes) score = rusboost.score(X_test, y_test) assert score > 0.7, "Failed with algorithm {} and score {}".format( algorithm, score) y_pred = rusboost.predict(X_test) assert y_pred.shape == y_test.shape
def fit(self, X, Y, sample_weight=None): import sklearn.tree self.n_estimators = int(self.n_estimators) self.learning_rate = float(self.learning_rate) self.max_depth = int(self.max_depth) base_estimator = sklearn.tree.DecisionTreeClassifier( max_depth=self.max_depth) from imblearn.ensemble import RUSBoostClassifier estimator = RUSBoostClassifier(base_estimator=base_estimator, n_estimators=self.n_estimators, learning_rate=self.learning_rate, algorithm=self.algorithm, random_state=self.random_state) estimator.fit(X, Y, sample_weight=sample_weight) self.estimator = estimator return self
def test_rusboost(imbalanced_dataset, algorithm): X, y = imbalanced_dataset X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y) classes = np.unique(y) n_estimators = 500 rusboost = RUSBoostClassifier(n_estimators=n_estimators, algorithm=algorithm, random_state=0) rusboost.fit(X_train, y_train) assert_array_equal(classes, rusboost.classes_) # check that we have an ensemble of samplers and estimators with a # consistent size assert len(rusboost.estimators_) > 1 assert len(rusboost.estimators_) == len(rusboost.samplers_) assert len(rusboost.pipelines_) == len(rusboost.samplers_) # each sampler in the ensemble should have different random state assert (len(set(sampler.random_state for sampler in rusboost.samplers_)) == len(rusboost.samplers_)) # each estimator in the ensemble should have different random state assert (len(set(est.random_state for est in rusboost.estimators_)) == len(rusboost.estimators_)) # check the consistency of the feature importances assert len(rusboost.feature_importances_) == imbalanced_dataset[0].shape[1] # check the consistency of the prediction outpus y_pred = rusboost.predict_proba(X_test) assert y_pred.shape[1] == len(classes) assert rusboost.decision_function(X_test).shape[1] == len(classes) score = rusboost.score(X_test, y_test) assert score > 0.7, "Failed with algorithm {} and score {}".format( algorithm, score) y_pred = rusboost.predict(X_test) assert y_pred.shape == y_test.shape
def test_rusboost_error(imbalanced_dataset, boosting_params, err_msg): rusboost = RUSBoostClassifier(**boosting_params) with pytest.raises(ValueError, match=err_msg): rusboost.fit(*imbalanced_dataset)
def test_balanced_random_forest_error(imbalanced_dataset, boosting_params, err_msg): rusboost = RUSBoostClassifier(**boosting_params) with pytest.raises(ValueError, message=err_msg): rusboost.fit(*imbalanced_dataset)
eec = EasyEnsembleClassifier(n_estimators=10, base_estimator=base_estimator, n_jobs=-1) eec.fit(X_train, y_train) y_pred_eec = eec.predict(X_test) print('Easy ensemble classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format( balanced_accuracy_score(y_test, y_pred_eec), geometric_mean_score(y_test, y_pred_eec))) cm_eec = confusion_matrix(y_test, y_pred_eec) fig, ax = plt.subplots(ncols=2) plot_confusion_matrix(cm_eec, classes=np.unique(satimage.target), ax=ax[0], title='Easy ensemble classifier') rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('RUSBoost classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format( balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) plot_confusion_matrix(cm_rusboost, classes=np.unique(satimage.target), ax=ax[1], title='RUSBoost classifier') plt.show()
# achieve worse performance. base_estimator = AdaBoostClassifier(n_estimators=10) eec = EasyEnsembleClassifier(n_estimators=10, base_estimator=base_estimator, n_jobs=-1) eec.fit(X_train, y_train) y_pred_eec = eec.predict(X_test) print('Easy ensemble classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}' .format(balanced_accuracy_score(y_test, y_pred_eec), geometric_mean_score(y_test, y_pred_eec))) cm_eec = confusion_matrix(y_test, y_pred_eec) fig, ax = plt.subplots(ncols=2) plot_confusion_matrix(cm_eec, classes=np.unique(satimage.target), ax=ax[0], title='Easy ensemble classifier') rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('RUSBoost classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}' .format(balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) plot_confusion_matrix(cm_rusboost, classes=np.unique(satimage.target), ax=ax[1], title='RUSBoost classifier') plt.show()
def test_balanced_random_forest_error(imbalanced_dataset, boosting_params, err_msg): rusboost = RUSBoostClassifier(**boosting_params) with pytest.raises(ValueError, message=err_msg): rusboost.fit(*imbalanced_dataset)
lin_clf = RUSBoostClassifier(base_estimator=single_clf, n_estimators=5000) # initialize booster sm = SMOTE(random_state=42) # perform cv5 precision_avg = [] recall_avg = [] fscore_avg = [] acc_avg = 0. for sp in cv5_ids: train_data, train_labels = full_data[sp[0]], labels[sp[0]] # train_data, train_labels = sm.fit_sample(train_data, train_labels) test_data, test_labels = full_data[sp[1]], labels[sp[1]] lin_clf.fit(train_data, train_labels) pred = lin_clf.predict(test_data) print(sp[1]) print(pred) print(test_labels) # metrics precision, recall, fscore, support = precision_recall_fscore_support( test_labels, pred, labels=[0, 1, 2], average=None) acc = float(sum(pred == test_labels)) / len(test_labels) print(precision, recall, fscore, support, acc) precision_avg.append(precision) recall_avg.append(recall) fscore_avg.append(fscore) acc_avg += acc precision, recall, fscore = np.mean(precision_avg, axis=0), np.mean( recall_avg, axis=0), np.mean(fscore_avg, axis=0)
def learning_model(year, class_weight): iters = 300 gap = 2 year_test = year data_test = reader.ordinary_data_reader('uscecchini28.csv', year_test, year_test) x_test = data_test.features y_test = data_test.labels test = np.c_[data_test.years, data_test.firms] ''' an if-else is used to judge whether the class_weight is None to prevent Exception from string concatenation a try-except for RusBoost with DecisionTreeClassifier using custom class_weight if we can find the right model trained last time on disk, we can directly use that model to predict the result without training twice otherwise, we have to train that model and save it on disk ''' # if class_weight is not None: # we use current_model_name to find/save the trained model with custom class_weight # current_model_name = class_weight + "_" + str(year_test) + ".m" # else: # current_model_name = str(year_test) + ".m" current_model_name = class_weight + "_" + str(year_test) + ".m" try: rusboost_model = joblib.load(current_model_name) except Exception as e: print('Running RUSBoost (training period: 1991-' + str(year_test - gap) + ', testing period: ' + str( year_test) + ', with ' + str(gap) + '-year gap)...') data_train = reader.ordinary_data_reader('uscecchini28.csv', 1991, year_test - gap) x_train = data_train.features y_train = data_train.labels newpaaer_train = data_train.newpaaers # formatter labels and newpaaers for the step: data_test.newpaaers(data_test.labels~=0) data_test.newpaaers = np.array(data_test.newpaaers) data_test.labels = np.array(data_test.labels) # replace the nan that should be remained in the array with 0 for i in range(len(data_test.newpaaers)): if np.isnan(data_test.newpaaers[i]): if data_test.labels[i] != 0: data_test.newpaaers[i] = 0 # replace all the nans remain in the array data_test.newpaaers = np.array([x for x in data_test.newpaaers if str(x) != 'nan']) # replace all the 0 back to nan for i in range(len(data_test.newpaaers)): if int(data_test.newpaaers[i]) == 0.0: data_test.newpaaers[i] = np.NaN # do the unique to get final result for newpaaer_test newpaaer_test = np.unique(data_test.newpaaers) ''' Caution: here we change the type of variable called y_train for matching the array index of formatted array newpaaer_train in the following loop ''' y_train = np.array(y_train) num_frauds = sum(y_train == 1) print(num_frauds) ''' here we use the function in1d to replace the function ismember used in matlab and a temp array for the other operation to handle serial frauds finish the step: y_train[ismember(newpaaer_train, newpaaer_test)] = 0 ''' temp_array = np.array(np.in1d(newpaaer_train, newpaaer_test)).astype(int) for i in range(len(temp_array)): if temp_array[i] == 1: y_train[i] = 0 # delete the temp array del temp_array num_frauds = num_frauds - sum(y_train == 1) print('Recode', num_frauds, 'overlapped frauds (i.e., change fraud label from 1 to 0).') start_time = time.perf_counter() rusboost_model = RUSBoostClassifier(DecisionTreeClassifier(min_samples_leaf=5, class_weight=class_weight), learning_rate=0.1, n_estimators=iters) rusboost_model.fit(x_train, y_train) end_time = time.perf_counter() t_train = end_time - start_time joblib.dump(rusboost_model, current_model_name) print(end_time - start_time) print('Training time: %.3f seconds' % t_train) start_time = time.perf_counter() predit = rusboost_model.predict(x_test) prob = rusboost_model.predict_proba(x_test) end_time = time.perf_counter() t_test = end_time - start_time print('Testing time %.3f seconds' % t_test) # test figures print("AUC: %.4f" % metrics.roc_auc_score(y_test, predit)) # np.set_printoptions(precision=4, threshold=8, edgeitems=4, linewidth=75, suppress=True, nanstr='nan', infstr='inf') print("precision: %.2f%%" % np.multiply(metrics.precision_score(y_test, predit, zero_division=0), 100)) print("recall: %.2f%%" % np.multiply(metrics.recall_score(y_test, predit), 100)) # dump part of the results(fraud probability) prob = np.around(np.delete(prob, 0, axis=1) * 100, decimals=5) data = np.c_[predit, prob] data = np.c_[test, data] file_data = pd.DataFrame(data) csv_file_name = 'data.csv' file_data.to_csv(csv_file_name, header=False, index=False)
base_estimator=DecisionTreeClassifier(random_state=0), random_state=42) rbc = RUSBoostClassifier(base_estimator=DecisionTreeClassifier(random_state=0), random_state=0) bbc_score = [] brfc_score = [] eec_score = [] rbc_score = [] for train_index, test_index in kf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] bbc.fit(X_train, y_train) brfc.fit(X_train, y_train) eec.fit(X_train, y_train) rbc.fit(X_train, y_train) y_pred_bbc = bbc.predict(X_test) y_pred_brfc = brfc.predict(X_test) y_pred_eec = eec.predict(X_test) y_pred_rbc = rbc.predict(X_test) bbc_score.append(balanced_accuracy_score(y_test, y_pred_bbc)) brfc_score.append(balanced_accuracy_score(y_test, y_pred_brfc)) eec_score.append(balanced_accuracy_score(y_test, y_pred_eec)) rbc_score.append(balanced_accuracy_score(y_test, y_pred_rbc)) print("\t Average score:\t\t Standard deviation:") print("bbc\t", sum(bbc_score) / float(len(bbc_score)), "\t", statistics.stdev(bbc_score)) print("brfc\t", sum(brfc_score) / float(len(brfc_score)), "\t",
#classifier = CUSBoostClassifier(**a) #classifier = AdaboostClassifier(**a) #classifier = RusBoost(depth=depth, n_estimators=estimators) #classifier = AdaboostNC_Classifier(**a) #classifier = CUSBoostNC_Classifier(**a) #classifier = RusBoost(**a) classifier = RUSBoostClassifier(DecisionTreeClassifier(max_depth=8), n_estimators=64) #classifier.fit(X_train, y_train, number_of_clusters, 0.5) #CUSBoost classifier #classifier.fit(X_train, y_train) #Adaboost classifier #classifier.fit(X_train, y_train, 0.5) #AdaboostNC classifier #classifier.fit(X_train, y_train, 6, 0.5) #classifier.fit(X_train, y_train, 6, fraction/100, 8) classifier.fit(X_train, y_train) predictions = classifier.predict_proba(X_test) prediction_ = classifier.predict(X_test) auc = roc_auc_score(y_test, predictions[:, 1]) f1 = f1_score(y_test, prediction_) accuracy = accuracy_score(y_test, prediction_) #aupr = average_precision_score(y_test, predictions[:, 1]) current_param_auc.append(auc) current_param_f1.append(f1) current_param_accuracy.append(accuracy)
# base_estimator=base_estimator, # n_jobs=-1) # eec.fit(X_train_seek, y_train_seek) # y_pred_eec = eec.predict(X_test_seek) # print('Easy ensemble classifier performance:') # print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}' # .format(balanced_accuracy_score(y_test_seek, y_pred_eec), # geometric_mean_score(y_test_seek, y_pred_eec))) # cm_eec = confusion_matrix(y_test_seek, y_pred_eec) # fig, ax = plt.subplots(ncols=2) # plot_confusion_matrix(cm_eec, classes=np.unique(dataset.target), ax=ax[0], # title='Easy ensemble classifier') base_estimator = AdaBoostClassifier(n_estimators=10) rusboost = RUSBoostClassifier(n_estimators=10, base_estimator=base_estimator) rusboost.fit(X_train, y_train) y_pred_rusboost = rusboost.predict(X_test) print('RUSBoost classifier performance:') print('Balanced accuracy: {:.2f} - Geometric mean {:.2f}'.format( balanced_accuracy_score(y_test, y_pred_rusboost), geometric_mean_score(y_test, y_pred_rusboost))) cm_rusboost = confusion_matrix(y_test, y_pred_rusboost) fig, ax = plt.subplots(ncols=2) plot_confusion_matrix(cm_rusboost, classes=np.unique(dataset.target), ax=ax[1], title='RUSBoost classifier') rusboost.fit(X_train_seek, y_train_seek) y_pred_rusboost_seek = rusboost.predict(X_test_seek)
#cellTypesTrue.append(lineE[int(len(lineE))-1]) exMpred.append(exLpred) #s.append("\n") exLpred = [] cellID.append(lineE[0]) #cellTypesTrue = np.array(cellTypesTrue) exMpred = np.array(exMpred) cellID = np.array(cellID) ################################### ##### Everything is ready for cell type prediction ##### rusboost = RUSBoostClassifier(random_state=0) rusboost.fit(exMtrain, cellTypesTrain) ##### Cell types prediction ##### cellTypesPred = rusboost.predict(exMpred) #accuracy_score = balanced_accuracy_score(cellTypesTrue, cellTypesPred) #print accuracy_score #classification_report(cellTypesTrue, cellTypesPred) ##### Checking performance ##### #confusionMatrix = confusion_matrix(cellTypesTrue, cellTypesPred) cellTypesProbs = rusboost.predict_proba(exMpred) #print confusionMatrix ##### Merging the cell types and probability score ##### cellID_Probs = np.concatenate((cellID[:, None], cellTypesProbs), axis=1)