def create_model_for_ensemble(x_train, y_train, params=None, times_to_repeat=1): def min_max_rescaling(values, min_, max_): return [(ith_element - min_) / (max_ - min_) for ith_element in values] if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_NO_AGGREGATED avg_threshold = [] avg_min = [] avg_max = [] avg_mean = [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) xg_reg = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) xg_reg.fit(_x_train, _y_train) y_pred = xg_reg.predict_proba(_x_val)[:, 1] min_ = min(y_pred) max_ = max(y_pred) mean_ = np.array(y_pred).mean() avg_min.append(min_) avg_max.append(max_) avg_mean.append(mean_) y_pred = min_max_rescaling(y_pred, min_, max_) avg_threshold.append( evaluation.find_best_threshold_fixed_fpr(_y_val, y_pred)) xg_reg = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) xg_reg.fit(x_train, y_train) return xg_reg, np.array(avg_threshold).mean(), np.array( avg_min).mean(), np.array(avg_max).mean(), np.array(avg_mean).mean()
def predict_test_based_on_sum(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised): y_val_pred_lstm, y_val_pred_rf, y_val_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_val, x_val_supervised) len_ = len(y_val_pred_lstm) def min_max_rescaling(values): min_ = min(values) max_ = max(values) return [(ith_element - min_) / (max_ - min_) for ith_element in values] y_val_pred_lstm = min_max_rescaling(y_val_pred_lstm) y_val_pred_rf = min_max_rescaling(y_val_pred_rf) y_val_pred_xgb = min_max_rescaling(y_val_pred_xgb) y_val_pred = y_val_pred_lstm * np.array( [0.333] * len_) + y_val_pred_rf * np.array( [0.333] * len_) + y_val_pred_xgb * np.array([0.333] * len_) threshold = evaluation.find_best_threshold_fixed_fpr(y_val, y_val_pred) y_pred_lstm, y_pred_rf, y_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_test, x_test_supervised) len_ = len(y_pred_lstm) y_pred_lstm = min_max_rescaling(y_pred_lstm) y_pred_rf = min_max_rescaling(y_pred_rf) y_pred_xgb = min_max_rescaling(y_pred_xgb) y_pred = y_pred_lstm * np.array([0.333] * len_) + y_pred_rf * np.array( [0.333] * len_) + y_pred_xgb * np.array([0.333] * len_) y_test_pred = evaluation.adjusted_classes(y_pred, threshold) return y_test_pred
def create_model_for_ensemble(x_train, y_train, params=None, times_to_repeat=1): def min_max_rescaling(values, min_, max_): return [(ith_element - min_) / (max_ - min_) for ith_element in values] if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_NO_AGGREGATED avg_threshold = [] avg_min = [] avg_max = [] avg_mean = [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set(x_train, y_train, 0.25) rf = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) rf.fit(_x_train, _y_train) y_pred = rf.predict_proba(_x_val)[:, 1] min_ = min(y_pred) max_ = max(y_pred) mean_ = np.array(y_pred).mean() avg_min.append(min_) avg_max.append(max_) avg_mean.append(mean_) y_pred = min_max_rescaling(y_pred, min_, max_) avg_threshold.append(evaluation.find_best_threshold_fixed_fpr(_y_val, y_pred)) rf = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) rf.fit(x_train, y_train) return rf, np.array(avg_threshold).mean(), np.array(avg_min).mean(), np.array(avg_max).mean(), np.array(avg_mean).mean()
def predict_test_based_on_voting(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test): y_val_pred_lstm, y_val_pred_rf, y_val_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_val, x_val_supervised) # get threshold for each model threshold_lstm = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_lstm) threshold_rf = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_rf) threshold_xgb = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_xgb) # predicting test set y_pred_lstm, y_pred_rf, y_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_test, x_test_supervised) y_pred_lstm = np.array( evaluation.adjusted_classes(y_pred_lstm, threshold_lstm)) y_pred_rf = np.array(evaluation.adjusted_classes(y_pred_rf, threshold_rf)) y_pred_xgb = np.array( evaluation.adjusted_classes(y_pred_xgb, threshold_xgb)) y_test_pred = [] lstm_indices_found = evaluation.get_indices(y_test, y_pred_lstm) rf_indices_found = evaluation.get_indices(y_test, y_pred_rf) xgboost_indices_found = evaluation.get_indices(y_test, y_pred_xgb) not_found_by_xgboost, not_found_by_rf, not_found_by_others = evaluation.get_jaccard_index( lstm_indices_found, rf_indices_found, xgboost_indices_found) for i in range(len(y_pred_lstm)): if y_pred_lstm[i] + y_pred_rf[i] + y_pred_xgb[i] >= 2: y_test_pred.append(1) else: y_test_pred.append(0) return y_test_pred, not_found_by_xgboost, not_found_by_rf, not_found_by_others
def create_fit_model(x_train, y_train, look_back, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_NO_AGGREGATED avg_threshold = 0 for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(_x_train, _y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) y_pred = model.predict(_x_val) avg_threshold += evaluation.find_best_threshold_fixed_fpr( _y_val, y_pred) # model = create_simple_model(x_train, y_train, look_back) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(x_train, y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) return model, avg_threshold / times_to_repeat
def create_model(x_train, y_train, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_NO_AGGREGATED thresholds = [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) xg_reg = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) xg_reg.fit(_x_train, _y_train) y_pred = xg_reg.predict_proba(_x_val)[:, 1] thresholds.append( evaluation.find_best_threshold_fixed_fpr(_y_val, y_pred)) print(np.array(thresholds).mean(), np.array(thresholds).std()) xg_reg = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) xg_reg.fit(x_train, y_train) return xg_reg, np.array(thresholds).mean()
def create_model(x_train, y_train, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_NO_AGGREGATED avg_threshold = 0 for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set(x_train, y_train, 0.25) rf = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) rf.fit(_x_train, _y_train) y_pred = rf.predict_proba(_x_val)[:, 1] avg_threshold += evaluation.find_best_threshold_fixed_fpr(_y_val, y_pred) rf = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) rf.fit(x_train, y_train) return rf, avg_threshold / times_to_repeat
def create_fit_model_for_ensemble(x_train, y_train, look_back, params=None, times_to_repeat=1): def min_max_rescaling(values, min_, max_): return [(ith_element - min_) / (max_ - min_) for ith_element in values] if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_NO_AGGREGATED avg_threshold = [] avg_min = [] avg_max = [] avg_mean = [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(_x_train, _y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) y_pred = model.predict(_x_val) min_ = min(y_pred) max_ = max(y_pred) mean_ = np.array(y_pred).mean() avg_min.append(min_) avg_max.append(max_) avg_mean.append(mean_) y_pred = min_max_rescaling(y_pred, min_, max_) avg_threshold.append( evaluation.find_best_threshold_fixed_fpr(_y_val, y_pred)) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(x_train, y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) return model, np.array(avg_threshold).mean(), np.array( avg_min).mean(), np.array(avg_max).mean(), np.array(avg_mean).mean()
def create_fit_model_for_ensemble_based_on_cdf(x_train, y_train, look_back, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_LSTM_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_LSTM_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_LSTM_OLD_DATASET_NO_AGGREGATED scales, locs, means, stds, thresholds = [], [], [], [], [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(_x_train, _y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) y_val_pred_lstm = model.predict(_x_val) ''' to get the shape of the distribution prova = y_val_pred_lstm.tolist() buckets = {} for i in range(0, 101): buckets[i / 100] = 0 for i in prova: buckets[round(i, 2)] += 1 plt.plot(list(buckets.keys()), ",", list(buckets.values()) plt.show() ''' loc_lstm, scale_lstm = stats.expon.fit(y_val_pred_lstm) samples_lstm = stats.expon.cdf(y_val_pred_lstm, scale=scale_lstm, loc=loc_lstm) lstm_mean = samples_lstm.mean() lstm_std = samples_lstm.std() threshold_lstm = evaluation.find_best_threshold_fixed_fpr( _y_val, samples_lstm) scales.append(scale_lstm) locs.append(loc_lstm) means.append(lstm_mean) stds.append(lstm_std) thresholds.append(threshold_lstm) model = create_model(params["layers"], params["dropout_rate"], look_back, len(x_train[0, 0])) model.fit(x_train, y_train, epochs=params["epochs"], verbose=1, batch_size=params["batch_size"]) return model, np.array(scales).mean(), np.array(locs).mean(), np.array( means).mean(), np.array(stds).mean(), np.array(thresholds).mean()
def repeat_experiment_n_times(lstm, rf, xg_reg, scenario, times_to_repeat=100, adversarial_attack=False, evasion_attack=False, is_white_box_attack=True, use_lstm_for_adversarial=False): tn_s = [] tp_s = [] fp_s = [] fn_s = [] f1_s = [] balanced_accuracies = [] precisions = [] recalls = [] aucpr_s = [] roc_aucs = [] num_decisions_taken_by_lstm,\ num_decisions_taken_by_rf, \ num_decisions_taken_by_xgb, \ num_decisions_correctly_taken_from_lstm, \ num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf = 0, 0, 0, 0, 0 for i in range(times_to_repeat): print("Iteration", i) x_test_set, y_test_set = sequences_crafting_for_classification.get_test_set( scenario=scenario) x_val, y_val, x_test, y_test = evaluation.get_val_test_set( x_test_set, y_test_set, val_size=0.25) x_val_supervised = x_val[:, len(x_val[0]) - 1, :] x_test_supervised = x_test[:, len(x_val[0]) - 1, :] if adversarial_attack or evasion_attack: # getting train set for training if is_white_box_attack: print("Using as training set, the real one - whitebox attack") dataset_type = REAL_DATASET else: print("Using as training set, the old one - blackbox attack") dataset_type = OLD_DATASET x_train, y_train = sequences_crafting_for_classification.get_train_set( dataset_type=dataset_type) x_train_supervised = x_train[:, look_back, :] if adversarial_attack: print("Crafting an adversarial attack") if not use_lstm_for_adversarial: print("The attacker will use a Multilayer perceptron") # training multilayer perceptron # todo: hyper param tuning multilayer perceptron adversarial_model = MultiLayerPerceptron.create_fit_model( x_train_supervised, y_train) # crafting adversarial samples x_test_supervised = x_test[:, len(x_test[0]) - 1, :] frauds = x_test_supervised[np.where(y_test == 1)] adversarial_samples = fgsm.craft_sample(frauds, adversarial_model, epsilon=0.01) x_test[np.where(y_test == 1), len(x_test[0]) - 1] = adversarial_samples x_test_supervised = x_test[:, len(x_test[0]) - 1, :] else: print("The attacker will use a LSTM network") # train the network using the right params if is_white_box_attack: if USING_AGGREGATED_FEATURES: params = BEST_PARAMS_LSTM_REAL_DATASET_AGGREGATED else: params = BEST_PARAMS_LSTM_REAL_DATASET_NO_AGGREGATED else: if USING_AGGREGATED_FEATURES: params = BEST_PARAMS_LSTM_OLD_DATASET_AGGREGATED else: params = BEST_PARAMS_LSTM_OLD_DATASET_NO_AGGREGATED adversarial_model = LSTM_classifier.create_fit_model( x_train, y_train, look_back, params=params) frauds = x_test[np.where(y_test == 1)] adversarial_samples = fgsm.craft_sample(frauds, adversarial_model, epsilon=0.01) x_test[np.where(y_test == 1)] = adversarial_samples x_test_supervised = x_test[:, len(x_test[0]) - 1, :] if evasion_attack: print("Crafting an evasion attack") # train the network using the right params if is_white_box_attack: if USING_AGGREGATED_FEATURES: params = BEST_PARAMS_RF_REAL_DATASET_AGGREGATED else: params = BEST_PARAMS_RF_REAL_DATASET_NO_AGGREGATED else: if USING_AGGREGATED_FEATURES: params = BEST_PARAMS_RF_OLD_DATASET_AGGREGATED else: params = BEST_PARAMS_RF_OLD_DATASET_NO_AGGREGATED # training the oracle oracle = RF.create_model(x_train_supervised, y_train, params=params) # get the oracle threshold y_val_pred_oracle = oracle.predict_proba(x_val_supervised) oracle_threshold = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_oracle[:, 1]) # if the oracle predicts the fraud as fraud -> discard it, otherwise inject in real bank system y_pred_oracle = rf.predict_proba(x_test_supervised) y_pred_oracle = y_pred_oracle[:, 1].ravel() y_pred_oracle = np.array( evaluation.adjusted_classes(y_pred_oracle, oracle_threshold)) x_test = x_test[(np.where(( (y_test == 1) & (y_pred_oracle == 0)) | (y_test == 0)))] y_test = y_test[(np.where(( (y_test == 1) & (y_pred_oracle == 0)) | (y_test == 0)))] x_test_supervised = x_test[:, len(x_test[0]) - 1, :] try: # a, b, c, d, e = 0, 0, 0, 0, 0 # y_test_pred, not_by_xgb, not_by_rf, not_found_by_others = predict_test_based_on_voting(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test) y_test_pred, a, b, c, d, e = predict_test_based_on_more_confident( lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test) # y_test_pred, a, b, c, d, e = predict_test_based_on_expon(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test) # y_test_pred = predict_test_based_on_sum(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised) # y_test_pred = predict_test_based_on_more_confident_and_majority_voting(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test) # not_found_by_xgboost += not_by_xgb # not_by_rf += not_by_rf # not_found_by_others += not_by_others y_test_pred = np.array(y_test_pred) confusion, f1, balanced_accuracy, precision, recall, aucpr, roc_auc = evaluation.get_performance( y_test, y_test_pred, threshold=True) tn = confusion[0, 0] tp = confusion[1, 1] fp = confusion[0, 1] fn = confusion[1, 0] tn_s.append(tn) tp_s.append(tp) fp_s.append(fp) fn_s.append(fn) f1_s.append(f1) num_decisions_taken_by_lstm += a num_decisions_taken_by_rf += b num_decisions_taken_by_xgb += c num_decisions_correctly_taken_from_lstm += d num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf += e balanced_accuracies.append(balanced_accuracy) precisions.append(precision) recalls.append(recall) aucpr_s.append(aucpr) roc_aucs.append(roc_auc) except RuntimeError: i -= 1 print("Num decisions taken from lstm: ", num_decisions_taken_by_lstm / times_to_repeat) print("Num decisions taken by rf: ", num_decisions_taken_by_rf / times_to_repeat) print("Num decisions taken by xgb: ", num_decisions_taken_by_xgb / times_to_repeat) print("Num decisions taken by lstm correctly taken: ", num_decisions_correctly_taken_from_lstm / times_to_repeat) print( "Num decisions taken by lstm correctly taken and not by others: ", num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf / times_to_repeat) evaluation.print_results( np.array(tn_s).mean(), np.array(fp_s).mean(), np.array(fn_s).mean(), np.array(tp_s).mean(), np.array(f1_s).mean(), np.array(balanced_accuracies).mean(), np.array(precisions).mean(), np.array(recalls).mean(), np.array(aucpr_s).mean(), np.array(roc_aucs).mean())
def predict_test_based_on_expon(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test): y_val_pred_lstm, y_val_pred_rf, y_val_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_val, x_val_supervised) ''' to get the shape of the distribution prova = y_val_pred_lstm.tolist() buckets = {} for i in range(0, 101): buckets[i / 100] = 0 for i in prova: buckets[round(i, 2)] += 1 plt.plot(list(buckets.keys()), ",", list(buckets.values()) plt.show() ''' # for each model, fit the exponential distribution loc_lstm, scale_lstm = stats.expon.fit(y_val_pred_lstm) loc_rf, scale_rf = stats.expon.fit(y_val_pred_rf) loc_xgb, scale_xgb = stats.expon.fit(y_val_pred_xgb) samples_lstm = stats.expon.cdf(y_val_pred_lstm, scale=scale_lstm, loc=loc_lstm) samples_rf = stats.expon.cdf(y_val_pred_rf, scale=scale_rf, loc=loc_rf) samples_xgb = stats.expon.cdf(y_val_pred_xgb, scale=scale_xgb, loc=loc_xgb) # get mean for each model lstm_mean = samples_lstm.mean() rf_mean = samples_rf.mean() xgb_mean = samples_xgb.mean() # get std for each model lstm_std = samples_lstm.std() rf_std = samples_rf.std() xgb_std = samples_xgb.std() threshold_lstm = evaluation.find_best_threshold_fixed_fpr( y_val, samples_lstm) threshold_rf = evaluation.find_best_threshold_fixed_fpr(y_val, samples_rf) threshold_xgb = evaluation.find_best_threshold_fixed_fpr( y_val, samples_xgb) y_pred_lstm, y_pred_rf, y_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_test, x_test_supervised) samples_lstm = stats.expon.cdf(y_pred_lstm, scale=scale_lstm, loc=loc_lstm) samples_rf = stats.expon.cdf(y_pred_rf, scale=scale_rf, loc=loc_rf) samples_xgb = stats.expon.cdf(y_pred_xgb, scale=scale_xgb, loc=loc_xgb) num_decisions_taken_by_lstm = 0 num_decisions_correctly_taken_from_lstm = 0 num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf = 0 num_decisions_taken_by_rf = 0 num_decisions_taken_by_xgb = 0 indices_decisions_taken_by_rf = [] y_test_pred = [] for i in range(len(y_pred_lstm)): max_value = max( abs(samples_lstm[i] - lstm_mean) / lstm_std, abs(samples_rf[i] - rf_mean) / rf_std, abs(samples_xgb[i] - xgb_mean) / xgb_std) if abs(samples_lstm[i] - lstm_mean) / lstm_std == max_value: lstm_output = 1 if samples_lstm[i] > threshold_lstm else 0 rf_output = 1 if samples_rf[i] > threshold_rf else 0 xgb_output = 1 if samples_xgb[i] > threshold_xgb else 0 if lstm_output == 1 and y_test[i] == 1 and ( rf_output == 0 or xgb_output == 0 ) or lstm_output == 0 and y_test[i] == 0 and (rf_output == 1 or xgb_output == 1): num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf += 1 if (lstm_output == 1 and y_test[i] == 1) or (lstm_output == 0 and y_test[i] == 0): num_decisions_correctly_taken_from_lstm += 1 num_decisions_taken_by_lstm += 1 y_test_pred.append(1 if samples_lstm[i] > threshold_lstm else 0) elif abs(samples_rf[i] - rf_mean) / rf_std == max_value: num_decisions_taken_by_rf += 1 rf_decision = 1 if samples_rf[i] > threshold_rf else 0 if rf_decision: indices_decisions_taken_by_rf.append(i) y_test_pred.append(1 if samples_rf[i] > threshold_rf else 0) elif abs(samples_xgb[i] - xgb_mean) / xgb_std == max_value: num_decisions_taken_by_xgb += 1 y_test_pred.append(1 if samples_xgb[i] > threshold_xgb else 0) else: print("errore") return y_test_pred, num_decisions_taken_by_lstm, num_decisions_taken_by_rf, num_decisions_taken_by_xgb, num_decisions_correctly_taken_from_lstm, num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf
def predict_test_based_on_more_confident_and_majority_voting( lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test): y_val_pred_lstm, y_val_pred_rf, y_val_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_val, x_val_supervised) # get threshold for each model threshold_lstm = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_lstm) threshold_rf = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_rf) threshold_xgb = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_xgb) # get mean for each model lstm_mean = np.array(y_val_pred_lstm).mean() rf_mean = np.array(y_val_pred_rf).mean() xgb_mean = np.array(y_val_pred_xgb).mean() # get std for each model lstm_std = np.array(y_val_pred_lstm).std() rf_std = np.array(y_val_pred_rf).std() xgb_std = np.array(y_val_pred_xgb).std() # predicting test set y_pred_lstm, y_pred_rf, y_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_test, x_test_supervised) y_test_pred = [] num_decisions_taken_by_lstm = 0 num_decisions_correctly_taken_from_lstm = 0 num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf = 0 num_decisions_taken_by_rf = 0 num_decisions_taken_by_xgb = 0 for i in range(len(y_pred_lstm)): max_value = max( abs(y_pred_lstm[i] - lstm_mean) / lstm_std, abs(y_pred_rf[i] - rf_mean) / rf_std, abs(y_pred_xgb[i] - xgb_mean) / xgb_std) lstm_output = 1 if y_pred_lstm[i] > threshold_lstm else 0 rf_output = 1 if y_pred_rf[i] > threshold_rf else 0 xgb_output = 1 if y_pred_xgb[i] > threshold_xgb else 0 if abs(y_pred_lstm[i] - lstm_mean) / lstm_std == max_value: if max_value > abs(y_pred_rf[i] - rf_mean) / rf_std + abs( y_pred_xgb[i] - xgb_mean) / xgb_std: num_decisions_taken_by_lstm += 1 y_test_pred.append(lstm_output) if lstm_output == 1 and y_test[i] == 1 and (rf_output == 0 or xgb_output == 0): num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf += 1 if lstm_output == 1 and y_test[i]: num_decisions_correctly_taken_from_lstm += 1 else: if lstm_output + rf_output + xgb_output >= 2: y_test_pred.append(1) else: y_test_pred.append(0) elif abs(y_pred_rf[i] - rf_mean) / rf_std == max_value: if max_value > abs(y_pred_lstm[i] - lstm_mean) / lstm_std + abs( y_pred_xgb[i] - xgb_mean) / xgb_std: y_test_pred.append(1 if y_pred_rf[i] > threshold_rf else 0) num_decisions_taken_by_rf += 1 else: if lstm_output + rf_output + xgb_output >= 2: y_test_pred.append(1) else: y_test_pred.append(0) else: if max_value > abs(y_pred_rf[i] - rf_mean) / rf_std + abs( y_pred_lstm[i] - lstm_mean) / lstm_std: num_decisions_taken_by_xgb += 1 y_test_pred.append(1 if y_pred_xgb[i] > threshold_xgb else 0) else: if lstm_output + rf_output + xgb_output >= 2: y_test_pred.append(1) else: y_test_pred.append(0) return y_test_pred
def predict_test_based_on_more_confident(lstm, rf, xg_reg, x_val, x_val_supervised, y_val, x_test, x_test_supervised, y_test): y_val_pred_lstm, y_val_pred_rf, y_val_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_val, x_val_supervised) def min_max_rescaling(values, min_, max_): return [(ith_element - min_) / (max_ - min_) for ith_element in values] lstm_min = min(y_val_pred_lstm) lstm_max = max(y_val_pred_lstm) rf_min = min(y_val_pred_rf) rf_max = max(y_val_pred_rf) xgb_min = min(y_val_pred_xgb) xgb_max = max(y_val_pred_xgb) y_val_pred_lstm = min_max_rescaling(y_val_pred_lstm, lstm_min, lstm_max) y_val_pred_rf = min_max_rescaling(y_val_pred_rf, rf_min, rf_max) y_val_pred_xgb = min_max_rescaling(y_val_pred_xgb, xgb_min, xgb_max) # get threshold for each model threshold_lstm = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_lstm) threshold_rf = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_rf) threshold_xgb = evaluation.find_best_threshold_fixed_fpr( y_val, y_val_pred_xgb) # get mean for each model lstm_mean = np.array(y_val_pred_lstm).mean() rf_mean = np.array(y_val_pred_rf).mean() xgb_mean = np.array(y_val_pred_xgb).mean() # get std for each model lstm_std = np.array(y_val_pred_lstm).std() rf_std = np.array(y_val_pred_rf).std() xgb_std = np.array(y_val_pred_xgb).std() # predicting test set y_pred_lstm, y_pred_rf, y_pred_xgb = get_predictions_for_each_model( lstm, rf, xg_reg, x_test, x_test_supervised) y_pred_lstm = min_max_rescaling(y_pred_lstm, lstm_min, lstm_max) y_pred_rf = min_max_rescaling(y_pred_rf, rf_min, rf_max) y_pred_xgb = min_max_rescaling(y_pred_xgb, xgb_min, xgb_max) y_test_pred = [] num_decisions_taken_by_lstm = 0 num_decisions_correctly_taken_from_lstm = 0 num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf = 0 num_decisions_taken_by_rf = 0 num_decisions_taken_by_xgb = 0 for i in range(len(y_pred_lstm)): max_value = max(abs(y_pred_lstm[i] - lstm_mean), abs(y_pred_rf[i] - rf_mean), abs(y_pred_xgb[i] - xgb_mean)) if abs(y_pred_lstm[i] - lstm_mean) == max_value: num_decisions_taken_by_lstm += 1 lstm_output = 1 if y_pred_lstm[i] > threshold_lstm else 0 rf_output = 1 if y_pred_rf[i] > threshold_rf else 0 xgb_output = 1 if y_pred_xgb[i] > threshold_xgb else 0 y_test_pred.append(lstm_output) if lstm_output == 1 and y_test[i] == 1 and (rf_output == 0 or xgb_output == 0): num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf += 1 if lstm_output == 1 and y_test[i]: num_decisions_correctly_taken_from_lstm += 1 elif abs(y_pred_rf[i] - rf_mean) == max_value: y_test_pred.append(1 if y_pred_rf[i] > threshold_rf else 0) num_decisions_taken_by_rf += 1 else: num_decisions_taken_by_xgb += 1 y_test_pred.append(1 if y_pred_xgb[i] > threshold_xgb else 0) return y_test_pred, num_decisions_taken_by_lstm, num_decisions_taken_by_rf, num_decisions_taken_by_xgb, num_decisions_correctly_taken_from_lstm, num_decisions_correctly_taken_from_lstm_and_not_from_xgb_or_rf
def create_fit_model_for_ensemble_based_on_cdf(x_train, y_train, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_XGBOOST_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_XGBOOST_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_XGBOOST_OLD_DATASET_NO_AGGREGATED scales, locs, means, stds, thresholds = [], [], [], [], [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set( x_train, y_train, 0.25) model = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) model.fit(x_train, y_train) y_val_pred_lstm = model.predict_proba(_x_val)[:, 1] ''' to get the shape of the distribution prova = y_val_pred_lstm.tolist() buckets = {} for i in range(0, 101): buckets[i / 100] = 0 for i in prova: buckets[round(i, 2)] += 1 plt.plot(list(buckets.keys()), ",", list(buckets.values()) plt.show() ''' loc, scale = stats.expon.fit(y_val_pred_lstm) samples = stats.expon.cdf(y_val_pred_lstm, scale=scale, loc=loc) mean = samples.mean() std = samples.std() threshold = evaluation.find_best_threshold_fixed_fpr(_y_val, samples) scales.append(scale) locs.append(loc) means.append(mean) stds.append(std) thresholds.append(threshold) xg_reg = xgb.XGBClassifier(subsample=params["subsample"], min_child_weight=params["min_child_weight"], max_depth=params["max_depth"], learning_rate=params["learning_rate"], gamma=params["gamma"], colsample_bytree=params["colsample_bytree"]) xg_reg.fit(x_train, y_train) return xg_reg, np.array(scales).mean(), np.array(locs).mean(), np.array( means).mean(), np.array(stds).mean(), np.array(thresholds).mean()
def create_fit_model_for_ensemble_based_on_cdf(x_train, y_train, params=None, times_to_repeat=1): if not params: if constants.USING_AGGREGATED_FEATURES: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_AGGREGATED else: if constants.DATASET_TYPE == constants.INJECTED_DATASET: params = constants.BEST_PARAMS_RF_NO_AGGREGATED if constants.DATASET_TYPE == constants.REAL_DATASET: params = constants.BEST_PARAMS_RF_REAL_DATASET_NO_AGGREGATED if constants.DATASET_TYPE == constants.OLD_DATASET: params = constants.BEST_PARAMS_RF_OLD_DATASET_NO_AGGREGATED scales, locs, means, stds, thresholds = [], [], [], [], [] for i in range(times_to_repeat): print("iteration: ", i, "of", times_to_repeat) _x_val, _y_val, _x_train, _y_train = evaluation.get_val_test_set(x_train, y_train, 0.25) model = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) model.fit(_x_train, _y_train) y_val_pred_lstm = model.predict_proba(_x_val)[:, 1] ''' to get the shape of the distribution prova = y_val_pred_lstm.tolist() buckets = {} for i in range(0, 101): buckets[i / 100] = 0 for i in prova: buckets[round(i, 2)] += 1 plt.plot(list(buckets.keys()), ",", list(buckets.values()) plt.show() ''' loc, scale = stats.expon.fit(y_val_pred_lstm) samples = stats.expon.cdf(y_val_pred_lstm, scale=scale, loc=loc) mean = samples.mean() std = samples.std() threshold = evaluation.find_best_threshold_fixed_fpr(_y_val, samples) scales.append(scale) locs.append(loc) means.append(mean) stds.append(std) thresholds.append(threshold) rf = RandomForestClassifier(n_estimators=params["n_estimators"], min_samples_split=params["min_samples_split"], min_samples_leaf=params["min_samples_leaf"], max_features=params["max_features"], max_depth=params["max_depth"], bootstrap=params["bootstrap"]) rf.fit(x_train, y_train) return rf, np.array(scales).mean(), np.array(locs).mean(), np.array(means).mean(), np.array(stds).mean(), np.array(thresholds).mean()