def computing_precise_vs_imprecise(in_path=None, ell_optimal=0.1, cv_n_fold=10, seeds=None, lib_path_server=None, model_type_precise='lda', model_type_imprecise='ilda', scaling=True): data = export_data_set('iris.data') if in_path is None else pd.read_csv( in_path) logger = create_logger("computing_precise_vs_imprecise", True) logger.info('Training dataset and models (%s, %s, %s, %s)', in_path, model_type_precise, model_type_imprecise, ell_optimal) X = data.iloc[:, :-1].values if scaling: X = normalize_minmax(X) y = np.array(data.iloc[:, -1].tolist()) seeds = generate_seeds(cv_n_fold) if seeds is None else seeds model_impr = __factory_model(model_type_imprecise, init_matlab=True, add_path_matlab=lib_path_server, DEBUG=False) model_prec = __factory_model_precise(model_type_precise, store_covariance=True) avg_imprecise, avg_precise, n_real_times = 0, 0, 0 for time in range(cv_n_fold): kf = KFold(n_splits=cv_n_fold, random_state=seeds[time], shuffle=True) imprecise_mean, precise_mean, n_real_fold = 0, 0, 0 for idx_train, idx_test in kf.split(y): X_cv_train, y_cv_train = X[idx_train], y[idx_train] X_cv_test, y_cv_test = X[idx_test], y[idx_test] model_impr.learn(X=X_cv_train, y=y_cv_train, ell=ell_optimal) model_prec.fit(X_cv_train, y_cv_train) n_real_tests, time_precise, time_imprecise = 0, 0, 0 n_test, _ = X_cv_test.shape for i, test in enumerate(X_cv_test): evaluate_imp, _ = model_impr.evaluate(test) evaluate = model_prec.predict([test]) if len(evaluate_imp) > 1: n_real_tests += 1 if y_cv_test[i] in evaluate_imp: time_imprecise += 1 if y_cv_test[i] in evaluate: time_precise += 1 logger.debug( "(time, iTest, ellOptimal, cautious, prediction, ground-truth)(%s, %s, %s, %s, %s, %s)", time, i, ell_optimal, evaluate_imp, evaluate, y_cv_test[i]) logger.debug( "(time, ellOptimal, nRealTests, timeImprecise, timePrecise) (%s, %s, %s, %s, %s)", time, ell_optimal, n_real_tests, time_imprecise, time_precise) if n_real_tests > 0: n_real_fold += 1 imprecise_mean += time_imprecise / n_real_tests precise_mean += time_precise / n_real_tests logger.debug("(time, nRealFold, imprecise, precise) (%s, %s, %s, %s)", time, n_real_fold, imprecise_mean, precise_mean) if n_real_fold > 0: n_real_times += 1 avg_imprecise += imprecise_mean / n_real_fold avg_precise += precise_mean / n_real_fold logger.debug("(dataset, models, imprec, prec) (%s, %s, %s, %s, %s)", in_path, model_type_imprecise, model_type_precise, avg_imprecise / n_real_times, avg_precise / n_real_times)
def performance_accuracy_noise_corrupted_test_data(in_train_paths=None, in_tests_paths=None, model_type_precise='lda', model_type_imprecise='ilda', ell_optimal=0.1, scaling=False, lib_path_server=None, nb_process=10): assert isinstance(in_train_paths, list), "Without training data, cannot create to model" assert isinstance( in_tests_paths, list), "Without training data, cannot performing accuracy" logger = create_logger("performance_accuracy_noise_corrupted_test_data", True) logger.info('Training dataset (%s, %s, %s)', in_train_paths, model_type_imprecise, ell_optimal) manager = ManagerWorkers(nb_process=nb_process) manager.executeAsync(model_type_imprecise, lib_path_server) versus = model_type_imprecise + "_vs_" + model_type_precise file_csv = open("results_" + versus + "_noise_accuracy.csv", 'w') writer = csv.writer(file_csv) model_precise = __factory_model_precise(model_type_precise, store_covariance=True) for in_train_path in in_train_paths: X_train, y_train = dataset_to_Xy(in_train_path, scaling=scaling) model_precise.fit(X_train, y_train) accuracies = dict({}) for in_test_path in in_tests_paths: X_test, y_test = dataset_to_Xy(in_test_path, scaling=scaling) _u65, _u80, _set = computing_training_testing_step( X_train, y_train, X_test, y_test, ell_optimal, manager, 0, 0, 0) evaluate = model_precise.predict(X_test) _acc = sum( 1 for k, j in zip(evaluate, y_test) if k == j) / len(y_test) logger.debug("accuracy-in_test_path (%s, %s, %s, %s, %s, %s)", ntpath.basename(in_train_path), ntpath.basename(in_test_path), ell_optimal, _u65, _u80, _acc) accuracies[ntpath.basename(in_test_path)] = [ ell_optimal, _u65, _u80, _set, _acc ] writer.writerow([ ntpath.basename(in_train_path), ntpath.basename(in_test_path), ell_optimal, _u65, _u80, _set, _acc ]) file_csv.flush() logger.debug("Partial-finish-accuracy-noise-corrupted_test %s: %s", ntpath.basename(in_train_path), accuracies) manager.poisonPillTraining() file_csv.close() logger.debug("Finish-accuracy-noise-corrupted_test")
def performance_hold_out(in_path=None, out_path=None, model_type='lda', test_pct=0.4, n_times=10, seeds=None, scaling=False): assert os.path.exists(in_path), "Without training data, not testing" assert os.path.exists(out_path), "Without output saving performance" logger = create_logger("performance_hold_out", True) logger.info('Training data set %s, test percentage %s, model_type %s', in_path, test_pct, model_type) data = pd.read_csv(in_path, header=None) X = data.iloc[:, :-1].values if scaling: X = normalize_minmax(X) y = data.iloc[:, -1].tolist() seeds = generate_seeds(n_times) if seeds is None else seeds logger.info('Seeds generated %s', seeds) file_csv = open(out_path, 'w') writer = csv.writer(file_csv) model = __factory_model_precise(model_type, store_covariance=True) mean_u65, mean_u80 = np.array([]), np.array([]) for i in range(0, n_times): X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=test_pct, random_state=seeds[i]) sum_u65, sum_u80 = 0, 0 model.fit(X_train, y_train) n, _ = X_test.shape for j, test in enumerate(X_test): evaluate = model.predict([test]) if y_test[j] in evaluate: sum_u65 += u65(evaluate) sum_u80 += u80(evaluate) logger.info("time, u65, u80 (%s, %s, %s)", i, sum_u65 / n, sum_u80 / n) mean_u65 = np.append(mean_u65, sum_u65 / n) mean_u80 = np.append(mean_u80, sum_u80 / n) writer.writerow([-999, i, mean_u65[i], mean_u80[i]]) file_csv.flush() file_csv.close() logger.info("[total:data-set:avgResults] (%s, %s)", np.mean(mean_u65), np.mean(mean_u80))
def performance_cv_accuracy(in_path=None, model_type='lda', cv_n_fold=10, seeds=None, scaling=False): assert os.path.exists(in_path), "Without training data, not testing" data = pd.read_csv(in_path, header=None) logger = create_logger("performance_cv_accuracy", True) logger.info('Training data set %s, cv_n_fold %s, model_type %s', in_path, cv_n_fold, model_type) X = data.iloc[:, :-1].values if scaling: X = normalize_minmax(X) y = np.array(data.iloc[:, -1].tolist()) avg_u65, avg_u80 = 0, 0 seeds = generate_seeds(cv_n_fold) if seeds is None else seeds logger.info('Seeds generated %s', seeds) for time in range(cv_n_fold): # Generation a random k-fold validation. kf = KFold(n_splits=cv_n_fold, random_state=seeds[time], shuffle=True) model = __factory_model_precise(model_type, store_covariance=True) mean_u65, mean_u80 = 0, 0 for idx_train, idx_test in kf.split(y): X_cv_train, y_cv_train = X[idx_train], y[idx_train] X_cv_test, y_cv_test = X[idx_test], y[idx_test] model.fit(X_cv_train, y_cv_train) n_test = len(idx_test) sum_u65, sum_u80 = 0, 0 for i, test in enumerate(X_cv_test): evaluate = model.predict([test]) logger.debug( "(testing, prediction, ground-truth) (%s, %s, %s)", i, evaluate, y_cv_test[i]) if y_cv_test[i] in evaluate: sum_u65 += u65(evaluate) sum_u80 += u80(evaluate) mean_u65 += sum_u65 / n_test mean_u80 += sum_u80 / n_test logger.info("Time, seed, u65, u80 (%s, %s, %s, %s)", time, seeds[time], mean_u65 / cv_n_fold, mean_u80 / cv_n_fold) avg_u65 += mean_u65 / cv_n_fold avg_u80 += mean_u80 / cv_n_fold logger.info("[Total:data-set:avgResults] (%s, %s, %s)", in_path, avg_u65 / cv_n_fold, avg_u80 / cv_n_fold)
def performance_qda_regularized(in_path=None, out_path=None, cv_n_fold=10, seeds=None, from_alpha=0, to_alpha=2.0, by_alpha=0.01, scaling=False): assert os.path.exists(in_path), "Without training data, not testing" assert os.path.exists(out_path), "Without output saving performance" data = pd.read_csv(in_path, header=None) logger = create_logger("performance_qda_regularized", True) logger.info('Training data set %s, cv_n_fold %s, model_type %s', in_path, cv_n_fold, "qda") X = data.iloc[:, :-1].values if scaling: X = normalize_minmax(X) y = np.array(data.iloc[:, -1].tolist()) seeds = generate_seeds(cv_n_fold) if seeds is None else seeds logger.info('Seeds generated %s', seeds) file_csv = open(out_path, 'a') writer = csv.writer(file_csv) alphas = np.arange(from_alpha, to_alpha, by_alpha) writer.writerow(alphas) qda_regularized = [None] * len(alphas) for idx, alpha in enumerate(alphas): qda_regularized[idx] = __factory_model_precise("qda", store_covariance=True, reg_param=alpha) # Generation a random k-fold validation. kf_second = KFold(n_splits=cv_n_fold, random_state=None, shuffle=True) ikfold, accuracy, best_alphas = 0, [0] * cv_n_fold, [0] * cv_n_fold for idx_learning, idx_testing in kf_second.split(y): X_training, y_training = X[idx_learning], y[idx_learning] X_testing, y_testing = X[idx_testing], y[idx_testing] kf = KFold(n_splits=cv_n_fold, random_state=seeds[ikfold], shuffle=True) acc_u80 = [0] * len(qda_regularized) for idx_train, idx_test in kf.split(y_training): X_cv_train, y_cv_train = X_training[idx_train], y_training[ idx_train] X_cv_test, y_cv_test = X_training[idx_test], y_training[idx_test] for model in qda_regularized: model.fit(X_cv_train, y_cv_train) n_test = len(idx_test) for i, test in enumerate(X_cv_test): for im, model in enumerate(qda_regularized): evaluate = model.predict([test]) if y_cv_test[i] in evaluate: acc_u80[im] += (u80(evaluate) / n_test) / cv_n_fold idx_best = np.argmax(acc_u80) logger.info("[1kfold:best_model:seed:u80] (%s, %s, %s, %s)", ikfold, alphas[idx_best], seeds[ikfold], acc_u80) writer.writerow(acc_u80) file_csv.flush() best_model = __factory_model_precise("qda", store_covariance=True, reg_param=alphas[idx_best]) best_model.fit(X_training, y_training) accuracy[ikfold], bn_test, best_alphas[ikfold] = 0, len( idx_testing), alphas[idx_best] for i, test in enumerate(X_testing): evaluate = best_model.predict([test]) if y_testing[i] in evaluate: accuracy[ikfold] += u80(evaluate) / bn_test logger.info("[2kfold:best_model:seed:accuracy] (%s, %s, %s)", ikfold, alphas[idx_best], accuracy[ikfold]) ikfold += 1 file_csv.close() logger.info("[total:data-set:avgResults] (%s, %s, %s, %s)", in_path, np.mean(accuracy), best_alphas, accuracy)