def evaluation(combinations, all_X, all_y, N, p, model='lasso'): train_acc = [] test_acc = [] coefficients = [] for k, comb in enumerate(combinations): num_train = math.ceil(PARAMS['n']*PARAMS['train_percent']) accuracy = np.zeros((N, 2)) coefs = np.zeros((N, p)) for i in range(0, N): X = all_X[k][i] y = all_y[k][i] X_train, X_test = X[0:num_train], X[num_train:n] y_train, y_test = y[0:num_train], y[num_train:n] if model == 'lasso': accuracy[i][0], accuracy[i][1], coefs[i] = lasso(X_train, y_train, X_test, y_test) elif model == 'dlda': accuracy[i][0], accuracy[i][1], coefs[i,:] = dlda(X_train, y_train, X_test, y_test) elif model == 'svm': accuracy[i][0], accuracy[i][1], coefs[i,:] = linearsvm(X_train, y_train, X_test, y_test) elif model == 'tc': accuracy[i][0], accuracy[i][1], coefs[i,:] = tournament_classifier(X_train, y_train, X_test, y_test) accuracy = np.mean(accuracy, axis = 0) train_acc.append(accuracy[0]) test_acc.append(accuracy[1]) coefficients.append(np.mean(coefs, axis = 0)) print('======combination: alpha = {}, a = {}, b = {} ======'.format(comb['alpha'], comb['a'], comb['b'])) print("{}: train accuracy= {} and test accuracy= {}".format(model, accuracy[0], accuracy[1])) with open(OUTPUT_DIR+'coef_{}.txt'.format(model), 'w') as file: for i, coef in enumerate(coefficients): file.write('alpha = {}, min value={}, max value={},'.format(combinations[i]['alpha'], combinations[i]['a'], combinations[i]['b'])) file.write(','.join(map(str, coef))) file.write('\n') return train_acc, test_acc
def evaluation(all_alpha, all_X, all_y, N, p, n, model='lasso'): train_acc = [] test_acc = [] coefficients = [] for a in all_alpha: num_train = math.ceil(n * PARAMS['train_percent']) accuracy = np.zeros((N, 2)) coefs = np.zeros((N, p)) for i in range(0, N): X = all_X[a][i] y = all_y[a][i] X_train, X_test = X[0:num_train], X[num_train:n] y_train, y_test = y[0:num_train], y[num_train:n] if model == 'lasso': accuracy[i][0], accuracy[i][1], coefs[i] = lasso( X_train, y_train, X_test, y_test) elif model == 'dlda': accuracy[i][0], accuracy[i][1], coefs[i, :] = dlda( X_train, y_train, X_test, y_test) elif model == 'svm': accuracy[i][0], accuracy[i][1], coefs[i, :] = linearsvm( X_train, y_train, X_test, y_test) elif model == 'tc': accuracy[i][0], accuracy[i][1], coefs[ i, :] = tournament_classifier(X_train, y_train, X_test, y_test) accuracy = np.mean(accuracy, axis=0) train_acc.append(accuracy[0]) test_acc.append(accuracy[1]) coefficients.append(np.mean(coefs, axis=0)) print('======alpha = {} ======'.format(a)) print("{}: train accuracy= {} and test accuracy= {}".format( model, accuracy[0], accuracy[1])) f = plt.figure(figsize=(15, 5)) plt.plot(all_alpha, train_acc, 'bo-', label='training accuracy') plt.plot(all_alpha, test_acc, 'ro-', label='test accuracy') plt.xlabel('alpha, probability that an entry is 0') plt.ylabel('accuracy') plt.legend(loc='upper right') plt.title('{}, n = {}, p = {} vs sparsity of covariance'.format( model, PARAMS['n'], PARAMS['p'])) f.savefig(OUTPUT_DIR + 'plot_{}.png'.format(model)) with open(OUTPUT_DIR + 'coef_{}.csv'.format(model), 'w') as file: for i, coef in enumerate(coefficients): file.write('alpha={},'.format(all_alpha[i])) file.write(','.join(map(str, coef))) file.write('\n') return train_acc, test_acc
def evaluation(all_n, all_X, all_y, N, p, model='lasso'): train_acc = [] test_acc = [] coefficients = [] for n in all_n: num_train = math.ceil(n*PARAMS['train_percent']) accuracy = np.zeros((N, 2)) coefs = np.zeros((N, p)) for i in range(0, N): X = all_X[n][i] y = all_y[n][i] X_train, X_test = X[0:num_train], X[num_train:n] y_train, y_test = y[0:num_train], y[num_train:n] if model == 'lasso': accuracy[i][0], accuracy[i][1], coefs[i] = lasso(X_train, y_train, X_test, y_test) elif model == 'dlda': accuracy[i][0], accuracy[i][1], coefs[i,:] = dlda(X_train, y_train, X_test, y_test) elif model == 'svm': accuracy[i][0], accuracy[i][1], coefs[i,:] = linearsvm(X_train, y_train, X_test, y_test) elif model == 'tc': accuracy[i][0], accuracy[i][1], coefs[i,:] = tournament_classifier(X_train, y_train, X_test, y_test) accuracy = np.mean(accuracy, axis = 0) train_acc.append(accuracy[0]) test_acc.append(accuracy[1]) coefficients.append(np.mean(coefs, axis = 0)) print('======n = {} p = {} ======'.format(n, PARAMS['p'])) print("{}: train accuracy= {} and test accuracy= {}".format(model, accuracy[0], accuracy[1])) # plot the accuracy data with respect to n for each model f = plt.figure(figsize=(15,5)) plt.plot(all_n, train_acc, 'bo-', label='training accuracy') plt.plot(all_n, test_acc, 'ro-', label='test accuracy') plt.xlabel('number of observatoins') plt.ylabel('accuracy') plt.legend(loc='upper right') plt.title('{}, p = {}'.format(model, PARAMS['p'])) f.savefig(OUTPUT_DIR+'plot_{}.png'.format(model)) with open(OUTPUT_DIR+'coef_np_ratio_{}.csv'.format(model), 'w') as file: for i, coef in enumerate(coefficients): file.write('n={},'.format(all_n[i])) file.write(','.join(map(str, coef))) file.write('\n') return train_acc, test_acc