def evaluation(combinations, all_X, all_y, N, p, model='lasso'):
train_acc = []
test_acc = []
coefficients = []
for k, comb in enumerate(combinations):
num_train = math.ceil(PARAMS['n']*PARAMS['train_percent'])
accuracy = np.zeros((N, 2))
coefs = np.zeros((N, p))
for i in range(0, N):
X = all_X[k][i]
y = all_y[k][i]
X_train, X_test = X[0:num_train], X[num_train:n]
y_train, y_test = y[0:num_train], y[num_train:n]
if model == 'lasso':
accuracy[i][0], accuracy[i][1], coefs[i] = lasso(X_train, y_train, X_test, y_test)
elif model == 'dlda':
accuracy[i][0], accuracy[i][1], coefs[i,:] = dlda(X_train, y_train, X_test, y_test)
elif model == 'svm':
accuracy[i][0], accuracy[i][1], coefs[i,:] = linearsvm(X_train, y_train, X_test, y_test)
elif model == 'tc':
accuracy[i][0], accuracy[i][1], coefs[i,:] = tournament_classifier(X_train, y_train, X_test, y_test)
accuracy = np.mean(accuracy, axis = 0)
train_acc.append(accuracy[0])
test_acc.append(accuracy[1])
coefficients.append(np.mean(coefs, axis = 0))
print('======combination: alpha = {}, a = {}, b = {} ======'.format(comb['alpha'], comb['a'], comb['b']))
print("{}: train accuracy= {} and test accuracy= {}".format(model, accuracy[0], accuracy[1]))
with open(OUTPUT_DIR+'coef_{}.txt'.format(model), 'w') as file:
for i, coef in enumerate(coefficients):
file.write('alpha = {}, min value={}, max value={},'.format(combinations[i]['alpha'], combinations[i]['a'], combinations[i]['b']))
file.write(','.join(map(str, coef)))
file.write('\n')
return train_acc, test_acc
def evaluation(all_alpha, all_X, all_y, N, p, n, model='lasso'):
train_acc = []
test_acc = []
coefficients = []
for a in all_alpha:
num_train = math.ceil(n * PARAMS['train_percent'])
accuracy = np.zeros((N, 2))
coefs = np.zeros((N, p))
for i in range(0, N):
X = all_X[a][i]
y = all_y[a][i]
X_train, X_test = X[0:num_train], X[num_train:n]
y_train, y_test = y[0:num_train], y[num_train:n]
if model == 'lasso':
accuracy[i][0], accuracy[i][1], coefs[i] = lasso(
X_train, y_train, X_test, y_test)
elif model == 'dlda':
accuracy[i][0], accuracy[i][1], coefs[i, :] = dlda(
X_train, y_train, X_test, y_test)
elif model == 'svm':
accuracy[i][0], accuracy[i][1], coefs[i, :] = linearsvm(
X_train, y_train, X_test, y_test)
elif model == 'tc':
accuracy[i][0], accuracy[i][1], coefs[
i, :] = tournament_classifier(X_train, y_train, X_test,
y_test)
accuracy = np.mean(accuracy, axis=0)
train_acc.append(accuracy[0])
test_acc.append(accuracy[1])
coefficients.append(np.mean(coefs, axis=0))
print('======alpha = {} ======'.format(a))
print("{}: train accuracy= {} and test accuracy= {}".format(
model, accuracy[0], accuracy[1]))
f = plt.figure(figsize=(15, 5))
plt.plot(all_alpha, train_acc, 'bo-', label='training accuracy')
plt.plot(all_alpha, test_acc, 'ro-', label='test accuracy')
plt.xlabel('alpha, probability that an entry is 0')
plt.ylabel('accuracy')
plt.legend(loc='upper right')
plt.title('{}, n = {}, p = {} vs sparsity of covariance'.format(
model, PARAMS['n'], PARAMS['p']))
f.savefig(OUTPUT_DIR + 'plot_{}.png'.format(model))
with open(OUTPUT_DIR + 'coef_{}.csv'.format(model), 'w') as file:
for i, coef in enumerate(coefficients):
file.write('alpha={},'.format(all_alpha[i]))
file.write(','.join(map(str, coef)))
file.write('\n')
return train_acc, test_acc
def evaluation(all_n, all_X, all_y, N, p, model='lasso'):
train_acc = []
test_acc = []
coefficients = []
for n in all_n:
num_train = math.ceil(n*PARAMS['train_percent'])
accuracy = np.zeros((N, 2))
coefs = np.zeros((N, p))
for i in range(0, N):
X = all_X[n][i]
y = all_y[n][i]
X_train, X_test = X[0:num_train], X[num_train:n]
y_train, y_test = y[0:num_train], y[num_train:n]
if model == 'lasso':
accuracy[i][0], accuracy[i][1], coefs[i] = lasso(X_train, y_train, X_test, y_test)
elif model == 'dlda':
accuracy[i][0], accuracy[i][1], coefs[i,:] = dlda(X_train, y_train, X_test, y_test)
elif model == 'svm':
accuracy[i][0], accuracy[i][1], coefs[i,:] = linearsvm(X_train, y_train, X_test, y_test)
elif model == 'tc':
accuracy[i][0], accuracy[i][1], coefs[i,:] = tournament_classifier(X_train, y_train, X_test, y_test)
accuracy = np.mean(accuracy, axis = 0)
train_acc.append(accuracy[0])
test_acc.append(accuracy[1])
coefficients.append(np.mean(coefs, axis = 0))
print('======n = {} p = {} ======'.format(n, PARAMS['p']))
print("{}: train accuracy= {} and test accuracy= {}".format(model, accuracy[0], accuracy[1]))
# plot the accuracy data with respect to n for each model
f = plt.figure(figsize=(15,5))
plt.plot(all_n, train_acc, 'bo-', label='training accuracy')
plt.plot(all_n, test_acc, 'ro-', label='test accuracy')
plt.xlabel('number of observatoins')
plt.ylabel('accuracy')
plt.legend(loc='upper right')
plt.title('{}, p = {}'.format(model, PARAMS['p']))
f.savefig(OUTPUT_DIR+'plot_{}.png'.format(model))
with open(OUTPUT_DIR+'coef_np_ratio_{}.csv'.format(model), 'w') as file:
for i, coef in enumerate(coefficients):
file.write('n={},'.format(all_n[i]))
file.write(','.join(map(str, coef)))
file.write('\n')
return train_acc, test_acc